├── manual ├── examples │ ├── tlp-stress-help.txt │ ├── tlp-stress-keyvalue.txt │ ├── info-key-value.txt │ ├── list-all.txt │ ├── easy-cass-stress-keyvalue.txt │ └── easy-cass-stress-help.txt ├── generate_examples.sh └── MANUAL.adoc ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── src ├── main │ ├── kotlin │ │ └── com │ │ │ └── rustyrazorblade │ │ │ └── easycassstress │ │ │ ├── Workload.kt │ │ │ ├── WorkloadParameter.kt │ │ │ ├── OperationStopException.kt │ │ │ ├── commands │ │ │ ├── IStressCommand.kt │ │ │ ├── Fields.kt │ │ │ ├── ListCommand.kt │ │ │ └── Info.kt │ │ │ ├── generators │ │ │ ├── Function.kt │ │ │ ├── FieldGenerator.kt │ │ │ ├── functions │ │ │ │ ├── Gaussian.kt │ │ │ │ ├── LastName.kt │ │ │ │ ├── FirstName.kt │ │ │ │ ├── USCities.kt │ │ │ │ ├── Random.kt │ │ │ │ └── Book.kt │ │ │ ├── ParsedFieldFunction.kt │ │ │ ├── FunctionLoader.kt │ │ │ └── Registry.kt │ │ │ ├── PopulateOption.kt │ │ │ ├── RowState.kt │ │ │ ├── PartitionKey.kt │ │ │ ├── DDLStatement.kt │ │ │ ├── converters │ │ │ ├── ConsistencyLevelConverter.kt │ │ │ ├── HumanReadableConverter.kt │ │ │ └── HumanReadableTimeConverter.kt │ │ │ ├── CoordinatorHostPredicate.kt │ │ │ ├── Util.kt │ │ │ ├── StressContext.kt │ │ │ ├── Main.kt │ │ │ ├── CommandLineParser.kt │ │ │ ├── profiles │ │ │ ├── Maps.kt │ │ │ ├── KeyValue.kt │ │ │ ├── Sets.kt │ │ │ ├── RangeScan.kt │ │ │ ├── CountersWide.kt │ │ │ ├── LWT.kt │ │ │ ├── AllowFiltering.kt │ │ │ ├── UdtTimeSeries.kt │ │ │ ├── MaterializedViews.kt │ │ │ ├── Locking.kt │ │ │ ├── BasicTimeSeries.kt │ │ │ ├── DSESearch.kt │ │ │ ├── IStressProfile.kt │ │ │ ├── RandomPartitionAccess.kt │ │ │ └── SAI.kt │ │ │ ├── Metrics.kt │ │ │ ├── OperationCallback.kt │ │ │ ├── PartitionKeyGenerator.kt │ │ │ ├── FileReporter.kt │ │ │ ├── Plugin.kt │ │ │ ├── RequestQueue.kt │ │ │ ├── SingleLineConsoleReporter.kt │ │ │ ├── ProfileRunner.kt │ │ │ ├── RateLimiterOptimizer.kt │ │ │ └── SchemaBuilder.kt │ └── resources │ │ └── log4j2.yaml └── test │ ├── kotlin │ └── com │ │ └── rustyrazorblade │ │ └── easycassstress │ │ ├── integration │ │ ├── FieldsTest.kt │ │ ├── FlagsTest.kt │ │ └── AllPluginsBasicTest.kt │ │ ├── generators │ │ ├── FirstNameTest.kt │ │ ├── USCitiesTest.kt │ │ ├── LastNameTest.kt │ │ ├── FunctionLoaderTest.kt │ │ ├── BookTest.kt │ │ ├── ParsedFieldFunctionTest.kt │ │ └── RegistryTest.kt │ │ ├── MainArgumentsTest.kt │ │ ├── CommandLineParserTest.kt │ │ ├── converters │ │ ├── HumanReadableConverterTest.kt │ │ ├── ConsistencyLevelConverterTest.kt │ │ └── HumanReadableTimeConverterTest.kt │ │ ├── PartitionKeyGeneratorTest.kt │ │ ├── RateLimiterOptimizerTest.kt │ │ ├── PluginTest.kt │ │ └── SchemaBuilderTest.kt │ └── resources │ └── log4j2-test.yaml ├── bin └── easy-cass-stress ├── .gitignore ├── docker-compose.yml ├── LICENSE.txt ├── .github └── workflows │ ├── gradle-publish-main-release.yml │ └── gradle-publish-disttar.yml ├── README.md ├── gradlew.bat ├── .circleci └── config.yml └── gradlew /manual/examples/tlp-stress-help.txt: -------------------------------------------------------------------------------- 1 | $ bin/easy-cass-stress 2 | -------------------------------------------------------------------------------- /manual/examples/tlp-stress-keyvalue.txt: -------------------------------------------------------------------------------- 1 | $ bin/easy-cass-stress run KeyValue -n 10000 2 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/george0st/easy-cass-stress/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/Workload.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | class Workload { 4 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/WorkloadParameter.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | annotation class WorkloadParameter(val description: String) -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/OperationStopException.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | class OperationStopException : Throwable() { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/commands/IStressCommand.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.commands 2 | 3 | interface IStressCommand { 4 | fun execute() 5 | } -------------------------------------------------------------------------------- /bin/easy-cass-stress: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | dir=$(dirname $0) 4 | BASE_STRESS_DIR="$(dirname "$dir")" 5 | cd $BASE_STRESS_DIR 6 | 7 | JAR=$(find build/libs -name '*-all.jar' | tail -n 1) 8 | 9 | java -jar $JAR "$@" 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.2-all.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/generators/Function.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators 2 | 3 | // all functions should be tagged 4 | annotation class Function(val name: String, 5 | val description : String) -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/PopulateOption.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | sealed class PopulateOption { 4 | class Standard : PopulateOption() 5 | class Custom(val rows: Long, val deletes: Boolean = true) : PopulateOption() 6 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/RowState.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | /** 4 | * The RowState is used to record the relevant state of a row at a given moment in time 5 | * This is designed to be used in the validation stage 6 | */ 7 | class RowState { 8 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/PartitionKey.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | /** 4 | * Will replace the current requirement that a PK can only be a text field 5 | */ 6 | class PartitionKey(val prefix: String, val id: Long) { 7 | 8 | fun getText(): String { 9 | return prefix + id.toString() 10 | } 11 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/DDLStatement.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | sealed class DDLStatement { 4 | class CreateTable() : DDLStatement() { 5 | 6 | } 7 | 8 | class Unknown : DDLStatement() 9 | 10 | companion object { 11 | fun parse(cql: String) : DDLStatement { 12 | return DDLStatement.Unknown() 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /manual/examples/info-key-value.txt: -------------------------------------------------------------------------------- 1 | $ bin/easy-cass-stress info KeyValue 2 | WARNING: sun.reflect.Reflection.getCallerClass is not supported. This will impact performance. 3 | CREATE TABLE IF NOT EXISTS keyvalue ( 4 | key text PRIMARY KEY, 5 | value text 6 | ) 7 | Default read rate: 0.5 (override with -r) 8 | 9 | No dynamic workload parameters. 10 | -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/integration/FieldsTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.integration 2 | 3 | import com.rustyrazorblade.easycassstress.commands.Fields 4 | import org.junit.jupiter.api.Test 5 | 6 | class FieldsTest { 7 | 8 | @Test 9 | fun testExecution() { 10 | // this is just to ensure things don't break 11 | Fields().execute() 12 | } 13 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/* 2 | *.iml 3 | *.pyc 4 | .DS_Store 5 | .vagrant.v1* 6 | .tags* 7 | /.gradle/ 8 | /out/ 9 | /build/ 10 | #gradle/ 11 | #gradlew.bat 12 | #settings.gradle 13 | *.log 14 | conf/logback.xml 15 | settings.gradle 16 | *.deb 17 | /metrics* 18 | /logs/ 19 | /*.csv 20 | easy-cass-stress.ipr 21 | easy-cass-stress.iws 22 | /*.rpm 23 | buildSrc/.gradle/ 24 | buildSrc/build/ 25 | .jira-url 26 | *.ipr 27 | *.iws 28 | .sdkmanrc 29 | -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/generators/FirstNameTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators 2 | 3 | import com.rustyrazorblade.easycassstress.generators.functions.FirstName 4 | import org.junit.jupiter.api.Test 5 | 6 | internal class FirstNameTest { 7 | @Test 8 | fun getNameTest() { 9 | val tmp = FirstName() 10 | val n = tmp.getText() 11 | println(n) 12 | 13 | } 14 | } -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/generators/USCitiesTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators 2 | 3 | import com.rustyrazorblade.easycassstress.generators.functions.USCities 4 | import org.junit.jupiter.api.Test 5 | 6 | internal class USCitiesTest { 7 | 8 | @Test 9 | fun getText() { 10 | val cities = USCities() 11 | for(i in 0..100000) 12 | cities.getText() 13 | } 14 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/converters/ConsistencyLevelConverter.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.converters 2 | 3 | import com.beust.jcommander.IStringConverter 4 | import com.datastax.driver.core.ConsistencyLevel 5 | 6 | class ConsistencyLevelConverter : IStringConverter { 7 | override fun convert(value: String?): ConsistencyLevel { 8 | return ConsistencyLevel.valueOf(value!!) 9 | } 10 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/CoordinatorHostPredicate.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import com.datastax.driver.core.Host 4 | import com.google.common.base.Predicate 5 | 6 | class CoordinatorHostPredicate : Predicate { 7 | override fun apply(input: Host?): Boolean { 8 | if(input == null) 9 | return false 10 | return input.tokens == null || input.tokens.size == 0 11 | } 12 | } 13 | 14 | -------------------------------------------------------------------------------- /manual/examples/list-all.txt: -------------------------------------------------------------------------------- 1 | $ bin/easy-cass-stress list 2 | WARNING: sun.reflect.Reflection.getCallerClass is not supported. This will impact performance. 3 | Available Workloads: 4 | 5 | AllowFiltering 6 | BasicTimeSeries 7 | CountersWide 8 | DSESearch 9 | KeyValue 10 | LWT 11 | Locking 12 | Maps 13 | MaterializedViews 14 | RandomPartitionAccess 15 | SAI 16 | Sets 17 | UdtTimeSeries 18 | 19 | You can run any of these workloads by running easy-cass-stress run WORKLOAD. 20 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | pandoc: 5 | image: jagregory/pandoc 6 | volumes: 7 | - ./:/source 8 | command: -s --toc manual/MANUAL.md -o docs/index.html 9 | 10 | docs: 11 | image: asciidoctor/docker-asciidoctor 12 | 13 | volumes: 14 | - ./manual:/documents 15 | - ./docs:/html 16 | 17 | command: asciidoctor -o /html/index.html MANUAL.adoc -a EASY_CASS_STRESS_VERSION=${EASY_CASS_STRESS_VERSION} 18 | 19 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/generators/FieldGenerator.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators 2 | 3 | class UnsupportedTypeException : Exception() 4 | 5 | interface FieldGenerator { 6 | fun getInt() : Int = throw UnsupportedTypeException() 7 | fun getFloat() : Float = throw UnsupportedTypeException() 8 | fun getText() : String = throw UnsupportedTypeException() 9 | 10 | fun getDescription() : String 11 | 12 | fun setParameters(params: List) 13 | } -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/generators/LastNameTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators 2 | 3 | import com.rustyrazorblade.easycassstress.generators.functions.LastName 4 | import org.apache.logging.log4j.kotlin.logger 5 | import org.junit.jupiter.api.Test 6 | 7 | internal class LastNameTest { 8 | val log = logger() 9 | 10 | @Test 11 | fun getNameTest() { 12 | val tmp = LastName() 13 | val n = tmp.getText() 14 | log.info { n } 15 | } 16 | } -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/MainArgumentsTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import com.rustyrazorblade.easycassstress.commands.Run 4 | import org.assertj.core.api.Assertions.assertThat 5 | import org.junit.jupiter.api.Test 6 | 7 | internal class MainArgumentsTest { 8 | @Test 9 | fun testPagingFlagWorks() { 10 | val run = Run("placeholder") 11 | val pageSize = 20000 12 | run.paging = pageSize 13 | assertThat(run.options.fetchSize).isEqualTo(pageSize) 14 | } 15 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/commands/Fields.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.commands 2 | 3 | import com.rustyrazorblade.easycassstress.generators.Registry 4 | 5 | 6 | class Fields : IStressCommand { 7 | override fun execute() { 8 | // show each generator 9 | val registry = Registry.create() 10 | 11 | for(func in registry.getFunctions()) { 12 | println("Generator: ${func.name}") 13 | 14 | print("Description:") 15 | println(func.description) 16 | 17 | } 18 | 19 | } 20 | } -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2019, The Last Pickle 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/generators/FunctionLoaderTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators 2 | 3 | import org.junit.jupiter.api.Assertions.* 4 | 5 | import org.assertj.core.api.Assertions.assertThat 6 | import org.junit.jupiter.api.Test 7 | 8 | internal class FunctionLoaderTest { 9 | 10 | @Test 11 | operator fun iterator() { 12 | } 13 | 14 | @Test 15 | fun getMap() { 16 | } 17 | 18 | @Test 19 | fun getInstance() { 20 | val function = FunctionLoader() 21 | } 22 | 23 | @Test 24 | fun getInstance1() { 25 | } 26 | } -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/generators/BookTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators 2 | 3 | import com.rustyrazorblade.easycassstress.generators.functions.Book 4 | import org.junit.jupiter.api.Test 5 | import org.assertj.core.api.Assertions.assertThat 6 | 7 | internal class BookTest { 8 | @Test 9 | fun bookSliceTest() { 10 | val b = Book() 11 | var previous = "" 12 | for(i in 1..10) { 13 | val tmp = b.getText() 14 | assertThat(tmp).isNotBlank().isNotEqualToIgnoringCase(previous) 15 | previous = tmp 16 | } 17 | 18 | 19 | } 20 | 21 | 22 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/Util.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import org.apache.commons.text.RandomStringGenerator 4 | import java.text.DecimalFormat 5 | 6 | /* 7 | Will be used later when adding configuration 8 | */ 9 | typealias Range = Pair 10 | 11 | 12 | /* 13 | Throwaway - will need to be thought out for 14 | */ 15 | fun randomString(length: Int) : String { 16 | val generator = RandomStringGenerator.Builder().withinRange(65, 90).build() 17 | return generator.generate(length) 18 | } 19 | 20 | fun round(num: Double) : Double { 21 | return DecimalFormat("##.##").format(num).toDouble() 22 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/commands/ListCommand.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.commands 2 | 3 | import com.beust.jcommander.Parameters 4 | import com.rustyrazorblade.easycassstress.Plugin 5 | 6 | @Parameters(commandDescription = "List all workloads.") 7 | class ListCommand : IStressCommand { 8 | override fun execute() { 9 | 10 | println("Available Workloads:\n") 11 | 12 | val plugins = Plugin.getPlugins() 13 | for((key, _) in plugins) { 14 | println("$key ") 15 | } 16 | println("\nYou can run any of these workloads by running easy-cass-stress run WORKLOAD.") 17 | 18 | } 19 | 20 | 21 | } -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/CommandLineParserTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import com.rustyrazorblade.easycassstress.commands.Run 4 | import org.assertj.core.api.Assertions.assertThat 5 | import org.junit.jupiter.api.Assertions.assertTrue 6 | import org.junit.jupiter.api.Test 7 | 8 | internal class CommandLineParserTest { 9 | @Test 10 | fun testBasicParser() { 11 | val args = arrayOf("run", "BasicTimeSeries") 12 | val result = com.rustyrazorblade.easycassstress.CommandLineParser.parse(args) 13 | assertThat(result.getParsedCommand()).isEqualToIgnoringCase("run") 14 | assertThat(result.getCommandInstance()).isInstanceOf(Run::class.java) 15 | } 16 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/StressContext.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import com.datastax.driver.core.ConsistencyLevel 4 | import com.datastax.driver.core.Session 5 | import com.google.common.util.concurrent.RateLimiter 6 | import com.rustyrazorblade.easycassstress.commands.Run 7 | import com.rustyrazorblade.easycassstress.generators.Registry 8 | import java.util.concurrent.Semaphore 9 | 10 | data class StressContext(val session: Session, 11 | val mainArguments: Run, 12 | val thread: Int, 13 | val metrics: Metrics, 14 | val registry: Registry, 15 | val rateLimiter: RateLimiter?) 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/Main.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import org.apache.logging.log4j.kotlin.logger 4 | 5 | 6 | fun main(argv: Array) { 7 | 8 | val log = logger("main") 9 | 10 | log.info { "Parsing $argv" } 11 | val parser = CommandLineParser.parse(argv) 12 | 13 | try { 14 | parser.execute() 15 | } catch (e: Exception) { 16 | log.error { "Crashed with error: " + e.message } 17 | println(e.message) 18 | e.printStackTrace() 19 | } finally { 20 | // we exit here to kill the console thread otherwise it waits forever. 21 | // I'm sure a reasonable fix exists, but I don't have time to look into it. 22 | System.exit(0) 23 | } 24 | 25 | } 26 | 27 | -------------------------------------------------------------------------------- /src/test/resources/log4j2-test.yaml: -------------------------------------------------------------------------------- 1 | Configuration: 2 | name: TestConfiguration 3 | 4 | appenders: 5 | Console: 6 | name: STDOUT 7 | PatternLayout: 8 | Pattern: "%d %p %C{1.} [%t] %m%n" 9 | 10 | File: 11 | - name: Debug 12 | fileName: logs/test-debug.log 13 | PatternLayout: 14 | Pattern: "%d %p %C{1.} [%t] %m%n" 15 | 16 | - name: Info 17 | fileName: logs/test-info.log 18 | PatternLayout: 19 | Pattern: "%d %p %C{1.} [%t] %m%n" 20 | 21 | 22 | Loggers: 23 | Root: 24 | level: debug 25 | AppenderRef: 26 | - ref: Debug 27 | 28 | 29 | logger: 30 | - 31 | name: com.rustyrazorblade.easycassstress 32 | level: debug 33 | AppenderRef: 34 | - ref: STDOUT 35 | - ref: Info 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/converters/HumanReadableConverterTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.converters 2 | 3 | import org.assertj.core.api.Assertions.assertThat 4 | import org.junit.jupiter.api.BeforeEach 5 | import org.junit.jupiter.api.Test 6 | 7 | import org.junit.jupiter.api.Assertions.* 8 | 9 | internal class HumanReadableConverterTest { 10 | 11 | lateinit var converter: HumanReadableConverter 12 | 13 | @BeforeEach 14 | fun setUp() { 15 | converter = HumanReadableConverter() 16 | } 17 | 18 | @Test 19 | fun convert() { 20 | assertThat(converter.convert("5k")).isEqualTo(5000L) 21 | assertThat(converter.convert("500")).isEqualTo(500L) 22 | assertThat(converter.convert("5m")).isEqualTo(5000000L) 23 | assertThat(converter.convert("5b")).isEqualTo(5000000000L) 24 | } 25 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/converters/HumanReadableConverter.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.converters 2 | 3 | import com.beust.jcommander.IStringConverter 4 | 5 | class HumanReadableConverter : IStringConverter { 6 | override fun convert(value: String?): Long { 7 | val regex = """(\d+)([BbMmKk]?)""".toRegex() 8 | val result = regex.find(value!!) 9 | 10 | return result?.groups?.let { 11 | val value = it[1]?.value 12 | val label = it[2] 13 | 14 | if(value == null) return 0L 15 | 16 | when(label?.value?.toLowerCase()) { 17 | "k" -> 1000L * value.toLong() 18 | "m" -> 1000000L * value.toLong() 19 | "b" -> 1000000000L * value.toLong() 20 | else -> value.toLong() 21 | } 22 | } ?: 0L 23 | } 24 | } -------------------------------------------------------------------------------- /manual/generate_examples.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # requires a running ccm cluster (otherwise certain tests will fail) 4 | 5 | set -x 6 | 7 | print_shell() { 8 | # params 9 | # $1 = name 10 | # #2 = command 11 | echo "running $1" 12 | 13 | printf "$ %s\n" "$2" > manual/examples/"${1}.txt" 14 | eval $2 >> manual/examples/"${1}.txt" 15 | echo "Sleeping" 16 | sleep 5 17 | } 18 | 19 | # help 20 | print_shell "easy-cass-stress-help" "bin/easy-cass-stress" 21 | 22 | # key value 23 | print_shell "easy-cass-stress-keyvalue" "bin/easy-cass-stress run KeyValue -n 10000" 24 | 25 | # info 26 | print_shell "info-key-value" "bin/easy-cass-stress info KeyValue" 27 | 28 | 29 | # list all workloads 30 | print_shell "list-all" "bin/easy-cass-stress list" 31 | 32 | print_shell "field-example-book" 'bin/easy-cass-stress run KeyValue --field.keyvalue.value="book(20,40)"' 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /src/main/resources/log4j2.yaml: -------------------------------------------------------------------------------- 1 | Configuration: 2 | 3 | Properties: 4 | property: 5 | - name: logs 6 | value: "${env:EASY_CASS_STRESS_LOG_DIR:-${sys:user.home}/.easy-cass-stress}" 7 | 8 | status: info 9 | name: StandardConfiguration 10 | 11 | thresholdFilter: 12 | level: info 13 | 14 | 15 | 16 | appenders: 17 | RollingFile: 18 | - name: File 19 | fileName: "${logs}/stress.log" 20 | 21 | filePattern: "${logs}/stress.%i.log.gz" 22 | 23 | PatternLayout: 24 | Pattern: "%d %p %C{1.} [%t] %m%n" 25 | 26 | policies: 27 | SizeBasedTriggeringPolicy: 28 | size: 10MB 29 | OnStartupTriggeringPolicy: {} 30 | 31 | Filters: 32 | ThresholdFilter: 33 | level: info 34 | 35 | 36 | Loggers: 37 | Root: 38 | level: info 39 | AppenderRef: 40 | ref: File 41 | 42 | -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/generators/ParsedFieldFunctionTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators 2 | 3 | 4 | import org.assertj.core.api.Assertions.assertThat 5 | import org.junit.jupiter.api.Test 6 | 7 | internal class ParsedFieldFunctionTest { 8 | @Test 9 | fun noArgumentParseTest() { 10 | val parsed = ParsedFieldFunction("book()") 11 | assertThat(parsed.name).isEqualTo("book") 12 | assertThat(parsed.args).hasSize(0) 13 | } 14 | 15 | @Test 16 | fun singleArgumentTest() { 17 | val parsed = ParsedFieldFunction("random(10)") 18 | assertThat(parsed.name).isEqualTo("random") 19 | assertThat(parsed.args).hasSize(1) 20 | assertThat(parsed.args[0]).isEqualTo("10") 21 | } 22 | 23 | @Test 24 | fun emptyFieldArgumentParseTest() { 25 | val args = ParsedFieldFunction.parseArguments("") 26 | assertThat(args).hasSize(0) 27 | } 28 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/generators/functions/Gaussian.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators.functions 2 | import com.rustyrazorblade.easycassstress.converters.HumanReadableConverter 3 | import com.rustyrazorblade.easycassstress.generators.Function 4 | import com.rustyrazorblade.easycassstress.generators.FieldGenerator 5 | 6 | 7 | @Function(name="gaussian", 8 | description = "Gaussian (normal) numerical data distribution") 9 | class Gaussian : FieldGenerator { 10 | var min: Long = 0 11 | var max: Long = 1000000 12 | 13 | override fun setParameters(params: List) { 14 | min = HumanReadableConverter().convert(params[0]) 15 | max = HumanReadableConverter().convert(params[1]) 16 | } 17 | 18 | 19 | override fun getDescription() = """ 20 | Generates numbers following a gaussian (normal) distribution. This is useful for simulating certain workloads which use certain values more than others. 21 | """.trimIndent() 22 | 23 | 24 | } -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/integration/FlagsTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.integration 2 | 3 | import com.datastax.driver.core.Cluster 4 | import com.rustyrazorblade.easycassstress.commands.Run 5 | import org.junit.jupiter.api.BeforeEach 6 | import org.junit.jupiter.api.Test 7 | 8 | /** 9 | * Simple tests for various flags that don't required dedicated testing 10 | */ 11 | class FlagsTest { 12 | val ip = System.getenv("EASY_CASS_STRESS_CASSANDRA_IP") ?: "127.0.0.1" 13 | 14 | val connection = Cluster.builder() 15 | .addContactPoint(ip) 16 | .build().connect() 17 | 18 | var keyvalue = Run("placeholder") 19 | 20 | 21 | @BeforeEach 22 | fun resetRunners() { 23 | keyvalue = keyvalue.apply { 24 | profile = "KeyValue" 25 | iterations = 100 26 | } 27 | } 28 | 29 | 30 | @Test 31 | fun csvTest() { 32 | keyvalue.apply { 33 | csvFile = "test.csv" 34 | }.execute() 35 | 36 | } 37 | } -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/generators/RegistryTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators 2 | 3 | import com.rustyrazorblade.easycassstress.generators.functions.Random 4 | import com.rustyrazorblade.easycassstress.generators.functions.USCities 5 | import org.junit.jupiter.api.* 6 | import org.assertj.core.api.Assertions.assertThat 7 | 8 | internal class RegistryTest { 9 | 10 | lateinit var registry: Registry 11 | 12 | @BeforeEach 13 | fun setUp() { 14 | registry = Registry.create() 15 | .setDefault("test", "city", USCities()) 16 | .setDefault("test", "age", Random().apply{ min=10; max=100 }) 17 | } 18 | 19 | 20 | @Test 21 | fun getOverriddenTypeTest() { 22 | assertThat(registry.getGenerator("test", "city")).isInstanceOf(USCities::class.java) 23 | registry.setOverride("test", "city", Random().apply{ min=10; max=100 }) 24 | 25 | assertThat(registry.getGenerator("test", "city")).isInstanceOf(Random::class.java) 26 | } 27 | } -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/converters/ConsistencyLevelConverterTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.converters 2 | 3 | import com.datastax.driver.core.ConsistencyLevel 4 | import org.assertj.core.api.Assertions.assertThat 5 | import org.junit.jupiter.api.BeforeEach 6 | import org.junit.jupiter.api.Test 7 | 8 | import org.junit.jupiter.api.Assertions.* 9 | 10 | import kotlin.test.assertFailsWith 11 | 12 | internal class ConsistencyLevelConverterTest { 13 | 14 | lateinit var converter: ConsistencyLevelConverter 15 | 16 | @BeforeEach 17 | fun setUp() { 18 | converter = ConsistencyLevelConverter() 19 | } 20 | 21 | @Test 22 | fun convert() { 23 | assertThat(converter.convert("LOCAL_ONE")).isEqualTo(ConsistencyLevel.LOCAL_ONE) 24 | assertThat(converter.convert("LOCAL_QUORUM")).isEqualTo(ConsistencyLevel.LOCAL_QUORUM) 25 | } 26 | 27 | @Test 28 | fun convertAndFail() { 29 | assertFailsWith {val cl = converter.convert("LOCAL")} 30 | } 31 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/generators/ParsedFieldFunction.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators 2 | 3 | import org.apache.logging.log4j.kotlin.logger 4 | 5 | /** 6 | * Helper class that parses the field function spec 7 | * Example: book(), random(10), random(10, 20) 8 | */ 9 | class ParsedFieldFunction(function: String) { 10 | 11 | val name : String 12 | val args: List 13 | 14 | companion object { 15 | val regex = """(^[a-z]+)\((.*)\)""".toRegex() 16 | val log = logger() 17 | 18 | internal fun parseArguments(s: String) : List { 19 | log.debug { "Parsing field arguments $s" } 20 | if(s.trim().isBlank()) return listOf() 21 | return s.split(",").map { it.trim() } 22 | } 23 | } 24 | 25 | init { 26 | val searchResult = regex.find(function)?.groupValues ?: throw Exception("Could not parse $function as a field function") 27 | 28 | name = searchResult[1] 29 | 30 | args = parseArguments(searchResult[2]) 31 | } 32 | 33 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/generators/functions/LastName.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators.functions 2 | 3 | import com.rustyrazorblade.easycassstress.generators.FieldGenerator 4 | import java.util.concurrent.ThreadLocalRandom 5 | import com.rustyrazorblade.easycassstress.generators.Function 6 | 7 | @Function(name="lastname", 8 | description = "Last names.") 9 | class LastName : FieldGenerator { 10 | val names = mutableListOf() 11 | 12 | init { 13 | val tmp = this::class.java.getResource("/names/last.txt") 14 | .readText() 15 | .split("\n") 16 | .map { it.split(" ").first() } 17 | names.addAll(tmp) 18 | } 19 | 20 | override fun setParameters(params: List) { 21 | 22 | } 23 | 24 | override fun getText(): String { 25 | val element = ThreadLocalRandom.current().nextInt(0, names.size) 26 | return names[element] 27 | } 28 | 29 | override fun getDescription() = """ 30 | Supplies common last names. 31 | """.trimIndent() 32 | 33 | } -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/PartitionKeyGeneratorTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import org.assertj.core.api.Assertions.assertThat 4 | import org.junit.jupiter.api.Test 5 | 6 | internal class PartitionKeyGeneratorTest { 7 | @Test 8 | fun basicKeyGenerationTest() { 9 | val p = PartitionKeyGenerator.random("test") 10 | val tmp = p.generateKey(1000000) 11 | val pk = tmp.take(1).toList()[0] 12 | assertThat(pk.getText()).contains("test") 13 | } 14 | 15 | @Test 16 | fun sequenceTest() { 17 | val p = PartitionKeyGenerator.sequence("test") 18 | val result = p.generateKey(10, 10).first() 19 | assertThat(result.id).isEqualTo(0L) 20 | 21 | 22 | } 23 | 24 | @Test 25 | fun testRepeatingSequence() { 26 | val p = PartitionKeyGenerator.sequence("test") 27 | val data = p.generateKey(10,2).take(5).toList().map { it.id.toInt() } 28 | assertThat(data).isEqualTo(listOf(0,1,2,0,1)) 29 | 30 | } 31 | 32 | @Test 33 | fun testNormal() { 34 | val p = PartitionKeyGenerator.normal("test") 35 | for (x in p.generateKey(1000, 1000)) { 36 | 37 | } 38 | 39 | } 40 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/generators/functions/FirstName.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators.functions 2 | 3 | import com.rustyrazorblade.easycassstress.generators.FieldGenerator 4 | import com.rustyrazorblade.easycassstress.generators.Function 5 | import java.util.concurrent.ThreadLocalRandom 6 | 7 | @Function(name="firstname", 8 | description = "First names.") 9 | class FirstName : FieldGenerator { 10 | 11 | override fun setParameters(params: List) { 12 | // nothing to do here 13 | } 14 | 15 | override fun getDescription() = """ 16 | Uses common first names, both male and female. 17 | """.trimIndent() 18 | 19 | 20 | val names = mutableListOf() 21 | 22 | init { 23 | 24 | for (s in arrayListOf("female", "male")) { 25 | val tmp = this::class.java.getResource("/names/female.txt") 26 | .readText() 27 | .split("\n") 28 | .map { it.split(" ").first() } 29 | names.addAll(tmp) 30 | } 31 | } 32 | 33 | override fun getText(): String { 34 | val element = ThreadLocalRandom.current().nextInt(0, names.size) 35 | return names[element] 36 | } 37 | 38 | } -------------------------------------------------------------------------------- /.github/workflows/gradle-publish-main-release.yml: -------------------------------------------------------------------------------- 1 | name: Release Workflow 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | build-and-release: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - name: Checkout code 14 | uses: actions/checkout@v4 15 | 16 | - name: Set up JDK 11 17 | uses: actions/setup-java@v4 18 | with: 19 | java-version: '11' 20 | distribution: 'temurin' 21 | 22 | - name: Build with Gradle 23 | run: ./gradlew distZip 24 | 25 | - name: Create Release 26 | id: create_release 27 | uses: actions/create-release@v1 28 | env: 29 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 30 | with: 31 | tag_name: ${{ github.ref }} 32 | release_name: Release ${{ github.ref }} 33 | draft: false 34 | prerelease: false 35 | 36 | - name: Upload Release Artifact 37 | uses: actions/upload-release-asset@v1 38 | env: 39 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 40 | with: 41 | upload_url: ${{ steps.create_release.outputs.upload_url }} 42 | asset_path: ./build/distributions/easy-cass-stress-6.0.0.zip 43 | asset_name: easy-cass-stress-6.0.0.zip 44 | asset_content_type: application/gzip 45 | -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/converters/HumanReadableTimeConverterTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.converters 2 | 3 | import com.datastax.driver.core.ConsistencyLevel 4 | import org.assertj.core.api.Assertions.assertThat 5 | import org.junit.jupiter.api.BeforeEach 6 | import org.junit.jupiter.api.Test 7 | 8 | import org.junit.jupiter.api.Assertions.* 9 | 10 | import kotlin.test.assertFailsWith 11 | 12 | internal class HumanReadableTimeConverterTest { 13 | 14 | lateinit var converter: HumanReadableTimeConverter 15 | 16 | @BeforeEach 17 | fun setUp() { 18 | converter = HumanReadableTimeConverter() 19 | } 20 | 21 | @Test 22 | fun convert() { 23 | assertThat(converter.convert("15m")).isEqualTo(15) 24 | assertThat(converter.convert("1h")).isEqualTo(60) 25 | assertThat(converter.convert("3h")).isEqualTo(180) 26 | assertThat(converter.convert("1d 1h")).isEqualTo(1500) 27 | assertThat(converter.convert("1h 5m")).isEqualTo(65) 28 | assertThat(converter.convert("3m 120s")).isEqualTo(5) 29 | assertThat(converter.convert("10m 1d 59s 2h")).isEqualTo(1570) 30 | assertThat(converter.convert("1d2h3m")).isEqualTo(1563) 31 | } 32 | 33 | @Test 34 | fun convertAndFail() { 35 | assertFailsWith {val cl = converter.convert("BLAh")} 36 | } 37 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/generators/functions/USCities.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators.functions 2 | 3 | import com.rustyrazorblade.easycassstress.generators.FieldGenerator 4 | import com.rustyrazorblade.easycassstress.generators.Function 5 | import java.util.concurrent.ThreadLocalRandom 6 | import kotlin.streams.toList 7 | 8 | data class City (val name:String, val stateShort:String, val stateFull:String, val cityAlias:String) { 9 | override fun toString() : String { 10 | return "$name, $stateShort" 11 | } 12 | } 13 | 14 | @Function(name="uscities", 15 | description = "US Cities") 16 | class USCities : FieldGenerator { 17 | private val cities : List 18 | private val size : Int 19 | init { 20 | 21 | val reader = this.javaClass.getResourceAsStream("/us_cities_states_counties.csv").bufferedReader() 22 | cities = reader.lines().skip(1).map { it.split("|") }.filter { it.size > 4 }.map { City(it[0], it[1], it[2], it[3]) }.toList() 23 | size = cities.count() 24 | 25 | } 26 | 27 | override fun setParameters(params: List) { 28 | 29 | } 30 | 31 | override fun getText(): String { 32 | 33 | val tmp = ThreadLocalRandom.current().nextInt(0, size) 34 | return cities[tmp].toString() 35 | } 36 | 37 | override fun getDescription() = """ 38 | Random US cities. 39 | """.trimIndent() 40 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/converters/HumanReadableTimeConverter.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.converters 2 | 3 | import com.beust.jcommander.IStringConverter 4 | import java.time.Duration 5 | 6 | 7 | class HumanReadableTimeConverter : IStringConverter { 8 | override fun convert(value: String?): Long { 9 | var duration = Duration.ofMinutes(0) 10 | 11 | val valueCharSequence = value!!.subSequence(0, value.length) 12 | /** 13 | * The duration is passed in via the value variable. It could contain multiple time values e.g. "1d 2h 3m 4s". 14 | * Parse the string using the following process: 15 | * 1. Find all occurrences of of an integer with a time unit and iterate through the matches. Note we need 16 | * to convert the value from a String to CharSequence so we can pass it to findAll. 17 | * 2. Iterate through the matched values. Add to the duration based on the units of each value. 18 | */ 19 | Regex("(?\\d+)(?[dhms])") 20 | .findAll(valueCharSequence) 21 | .forEach { 22 | val quantity = it.groups["num"]!!.value.toLong() 23 | when (it.groups["str"]!!.value) { 24 | "d" -> duration = duration.plusDays(quantity) 25 | "h" -> duration = duration.plusHours(quantity) 26 | "m" -> duration = duration.plusMinutes(quantity) 27 | "s" -> duration = duration.plusSeconds(quantity) 28 | } 29 | } 30 | 31 | if (duration.isZero) 32 | throw IllegalArgumentException("Value ${value} resulted in 0 time duration") 33 | 34 | return duration.toMinutes() 35 | } 36 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/generators/functions/Random.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators.functions 2 | 3 | import com.rustyrazorblade.easycassstress.converters.HumanReadableConverter 4 | import com.rustyrazorblade.easycassstress.generators.FieldGenerator 5 | import org.apache.commons.text.RandomStringGenerator 6 | import java.util.concurrent.ThreadLocalRandom 7 | import com.rustyrazorblade.easycassstress.generators.Function 8 | 9 | 10 | @Function(name="random", 11 | description = "Random numbers.") 12 | class Random : FieldGenerator { 13 | 14 | 15 | 16 | 17 | var min = 0L 18 | var max = 100000L 19 | 20 | override fun setParameters(params: List) { 21 | min = HumanReadableConverter().convert(params[0]) 22 | max = HumanReadableConverter().convert(params[1]) 23 | } 24 | 25 | override fun getInt(): Int { 26 | if(min > Int.MAX_VALUE || max > Int.MAX_VALUE) 27 | throw Exception("Int larger than Int.MAX_VALUE requested, use a long instead") 28 | 29 | return ThreadLocalRandom.current().nextInt(min.toInt(), max.toInt()) 30 | } 31 | 32 | override fun getText(): String { 33 | val length = ThreadLocalRandom.current().nextInt(min.toInt(), max.toInt()) 34 | 35 | val generator = RandomStringGenerator.Builder().withinRange(65, 90).build() 36 | return generator.generate(length) 37 | } 38 | 39 | companion object { 40 | fun create(min: Long, max: Long) = Random() 41 | .apply { 42 | this.min = min 43 | this.max = max 44 | } 45 | } 46 | 47 | override fun getDescription() = """ 48 | Completely random data with even distribution. 49 | """.trimIndent() 50 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/generators/functions/Book.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators.functions 2 | 3 | import com.rustyrazorblade.easycassstress.generators.FieldGenerator 4 | import com.rustyrazorblade.easycassstress.generators.Function 5 | import java.util.concurrent.ThreadLocalRandom 6 | 7 | @Function(name="book", 8 | description = "Picks random sections of books.") 9 | class Book : FieldGenerator { 10 | 11 | var min: Int = 20 12 | var max: Int = 50 13 | 14 | override fun setParameters(params: List) { 15 | min = params[0].toInt() 16 | max = params[1].toInt() 17 | } 18 | 19 | 20 | override fun getDescription() = """ 21 | Uses random sections of open books to provide real world text data. 22 | """.trimIndent() 23 | 24 | companion object { 25 | fun create(min: Int, max: Int) : Book { 26 | val b = Book() 27 | b.setParameters(arrayListOf(min.toString(), max.toString())) 28 | return b 29 | } 30 | } 31 | 32 | // all the content from books will go here 33 | val content = mutableListOf() 34 | 35 | init { 36 | 37 | val files = listOf("alice.txt", "moby-dick.txt", "war.txt") 38 | for(f in files) { 39 | val tmp = this::class.java.getResource("/books/$f").readText() 40 | val splitContent = tmp.split("\\s+".toRegex()) 41 | content.addAll(splitContent) 42 | } 43 | } 44 | 45 | override fun getText(): String { 46 | // first get the length 47 | val length = ThreadLocalRandom.current().nextInt(min, max) 48 | val start = ThreadLocalRandom.current().nextInt(0, content.size-length) 49 | 50 | return content.subList(start, start + length).joinToString(" ") 51 | } 52 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/commands/Info.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.commands 2 | 3 | import com.beust.jcommander.Parameter 4 | import com.beust.jcommander.Parameters 5 | import com.rustyrazorblade.easycassstress.Plugin 6 | import com.github.ajalt.mordant.TermColors 7 | 8 | 9 | @Parameters(commandDescription = "Get details of a specific workload.") 10 | class Info : IStressCommand { 11 | @Parameter(required = true) 12 | var profile = "" 13 | 14 | override fun execute() { 15 | // Description 16 | // Schema 17 | // Field options 18 | val plugin = Plugin.getPlugins().get(profile)!! 19 | 20 | 21 | for(cql in plugin.instance.schema()) { 22 | println(cql) 23 | } 24 | 25 | println("Default read rate: ${plugin.instance.getDefaultReadRate()} (override with -r)\n") 26 | 27 | 28 | val params = plugin.getCustomParams() 29 | 30 | if(params.size > 0) { 31 | 32 | println("Dynamic workload parameters (override with --workload.name=X)\n") 33 | // TODO: Show dynamic parameters 34 | 35 | val cols = arrayOf(0, 0, 0) 36 | cols[0] = params.map { it.name.length }.max()?:0 + 1 37 | cols[1] = params.map { it.description.length }.max() ?: 0 + 1 38 | cols[2] = params.map { it.type.length }.max() ?: 0 + 1 39 | 40 | with(TermColors()) { 41 | println("${underline("Name".padEnd(cols[0]))} | ${underline("Description".padEnd(cols[1]))} | ${underline("Type".padEnd(cols[2]))}") 42 | } 43 | 44 | for(row in params) { 45 | println("${row.name.padEnd(cols[0])} | ${row.description.padEnd(cols[1])} | ${row.type.padEnd(cols[2])}") 46 | } 47 | } else { 48 | println("No dynamic workload parameters.") 49 | } 50 | 51 | 52 | } 53 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/CommandLineParser.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import com.beust.jcommander.JCommander 4 | import com.beust.jcommander.Parameter 5 | import com.rustyrazorblade.easycassstress.commands.* 6 | 7 | class MainArgs { 8 | 9 | @Parameter(names = ["--help", "-h"], description = "Shows this help.") 10 | var help = false 11 | 12 | } 13 | 14 | class CommandLineParser(val jCommander: JCommander, 15 | val commands: Map) { 16 | 17 | 18 | companion object { 19 | fun parse(arguments: Array): com.rustyrazorblade.easycassstress.CommandLineParser { 20 | 21 | // JCommander set up 22 | val jcommander = JCommander.newBuilder().programName("easy-cass-stress") 23 | val args = com.rustyrazorblade.easycassstress.MainArgs() 24 | 25 | // needed to get help 26 | jcommander.addObject(args) 27 | // subcommands 28 | 29 | val commands = mapOf( 30 | "run" to Run(arguments.joinToString(" ")), 31 | "info" to Info(), 32 | "list" to ListCommand(), 33 | "fields" to Fields()) 34 | 35 | for(x in commands.entries) { 36 | jcommander.addCommand(x.key, x.value) 37 | } 38 | 39 | val jc = jcommander.build() 40 | jc.parse(*arguments) 41 | 42 | if (jc.parsedCommand == null) { 43 | jc.usage() 44 | System.exit(0) 45 | } 46 | return com.rustyrazorblade.easycassstress.CommandLineParser(jc, commands) 47 | } 48 | } 49 | 50 | fun execute() { 51 | getCommandInstance().execute() 52 | } 53 | 54 | fun getParsedCommand() : String { 55 | return jCommander.parsedCommand 56 | } 57 | 58 | fun getCommandInstance() : IStressCommand { 59 | return commands[getParsedCommand()]!! 60 | 61 | } 62 | 63 | 64 | } 65 | 66 | -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/integration/AllPluginsBasicTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.integration 2 | 3 | import com.datastax.driver.core.Cluster 4 | import com.rustyrazorblade.easycassstress.Plugin 5 | import com.rustyrazorblade.easycassstress.commands.Run 6 | import org.junit.jupiter.api.AfterEach 7 | import org.junit.jupiter.api.BeforeEach 8 | import org.junit.jupiter.params.ParameterizedTest 9 | import org.junit.jupiter.params.provider.MethodSource 10 | 11 | 12 | @Retention(AnnotationRetention.RUNTIME) 13 | @MethodSource("getPlugins") 14 | annotation class AllPlugins 15 | 16 | /** 17 | * This test grabs every plugin and ensures it can run against localhost 18 | * Next step is to start up a docker container with Cassandra 19 | * Baby steps. 20 | */ 21 | class AllPluginsBasicTest { 22 | 23 | val ip = System.getenv("EASY_CASS_STRESS_CASSANDRA_IP") ?: "127.0.0.1" 24 | 25 | val connection = Cluster.builder() 26 | .addContactPoint(ip) 27 | .build().connect() 28 | 29 | lateinit var run : Run 30 | 31 | var prometheusPort = 9600 32 | 33 | /** 34 | * Annotate a test with @AllPlugins 35 | */ 36 | companion object { 37 | @JvmStatic 38 | fun getPlugins() = Plugin.getPlugins().values.filter { 39 | it.name != "Demo" 40 | } 41 | 42 | } 43 | 44 | 45 | @BeforeEach 46 | fun cleanup() { 47 | connection.execute("DROP KEYSPACE IF EXISTS easy_cass_stress") 48 | run = Run("placeholder") 49 | } 50 | 51 | @AfterEach 52 | fun shutdownMetrics() { 53 | 54 | 55 | } 56 | 57 | @AllPlugins 58 | @ParameterizedTest(name = "run test {0}") 59 | fun runEachTest(plugin: Plugin) { 60 | 61 | run.apply { 62 | host = ip 63 | profile = plugin.name 64 | iterations = 1000 65 | rate = 100L 66 | partitionValues = 1000 67 | prometheusPort = prometheusPort++ 68 | threads = 2 69 | }.execute() 70 | } 71 | 72 | 73 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/profiles/Maps.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.profiles 2 | 3 | import com.beust.jcommander.Parameter 4 | import com.datastax.driver.core.PreparedStatement 5 | import com.datastax.driver.core.Session 6 | import com.rustyrazorblade.easycassstress.PartitionKey 7 | import com.rustyrazorblade.easycassstress.StressContext 8 | import com.rustyrazorblade.easycassstress.profiles.IStressProfile 9 | import com.rustyrazorblade.easycassstress.profiles.IStressRunner 10 | import com.rustyrazorblade.easycassstress.profiles.Operation 11 | 12 | 13 | class Maps : IStressProfile { 14 | 15 | lateinit var insert : PreparedStatement 16 | lateinit var select : PreparedStatement 17 | lateinit var delete : PreparedStatement 18 | 19 | override fun prepare(session: Session) { 20 | insert = session.prepare("UPDATE map_stress SET data[?] = ? WHERE id = ?") 21 | select = session.prepare("SELECT * from map_stress WHERE id = ?") 22 | delete = session.prepare("DELETE from map_stress WHERE id = ?") 23 | } 24 | 25 | override fun schema(): List { 26 | val query = """ CREATE TABLE IF NOT EXISTS map_stress (id text, data map, primary key (id)) """ 27 | return listOf(query) 28 | } 29 | 30 | 31 | override fun getRunner(context: StressContext): IStressRunner { 32 | return object : IStressRunner { 33 | override fun getNextMutation(partitionKey: PartitionKey): Operation { 34 | return Operation.Mutation(insert.bind("key", "value", partitionKey.getText())) 35 | } 36 | 37 | override fun getNextSelect(partitionKey: PartitionKey): Operation { 38 | val b = select.bind(partitionKey.getText()) 39 | return Operation.SelectStatement(b) 40 | } 41 | 42 | override fun getNextDelete(partitionKey: PartitionKey): Operation { 43 | val b = delete.bind(partitionKey.getText()) 44 | return Operation.Deletion(b) 45 | } 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/Metrics.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import java.util.concurrent.TimeUnit 4 | 5 | import com.codahale.metrics.MetricRegistry 6 | import com.codahale.metrics.ScheduledReporter 7 | 8 | import io.prometheus.client.CollectorRegistry 9 | import io.prometheus.client.dropwizard.DropwizardExports 10 | import io.prometheus.client.exporter.HTTPServer 11 | 12 | import org.HdrHistogram.SynchronizedHistogram 13 | import java.util.Optional 14 | import javax.swing.text.html.Option 15 | 16 | class Metrics(val metricRegistry: MetricRegistry, val reporters: List, httpPort : Int) { 17 | 18 | val server: Optional 19 | 20 | 21 | fun startReporting() { 22 | for(reporter in reporters) 23 | reporter.start(3, TimeUnit.SECONDS) 24 | } 25 | 26 | fun shutdown() { 27 | server.map { it.close() } 28 | 29 | for(reporter in reporters) { 30 | reporter.stop() 31 | } 32 | } 33 | 34 | fun resetErrors() { 35 | metricRegistry.remove("errors") 36 | errors = metricRegistry.meter("errors") 37 | } 38 | 39 | init { 40 | server = if (httpPort > 0) { 41 | CollectorRegistry.defaultRegistry.register(DropwizardExports(metricRegistry)) 42 | Optional.of(HTTPServer(httpPort)) 43 | } else { 44 | println("Not setting up prometheus endpoint.") 45 | Optional.empty() 46 | } 47 | 48 | } 49 | 50 | 51 | var errors = metricRegistry.meter("errors") 52 | val mutations = metricRegistry.timer("mutations") 53 | val selects = metricRegistry.timer("selects") 54 | val deletions = metricRegistry.timer("deletions") 55 | 56 | val populate = metricRegistry.timer("populateMutations") 57 | 58 | // Using a synchronized histogram for now, we may need to change this later if it's a perf bottleneck 59 | val mutationHistogram = SynchronizedHistogram(2) 60 | val selectHistogram = SynchronizedHistogram(2) 61 | val deleteHistogram = SynchronizedHistogram(2) 62 | } 63 | -------------------------------------------------------------------------------- /manual/examples/easy-cass-stress-keyvalue.txt: -------------------------------------------------------------------------------- 1 | $ bin/easy-cass-stress run KeyValue -n 10000 2 | WARNING: sun.reflect.Reflection.getCallerClass is not supported. This will impact performance. 3 | Creating easy_cass_stress: 4 | CREATE KEYSPACE 5 | IF NOT EXISTS easy_cass_stress 6 | WITH replication = {'class': 'SimpleStrategy', 'replication_factor':3 } 7 | 8 | Creating schema 9 | Executing 10000 operations with consistency level LOCAL_ONE 10 | Connected 11 | Creating Tables 12 | CREATE TABLE IF NOT EXISTS keyvalue ( 13 | key text PRIMARY KEY, 14 | value text 15 | ) WITH caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'} AND default_time_to_live = 0 16 | Preparing queries 17 | Initializing metrics 18 | Stepping rate limiter by 500.0 to 5000.0 19 | Connecting 20 | Creating generator random 21 | 1 threads prepared. 22 | Starting main runner 23 | [Thread 0]: Running the profile for 10000 iterations... 24 | Writes Reads Deletes Errors 25 | Count Latency (p99) 1min (req/s) | Count Latency (p99) 1min (req/s) | Count Latency (p99) 1min (req/s) | Count 1min (errors/s) 26 | 624 35.88 0 | 597 34.79 0 | 0 0 0 | 0 0 27 | 1345 11.63 218 | 1375 8.61 224.4 | 0 0 0 | 0 0 28 | 2143 0.75 218 | 2075 0.67 224.4 | 0 0 0 | 0 0 29 | 3378 0.62 221.29 | 3342 0.64 225.71 | 0 0 0 | 0 0 30 | 4862 0.64 243.13 | 4858 0.57 247.99 | 0 0 0 | 0 0 31 | 4998 0.63 243.13 | 5002 0.62 247.99 | 0 0 0 | 0 0 32 | Stress complete, 1. 33 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/OperationCallback.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import com.datastax.driver.core.ResultSet 4 | import com.google.common.util.concurrent.FutureCallback 5 | import com.rustyrazorblade.easycassstress.profiles.IStressRunner 6 | import com.rustyrazorblade.easycassstress.profiles.Operation 7 | import org.apache.logging.log4j.kotlin.logger 8 | 9 | /** 10 | * Callback after a mutation or select 11 | * This was moved out of the inline ProfileRunner to make populate mode easier 12 | * as well as reduce clutter 13 | */ 14 | class OperationCallback(val context: StressContext, 15 | val runner: IStressRunner, 16 | val op: Operation, 17 | val paginate: Boolean = false) : FutureCallback { 18 | 19 | companion object { 20 | val log = logger() 21 | } 22 | 23 | override fun onFailure(t: Throwable) { 24 | context.metrics.errors.mark() 25 | log.error { t } 26 | 27 | } 28 | 29 | override fun onSuccess(result: ResultSet) { 30 | // maybe paginate 31 | if (paginate) { 32 | var tmp = result 33 | while (!tmp.isFullyFetched) { 34 | tmp = result.fetchMoreResults().get() 35 | } 36 | } 37 | 38 | val time = op.startTime.stop() 39 | 40 | // we log to the HDR histogram and do the callback for mutations 41 | // might extend this to select, but I can't see a reason for it now 42 | when (op) { 43 | is Operation.Mutation -> { 44 | context.metrics.mutationHistogram.recordValue(time) 45 | runner.onSuccess(op, result) 46 | } 47 | 48 | is Operation.Deletion -> { 49 | context.metrics.deleteHistogram.recordValue(time) 50 | } 51 | 52 | is Operation.SelectStatement -> { 53 | context.metrics.selectHistogram.recordValue(time) 54 | } 55 | is Operation.Stop -> { 56 | throw OperationStopException() 57 | } 58 | } 59 | 60 | } 61 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # easy-cass-stress: A workload centric stress tool and framework designed for ease of use. 2 | 3 | This project is a work in progress. 4 | 5 | cassandra-stress is a configuration-based tool for doing benchmarks and testing simple data models for Apache Cassandra. 6 | Unfortunately, it can be challenging to configure a workload. There are fairly common data models and workloads seen on Apache Cassandra. 7 | This tool aims to provide a means of executing configurable, pre-defined profiles. 8 | 9 | Full docs are here: https://rustyrazorblade.github.io/easy-cass-stress/ 10 | 11 | # Installation 12 | 13 | The easiest way to get started on Linux is to use system packages. 14 | Instructions for installation can be found here: https://rustyrazorblade.github.io/easy-cass-stress/#_installation 15 | 16 | 17 | # Building 18 | 19 | Clone this repo, then build with gradle: 20 | 21 | git clone https://github.com/rustyrazorblade/easy-cass-stress.git 22 | cd easy-cass-stress 23 | ./gradlew shadowJar 24 | 25 | Use the shell script wrapper to start and get help: 26 | 27 | bin/easy-cass-stress -h 28 | 29 | # Examples 30 | 31 | Time series workload with a billion operations: 32 | 33 | bin/easy-cass-stress run BasicTimeSeries -i 1B 34 | 35 | Key value workload with a million operations across 5k partitions, 50:50 read:write ratio: 36 | 37 | bin/easy-cass-stress run KeyValue -i 1M -p 5k -r .5 38 | 39 | 40 | Time series workload, using TWCS: 41 | 42 | bin/easy-cass-stress run BasicTimeSeries -i 10M --compaction "{'class':'TimeWindowCompactionStrategy', 'compaction_window_size': 1, 'compaction_window_unit': 'DAYS'}" 43 | 44 | Time series workload with a run lasting 1h and 30mins: 45 | 46 | bin/easy-cass-stress run BasicTimeSeries -d "1h30m" 47 | 48 | Time series workload with Cassandra Authentication enabled: 49 | 50 | bin/easy-cass-stress run BasicTimeSeries -d '30m' -U '' -P '' 51 | **Note**: The quotes are mandatory around the username/password 52 | if they contain special chararacters, which is pretty common for password 53 | 54 | # Generating docs 55 | 56 | Docs are served out of /docs and can be rebuild using `./gradlew docs`. -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/RateLimiterOptimizerTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import com.google.common.util.concurrent.RateLimiter 4 | import io.mockk.every 5 | import io.mockk.junit5.MockKExtension 6 | import org.junit.jupiter.api.Test 7 | import io.mockk.mockk 8 | import io.mockk.spyk 9 | import org.assertj.core.api.Assertions.* 10 | import org.junit.jupiter.api.extension.ExtendWith 11 | import java.util.* 12 | 13 | @ExtendWith(MockKExtension::class) 14 | class RateLimiterOptimizerTest { 15 | 16 | var rateLimiter: RateLimiter = RateLimiter.create(1000.0) 17 | val metrics = mockk() 18 | 19 | fun pair(current: Double, max: Long) : Optional> { 20 | return Optional.of(Pair(current, max)) 21 | } 22 | 23 | @Test 24 | fun testSimpleReadLimitRaise() { 25 | 26 | 27 | val optimizer = spyk(RateLimiterOptimizer(rateLimiter, metrics, 100, 100)) 28 | every { optimizer.getCurrentAndMaxLatency() } returns pair(10.0, 50) 29 | every { optimizer.getTotalOperations() } returns 100 30 | 31 | val newRate = optimizer.execute() 32 | assertThat(newRate).isGreaterThan(1000.0) 33 | } 34 | 35 | @Test 36 | fun testSimpleLimitLower() { 37 | 38 | val maxLatency = 100L 39 | val optimizer = spyk(RateLimiterOptimizer(rateLimiter, metrics, maxLatency, maxLatency)) 40 | every { optimizer.getCurrentAndMaxLatency() } returns pair(110.0, maxLatency) 41 | every { optimizer.getTotalOperations() } returns 100 42 | 43 | val newRate = optimizer.execute() 44 | assertThat(newRate).isLessThan(1000.0) 45 | } 46 | 47 | // Current limiter: 10.0 latency 1.4934458E7, max: 50 adjustment factor: 2.5109716067365823E-6 48 | @Test 49 | fun testLowInitialRate() { 50 | val maxLatency = 50L 51 | rateLimiter = RateLimiter.create(10.0) 52 | 53 | val optimizer = spyk(RateLimiterOptimizer(rateLimiter, metrics, maxLatency, maxLatency)) 54 | every { optimizer.getCurrentAndMaxLatency() } returns pair(1.0, maxLatency) 55 | every { optimizer.getTotalOperations() } returns 100 56 | 57 | val newRate = optimizer.execute() 58 | assertThat(newRate).isGreaterThan(10.0) 59 | } 60 | 61 | 62 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/PartitionKeyGenerator.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import java.util.concurrent.ThreadLocalRandom 4 | 5 | import org.apache.commons.math3.random.RandomDataGenerator 6 | 7 | /** 8 | * Accepts a function that generates numbers. 9 | * returns a partition key using a prefix 10 | * for any normal case there should be a compaction object function 11 | * that should create a generator that has a function with all the logic inside it 12 | */ 13 | 14 | class PartitionKeyGenerator( 15 | val genFunc: (max: Long) -> Long, 16 | val prefix: String) { 17 | /** 18 | * 19 | */ 20 | companion object { 21 | /** 22 | * 23 | */ 24 | fun random(prefix: String = "test") : PartitionKeyGenerator { 25 | return PartitionKeyGenerator({max -> ThreadLocalRandom.current().nextLong(0, max) }, prefix) 26 | } 27 | 28 | /** 29 | * 30 | */ 31 | fun sequence(prefix: String = "test") : PartitionKeyGenerator { 32 | var current = 0L 33 | return PartitionKeyGenerator( 34 | { 35 | max -> 36 | if(current > max) 37 | current = 0 38 | current++ 39 | }, prefix) 40 | } 41 | 42 | /** 43 | * Gaussian distribution 44 | */ 45 | fun normal(prefix: String = "test") : PartitionKeyGenerator { 46 | val generator = RandomDataGenerator() 47 | return PartitionKeyGenerator({ max -> 48 | var result = 0L 49 | while(true) { 50 | val mid = (max / 2).toDouble() 51 | result = generator.nextGaussian(mid, mid / 4.0).toLong() 52 | if(result in 0..max) 53 | break 54 | } 55 | result 56 | }, prefix) 57 | } 58 | } 59 | 60 | 61 | fun generateKey(total: Long, maxId: Long = 100000) = sequence { 62 | var i : Long = 0 63 | while(true) { 64 | val tmp = genFunc(maxId) 65 | 66 | val result = PartitionKey(prefix, tmp) 67 | yield(result) 68 | i++ 69 | 70 | if(i == total) 71 | break 72 | } 73 | } 74 | 75 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/generators/FunctionLoader.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators 2 | 3 | import org.apache.logging.log4j.kotlin.logger 4 | import org.reflections.Reflections 5 | 6 | data class FunctionDescription(val name: String, 7 | val description: String) 8 | 9 | class AnnotationMissingException(val name: Class) : Exception() 10 | 11 | /** 12 | * Finds all the available functions and tracks them by name 13 | */ 14 | class FunctionLoader : Iterable { 15 | 16 | override fun iterator(): Iterator { 17 | return object : Iterator { 18 | val iter = map.iterator() 19 | 20 | override fun hasNext() = iter.hasNext() 21 | 22 | override fun next(): FunctionDescription { 23 | val tmp = iter.next() 24 | val annotation = tmp.value.getAnnotation(Function::class.java) ?: throw AnnotationMissingException(tmp.value) 25 | 26 | return FunctionDescription(annotation.name, annotation.description) 27 | } 28 | 29 | } 30 | } 31 | 32 | class FunctionNotFound(val name: String) : Exception() 33 | 34 | val map : MutableMap> = mutableMapOf() 35 | 36 | init { 37 | val r = Reflections("com.rustyrazorblade.easycassstress") 38 | log.debug { "Getting FieldGenerator subtypes" } 39 | val modules = r.getSubTypesOf(FieldGenerator::class.java) 40 | 41 | 42 | modules.forEach { 43 | log.debug { "Getting annotations for $it" } 44 | 45 | val annotation = it.getAnnotation(Function::class.java) ?: throw AnnotationMissingException(it) 46 | 47 | val name = annotation.name 48 | map[name] = it 49 | } 50 | } 51 | 52 | companion object { 53 | val log = logger() 54 | 55 | } 56 | 57 | 58 | /** 59 | * Returns an instance of the requested class 60 | */ 61 | fun getInstance(name: String) : FieldGenerator { 62 | val tmp = map[name] 63 | val result = tmp?.newInstance() ?: throw FunctionNotFound(name) 64 | return result 65 | } 66 | 67 | /** 68 | * 69 | */ 70 | fun getInstance(func: ParsedFieldFunction) : FieldGenerator { 71 | val tmp = getInstance(func.name) 72 | tmp.setParameters(func.args) 73 | return tmp 74 | } 75 | 76 | 77 | 78 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/profiles/KeyValue.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.profiles 2 | 3 | import com.datastax.driver.core.PreparedStatement 4 | import com.datastax.driver.core.Session 5 | import com.rustyrazorblade.easycassstress.PartitionKey 6 | import com.rustyrazorblade.easycassstress.StressContext 7 | import com.rustyrazorblade.easycassstress.generators.FieldGenerator 8 | import com.rustyrazorblade.easycassstress.generators.Field 9 | import com.rustyrazorblade.easycassstress.generators.FieldFactory 10 | import com.rustyrazorblade.easycassstress.generators.functions.Random 11 | 12 | 13 | class KeyValue : IStressProfile { 14 | 15 | lateinit var insert: PreparedStatement 16 | lateinit var select: PreparedStatement 17 | lateinit var delete: PreparedStatement 18 | 19 | 20 | override fun prepare(session: Session) { 21 | insert = session.prepare("INSERT INTO keyvalue (key, value) VALUES (?, ?)") 22 | select = session.prepare("SELECT * from keyvalue WHERE key = ?") 23 | delete = session.prepare("DELETE from keyvalue WHERE key = ?") 24 | } 25 | 26 | override fun schema(): List { 27 | val table = """CREATE TABLE IF NOT EXISTS keyvalue ( 28 | key text PRIMARY KEY, 29 | value text 30 | )""".trimIndent() 31 | return listOf(table) 32 | } 33 | 34 | override fun getDefaultReadRate(): Double { 35 | return 0.5 36 | } 37 | 38 | override fun getRunner(context: StressContext): IStressRunner { 39 | 40 | val value = context.registry.getGenerator("keyvalue", "value") 41 | 42 | return object : IStressRunner { 43 | 44 | override fun getNextSelect(partitionKey: PartitionKey): Operation { 45 | val bound = select.bind(partitionKey.getText()) 46 | return Operation.SelectStatement(bound) 47 | } 48 | 49 | override fun getNextMutation(partitionKey: PartitionKey): Operation { 50 | val data = value.getText() 51 | val bound = insert.bind(partitionKey.getText(), data) 52 | 53 | return Operation.Mutation(bound) 54 | } 55 | 56 | override fun getNextDelete(partitionKey: PartitionKey): Operation { 57 | val bound = delete.bind(partitionKey.getText()) 58 | return Operation.Deletion(bound) 59 | } 60 | } 61 | } 62 | 63 | override fun getFieldGenerators(): Map { 64 | val kv = FieldFactory("keyvalue") 65 | return mapOf(kv.getField("value") to Random().apply{min=100; max=200}) 66 | } 67 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/profiles/Sets.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.profiles 2 | 3 | import com.datastax.driver.core.PreparedStatement 4 | import com.datastax.driver.core.Session 5 | import com.rustyrazorblade.easycassstress.PartitionKey 6 | import com.rustyrazorblade.easycassstress.StressContext 7 | import com.rustyrazorblade.easycassstress.generators.Field 8 | import com.rustyrazorblade.easycassstress.generators.FieldGenerator 9 | import com.rustyrazorblade.easycassstress.generators.functions.Random 10 | 11 | class Sets : IStressProfile { 12 | 13 | lateinit var insert : PreparedStatement 14 | lateinit var update : PreparedStatement 15 | lateinit var select : PreparedStatement 16 | lateinit var deleteElement : PreparedStatement 17 | 18 | override fun prepare(session: Session) { 19 | insert = session.prepare("INSERT INTO sets (key, values) VALUES (?, ?)") 20 | update = session.prepare("UPDATE sets SET values = values + ? WHERE key = ?") 21 | select = session.prepare("SELECT * from sets WHERE key = ?") 22 | deleteElement = session.prepare("UPDATE sets SET values = values - ? WHERE key = ?") 23 | } 24 | 25 | override fun schema(): List { 26 | return listOf(""" 27 | CREATE TABLE IF NOT EXISTS sets ( 28 | |key text primary key, 29 | |values set 30 | |) 31 | """.trimMargin()) 32 | } 33 | 34 | override fun getRunner(context: StressContext): IStressRunner { 35 | val payload = context.registry.getGenerator("sets", "values") 36 | 37 | return object : IStressRunner { 38 | 39 | override fun getNextMutation(partitionKey: PartitionKey): Operation { 40 | val value = payload.getText() 41 | val bound = update.bind() 42 | .setSet(0, setOf(value)) 43 | .setString(1, partitionKey.getText()) 44 | 45 | return Operation.Mutation(bound) 46 | } 47 | 48 | override fun getNextSelect(partitionKey: PartitionKey): Operation { 49 | val bound = select.bind(partitionKey.getText()) 50 | return Operation.SelectStatement(bound) 51 | } 52 | 53 | override fun getNextDelete(partitionKey: PartitionKey): Operation { 54 | val bound = deleteElement.bind(setOf(partitionKey.getText()), partitionKey.getText()) 55 | return Operation.Deletion(bound) 56 | } 57 | 58 | } 59 | } 60 | 61 | override fun getFieldGenerators(): Map { 62 | return mapOf(Field("sets", "values") to Random().apply{ min=6; max=16}) 63 | } 64 | } -------------------------------------------------------------------------------- /.github/workflows/gradle-publish-disttar.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. 2 | # They are provided by a third-party and are governed by 3 | # separate terms of service, privacy policy, and support 4 | # documentation. 5 | # This workflow will build a package using Gradle and then publish it to GitHub packages when a release is created 6 | # For more information see: https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#Publishing-using-gradle 7 | 8 | name: Gradle Package 9 | 10 | on: 11 | release: 12 | types: [created] 13 | 14 | jobs: 15 | build: 16 | 17 | runs-on: ubuntu-latest 18 | permissions: 19 | contents: read 20 | packages: write 21 | 22 | steps: 23 | - uses: actions/checkout@v4 24 | - name: Set up JDK 11 25 | uses: actions/setup-java@v4 26 | with: 27 | java-version: '11' 28 | distribution: 'temurin' 29 | server-id: github # Value of the distributionManagement/repository/id field of the pom.xml 30 | settings-path: ${{ github.workspace }} # location for the settings.xml file 31 | 32 | - name: Build with Gradle 33 | uses: gradle/gradle-build-action@bd5760595778326ba7f1441bcf7e88b49de61a25 # v2.6.0 34 | with: 35 | arguments: disttar 36 | 37 | - name: Upload a Build Artifact 38 | uses: actions/upload-artifact@v4.3.0 39 | with: 40 | # Artifact name 41 | #name: # optional, default is artifact 42 | # A file, directory or wildcard pattern that describes what to upload 43 | #path: 44 | # The desired behavior if no files are found using the provided path. 45 | #Available Options: 46 | #warn: Output a warning but do not fail the action 47 | #error: Fail the action with an error message 48 | #ignore: Do not output any warnings or errors, the action does not fail 49 | 50 | #if-no-files-found: # optional, default is warn 51 | # Duration after which artifact will expire in days. 0 means using default retention. 52 | # Minimum 1 day. Maximum 90 days unless changed from the repository settings page. 53 | 54 | retention-days: 0 # optional 55 | # The level of compression for Zlib to be applied to the artifact archive. The value can range from 0 to 9: - 0: No compression - 1: Best speed - 6: Default compression (same as GNU Gzip) - 9: Best compression Higher levels will result in better compression, but will take longer to complete. For large files that are not easily compressed, a value of 0 is recommended for significantly faster uploads. 56 | 57 | #compression-level: # optional, default is 6 58 | # If true, an artifact with a matching name will be deleted before a new one is uploaded. If false, the action will fail if an artifact for the given name already exists. Does not fail if the artifact does not exist. 59 | 60 | overwrite: true # optional, default is false 61 | 62 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/profiles/RangeScan.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.profiles 2 | 3 | import com.datastax.driver.core.PreparedStatement 4 | import com.datastax.driver.core.Session 5 | import com.datastax.driver.core.TokenRange 6 | import com.rustyrazorblade.easycassstress.PartitionKey 7 | import com.rustyrazorblade.easycassstress.StressContext 8 | import com.rustyrazorblade.easycassstress.WorkloadParameter 9 | import org.apache.logging.log4j.kotlin.logger 10 | 11 | /** 12 | * this is a bit of an oddball workout because it doesn't support writes. 13 | */ 14 | 15 | class RangeScan : IStressProfile { 16 | private lateinit var ranges: List 17 | 18 | @WorkloadParameter("Table to perform full scan against. Does not support writes of any kind.") 19 | var table = "system.local" 20 | 21 | @WorkloadParameter("Number of ranges (splits) to subdivide each token range into. Ignored by default. Default is to scan the entire table without ranges.") 22 | var splits: Int = 1 23 | 24 | lateinit var select: PreparedStatement 25 | 26 | var logger = logger() 27 | override fun prepare(session: Session) { 28 | val rq = if (splits > 1) { 29 | ranges = session.cluster.metadata.tokenRanges.flatMap { it.splitEvenly(splits) } 30 | val tmp = table.split(".") 31 | var partitionKeys = session.cluster.metadata.getKeyspace(tmp[0]) 32 | .getTable(tmp[1]) 33 | .partitionKey.map { it.name } 34 | .joinToString(", ") 35 | logger.info("Using splits on $partitionKeys") 36 | " WHERE token($partitionKeys) > ? AND token($partitionKeys) < ?" 37 | } else { 38 | logger.info("Not using splits because workload.splits parameter=$splits") 39 | "" 40 | } 41 | val s = "SELECT * from $table $rq" 42 | logger.info("Preparing range query: $s") 43 | 44 | select = session.prepare(s) 45 | } 46 | 47 | override fun schema(): List { 48 | return listOf() 49 | } 50 | override fun getDefaultReadRate(): Double { 51 | return 1.0 52 | } 53 | override fun getRunner(context: StressContext): IStressRunner { 54 | return object : IStressRunner { 55 | override fun getNextMutation(partitionKey: PartitionKey): Operation { 56 | // we need the ability to say a workload doesn't support mutations 57 | TODO("Not yet implemented") 58 | } 59 | 60 | override fun getNextSelect(partitionKey: PartitionKey): Operation { 61 | return if (splits > 1) { 62 | val tmp = ranges.random() 63 | Operation.SelectStatement(select.bind(tmp.start.value, tmp.end.value)) 64 | } else { 65 | Operation.SelectStatement(select.bind()) 66 | } 67 | } 68 | 69 | override fun getNextDelete(partitionKey: PartitionKey): Operation { 70 | // we need the ability to say a workload doesn't support deletes 71 | TODO("Not yet implemented") 72 | } 73 | 74 | } 75 | } 76 | 77 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/profiles/CountersWide.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.profiles 2 | 3 | import com.datastax.driver.core.PreparedStatement 4 | import com.datastax.driver.core.Session 5 | import com.rustyrazorblade.easycassstress.PartitionKey 6 | import com.rustyrazorblade.easycassstress.StressContext 7 | import com.rustyrazorblade.easycassstress.WorkloadParameter 8 | import java.util.concurrent.ThreadLocalRandom 9 | import kotlin.math.roundToLong 10 | 11 | class CountersWide : IStressProfile { 12 | 13 | lateinit var increment: PreparedStatement 14 | lateinit var selectOne: PreparedStatement 15 | lateinit var selectAll: PreparedStatement 16 | lateinit var deleteOne: PreparedStatement 17 | 18 | @WorkloadParameter("Total rows per partition.") 19 | var rowsPerPartition = 10000 20 | 21 | override fun prepare(session: Session) { 22 | increment = session.prepare("UPDATE counter_wide SET value = value + 1 WHERE key = ? and cluster = ?") 23 | selectOne = session.prepare("SELECT * from counter_wide WHERE key = ? AND cluster = ?") 24 | selectAll = session.prepare("SELECT * from counter_wide WHERE key = ?") 25 | deleteOne = session.prepare("DELETE from counter_wide WHERE key = ? AND cluster = ?") 26 | } 27 | 28 | override fun schema(): List { 29 | return listOf("""CREATE TABLE IF NOT EXISTS counter_wide ( 30 | | key text, 31 | | cluster bigint, 32 | | value counter, 33 | | primary key(key, cluster)) 34 | """.trimMargin()) 35 | 36 | } 37 | 38 | override fun getRunner(context: StressContext): IStressRunner { 39 | 40 | // for now i'm just going to hardcode this at 10K items 41 | // later when a profile can accept dynamic parameters i'll make it configurable 42 | 43 | 44 | return object : IStressRunner { 45 | 46 | var iterations = 0L 47 | 48 | override fun getNextMutation(partitionKey: PartitionKey): Operation { 49 | 50 | val clusteringKey = (ThreadLocalRandom.current().nextGaussian() * rowsPerPartition.toDouble()).roundToLong() 51 | val tmp = increment.bind(partitionKey.getText(), clusteringKey) 52 | return Operation.Mutation(tmp) 53 | } 54 | 55 | override fun getNextSelect(partitionKey: PartitionKey): Operation { 56 | iterations++ 57 | 58 | if (iterations % 2 == 0L) { 59 | val clusteringKey = (ThreadLocalRandom.current().nextGaussian() * rowsPerPartition.toDouble()).roundToLong() 60 | return Operation.SelectStatement(selectOne.bind(partitionKey.getText(), clusteringKey)) 61 | } 62 | 63 | return Operation.SelectStatement(selectAll.bind(partitionKey.getText())) 64 | } 65 | 66 | override fun getNextDelete(partitionKey: PartitionKey): Operation { 67 | val clusteringKey = (ThreadLocalRandom.current().nextGaussian() * rowsPerPartition.toDouble()).roundToLong() 68 | return Operation.Deletion(deleteOne.bind(partitionKey.getText(), clusteringKey)) 69 | } 70 | } 71 | } 72 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/profiles/LWT.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.profiles 2 | 3 | import com.datastax.driver.core.PreparedStatement 4 | import com.datastax.driver.core.ResultSet 5 | import com.datastax.driver.core.Session 6 | import com.rustyrazorblade.easycassstress.PartitionKey 7 | import com.rustyrazorblade.easycassstress.StressContext 8 | 9 | class LWT : IStressProfile { 10 | 11 | lateinit var insert : PreparedStatement 12 | lateinit var update: PreparedStatement 13 | lateinit var select: PreparedStatement 14 | lateinit var delete: PreparedStatement 15 | lateinit var deletePartition: PreparedStatement 16 | 17 | override fun schema(): List { 18 | return arrayListOf("""CREATE TABLE IF NOT EXISTS lwt (id text primary key, value int) """) 19 | } 20 | 21 | override fun prepare(session: Session) { 22 | insert = session.prepare("INSERT INTO lwt (id, value) VALUES (?, ?) IF NOT EXISTS") 23 | update = session.prepare("UPDATE lwt SET value = ? WHERE id = ? IF value = ?") 24 | select = session.prepare("SELECT * from lwt WHERE id = ?") 25 | delete = session.prepare("DELETE from lwt WHERE id = ? IF value = ?") 26 | deletePartition = session.prepare("DELETE from lwt WHERE id = ? IF EXISTS") 27 | } 28 | 29 | 30 | override fun getRunner(context: StressContext): IStressRunner { 31 | data class CallbackPayload(val id: String, val value: Int) 32 | 33 | return object : IStressRunner { 34 | val state = mutableMapOf() 35 | 36 | override fun getNextMutation(partitionKey: PartitionKey): Operation { 37 | val currentValue = state[partitionKey.getText()] 38 | val newValue: Int 39 | 40 | val mutation = if(currentValue != null) { 41 | newValue = currentValue + 1 42 | update.bind(0, partitionKey.getText(), newValue) 43 | } else { 44 | newValue = 0 45 | insert.bind(partitionKey.getText(), newValue) 46 | } 47 | val payload = CallbackPayload(partitionKey.getText(), newValue) 48 | return Operation.Mutation(mutation, payload) 49 | } 50 | 51 | override fun getNextSelect(partitionKey: PartitionKey): Operation { 52 | return Operation.SelectStatement(select.bind(partitionKey.getText())) 53 | } 54 | 55 | override fun getNextDelete(partitionKey: PartitionKey): Operation { 56 | val currentValue = state[partitionKey.getText()] 57 | val newValue: Int 58 | 59 | val deletion = if(currentValue != null) { 60 | delete.bind(partitionKey.getText(), currentValue) 61 | } else { 62 | deletePartition.bind(partitionKey.getText()) 63 | } 64 | return Operation.Deletion(deletion) 65 | } 66 | 67 | override fun onSuccess(op: Operation.Mutation, result: ResultSet?) { 68 | val payload = op.callbackPayload!! as CallbackPayload 69 | state[payload.id] = payload.value 70 | 71 | } 72 | 73 | } 74 | } 75 | } -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/PluginTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import com.datastax.driver.core.BoundStatement 4 | import com.datastax.driver.core.Session 5 | import com.rustyrazorblade.easycassstress.profiles.IStressProfile 6 | import com.rustyrazorblade.easycassstress.profiles.IStressRunner 7 | import com.rustyrazorblade.easycassstress.profiles.Operation 8 | import io.mockk.mockk 9 | 10 | import org.assertj.core.api.Assertions.assertThat 11 | import org.assertj.core.api.Assertions.assertThatExceptionOfType 12 | import org.junit.jupiter.api.BeforeEach 13 | import org.junit.jupiter.api.Test 14 | 15 | 16 | internal class PluginTest { 17 | 18 | lateinit var plugin : Plugin 19 | 20 | @BeforeEach 21 | fun setPlugin() { 22 | plugin = Plugin.getPlugins()["Demo"]!! 23 | } 24 | 25 | class Demo : IStressProfile { 26 | 27 | @WorkloadParameter("Number of rows for each") 28 | var rows : Int = 100 29 | 30 | @WorkloadParameter("First name of person") 31 | var name : String = "Jon" 32 | 33 | var notWorkloadParameter : String = "oh nooo" 34 | 35 | override fun prepare(session: Session) = Unit 36 | override fun schema(): List = listOf() 37 | 38 | override fun getRunner(context: StressContext): IStressRunner { 39 | return object : IStressRunner { 40 | override fun getNextMutation(partitionKey: PartitionKey): Operation { 41 | val b = mockk() 42 | return Operation.Mutation(b) 43 | } 44 | 45 | override fun getNextSelect(partitionKey: PartitionKey): Operation { 46 | val b = mockk() 47 | return Operation.SelectStatement(b) 48 | } 49 | 50 | override fun getNextDelete(partitionKey: PartitionKey): Operation { 51 | val b = mockk() 52 | return Operation.Deletion(b) 53 | } 54 | } 55 | } 56 | 57 | } 58 | 59 | // simple test, but ¯\_(ツ)_/¯ 60 | // we should have at least 2 plugins 61 | @Test 62 | fun testGetPlugins() { 63 | val tmp = Plugin.getPlugins() 64 | assertThat(tmp.count()).isGreaterThan(1) 65 | } 66 | 67 | @Test 68 | fun testApplyDynamicSettings() { 69 | 70 | val fields = mapOf("rows" to "10", 71 | "name" to "Anthony") 72 | 73 | plugin.applyDynamicSettings(fields) 74 | 75 | val instance = plugin.instance as Demo 76 | 77 | assertThat(instance.rows).isEqualTo(10) 78 | assertThat(instance.name).isEqualTo("Anthony") 79 | } 80 | 81 | 82 | @Test 83 | fun testGetProperty() { 84 | val prop = plugin.getProperty("name") 85 | assertThat(prop.name).isEqualTo("name") 86 | } 87 | 88 | @Test 89 | fun testGetNonexistentPropertyThrowsException() { 90 | assertThatExceptionOfType(NoSuchElementException::class.java).isThrownBy { 91 | plugin.getProperty("NOT_A_REAL_PROPERTY_OH_NOES") 92 | } 93 | } 94 | 95 | @Test 96 | fun testGetCustomParams() { 97 | val params = plugin.getCustomParams() 98 | println(params) 99 | } 100 | 101 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/generators/Registry.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.generators 2 | 3 | import org.apache.logging.log4j.kotlin.logger 4 | 5 | data class Field(val table: String, val field: String) 6 | 7 | class FieldFactory(private val table: String) { 8 | fun getField(field: String) : Field = Field(table, field) 9 | } 10 | 11 | 12 | 13 | /** 14 | * Registry for data generators 15 | * When the original schema is created, the registry will be set up with default generators for each field 16 | * A generator option can be overridden on the command line as a dynamic flag with field.* 17 | * The idea here is we should be able to customize the data we increment without custom coding 18 | * for instance, I could use random(1, 100) to be an int field of 1-100 or a text field of 1-100 characters. 19 | * book(10, 100) is a random selection of 10-100 words from a bunch of open source licensed books 20 | * Ideally we have enough here to simulate a lot (call it 90%) of common workloads 21 | * 22 | */ 23 | class Registry(val defaults: MutableMap = mutableMapOf(), 24 | val overrides: MutableMap = mutableMapOf()) { 25 | 26 | companion object { 27 | 28 | val log = logger() 29 | 30 | val functionLoader = FunctionLoader() 31 | 32 | fun create(defaults: MutableMap) : Registry { 33 | return Registry(defaults) 34 | } 35 | 36 | fun create() : Registry { 37 | return Registry() 38 | } 39 | 40 | } 41 | 42 | fun getFunctions() : Iterator = 43 | functionLoader.iterator() 44 | 45 | /** 46 | * Sets the default generator for a table / field pair 47 | * Not all generators work on all fields 48 | */ 49 | fun setDefault(table: String, field: String, generator: FieldGenerator) : Registry { 50 | val f = Field(table, field) 51 | return this.setDefault(f, generator) 52 | } 53 | 54 | 55 | fun setDefault(field: Field, generator: FieldGenerator) : Registry { 56 | defaults[field] = generator 57 | return this 58 | } 59 | 60 | /** 61 | * Overrides the default generator for a table / field pair 62 | * Not all generators work on all fields 63 | * 64 | * @param table table that's affected 65 | * @param field field that's affected 66 | */ 67 | fun setOverride(table: String, field: String, generator: FieldGenerator) : Registry { 68 | 69 | val f = Field(table, field) 70 | 71 | return this.setOverride(f, generator) 72 | } 73 | 74 | fun setOverride(field: Field, generator: FieldGenerator) : Registry { 75 | overrides[field] = generator 76 | return this 77 | } 78 | 79 | fun setOverride(table: String, field: String, parsedField: ParsedFieldFunction) : Registry { 80 | val instance = functionLoader.getInstance(parsedField) 81 | return setOverride(table, field, instance) 82 | } 83 | 84 | fun getGenerator(table: String, field: String) : FieldGenerator { 85 | log.info("Getting generator for $table.$field") 86 | val tmp = Field(table, field) 87 | if(tmp in overrides) 88 | return overrides[tmp]!! 89 | return defaults[tmp]!! 90 | } 91 | 92 | } 93 | 94 | class FieldNotFoundException(message: String) : Throwable() { 95 | 96 | } 97 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/profiles/AllowFiltering.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.profiles 2 | 3 | import com.datastax.driver.core.PreparedStatement 4 | import com.datastax.driver.core.Session 5 | import com.rustyrazorblade.easycassstress.PartitionKey 6 | import com.rustyrazorblade.easycassstress.StressContext 7 | import com.rustyrazorblade.easycassstress.WorkloadParameter 8 | import com.rustyrazorblade.easycassstress.generators.Field 9 | import com.rustyrazorblade.easycassstress.generators.FieldFactory 10 | import com.rustyrazorblade.easycassstress.generators.FieldGenerator 11 | import com.rustyrazorblade.easycassstress.generators.functions.Random 12 | import java.util.concurrent.ThreadLocalRandom 13 | 14 | class AllowFiltering : IStressProfile { 15 | 16 | @WorkloadParameter(description = "Number of rows per partition") 17 | var rows = 100 18 | 19 | @WorkloadParameter(description = "Max Value of the value field. Lower values will return more results.") 20 | var maxValue = 100 21 | 22 | lateinit var insert : PreparedStatement 23 | lateinit var select: PreparedStatement 24 | lateinit var delete: PreparedStatement 25 | 26 | override fun prepare(session: Session) { 27 | insert = session.prepare("INSERT INTO allow_filtering (partition_id, row_id, value, payload) values (?, ?, ?, ?)") 28 | select = session.prepare("SELECT * from allow_filtering WHERE partition_id = ? and value = ? ALLOW FILTERING") 29 | delete = session.prepare("DELETE from allow_filtering WHERE partition_id = ? and row_id = ?") 30 | } 31 | 32 | override fun schema(): List { 33 | return listOf("""CREATE TABLE IF NOT EXISTS allow_filtering ( 34 | |partition_id text, 35 | |row_id int, 36 | |value int, 37 | |payload text, 38 | |primary key (partition_id, row_id) 39 | |) 40 | """.trimMargin()) 41 | } 42 | 43 | override fun getRunner(context: StressContext): IStressRunner { 44 | 45 | val payload = context.registry.getGenerator("allow_filtering", "payload") 46 | val random = ThreadLocalRandom.current() 47 | 48 | return object : IStressRunner { 49 | override fun getNextMutation(partitionKey: PartitionKey): Operation { 50 | val rowId = random.nextInt(0, rows) 51 | val value = random.nextInt(0, maxValue) 52 | 53 | val bound = insert.bind(partitionKey.getText(), rowId, value, payload.getText()) 54 | return Operation.Mutation(bound) 55 | 56 | } 57 | 58 | override fun getNextSelect(partitionKey: PartitionKey): Operation { 59 | val value = random.nextInt(0, maxValue) 60 | val bound = select.bind(partitionKey.getText(), value) 61 | return Operation.SelectStatement(bound) 62 | } 63 | 64 | override fun getNextDelete(partitionKey: PartitionKey): Operation { 65 | val rowId = random.nextInt(0, rows) 66 | val bound = delete.bind(partitionKey.getText(), rowId) 67 | return Operation.Deletion(bound) 68 | } 69 | } 70 | } 71 | 72 | override fun getFieldGenerators(): Map { 73 | val af = FieldFactory("allow_filtering") 74 | return mapOf(af.getField("payload") to Random().apply{ min = 0; max = 1}) 75 | 76 | } 77 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/FileReporter.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import com.codahale.metrics.* 4 | import com.codahale.metrics.Timer 5 | import java.io.BufferedOutputStream 6 | import java.io.BufferedWriter 7 | import java.io.File 8 | import java.io.FileWriter 9 | import java.text.SimpleDateFormat 10 | import java.time.Instant 11 | import java.util.* 12 | import java.util.concurrent.TimeUnit 13 | import java.util.zip.GZIPOutputStream 14 | 15 | 16 | class FileReporter(registry: MetricRegistry, outputFileName: String, command: String) : ScheduledReporter(registry, 17 | "file-reporter", 18 | MetricFilter.ALL, 19 | TimeUnit.SECONDS, 20 | TimeUnit.MILLISECONDS 21 | ) { 22 | 23 | // date 24h time 24 | // Thu-14Mar19-13.30.00 25 | private val startTime = Date() 26 | 27 | private val opHeaders = listOf("Count", "Latency (p99)", "1min (req/s)").joinToString(",", postfix = ",") 28 | private val errorHeaders = listOf("Count", "1min (errors/s)").joinToString(",") 29 | 30 | val outputFile = File(outputFileName) 31 | val buffer : BufferedWriter 32 | 33 | init { 34 | 35 | buffer = if(outputFileName.endsWith(".gz")) GZIPOutputStream(outputFile.outputStream()).bufferedWriter() else outputFile.bufferedWriter() 36 | 37 | buffer.write("# easy-cass-stress run at $startTime") 38 | buffer.newLine() 39 | buffer.write("# $command") 40 | buffer.newLine() 41 | 42 | buffer.write(",,Mutations,,,") 43 | buffer.write("Reads,,,") 44 | buffer.write("Deletes,,,") 45 | buffer.write("Errors,") 46 | buffer.newLine() 47 | 48 | buffer.write("Timestamp, Elapsed Time,") 49 | buffer.write(opHeaders) 50 | buffer.write(opHeaders) 51 | buffer.write(opHeaders) 52 | buffer.write(errorHeaders) 53 | buffer.newLine() 54 | } 55 | 56 | private fun Timer.getMetricsList(): List { 57 | val duration = convertDuration(this.snapshot.get99thPercentile()) 58 | 59 | return listOf(this.count, duration, this.oneMinuteRate) 60 | } 61 | 62 | override fun report(gauges: SortedMap>?, 63 | counters: SortedMap?, 64 | histograms: SortedMap?, 65 | meters: SortedMap?, 66 | timers: SortedMap?) { 67 | 68 | val timestamp = Instant.now().toString() 69 | val elapsedTime = Instant.now().minusMillis(startTime.time).toEpochMilli() / 1000 70 | 71 | buffer.write(timestamp + "," + elapsedTime + ",") 72 | 73 | val writeRow = timers!!["mutations"]!! 74 | .getMetricsList() 75 | .joinToString(",", postfix = ",") 76 | 77 | buffer.write(writeRow) 78 | 79 | val readRow = timers["selects"]!! 80 | .getMetricsList() 81 | .joinToString(",", postfix = ",") 82 | 83 | buffer.write(readRow) 84 | 85 | val deleteRow = timers["deletions"]!! 86 | .getMetricsList() 87 | .joinToString(",", postfix = ",") 88 | 89 | buffer.write(deleteRow) 90 | 91 | val errors = meters!!["errors"]!! 92 | val errorRow = listOf(errors.count, errors.oneMinuteRate) 93 | .joinToString(",", postfix = "\n") 94 | 95 | buffer.write(errorRow) 96 | } 97 | 98 | override fun stop() { 99 | buffer.flush() 100 | buffer.close() 101 | super.stop() 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/profiles/UdtTimeSeries.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.profiles 2 | 3 | import com.datastax.driver.core.PreparedStatement 4 | import com.datastax.driver.core.Session 5 | import com.datastax.driver.core.utils.UUIDs 6 | import com.rustyrazorblade.easycassstress.PartitionKey 7 | import com.rustyrazorblade.easycassstress.StressContext 8 | import com.rustyrazorblade.easycassstress.WorkloadParameter 9 | import com.rustyrazorblade.easycassstress.generators.* 10 | import com.rustyrazorblade.easycassstress.generators.functions.Random 11 | 12 | 13 | /** 14 | * Create a simple time series use case with some number of partitions 15 | * TODO make it use TWCS 16 | */ 17 | class UdtTimeSeries : IStressProfile { 18 | 19 | override fun schema(): List { 20 | val queryUdt = """CREATE TYPE IF NOT EXISTS sensor_data_details ( 21 | data1 text, 22 | data2 text, 23 | data3 text 24 | )""".trimIndent() 25 | 26 | val queryTable = """CREATE TABLE IF NOT EXISTS sensor_data_udt ( 27 | sensor_id text, 28 | timestamp timeuuid, 29 | data frozen, 30 | primary key(sensor_id, timestamp)) 31 | WITH CLUSTERING ORDER BY (timestamp DESC) 32 | """.trimIndent() 33 | 34 | return listOf(queryUdt, queryTable) 35 | } 36 | 37 | lateinit var insert: PreparedStatement 38 | lateinit var getPartitionHead: PreparedStatement 39 | lateinit var deletePartitionHead: PreparedStatement 40 | 41 | @WorkloadParameter("Limit select to N rows.") 42 | var limit = 500 43 | 44 | override fun prepare(session: Session) { 45 | insert = session.prepare("INSERT INTO sensor_data_udt (sensor_id, timestamp, data) VALUES (?, ?, ?)") 46 | getPartitionHead = session.prepare("SELECT * from sensor_data_udt WHERE sensor_id = ? LIMIT ?") 47 | deletePartitionHead = session.prepare("DELETE from sensor_data_udt WHERE sensor_id = ?") 48 | } 49 | 50 | /** 51 | * need to fix custom arguments 52 | */ 53 | override fun getRunner(context: StressContext): IStressRunner { 54 | 55 | val dataField = context.registry.getGenerator("sensor_data", "data") 56 | 57 | return object : IStressRunner { 58 | 59 | val udt = context.session.cluster.getMetadata().getKeyspace(context.session.loggedKeyspace).getUserType("sensor_data_details") 60 | 61 | override fun getNextSelect(partitionKey: PartitionKey): Operation { 62 | val bound = getPartitionHead.bind(partitionKey.getText(), limit) 63 | return Operation.SelectStatement(bound) 64 | } 65 | 66 | override fun getNextMutation(partitionKey: PartitionKey) : Operation { 67 | val data = dataField.getText() 68 | val chunks = data.chunked(data.length/3) 69 | val udtValue = udt.newValue().setString("data1", chunks[0]).setString("data2", chunks[1]).setString("data3", chunks[2]) 70 | val timestamp = UUIDs.timeBased() 71 | val bound = insert.bind(partitionKey.getText(),timestamp, udtValue) 72 | return Operation.Mutation(bound) 73 | } 74 | 75 | override fun getNextDelete(partitionKey: PartitionKey): Operation { 76 | val bound = deletePartitionHead.bind(partitionKey.getText()) 77 | return Operation.Deletion(bound) 78 | } 79 | } 80 | } 81 | 82 | override fun getFieldGenerators(): Map { 83 | return mapOf(Field("sensor_data", "data") to Random().apply {min=100; max=200}) 84 | } 85 | 86 | 87 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/profiles/MaterializedViews.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.profiles 2 | 3 | import com.datastax.driver.core.PreparedStatement 4 | import com.datastax.driver.core.Session 5 | import com.rustyrazorblade.easycassstress.PartitionKey 6 | import com.rustyrazorblade.easycassstress.StressContext 7 | import com.rustyrazorblade.easycassstress.generators.* 8 | import com.rustyrazorblade.easycassstress.generators.functions.FirstName 9 | import com.rustyrazorblade.easycassstress.generators.functions.LastName 10 | import com.rustyrazorblade.easycassstress.generators.functions.USCities 11 | import java.util.concurrent.ThreadLocalRandom 12 | 13 | class MaterializedViews : IStressProfile { 14 | 15 | override fun prepare(session: Session) { 16 | insert = session.prepare("INSERT INTO person (name, age, city) values (?, ?, ?)") 17 | select_base = session.prepare("SELECT * FROM person WHERE name = ?") 18 | select_by_age = session.prepare("SELECT * FROM person_by_age WHERE age = ?") 19 | select_by_city = session.prepare("SELECT * FROM person_by_city WHERE city = ?") 20 | delete_base = session.prepare("DELETE FROM person WHERE name = ?") 21 | 22 | 23 | } 24 | 25 | override fun schema(): List = listOf("""CREATE TABLE IF NOT EXISTS person 26 | | (name text, age int, city text, primary key(name))""".trimMargin(), 27 | 28 | """CREATE MATERIALIZED VIEW IF NOT EXISTS person_by_age AS 29 | |SELECT age, name, city FROM person 30 | |WHERE age IS NOT NULL AND name IS NOT NULL 31 | |PRIMARY KEY (age, name)""".trimMargin(), 32 | 33 | """CREATE MATERIALIZED VIEW IF NOT EXISTS person_by_city AS 34 | |SELECT city, name, age FROM person 35 | |WHERE city IS NOT NULL AND name IS NOT NULL 36 | |PRIMARY KEY (city, name) """.trimMargin()) 37 | 38 | override fun getRunner(context: StressContext): IStressRunner { 39 | 40 | return object : IStressRunner { 41 | var select_count = 0L 42 | 43 | val cities = context.registry.getGenerator("person", "city") 44 | 45 | override fun getNextMutation(partitionKey: PartitionKey): Operation { 46 | val num = ThreadLocalRandom.current().nextInt(1, 110) 47 | return Operation.Mutation(insert.bind(partitionKey.getText(), num, cities.getText())) 48 | } 49 | 50 | override fun getNextSelect(partitionKey: PartitionKey): Operation { 51 | val num = ThreadLocalRandom.current().nextInt(1, 110) 52 | val result = when(select_count % 2L) { 53 | 0L -> 54 | Operation.SelectStatement(select_by_age.bind(num)) 55 | else -> 56 | Operation.SelectStatement(select_by_city.bind("test")) 57 | 58 | } 59 | select_count++ 60 | return result 61 | } 62 | 63 | override fun getNextDelete(partitionKey: PartitionKey): Operation { 64 | return Operation.Deletion(delete_base.bind(partitionKey.getText())) 65 | } 66 | } 67 | } 68 | 69 | override fun getFieldGenerators(): Map { 70 | val person = FieldFactory("person") 71 | return mapOf(person.getField("firstname") to FirstName(), 72 | person.getField("lastname") to LastName(), 73 | person.getField("city") to USCities() 74 | ) 75 | } 76 | 77 | 78 | lateinit var insert : PreparedStatement 79 | lateinit var select_base : PreparedStatement 80 | lateinit var select_by_age : PreparedStatement 81 | lateinit var select_by_city : PreparedStatement 82 | lateinit var delete_base : PreparedStatement 83 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/profiles/Locking.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.profiles 2 | 3 | import com.datastax.driver.core.PreparedStatement 4 | import com.datastax.driver.core.Session 5 | import com.rustyrazorblade.easycassstress.* 6 | import com.rustyrazorblade.easycassstress.commands.Run 7 | import org.apache.logging.log4j.kotlin.logger 8 | import java.util.* 9 | import java.util.concurrent.ConcurrentHashMap 10 | 11 | /** 12 | * Note: currently broken :( 13 | * Warning: this workload is under development and should not be used as a reference across multiple easy-cass-stress runs with 14 | * different versions of easy-cass-stress as the implementation may change! 15 | * 16 | * Load test for a case where we have a dataset that requires LWT for a status update type workload 17 | * This could be a lock on status or a state machine in the real world 18 | * 19 | * For this test, we'll use the following states 20 | * 21 | * 0: normal 22 | * 1: temporarily locked 23 | */ 24 | class Locking : IStressProfile { 25 | 26 | 27 | 28 | lateinit var insert: PreparedStatement 29 | lateinit var update: PreparedStatement 30 | lateinit var select: PreparedStatement 31 | lateinit var delete: PreparedStatement 32 | 33 | var log = logger() 34 | 35 | override fun prepare(session: Session) { 36 | insert = session.prepare("INSERT INTO lwtupdates (item_id, name, status) VALUES (?, ?, 0)") 37 | update = session.prepare("UPDATE lwtupdates set status = ? WHERE item_id = ? IF status = ?") 38 | select = session.prepare("SELECT * from lwtupdates where item_id = ?") 39 | delete = session.prepare("DELETE from lwtupdates where item_id = ? IF EXISTS") 40 | } 41 | 42 | override fun schema(): List { 43 | val query = """ 44 | CREATE TABLE IF NOT EXISTS lwtupdates ( 45 | item_id text primary key, 46 | name text, 47 | status int 48 | ) 49 | """.trimIndent() 50 | return listOf(query) 51 | } 52 | 53 | override fun getPopulateOption(args: Run) : PopulateOption = PopulateOption.Custom(args.partitionValues, deletes = false) 54 | 55 | override fun getPopulatePartitionKeyGenerator(): Optional { 56 | return Optional.of(PartitionKeyGenerator.sequence("test")) 57 | } 58 | 59 | 60 | override fun getRunner(context: StressContext): IStressRunner { 61 | return object : IStressRunner { 62 | 63 | // this test can't do more than 2 billion partition keys 64 | 65 | val state : ConcurrentHashMap = ConcurrentHashMap(context.mainArguments.partitionValues.toInt()) 66 | 67 | override fun getNextMutation(partitionKey: PartitionKey): Operation { 68 | val currentState = state.getOrDefault(partitionKey.getText(), 0) 69 | 70 | val newState = when(currentState) { 71 | 0 -> 1 72 | else -> 0 73 | } 74 | 75 | log.trace{"Updating ${partitionKey.getText()} to $newState"} 76 | 77 | val bound = update.bind(newState, partitionKey.getText(), newState) 78 | state[partitionKey.getText()] = newState 79 | return Operation.Mutation(bound) 80 | } 81 | 82 | override fun getNextSelect(partitionKey: PartitionKey): Operation { 83 | val bound = select.bind(partitionKey.getText()) 84 | return Operation.SelectStatement(bound) 85 | } 86 | 87 | override fun getNextDelete(partitionKey: PartitionKey): Operation { 88 | val bound = delete.bind(partitionKey.getText()) 89 | return Operation.Deletion(bound) 90 | } 91 | 92 | override fun getNextPopulate(partitionKey: PartitionKey): Operation { 93 | val bound = insert.bind(partitionKey.getText(), "test") 94 | return Operation.Mutation(bound) 95 | } 96 | 97 | 98 | 99 | 100 | } 101 | } 102 | 103 | 104 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/Plugin.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import com.rustyrazorblade.easycassstress.profiles.IStressProfile 4 | import org.apache.logging.log4j.kotlin.logger 5 | import org.reflections.Reflections 6 | import java.lang.Exception 7 | import java.util.* 8 | import kotlin.reflect.KMutableProperty 9 | import kotlin.reflect.KProperty1 10 | import kotlin.reflect.full.* 11 | import kotlin.reflect.jvm.javaType 12 | 13 | /** 14 | * Wrapper for Stress Profile Plugins 15 | * Anything found in the class path will be returned. 16 | * TODO: Add a caching layer to prevent absurdly slow 17 | * reflection time 18 | */ 19 | 20 | data class Plugin (val name: String, 21 | val cls: Class, 22 | val instance: IStressProfile) { 23 | 24 | data class WorkloadParameterType(val name: String, val description: String, val type: String) 25 | 26 | override fun toString() = name 27 | 28 | companion object { 29 | 30 | val log = logger() 31 | 32 | fun getPlugins() : Map { 33 | val r = Reflections("com.rustyrazorblade.easycassstress") 34 | val modules = r.getSubTypesOf(IStressProfile::class.java) 35 | 36 | 37 | var result = sortedMapOf() 38 | 39 | for(m in modules) { 40 | val instance = m.getConstructor().newInstance() 41 | // val args = instance.getArguments() 42 | val tmp = Plugin(m.simpleName, m, instance) 43 | result[m.simpleName] = tmp 44 | } 45 | 46 | return result 47 | } 48 | 49 | 50 | 51 | } 52 | 53 | /** 54 | * Takes the parameters passed in via the dynamic --workload. flag 55 | * and assigns the values to the instance 56 | */ 57 | fun applyDynamicSettings(workloadParameters: Map) { 58 | 59 | for((key, value) in workloadParameters) { 60 | var prop = getProperty(key) as KMutableProperty<*> 61 | val annotation = prop.findAnnotation() 62 | log.debug("Annotation for $key found: $annotation") 63 | 64 | // Int 65 | if(prop.returnType.isSubtypeOf(Int::class.createType())) { 66 | log.debug("Found the type, we have an int, setting the value") 67 | prop.setter.call(instance, value.toInt()) 68 | continue 69 | } 70 | 71 | // String 72 | if(prop.returnType.isSubtypeOf(String::class.createType())) { 73 | log.debug("Found the type, we have a String, setting the value") 74 | prop.setter.call(instance, value) 75 | } 76 | 77 | // Boolean 78 | if(prop.returnType.isSubtypeOf(Boolean::class.createType())) { 79 | log.debug("Found the type, we have a Boolean, setting the value") 80 | prop.setter.call(instance, value.toBoolean()) 81 | } 82 | 83 | } 84 | 85 | } 86 | 87 | fun getProperty(name: String) = 88 | instance::class 89 | .declaredMemberProperties 90 | .filter { it.name == name } 91 | .first() 92 | 93 | 94 | fun getAnnotation(field: KProperty1): Optional { 95 | val tmp = field.annotations.filter { it is WorkloadParameter } 96 | 97 | return if(tmp.size == 1) 98 | Optional.of(tmp.first()) 99 | else 100 | Optional.empty() 101 | 102 | } 103 | 104 | /** 105 | * Returns the name and description 106 | * This code is a bit hairy... 107 | */ 108 | fun getCustomParams() : List { 109 | val result = mutableListOf() 110 | 111 | for(prop in instance::class.declaredMemberProperties) { 112 | 113 | (prop.annotations.firstOrNull { it.annotationClass == WorkloadParameter::class } as? WorkloadParameter)?.run { 114 | result.add(WorkloadParameterType(prop.name, description, prop.returnType.toString() )) 115 | } 116 | 117 | } 118 | return result 119 | } 120 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/profiles/BasicTimeSeries.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.profiles 2 | 3 | import com.datastax.driver.core.PreparedStatement 4 | import com.datastax.driver.core.Session 5 | import com.datastax.driver.core.VersionNumber 6 | import com.datastax.driver.core.utils.UUIDs 7 | import com.rustyrazorblade.easycassstress.PartitionKey 8 | import com.rustyrazorblade.easycassstress.StressContext 9 | import com.rustyrazorblade.easycassstress.WorkloadParameter 10 | import com.rustyrazorblade.easycassstress.generators.* 11 | import com.rustyrazorblade.easycassstress.generators.functions.Random 12 | import java.lang.UnsupportedOperationException 13 | import java.sql.Timestamp 14 | import java.time.LocalDateTime 15 | 16 | 17 | /** 18 | * Create a simple time series use case with some number of partitions 19 | * TODO make it use TWCS 20 | */ 21 | class BasicTimeSeries : IStressProfile { 22 | 23 | override fun schema(): List { 24 | val query = """CREATE TABLE IF NOT EXISTS sensor_data ( 25 | sensor_id text, 26 | timestamp timeuuid, 27 | data text, 28 | primary key(sensor_id, timestamp)) 29 | WITH CLUSTERING ORDER BY (timestamp DESC) 30 | """.trimIndent() 31 | 32 | return listOf(query) 33 | } 34 | 35 | lateinit var prepared: PreparedStatement 36 | lateinit var getPartitionHead: PreparedStatement 37 | lateinit var delete: PreparedStatement 38 | lateinit var cassandraVersion: VersionNumber 39 | 40 | 41 | @WorkloadParameter("Number of rows to fetch back on SELECT queries") 42 | var limit = 500 43 | 44 | @WorkloadParameter("Deletion range in seconds. Range tombstones will cover all rows older than the given value.") 45 | var deleteDepth = 30 46 | 47 | @WorkloadParameter("Insert TTL") 48 | var ttl = 0 49 | 50 | override fun prepare(session: Session) { 51 | println("Using a limit of $limit for reads and deleting data older than $deleteDepth seconds (if enabled).") 52 | cassandraVersion = session.cluster.metadata.allHosts.map { host -> host.cassandraVersion }.min()!! 53 | var ttlStr = if (ttl > 0) { 54 | " USING TTL $ttl" 55 | } else "" 56 | 57 | prepared = session.prepare("INSERT INTO sensor_data (sensor_id, timestamp, data) VALUES (?, ?, ?) $ttlStr") 58 | getPartitionHead = session.prepare("SELECT * from sensor_data WHERE sensor_id = ? LIMIT ?") 59 | if (cassandraVersion.compareTo(VersionNumber.parse("3.0")) >= 0) { 60 | delete = session.prepare("DELETE from sensor_data WHERE sensor_id = ? and timestamp < maxTimeuuid(?)") 61 | } else { 62 | throw UnsupportedOperationException("Cassandra version $cassandraVersion does not support range deletes (only available in 3.0+).") 63 | } 64 | } 65 | 66 | /** 67 | * need to fix custom arguments 68 | */ 69 | override fun getRunner(context: StressContext): IStressRunner { 70 | 71 | val dataField = context.registry.getGenerator("sensor_data", "data") 72 | 73 | return object : IStressRunner { 74 | 75 | override fun getNextSelect(partitionKey: PartitionKey): Operation { 76 | val bound = getPartitionHead.bind(partitionKey.getText(), limit) 77 | return Operation.SelectStatement(bound) 78 | } 79 | 80 | override fun getNextMutation(partitionKey: PartitionKey) : Operation { 81 | val data = dataField.getText() 82 | val timestamp = UUIDs.timeBased() 83 | val bound = prepared.bind(partitionKey.getText(),timestamp, data) 84 | return Operation.Mutation(bound) 85 | } 86 | 87 | override fun getNextDelete(partitionKey: PartitionKey): Operation { 88 | val bound = delete.bind(partitionKey.getText(), Timestamp.valueOf(LocalDateTime.now().minusSeconds(deleteDepth.toLong()))) 89 | return Operation.Deletion(bound) 90 | } 91 | } 92 | } 93 | 94 | override fun getFieldGenerators(): Map { 95 | return mapOf(Field("sensor_data", "data") to Random().apply { min=100; max=200 }) 96 | } 97 | 98 | 99 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/profiles/DSESearch.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.profiles 2 | 3 | import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper 4 | 5 | import com.datastax.driver.core.PreparedStatement 6 | import com.datastax.driver.core.Session 7 | import com.fasterxml.jackson.annotation.JsonInclude 8 | import com.rustyrazorblade.easycassstress.PartitionKey 9 | import com.rustyrazorblade.easycassstress.StressContext 10 | import com.rustyrazorblade.easycassstress.WorkloadParameter 11 | import com.rustyrazorblade.easycassstress.generators.Field 12 | import com.rustyrazorblade.easycassstress.generators.FieldFactory 13 | import com.rustyrazorblade.easycassstress.generators.FieldGenerator 14 | import com.rustyrazorblade.easycassstress.generators.functions.Book 15 | import com.rustyrazorblade.easycassstress.generators.functions.Random 16 | import java.util.concurrent.ThreadLocalRandom 17 | 18 | 19 | class DSESearch : IStressProfile { 20 | 21 | val TABLE: String = "dse_search" 22 | val MIN_VALUE_TEXT_SIZE=5 23 | val MAX_VALUE_TEXT_SIZE=10 24 | 25 | lateinit var insert: PreparedStatement 26 | lateinit var select: PreparedStatement 27 | lateinit var delete: PreparedStatement 28 | 29 | val mapper = jacksonObjectMapper() 30 | 31 | @WorkloadParameter("Enable global queries.") 32 | var global = false 33 | 34 | @WorkloadParameter(description = "Max rows per partition") 35 | var rows = 10000 36 | 37 | override fun prepare(session: Session) { 38 | insert = session.prepare("INSERT INTO $TABLE (key, c, value_text) VALUES (?, ?, ?)") 39 | select = session.prepare("SELECT key, c, value_text from $TABLE WHERE solr_query = ?") 40 | 41 | delete = session.prepare("DELETE from $TABLE WHERE key = ? and c = ?") 42 | } 43 | 44 | override fun schema(): List { 45 | return listOf("""CREATE TABLE IF NOT EXISTS $TABLE ( 46 | key text, 47 | c int, 48 | value_text text, 49 | PRIMARY KEY (key, c) 50 | )""".trimIndent(), 51 | """CREATE SEARCH INDEX IF NOT EXISTS ON $TABLE WITH COLUMNS value_text 52 | """.trimIndent()) 53 | } 54 | 55 | override fun getRunner(context: StressContext): IStressRunner { 56 | val value = context.registry.getGenerator(TABLE, "value_text") 57 | val regex = "[^a-zA-Z0-9]".toRegex() 58 | 59 | @JsonInclude(JsonInclude.Include.NON_EMPTY) 60 | data class SolrQuery( 61 | var q: String, 62 | var fq: String 63 | ) 64 | 65 | return object : IStressRunner { 66 | 67 | val c_id = ThreadLocalRandom.current() 68 | val nextRowId : Int get() = c_id.nextInt(0, rows) 69 | 70 | override fun getNextMutation(partitionKey: PartitionKey): Operation { 71 | val bound = insert.bind(partitionKey.getText(), nextRowId, value.getText()) 72 | return Operation.Mutation(bound) 73 | } 74 | 75 | override fun getNextSelect(partitionKey: PartitionKey): Operation { 76 | val valueValue = value.getText().substringBeforeLast(" ") 77 | .replace(regex, " ") 78 | .trim() 79 | 80 | 81 | val query = SolrQuery(q= "value_text:($valueValue)", 82 | fq = if (!global) "key:${partitionKey.getText()}" else "" 83 | ) 84 | 85 | val queryString = mapper.writeValueAsString(query) 86 | 87 | val bound = select.bind(queryString) 88 | return Operation.SelectStatement(bound) 89 | } 90 | 91 | override fun getNextDelete(partitionKey: PartitionKey) = 92 | Operation.Deletion(delete.bind(partitionKey.getText(), nextRowId)) 93 | 94 | 95 | } 96 | } 97 | 98 | override fun getFieldGenerators(): Map { 99 | val search = FieldFactory(TABLE) 100 | return mapOf(Field(TABLE, "value_text") to Book().apply { min=MIN_VALUE_TEXT_SIZE; max=MAX_VALUE_TEXT_SIZE }, 101 | Field(TABLE, "value_int") to Random().apply {min=0; max=100}) 102 | } 103 | 104 | 105 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/profiles/IStressProfile.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.profiles 2 | 3 | import com.datastax.driver.core.Session 4 | import com.datastax.driver.core.BoundStatement 5 | import com.datastax.driver.core.ResultSet 6 | import com.rustyrazorblade.easycassstress.PartitionKey 7 | import com.rustyrazorblade.easycassstress.PopulateOption 8 | import com.rustyrazorblade.easycassstress.StressContext 9 | import com.rustyrazorblade.easycassstress.commands.Run 10 | import com.rustyrazorblade.easycassstress.generators.FieldGenerator 11 | import com.rustyrazorblade.easycassstress.generators.Field 12 | import com.codahale.metrics.Timer.Context 13 | import com.rustyrazorblade.easycassstress.PartitionKeyGenerator 14 | import java.util.* 15 | 16 | interface IStressRunner { 17 | fun getNextMutation(partitionKey: PartitionKey) : Operation 18 | fun getNextSelect(partitionKey: PartitionKey) : Operation 19 | fun getNextDelete(partitionKey: PartitionKey) : Operation 20 | 21 | /** 22 | * Populate phase will typically just perform regular mutations. 23 | * However, certain workloads may need custom setup. 24 | * @see Locking 25 | **/ 26 | fun getNextPopulate(partitionKey: PartitionKey) : Operation { 27 | return getNextMutation(partitionKey) 28 | } 29 | /** 30 | * Callback after a query executes successfully. 31 | * Will be used for state tracking on things like LWTs as well as provides an avenue for future work 32 | * doing post-workload correctness checks 33 | */ 34 | fun onSuccess(op: Operation.Mutation, result: ResultSet?) { } 35 | 36 | } 37 | 38 | /** 39 | * Stress profile interface. A stress profile defines the schema, prepared 40 | * statements, and queries that will be executed. It should be fairly trivial 41 | * to imp 42 | */ 43 | interface IStressProfile { 44 | /** 45 | * Handles any prepared statements that are needed 46 | * the class should track all prepared statements internally 47 | * and pass them on to the Runner 48 | */ 49 | fun prepare(session: Session) 50 | /** 51 | * returns a bunch of DDL statements 52 | * this can be any valid DDL such as 53 | * CREATE table, index, materialized view, etc 54 | * for most tests this is probably a single table 55 | * it's OK to put a clustering order in, but otherwise the schema 56 | * should not specify any other options here because they can all 57 | * but supplied on the command line. 58 | * 59 | * I may introduce a means of supplying default values, because 60 | * there are plenty of use cases where you would want a specific 61 | * compaction strategy most of the time (like a time series, or a cache) 62 | */ 63 | fun schema(): List 64 | 65 | /** 66 | * returns an instance of the stress runner for this particular class 67 | * This was done to allow a single instance of an IStress profile to be 68 | * generated, and passed to the ProfileRunner. 69 | * The issue is that the profile needs to generate a single schema 70 | * but then needs to create multiple stress runners 71 | * this allows the code to be a little cleaner 72 | */ 73 | fun getRunner(context: StressContext): IStressRunner 74 | 75 | /** 76 | * returns a map of generators cooresponding to the different fields 77 | * it's required to specify all fields that use a generator 78 | * some fields don't, like TimeUUID or the first partition key 79 | * This is optional, but encouraged 80 | * 81 | * A profile can technically do whatever it wants, no one is obligated to use the generator 82 | * Using this does give the flexibility of specifying a different generator however 83 | * In the case of text fields, this is VERY strongly encouraged to allow for more flexibility with the size 84 | * of the text payload 85 | */ 86 | fun getFieldGenerators() : Map = mapOf() 87 | 88 | fun getDefaultReadRate() : Double { return .01 } 89 | 90 | fun getPopulateOption(args: Run) : PopulateOption = PopulateOption.Standard() 91 | 92 | fun getPopulatePartitionKeyGenerator(): Optional = 93 | Optional.empty() 94 | 95 | 96 | } 97 | 98 | 99 | sealed class Operation(val bound: BoundStatement?) { 100 | // we're going to track metrics on the mutations differently 101 | // inserts will also carry data that might be saved for later validation 102 | // clustering keys won't be realistic to compute in the framework 103 | lateinit var startTime: Context 104 | 105 | class Mutation(bound: BoundStatement, val callbackPayload: Any? = null) : Operation(bound) 106 | 107 | class SelectStatement(bound: BoundStatement): Operation(bound) 108 | 109 | class Deletion(bound: BoundStatement): Operation(bound) 110 | 111 | class Stop : Operation(null) 112 | 113 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/RequestQueue.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import com.rustyrazorblade.easycassstress.profiles.IStressRunner 4 | import com.rustyrazorblade.easycassstress.profiles.Operation 5 | import com.codahale.metrics.Timer 6 | import org.apache.logging.log4j.kotlin.logger 7 | import java.time.LocalDateTime 8 | import java.util.concurrent.ArrayBlockingQueue 9 | import java.util.concurrent.ThreadLocalRandom 10 | import kotlin.concurrent.thread 11 | 12 | /** 13 | * Request adds ability to set a proper rate limiter and addresses 14 | * the issue of coordinated omission. 15 | */ 16 | class RequestQueue( 17 | 18 | private val partitionKeyGenerator: PartitionKeyGenerator, 19 | context: StressContext, 20 | totalValues: Long, 21 | duration: Long, 22 | runner: IStressRunner, 23 | readRate: Double, 24 | deleteRate: Double, 25 | populatePhase: Boolean = false, 26 | ) { 27 | 28 | val queue = ArrayBlockingQueue(context.mainArguments.queueDepth.toInt(), true); 29 | var generatorThread : Thread 30 | 31 | companion object { 32 | val log = logger() 33 | } 34 | 35 | init { 36 | 37 | generatorThread = thread(start=false) { 38 | // hack to ensure we don't start before the process is ready 39 | 40 | val desiredEndTime = LocalDateTime.now().plusMinutes(duration) 41 | var executed = 0L 42 | log.info("populate=$populatePhase total values: $totalValues, duration: $duration") 43 | 44 | // we're using a separate timer for populate phase 45 | // regardless of the operation performed 46 | fun getTimer(operation: Operation) : Timer { 47 | return if (populatePhase) 48 | context.metrics.populate 49 | else when (operation) { 50 | is Operation.SelectStatement -> context.metrics.selects 51 | is Operation.Mutation -> context.metrics.mutations 52 | is Operation.Deletion -> context.metrics.deletions 53 | is Operation.Stop -> throw OperationStopException() 54 | } 55 | } 56 | 57 | for (key in partitionKeyGenerator.generateKey(totalValues, context.mainArguments.partitionValues)) { 58 | if (duration > 0 && desiredEndTime.isBefore(LocalDateTime.now())) { 59 | log.info("Reached duration, ending") 60 | break 61 | } 62 | 63 | if (totalValues > 0 && executed == totalValues) { 64 | log.info("Reached total values $totalValues") 65 | break 66 | } 67 | 68 | // check if we hit our limit 69 | // get next thing from the profile 70 | // thing could be a statement, or it could be a failure command 71 | // certain profiles will want to deterministically inject failures 72 | // others can be randomly injected by the runner 73 | // I should be able to just tell the runner to inject gossip failures in any test 74 | // without having to write that code in the profile 75 | 76 | val nextOp = ThreadLocalRandom.current().nextInt(0, 100) 77 | 78 | context.rateLimiter?.run { 79 | acquire(1) 80 | } 81 | 82 | // we only do the mutations in non-populate run 83 | val op = if ( !populatePhase && readRate * 100 > nextOp) { 84 | runner.getNextSelect(key) 85 | } else if (deleteRate > 0.0 && (readRate * 100) + (deleteRate * 100) > nextOp) { 86 | // we might be in a populate phase but only if the user specifically requested it 87 | runner.getNextDelete(key) 88 | } else if (populatePhase) { 89 | // at this point we're either populating or we're in a normal run 90 | runner.getNextPopulate(key) 91 | } else { 92 | runner.getNextMutation(key) 93 | } 94 | 95 | 96 | 97 | op.startTime=getTimer(op).time() 98 | 99 | if(!queue.offer(op)) { 100 | context.metrics.errors.mark() 101 | } 102 | executed++ 103 | } 104 | 105 | // wait for the queue to drain 106 | 107 | log.info("Finished queuing requests, waiting for queue to empty. $executed executed") 108 | Thread.sleep(1000) 109 | while (queue.size > 0) { 110 | Thread.sleep(1000) 111 | } 112 | queue.add(Operation.Stop()) 113 | } 114 | } 115 | 116 | fun getNextOperation() = sequence { 117 | while (generatorThread.isAlive) { 118 | when (val tmp = queue.take()) { 119 | is Operation.Stop -> break 120 | else -> yield(tmp) 121 | } 122 | } 123 | } 124 | 125 | fun start() { 126 | generatorThread.start() 127 | } 128 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/profiles/RandomPartitionAccess.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.profiles 2 | 3 | import com.datastax.driver.core.PreparedStatement 4 | import com.datastax.driver.core.Session 5 | import com.rustyrazorblade.easycassstress.PartitionKey 6 | import com.rustyrazorblade.easycassstress.StressContext 7 | import com.rustyrazorblade.easycassstress.WorkloadParameter 8 | import com.rustyrazorblade.easycassstress.generators.Field 9 | import com.rustyrazorblade.easycassstress.generators.FieldFactory 10 | import com.rustyrazorblade.easycassstress.generators.FieldGenerator 11 | import com.rustyrazorblade.easycassstress.generators.functions.Random 12 | import java.util.concurrent.ThreadLocalRandom 13 | 14 | class RandomPartitionAccess : IStressProfile { 15 | 16 | @WorkloadParameter(description = "Number of rows per partition, defaults to 100") 17 | var rows = 100 18 | 19 | @WorkloadParameter("Select random row or the entire partition. Acceptable values: row, partition") 20 | var select = "row" 21 | 22 | @WorkloadParameter("Delete random row or the entire partition. Acceptable values: row, partition") 23 | var delete = "row" 24 | 25 | lateinit var insert_query : PreparedStatement 26 | lateinit var select_query : PreparedStatement 27 | lateinit var delete_query : PreparedStatement 28 | 29 | override fun prepare(session: Session) { 30 | insert_query = session.prepare("INSERT INTO random_access (partition_id, row_id, value) values (?, ?, ?)") 31 | 32 | select_query = when(select) { 33 | 34 | "partition" -> { 35 | println("Preparing full partition reads") 36 | session.prepare("SELECT * from random_access WHERE partition_id = ?") 37 | } 38 | "row" -> { 39 | println("Preparing single row reads") 40 | session.prepare("SELECT * from random_access WHERE partition_id = ? and row_id = ?") 41 | } 42 | else -> 43 | throw RuntimeException("select must be row or partition.") 44 | } 45 | 46 | delete_query = when(delete) { 47 | 48 | "partition" -> { 49 | println("Preparing full partition deletes") 50 | session.prepare("DELETE FROM random_access WHERE partition_id = ?") 51 | } 52 | "row" -> { 53 | println("Preparing single row deletes") 54 | session.prepare("DELETE FROM random_access WHERE partition_id = ? and row_id = ?") 55 | } 56 | else -> 57 | throw RuntimeException("select must be row or partition.") 58 | } 59 | } 60 | 61 | override fun schema(): List { 62 | return listOf("""CREATE TABLE IF NOT EXISTS random_access ( 63 | partition_id text, 64 | row_id int, 65 | value text, 66 | primary key (partition_id, row_id) 67 | )""") 68 | } 69 | 70 | override fun getRunner(context: StressContext) : IStressRunner { 71 | 72 | println("Using $rows rows per partition") 73 | 74 | return object : IStressRunner { 75 | 76 | val value = context.registry.getGenerator("random_access", "value") 77 | val random = ThreadLocalRandom.current() 78 | 79 | 80 | override fun getNextMutation(partitionKey: PartitionKey): Operation { 81 | val rowId = random.nextInt(0, rows) 82 | val bound = insert_query.bind(partitionKey.getText(), 83 | rowId, value.getText()) 84 | return Operation.Mutation(bound) 85 | } 86 | 87 | override fun getNextSelect(partitionKey: PartitionKey): Operation { 88 | val bound = when(select) { 89 | 90 | "partition" -> 91 | select_query.bind(partitionKey.getText()) 92 | "row" -> { 93 | val rowId = random.nextInt(0, rows) 94 | select_query.bind(partitionKey.getText(), rowId) 95 | } 96 | else -> throw RuntimeException("not even sure how you got here") 97 | 98 | } 99 | return Operation.SelectStatement(bound) 100 | 101 | } 102 | 103 | override fun getNextDelete(partitionKey: PartitionKey): Operation { 104 | val bound = when(delete) { 105 | "partition" -> 106 | delete_query.bind(partitionKey.getText()) 107 | "row" -> { 108 | val rowId = random.nextInt(0, rows) 109 | delete_query.bind(partitionKey.getText(), rowId) 110 | } 111 | else -> throw RuntimeException("not even sure how you got here") 112 | 113 | } 114 | return Operation.Deletion(bound) 115 | } 116 | } 117 | } 118 | 119 | override fun getFieldGenerators(): Map { 120 | val ra = FieldFactory("random_access") 121 | return mapOf(ra.getField("value") to Random().apply{ min=100; max=200}) 122 | } 123 | 124 | } -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Java Gradle CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-java/ for more details 4 | # 5 | version: 2.1 6 | 7 | aliases: 8 | base_job8: &base_job8 9 | docker: 10 | # specify the version you desire here 11 | - image: circleci/openjdk:8-jdk 12 | 13 | working_directory: ~/repo 14 | 15 | environment: 16 | TERM: dumb 17 | 18 | base_job11: &base_job11 19 | docker: 20 | # specify the version you desire here 21 | - image: circleci/openjdk:11-jdk 22 | 23 | working_directory: ~/repo 24 | 25 | environment: 26 | TERM: dumb 27 | 28 | commands: 29 | install_ccm: 30 | description: "Installs and starts CCM" 31 | parameters: 32 | version: 33 | type: string 34 | default: "2.2.14" 35 | cluster_name: 36 | type: string 37 | default: "test" 38 | 39 | steps: 40 | - restore_cache: 41 | keys: 42 | - ccm 43 | 44 | - run: sudo apt-get update -qq 45 | - run: sudo apt-get install -y libjna-java python-dev python3-pip libyaml-dev nodejs 46 | - run: sudo pip install pyYaml ccm 47 | - run: ccm create test -n 1 -v << parameters.version >> 48 | - run: 49 | name: "Adjust cluster parameters" 50 | command: | 51 | for i in `seq 1 1` ; do 52 | sed -i 's/#MAX_HEAP_SIZE="4G"/MAX_HEAP_SIZE="256m"/' ~/.ccm/test/node$i/conf/cassandra-env.sh 53 | sed -i 's/#HEAP_NEWSIZE="800M"/HEAP_NEWSIZE="128m"/' ~/.ccm/test/node$i/conf/cassandra-env.sh 54 | sed -i 's/num_tokens: 256/num_tokens: 1/' ~/.ccm/test/node$i/conf/cassandra.yaml 55 | echo 'phi_convict_threshold: 16' >> ~/.ccm/test/node$i/conf/cassandra.yaml 56 | sed -i 's/concurrent_reads: 32/concurrent_reads: 4/' ~/.ccm/test/node$i/conf/cassandra.yaml 57 | sed -i 's/concurrent_writes: 32/concurrent_writes: 4/' ~/.ccm/test/node$i/conf/cassandra.yaml 58 | sed -i 's/concurrent_counter_writes: 32/concurrent_counter_writes: 4/' ~/.ccm/test/node$i/conf/cassandra.yaml 59 | sed -i 's/# file_cache_size_in_mb: 512/file_cache_size_in_mb: 1/' ~/.ccm/test/node$i/conf/cassandra.yaml 60 | sed -i 's/enable_materialized_views: false/enable_materialized_views: true/' ~/.ccm/test/node$i/conf/cassandra.yaml 61 | done 62 | ccm start -v 63 | sleep 5 64 | ccm status 65 | ccm checklogerror 66 | ./gradlew test 67 | 68 | gradle_test: 69 | description: "Run gradle test and deal with caches" 70 | steps: 71 | - restore_cache: 72 | keys: 73 | - gradle-deps-{{ checksum "build.gradle" }} 74 | - run: ./gradlew test -i 75 | - save_cache: 76 | key: gradle-deps-{{ checksum "build.gradle" }} 77 | paths: 78 | - /home/circleci/.gradle/ 79 | 80 | build_packages: 81 | description: "Build packages through gradle" 82 | steps: 83 | - run: ./gradlew buildAll 84 | 85 | jobs: 86 | 87 | build_with_java8: 88 | <<: *base_job8 89 | 90 | steps: 91 | - checkout 92 | - run: sudo apt-get update -qq 93 | - run: sudo apt-get install -y libjna-java python-dev python3-pip libyaml-dev nodejs 94 | - run: sudo pip install pyYaml ccm 95 | 96 | - run: 97 | name: Download CCM for Cache 98 | command: | 99 | sudo apt-get update -qq 100 | sudo apt-get install -y libjna-java python-dev python3-pip libyaml-dev nodejs 101 | sudo pip install pyYaml ccm 102 | ccm create cassandra_30 --no-switch -v 3.0.17 103 | ccm create cassandra_311 --no-switch -v 3.11.4 104 | 105 | - save_cache: 106 | key: ccm 107 | paths: 108 | - ~/.ccm/repository 109 | 110 | - run: ./gradlew testClasses 111 | 112 | build_with_java11: 113 | <<: *base_job11 114 | 115 | steps: 116 | - checkout 117 | - run: sudo apt-get update -qq 118 | - run: sudo apt-get install -y libjna-java python-dev python3-pip libyaml-dev nodejs 119 | - run: sudo pip3 install pyYaml ccm 120 | 121 | - run: 122 | name: Download CCM for Cache 123 | command: | 124 | sudo apt-get update -qq 125 | sudo apt-get install -y libjna-java python-dev python3-pip libyaml-dev nodejs 126 | sudo pip3 install pyYaml ccm 127 | ccm create cassandra_40 --no-switch -v 4.0.6 128 | 129 | - save_cache: 130 | key: ccm 131 | paths: 132 | - ~/.ccm/repository 133 | 134 | - run: ./gradlew testClasses 135 | 136 | cassandra_30: 137 | <<: *base_job8 138 | 139 | steps: 140 | - checkout 141 | - install_ccm: 142 | version: 3.0.17 143 | 144 | cassandra_311: 145 | <<: *base_job8 146 | 147 | steps: 148 | - checkout 149 | - install_ccm: 150 | version: 3.11.4 151 | 152 | cassandra_40: 153 | <<: *base_job8 154 | 155 | steps: 156 | - checkout 157 | - install_ccm: 158 | version: 4.0.6 159 | 160 | build_packages: 161 | <<: *base_job8 162 | 163 | steps: 164 | - checkout 165 | - build_packages 166 | 167 | workflows: 168 | version: 2.1 169 | 170 | cassandra_30_java8: 171 | jobs: 172 | - build_with_java8 173 | - cassandra_30 174 | 175 | cassandra_311_java8: 176 | jobs: 177 | - build_with_java8 178 | - cassandra_311 179 | 180 | cassandra_40: 181 | jobs: 182 | - build_with_java11 183 | - cassandra_40 184 | 185 | build_everything: 186 | jobs: 187 | - build_packages -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/SingleLineConsoleReporter.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import com.codahale.metrics.* 4 | import com.codahale.metrics.Timer 5 | import java.text.DecimalFormat 6 | import java.util.* 7 | import java.util.concurrent.TimeUnit 8 | import java.util.concurrent.atomic.AtomicInteger 9 | import com.github.ajalt.mordant.TermColors 10 | import org.apache.logging.log4j.kotlin.logger 11 | 12 | class SingleLineConsoleReporter(registry: MetricRegistry) : ScheduledReporter(registry, 13 | "single-line-console-reporter", 14 | MetricFilter.ALL, 15 | TimeUnit.SECONDS, 16 | TimeUnit.MILLISECONDS 17 | ) { 18 | 19 | val logger = logger() 20 | var lines = 0L 21 | 22 | var opHeaders = listOf("Count", "Latency (p99)", "1min (req/s)") 23 | var width = mutableMapOf( ).withDefault { 0 } 24 | 25 | // initialize all the headers 26 | // it's ok if this isn't perfect, it just has to work for the first round of headers 27 | init { 28 | 29 | for ((i, h) in opHeaders.withIndex()) { 30 | getWidth(i, h) // first pass - writes 31 | getWidth(i + opHeaders.size, h) // second pass - reads 32 | } 33 | } 34 | 35 | 36 | val formatter = DecimalFormat("##.##") 37 | 38 | 39 | val termColors = TermColors() 40 | 41 | override fun report(gauges: SortedMap>?, 42 | counters: SortedMap?, 43 | histograms: SortedMap?, 44 | meters: SortedMap?, 45 | timers: SortedMap?) { 46 | 47 | 48 | if(lines % 10L == 0L) 49 | printHeader() 50 | 51 | val state = AtomicInteger() 52 | 53 | // this is a little weird, but we should show the same headers for writes & selects 54 | val queries = listOf(timers!!["mutations"]!!, timers["selects"]!!, timers["deletions"]!!) 55 | 56 | for(queryType in queries) { 57 | with(queryType) { 58 | printColumn(count, state.getAndIncrement()) 59 | 60 | val duration = convertDuration(snapshot.get99thPercentile()) 61 | 62 | printColumn(duration, state.getAndIncrement()) 63 | printColumn(formatter.format(oneMinuteRate), state.getAndIncrement()) 64 | 65 | } 66 | print(" | ") 67 | } 68 | 69 | val errors = meters!!["errors"]!! 70 | printColumn(errors.count, state.getAndIncrement()) 71 | printColumn(formatter.format(errors.oneMinuteRate), state.getAndIncrement()) 72 | 73 | 74 | println() 75 | lines++ 76 | } 77 | 78 | /* 79 | Helpers for printing the column with correct spacing 80 | */ 81 | fun printColumn(value: Double, index: Int) { 82 | // round to 2 decimal places 83 | val tmp = DecimalFormat("##.##").format(value) 84 | 85 | printColumn(tmp, index) 86 | } 87 | 88 | fun printColumn(value: Long, index: Int) { 89 | printColumn(value.toString(), index) 90 | } 91 | 92 | fun printColumn(value: Int, index: Int) { 93 | printColumn(value.toString(), index) 94 | } 95 | 96 | fun printColumn(value: String, index: Int) { 97 | val width = getWidth(index, value) 98 | val tmp = value.padStart(width) 99 | print(tmp) 100 | } 101 | 102 | fun printHeader() { 103 | 104 | var widthOfEachOperation = 0 105 | 106 | for(i in 0..opHeaders.size) { 107 | widthOfEachOperation += getWidth(i) 108 | } 109 | 110 | val paddingEachSide = (widthOfEachOperation - "Writes".length) / 2 - 1 111 | 112 | print(" ".repeat(paddingEachSide)) 113 | print( termColors.blue("Writes")) 114 | print(" ".repeat(paddingEachSide)) 115 | 116 | print(" ".repeat(paddingEachSide)) 117 | print(termColors.blue("Reads")) 118 | print(" ".repeat(paddingEachSide)) 119 | 120 | print(" ".repeat(paddingEachSide)) 121 | print(termColors.blue("Deletes")) 122 | print(" ".repeat(paddingEachSide)) 123 | 124 | print(" ".repeat(6)) 125 | print(termColors.red("Errors")) 126 | 127 | println() 128 | var i = 0 129 | 130 | for(x in 0..2) { 131 | 132 | for (h in opHeaders) { 133 | 134 | val colWidth = getWidth(i, h) 135 | val required = colWidth - h.length 136 | 137 | val tmp = " ".repeat(required) + termColors.underline(h) 138 | 139 | 140 | 141 | print(tmp) 142 | i++ 143 | } 144 | print(" | ") 145 | 146 | } 147 | 148 | val errorHeaders = arrayListOf("Count", "1min (errors/s)") 149 | // TODO: refactor this + the above loop to be a single function 150 | for (h in errorHeaders) { 151 | 152 | val colWidth = getWidth(i, h) 153 | val required = colWidth - h.length 154 | 155 | val tmp = " ".repeat(required) + termColors.underline(h) 156 | 157 | print(tmp) 158 | i++ 159 | } 160 | 161 | 162 | println() 163 | 164 | } 165 | 166 | /** 167 | * Gets the width for a column, resizing the column if necessary 168 | */ 169 | fun getWidth(i: Int, value : String = "") : Int { 170 | val tmp = width.getValue(i) 171 | if(value.length > tmp) { 172 | 173 | logger.debug("Resizing column[$i] to ${value.length}") 174 | // give a little extra padding in case the number grows quickly 175 | width.set(i, value.length + 2) 176 | } 177 | return width.getValue(i) 178 | } 179 | 180 | 181 | 182 | } 183 | -------------------------------------------------------------------------------- /src/test/kotlin/com/rustyrazorblade/easycassstress/SchemaBuilderTest.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import org.apache.logging.log4j.kotlin.logger 4 | import org.assertj.core.api.Assertions.assertThat 5 | import org.junit.jupiter.api.BeforeEach 6 | import org.junit.jupiter.api.Test 7 | import kotlin.test.assertFails 8 | import kotlin.test.fail 9 | 10 | internal class SchemaBuilderTest { 11 | var log = logger() 12 | 13 | lateinit var createTable : SchemaBuilder 14 | 15 | @BeforeEach 16 | fun setUp() { 17 | val statement = """CREATE TABLE test ( 18 | | id int primary key, 19 | | name text 20 | | ) """.trimMargin() 21 | 22 | createTable = SchemaBuilder.create(statement) 23 | } 24 | 25 | 26 | @Test 27 | fun compactionTest() { 28 | val result = createTable.withCompaction("{ 'class': 'LeveledCompactionStrategy', 'sstable_size_in_mb': 100}").build() 29 | assertThat(result).contains("sstable_size_in_mb': 100") 30 | assertThat(result).doesNotContain("compression") 31 | assertThat(result.toLowerCase()).containsOnlyOnce("with") 32 | } 33 | 34 | @Test 35 | fun compressionTest() { 36 | val c = "{'enabled':false}" 37 | val result = createTable.withCompression(c).build() 38 | assertThat(result).contains(c) 39 | } 40 | 41 | @Test 42 | fun clusteringOrderTest() { 43 | val base = """CREATE TABLE IF NOT EXISTS sensor_data ( 44 | |sensor_id int, 45 | |timestamp timeuuid, 46 | |data text, 47 | |primary key(sensor_id, timestamp)) 48 | |WITH CLUSTERING ORDER BY (timestamp DESC) 49 | | 50 | |""".trimMargin() 51 | 52 | val query = SchemaBuilder.create(base) 53 | .withCompression("{'enabled':enabled}") 54 | .build() 55 | 56 | assertThat(query.toLowerCase()).containsOnlyOnce("with") 57 | 58 | } 59 | 60 | @Test 61 | fun createTypeShouldNotHaveWithClause() { 62 | val query = """CREATE TYPE IF NOT EXISTS sensor_data_details ( 63 | data1 text, 64 | data2 text, 65 | data3 text 66 | )""" 67 | 68 | val result = SchemaBuilder.create(query) 69 | .withKeyCache("NONE") 70 | .build() 71 | 72 | assertThat(result).doesNotContain("WITH") 73 | } 74 | 75 | @Test 76 | fun ensureRegexFailsOnStupid() { 77 | var result = createTable.compactionShortcutRegex.find("stcsf") 78 | assertThat(result).isNull() 79 | } 80 | 81 | @Test 82 | fun ensureRegexMatchesBasic() { 83 | val result = createTable.compactionShortcutRegex.find("stcs")!!.groupValues 84 | assertThat(result[1]).isEqualTo("stcs") 85 | 86 | val result2 = createTable.compactionShortcutRegex.find("lcs")!!.groupValues 87 | assertThat(result2[1]).isEqualTo("lcs") 88 | 89 | val result3 = createTable.compactionShortcutRegex.find("twcs")!!.groupValues 90 | assertThat(result3[1]).isEqualTo("twcs") 91 | } 92 | 93 | 94 | @Test 95 | fun ensureRegexMatchesSTCSWithParams() { 96 | val result = createTable.compactionShortcutRegex.find("stcs,4,48")!!.groupValues 97 | assertThat(result[2]).isEqualTo(",4,48") 98 | } 99 | 100 | @Test 101 | fun testParseStcsComapctionReturnsStcs() { 102 | when (val compaction = createTable.parseCompaction("stcs")) { 103 | is SchemaBuilder.Compaction.STCS -> Unit 104 | else -> { 105 | fail("Expecting STCS, Got $compaction") 106 | } 107 | 108 | } 109 | } 110 | 111 | @Test 112 | fun testParseLCS() { 113 | when (val compaction = createTable.parseCompaction("lcs,120,8")) { 114 | is SchemaBuilder.Compaction.LCS -> { 115 | assertThat(compaction.fanoutSize).isEqualTo(8) 116 | assertThat(compaction.sstableSizeInMb).isEqualTo(120) 117 | } 118 | else -> { 119 | fail("Expecting LCS, Got $compaction") 120 | } 121 | 122 | } 123 | 124 | } 125 | @Test 126 | fun testParseTWCS() { 127 | val compaction = createTable.parseCompaction("twcs,1,days") 128 | when (compaction) { 129 | is SchemaBuilder.Compaction.TWCS -> { 130 | assertThat(compaction.windowSize).isEqualTo(1) 131 | assertThat(compaction.windowUnit).isEqualTo(SchemaBuilder.WindowUnit.DAYS) 132 | } 133 | else -> { 134 | fail("Expecting TWCS, 1 DAYS, Got $compaction") 135 | } 136 | } 137 | val cql = compaction.toCQL() 138 | 139 | } 140 | 141 | @Test 142 | fun testParseUCS() { 143 | val compaction = createTable.parseCompaction("ucs,T4") 144 | when (compaction) { 145 | is SchemaBuilder.Compaction.UCS -> { 146 | assertThat(compaction.scalingParameters).isEqualTo("T4") 147 | } 148 | else -> { 149 | fail("Expecting UC Got $compaction") 150 | } 151 | } 152 | } 153 | 154 | @Test 155 | fun tesstFullCompactionShortcut() { 156 | val result = createTable.withCompaction("lcs").build() 157 | assertThat(result).contains("LeveledCompactionStrategy") 158 | assertThat(result).doesNotContain("null") 159 | 160 | } 161 | 162 | @Test 163 | fun testTWCSEmptyWindow() { 164 | val result = createTable.withCompaction("twcs").build() 165 | assertThat(result).doesNotContain("compaction_window_unit") 166 | } 167 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/ProfileRunner.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import com.google.common.util.concurrent.Futures 4 | import com.google.common.util.concurrent.MoreExecutors 5 | import com.rustyrazorblade.easycassstress.profiles.IStressProfile 6 | import org.apache.logging.log4j.kotlin.logger 7 | import java.time.Duration 8 | import java.time.LocalTime 9 | import java.time.format.DateTimeFormatter 10 | import java.time.format.FormatStyle 11 | 12 | class PartitionKeyGeneratorException(e: String) : Exception() 13 | 14 | /** 15 | * Single threaded profile runner. 16 | * One profile runner should be created per thread 17 | * Logs all errors along the way 18 | * Keeps track of useful metrics, per thread 19 | */ 20 | class ProfileRunner(val context: StressContext, 21 | val profile: IStressProfile, 22 | val partitionKeyGenerator: PartitionKeyGenerator) { 23 | 24 | companion object { 25 | fun create(context: StressContext, profile: IStressProfile) : ProfileRunner { 26 | 27 | val partitionKeyGenerator = getGenerator(context, context.mainArguments.partitionKeyGenerator) 28 | 29 | return ProfileRunner(context, profile, partitionKeyGenerator) 30 | } 31 | 32 | fun getGenerator(context: StressContext, name: String) : PartitionKeyGenerator { 33 | val prefix = context.mainArguments.id + "." + context.thread + "." 34 | println("Creating generator $name") 35 | val partitionKeyGenerator = when(name) { 36 | "normal" -> PartitionKeyGenerator.normal(prefix) 37 | "random" -> PartitionKeyGenerator.random(prefix) 38 | "sequence" -> PartitionKeyGenerator.sequence(prefix) 39 | else -> throw PartitionKeyGeneratorException("not a valid generator") 40 | } 41 | return partitionKeyGenerator 42 | } 43 | 44 | val log = logger() 45 | } 46 | 47 | 48 | val readRate: Double 49 | 50 | init { 51 | val tmp = context.mainArguments.readRate 52 | 53 | if(tmp != null) { 54 | readRate = tmp 55 | } 56 | else { 57 | readRate = profile.getDefaultReadRate() 58 | } 59 | 60 | // check unsupported operations 61 | } 62 | 63 | val deleteRate: Double 64 | 65 | init { 66 | val tmp = context.mainArguments.deleteRate 67 | 68 | if(tmp != null) { 69 | deleteRate = tmp 70 | } 71 | else { 72 | deleteRate = 0.0 73 | } 74 | } 75 | 76 | fun print(message: String) { 77 | println("[Thread ${context.thread}]: $message") 78 | 79 | } 80 | 81 | 82 | /** 83 | 84 | */ 85 | fun run() { 86 | 87 | if (context.mainArguments.duration == 0L) { 88 | print("Running the profile for ${context.mainArguments.iterations} iterations...") 89 | } else { 90 | val startTime = LocalTime.now() 91 | val endTime = startTime.plus(Duration.ofMinutes(context.mainArguments.duration)) 92 | val formatter = DateTimeFormatter.ofLocalizedTime(FormatStyle.SHORT) 93 | 94 | print("Running the profile for ${context.mainArguments.duration}min (start: ${formatter.format(startTime)} end: ${formatter.format(endTime)})") 95 | } 96 | executeOperations(context.mainArguments.iterations, context.mainArguments.duration) 97 | } 98 | 99 | /** 100 | * Used for both pre-populating data and for performing the actual runner 101 | */ 102 | private fun executeOperations(iterations: Long, duration: Long) { 103 | // create a semaphore local to the thread to limit the query concurrency 104 | val runner = profile.getRunner(context) 105 | 106 | 107 | // we use MAX_VALUE since it's essentially infinite if we give a duration 108 | val totalValues = if (duration > 0) Long.MAX_VALUE else iterations 109 | 110 | // if we have a custom generator for the populate phase we'll use that 111 | 112 | val queue = RequestQueue(partitionKeyGenerator, context, totalValues, duration, runner, readRate, deleteRate) 113 | 114 | queue.start() 115 | 116 | // pull requests off the queue instead of using generateKey 117 | // move the getNextOperation into the queue thing 118 | for (op in queue.getNextOperation()) { 119 | val future = context.session.executeAsync(op.bound) 120 | Futures.addCallback(future, OperationCallback(context, runner, op, paginate = context.mainArguments.paginate), MoreExecutors.directExecutor()) 121 | } 122 | 123 | } 124 | 125 | 126 | /** 127 | * Prepopulates the database with numRows 128 | * Mutations only, does not count towards the normal metrics 129 | * Records all timers in the populateMutations metrics 130 | * Can (and should) be graphed separately 131 | */ 132 | fun populate(numRows: Long, deletes:Boolean = true) { 133 | 134 | val runner = profile.getRunner(context) 135 | 136 | val populatePartitionKeyGenerator = profile.getPopulatePartitionKeyGenerator().orElse(partitionKeyGenerator) 137 | 138 | val queue = RequestQueue(populatePartitionKeyGenerator, context, numRows, 0, runner, 0.0, 139 | if (deletes) deleteRate else 0.0, 140 | populatePhase = true) 141 | queue.start() 142 | 143 | try { 144 | for (op in queue.getNextOperation()) { 145 | val future = context.session.executeAsync(op.bound) 146 | Futures.addCallback( 147 | future, 148 | OperationCallback(context, runner, op, false), 149 | MoreExecutors.directExecutor() 150 | ) 151 | } 152 | } catch (_: OperationStopException) { 153 | log.info("Received Stop signal") 154 | Thread.sleep(3000) 155 | } catch (e: Exception) { 156 | log.warn("Received unknown exception ${e.message}") 157 | throw e 158 | } 159 | } 160 | 161 | 162 | fun prepare() { 163 | profile.prepare(context.session) 164 | } 165 | 166 | 167 | } -------------------------------------------------------------------------------- /manual/examples/easy-cass-stress-help.txt: -------------------------------------------------------------------------------- 1 | $ bin/easy-cass-stress 2 | WARNING: sun.reflect.Reflection.getCallerClass is not supported. This will impact performance. 3 | Usage: easy-cass-stress [options] [command] [command options] 4 | Options: 5 | --help, -h 6 | Shows this help. 7 | Default: false 8 | Commands: 9 | run Run a easy-cass-stress profile 10 | Usage: run [options] 11 | Options: 12 | --cl 13 | Consistency level for reads/writes (Defaults to LOCAL_ONE, set 14 | custom default with EASY_CASS_STRESS_CONSISTENCY_LEVEL). 15 | Default: LOCAL_ONE 16 | Possible Values: [ANY, ONE, TWO, THREE, QUORUM, ALL, LOCAL_QUORUM, EACH_QUORUM, SERIAL, LOCAL_SERIAL, LOCAL_ONE] 17 | --compaction 18 | Compaction option to use. Double quotes will auto convert to 19 | single for convenience. A shorthand is also available: stcs, lcs, 20 | twcs. See the full documentation for all possibilities. 21 | Default: 22 | --compression 23 | Compression options 24 | Default: 25 | --concurrency, -c 26 | DEPRECATED. Concurrent queries allowed. Increase for larger 27 | clusters. This flag is deprecated and does nothing. 28 | Default: 100 29 | --coordinatoronly, --co 30 | Coordinator only made. This will cause easy-cass-stress to round 31 | robin between nodes without tokens. Requires using 32 | -Djoin_ring=false in cassandra-env.sh. When using this option you 33 | must only provide a coordinator to --host. 34 | Default: false 35 | --core-connections 36 | Sets the number of core connections per host 37 | Default: 4 38 | --cql 39 | Additional CQL to run after the schema is created. Use for DDL 40 | modifications such as creating indexes. 41 | Default: [] 42 | --csv 43 | Write metrics to this file in CSV format. 44 | Default: 45 | --dc 46 | The data center to which requests should be sent 47 | Default: 48 | --deleterate, --deletes 49 | Deletion Rate, 0-1. Workloads may have their own defaults. 50 | Default is dependent on workload. 51 | --drop 52 | Drop the keyspace before starting. 53 | Default: false 54 | --duration, -d 55 | Duration of the stress test. Expressed in format 1d 3h 15m 56 | Default: 0 57 | --field. 58 | Override a field's data generator. Example usage: 59 | --field.tablename.fieldname='book(100,200)' 60 | Syntax: --field.key=value 61 | Default: {} 62 | --hdr 63 | Print HDR Histograms using this prefix 64 | Default: 65 | -h, --help 66 | Show this help 67 | --host 68 | Default: 127.0.0.1 69 | --id 70 | Identifier for this run, will be used in partition keys. Make 71 | unique for when starting concurrent runners. 72 | Default: 001 73 | --iterations, -i, -n 74 | Number of operations to run. 75 | Default: 0 76 | --keycache 77 | Key cache setting 78 | Default: ALL 79 | --keyspace 80 | Keyspace to use 81 | Default: easy_cass_stress 82 | --max-connections 83 | Sets the number of max connections per host 84 | Default: 8 85 | --max-requests 86 | Sets the max requests per connection 87 | Default: 32768 88 | --maxrlat 89 | Max Read Latency 90 | --maxwlat 91 | 92 | --no-schema 93 | Skips schema creation 94 | Default: false 95 | --paginate 96 | Paginate through the entire partition before completing 97 | Default: false 98 | --paging 99 | Override the driver's default page size. 100 | --partitiongenerator, --pg 101 | Method of generating partition keys. Supports random, normal 102 | (gaussian), and sequence. 103 | Default: random 104 | --partitions, -p 105 | Max value of integer component of first partition key. 106 | Default: 1000000 107 | --password, -P 108 | Default: cassandra 109 | --populate 110 | Pre-population the DB with N rows before starting load test. 111 | Default: 0 112 | --port 113 | Override the cql port. Defaults to 9042. 114 | Default: 9042 115 | --prometheusport 116 | Override the default prometheus port. 117 | Default: 9500 118 | --queue 119 | Queue Depth. 2x the rate by default. 120 | Default: 10000 121 | --rate 122 | Rate limiter, accepts human numbers. 0 = disabled 123 | Default: 5000 124 | --readrate, --reads, -r 125 | Read Rate, 0-1. Workloads may have their own defaults. Default 126 | is dependent on workload. 127 | --replication 128 | Replication options 129 | Default: {'class': 'SimpleStrategy', 'replication_factor':3 } 130 | --rowcache 131 | Row cache setting 132 | Default: NONE 133 | --ssl 134 | Enable SSL 135 | Default: false 136 | --threads, -t 137 | Threads to run 138 | Default: 1 139 | --ttl 140 | Table level TTL, 0 to disable. 141 | Default: 0 142 | --username, -U 143 | Default: cassandra 144 | --workload., -w. 145 | Override workload specific parameters. 146 | Syntax: --workload.key=value 147 | Default: {} 148 | 149 | info Get details of a specific workload. 150 | Usage: info 151 | 152 | list List all workloads. 153 | Usage: list 154 | 155 | fields null 156 | Usage: fields 157 | 158 | 159 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/profiles/SAI.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress.profiles 2 | 3 | import com.datastax.driver.core.PreparedStatement 4 | import com.datastax.driver.core.Session 5 | import com.rustyrazorblade.easycassstress.PartitionKey 6 | import com.rustyrazorblade.easycassstress.StressContext 7 | import com.rustyrazorblade.easycassstress.WorkloadParameter 8 | import com.rustyrazorblade.easycassstress.generators.Field 9 | import com.rustyrazorblade.easycassstress.generators.FieldGenerator 10 | import com.rustyrazorblade.easycassstress.generators.functions.Book 11 | import com.rustyrazorblade.easycassstress.generators.functions.LastName 12 | import com.rustyrazorblade.easycassstress.generators.functions.Random 13 | import org.apache.logging.log4j.kotlin.logger 14 | import java.util.concurrent.Executors 15 | import java.util.concurrent.ThreadLocalRandom 16 | import java.util.concurrent.ThreadPoolExecutor 17 | 18 | /** 19 | * Executes a SAI workload with queries restricted to a single partition, 20 | * which is the primary workload targeted by SAI indexes. 21 | */ 22 | 23 | const val TABLE : String = "sai" 24 | const val MIN_VALUE_TEXT_SIZE=1 25 | const val MAX_VALUE_TEXT_SIZE=2 26 | 27 | class SAI : IStressProfile { 28 | 29 | @WorkloadParameter(description = "Operator to use for SAI queries, defaults to equality = search.") 30 | var intCompare = "=" 31 | 32 | @WorkloadParameter(description = "Logic operator combining multiple predicates. Not yet supported." ) 33 | var operator = "AND" 34 | 35 | @WorkloadParameter(description = "Max rows per partition") 36 | var rows = 10000 37 | 38 | @WorkloadParameter(description = "Enable global queries with true to query the entire cluster.") 39 | var global = false 40 | 41 | @WorkloadParameter(description = "Fields to index, comma separated") 42 | var indexFields = "value_int,value_text" 43 | 44 | @WorkloadParameter(description = "Fields to search, comma separated") 45 | var searchFields = "value_text" 46 | 47 | @WorkloadParameter(description = "Limit count.") 48 | var limit = 0 49 | 50 | lateinit var insert: PreparedStatement 51 | lateinit var select: PreparedStatement 52 | lateinit var delete: PreparedStatement 53 | 54 | // mutable sets are backed by a LinkedHashSet, so we can preserve order 55 | lateinit var indexFieldsSet : Set 56 | lateinit var searchFieldsSet : Set 57 | 58 | val log = logger() 59 | override fun prepare(session: Session) { 60 | println("Preparing workload with global=$global") 61 | 62 | indexFieldsSet = indexFields.split("\\s*,\\s*".toRegex()).toSet() 63 | searchFieldsSet = searchFields.split("\\s*,\\s*".toRegex()).toSet() 64 | 65 | insert = session.prepare("INSERT INTO $TABLE (partition_id, c_id, value_text, value_int) VALUES (?, ?, ?, ?)") 66 | // todo make the operator configurable with a workload parameter 67 | 68 | val parts = mutableListOf() 69 | 70 | // if we're doing a global query we skip the partition key 71 | if (!global) { 72 | parts.add("partition_id = ?") 73 | } 74 | 75 | if (searchFieldsSet.contains("value_text")) { 76 | parts.add("value_text = ?") 77 | } 78 | if (searchFieldsSet.contains("value_int")) { 79 | parts.add("value_int $intCompare ?") 80 | } 81 | 82 | val limitClause = if (limit > 0) " LIMIT $limit " else "" 83 | val selectQuery = "SELECT * from $TABLE WHERE " + parts.joinToString(" $operator ") + limitClause 84 | println("Preparing $selectQuery") 85 | select = session.prepare(selectQuery) 86 | 87 | delete = session.prepare("DELETE from $TABLE WHERE partition_id = ? AND c_id = ?") 88 | } 89 | 90 | override fun schema(): List { 91 | val result = mutableListOf( 92 | """ 93 | CREATE TABLE IF NOT EXISTS $TABLE ( 94 | partition_id text, 95 | c_id int, 96 | value_text text, 97 | value_int int, 98 | primary key (partition_id, c_id) 99 | ) 100 | """.trimIndent() 101 | ) 102 | if (indexFields.contains("value_text") ) { 103 | result.add("CREATE INDEX IF NOT EXISTS ON $TABLE (value_text) USING 'sai'") 104 | } 105 | if (indexFields.contains("value_int")) { 106 | result.add("CREATE INDEX IF NOT EXISTS ON $TABLE (value_int) USING 'sai'") 107 | } 108 | return result 109 | } 110 | 111 | override fun getRunner(context: StressContext): IStressRunner { 112 | return object : IStressRunner { 113 | 114 | val c_id = ThreadLocalRandom.current() 115 | // use nextRowId 116 | val nextRowId : Int get() = c_id.nextInt(0, rows) 117 | 118 | // generator for the value field 119 | val value_text = context.registry.getGenerator(TABLE, "value_text") 120 | val value_int = context.registry.getGenerator(TABLE, "value_int") 121 | 122 | override fun getNextMutation(partitionKey: PartitionKey): Operation { 123 | val bound = insert.bind(partitionKey.getText(), nextRowId, value_text.getText(), value_int.getInt()) 124 | return Operation.Mutation(bound) 125 | } 126 | 127 | override fun getNextSelect(partitionKey: PartitionKey): Operation { 128 | // first bind the partition key 129 | val boundValues = mutableListOf() 130 | 131 | if (!global) { 132 | boundValues.add(partitionKey.getText()) 133 | } 134 | 135 | if (searchFieldsSet.contains("value_text")) { 136 | boundValues.add(value_text.getText()) 137 | } 138 | 139 | if (searchFieldsSet.contains("value_int")) { 140 | boundValues.add(value_int.getInt()) 141 | } 142 | 143 | val boundStatement = select.bind(*boundValues.toTypedArray()) 144 | return Operation.SelectStatement(boundStatement) 145 | } 146 | 147 | override fun getNextDelete(partitionKey: PartitionKey) = 148 | Operation.Deletion(delete.bind(partitionKey.getText(), nextRowId)) 149 | 150 | } 151 | } 152 | 153 | override fun getFieldGenerators(): Map { 154 | return mapOf(Field(TABLE, "value_text") to LastName(), 155 | Field(TABLE, "value_int") to Random().apply{ min=0; max=10000}) 156 | } 157 | } -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/RateLimiterOptimizer.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import com.google.common.util.concurrent.RateLimiter 4 | import org.apache.logging.log4j.kotlin.logger 5 | import java.lang.Math.cbrt 6 | import java.lang.Math.min 7 | import java.util.* 8 | import java.util.concurrent.TimeUnit 9 | import kotlin.math.sqrt 10 | 11 | /** 12 | * 13 | */ 14 | class RateLimiterOptimizer(val rateLimiter: RateLimiter, 15 | val metrics: Metrics, 16 | val maxReadLatency: Long?, 17 | val maxWriteLatency: Long?) { 18 | companion object { 19 | val log = logger() 20 | } 21 | val durationFactor = 1.0 / TimeUnit.MILLISECONDS.toNanos(1) 22 | 23 | val initial: Double = rateLimiter.rate 24 | val stepValue = initial / 10.0 25 | var isStepPhase = true 26 | 27 | init { 28 | println("Stepping rate limiter by $stepValue to $initial") 29 | } 30 | 31 | /** 32 | * Updates the rate limiter to use the new value and returns the new rate limit 33 | */ 34 | fun execute(): Double { 35 | if (isStepPhase) { 36 | log.info("Stepping rate limiter by $stepValue") 37 | val newValue = min(rateLimiter.rate + stepValue, initial) 38 | if (newValue == initial) { 39 | log.info("Moving to optimization phase") 40 | isStepPhase = false 41 | } 42 | rateLimiter.rate = newValue 43 | log.info("New rate limiter value: ${rateLimiter.rate}") 44 | return rateLimiter.rate 45 | } 46 | 47 | // determine the latency number that's closest to it's limit 48 | getCurrentAndMaxLatency().map { 49 | val newLimit = getNextValue(rateLimiter.rate, it.first, it.second) 50 | return@map if (newLimit == rateLimiter.rate) { 51 | log.info("Optimizer has nothing to do") 52 | newLimit 53 | } else { 54 | // don't increase the rate limiter if we're not within 20% of the target 55 | val currentThroughput = getCurrentTotalThroughput() 56 | if (newLimit > rateLimiter.rate && currentThroughput < rateLimiter.rate * .9) { 57 | 58 | log.info("Not increasing rate limiter, not within 10% of the current limit (current: ${currentThroughput} vs actual:${rateLimiter.rate})") 59 | return@map rateLimiter.rate 60 | } 61 | // if we're decreasing the limit, we want to make sure we don't lower it too quickly, 62 | // overwise we oscillate between too high and too low 63 | if (newLimit < rateLimiter.rate && currentThroughput < rateLimiter.rate * .9) { 64 | log.info("Not decreasing rate limiter, current throughput is above current limit (current: ${currentThroughput} vs actual: ${rateLimiter.rate})") 65 | } 66 | log.info("Updating rate limiter from ${rateLimiter.rate} to ${newLimit}") 67 | rateLimiter.rate = newLimit 68 | rateLimiter.rate 69 | } 70 | }.orElse( 71 | return rateLimiter.rate 72 | ) 73 | } 74 | 75 | /** 76 | * Added to prevent the rate limiter from acting when queries aren't running, generally during populate phase 77 | */ 78 | fun getTotalOperations() : Long { 79 | return metrics.mutations.count + metrics.selects.count 80 | } 81 | 82 | fun getCurrentTotalThroughput() : Double { 83 | return metrics.mutations.oneMinuteRate + 84 | metrics.selects.oneMinuteRate + 85 | metrics.deletions.oneMinuteRate + 86 | metrics.populate.oneMinuteRate 87 | } 88 | 89 | /** 90 | * Returns current, target Pair 91 | */ 92 | fun getCurrentAndMaxLatency() : Optional> { 93 | if (maxWriteLatency == null && maxReadLatency == null) { 94 | return Optional.empty() 95 | } 96 | // if we're in the populate phase 97 | if (metrics.mutations.count == 0L && 98 | metrics.selects.count == 0L && 99 | metrics.deletions.count == 0L && 100 | metrics.populate.count > 0L ) 101 | if (maxWriteLatency != null) { 102 | log.info("In populate phase, using populate latency") 103 | return Optional.of(Pair(getPopulateLatency(), maxWriteLatency)) 104 | } else { 105 | return Optional.empty>() 106 | } 107 | if (maxReadLatency == null) { 108 | return Optional.of(Pair(getWriteLatency(), maxWriteLatency!!)) 109 | } 110 | if (maxWriteLatency == null) { 111 | return Optional.of(Pair(getReadLatency(), maxReadLatency)) 112 | } 113 | 114 | val rLatP = getReadLatency() / maxReadLatency 115 | val wLatP = getWriteLatency() / maxWriteLatency 116 | 117 | // if either is over, return the one that's the most % over 118 | return if (rLatP > wLatP) { 119 | Optional.of(Pair(getReadLatency(), maxReadLatency)) 120 | } else Optional.of(Pair(getWriteLatency(), maxWriteLatency)) 121 | } 122 | 123 | /** 124 | * Provide the new value for the rate limiter 125 | * If we're over, we significantly reduce traffic 126 | * If we're within 95% of our target we increase by 1 (just to do something) 127 | * if we're under, we increase by up to 2x 128 | */ 129 | fun getNextValue(currentRateLimiterValue: Double, currentLatency: Double, maxLatency: Long): Double { 130 | // we set our max increase relative to the total latency we can tolerate, at most increasing by 5% 131 | // small latency requirements (< 10ms) should barely adjust the throughput b/c it's so sensitive 132 | var maxIncrease = (1.0 + sqrt(maxLatency.toDouble()) / 100.0).coerceAtMost(1.05) 133 | 134 | if (currentLatency > maxLatency) { 135 | log.info("Current Latency ($currentLatency) over Max Latency ($maxLatency) reducing throughput by 10%") 136 | return currentRateLimiterValue * .90 137 | } 138 | else if (currentLatency / maxLatency.toDouble() > .90) { 139 | log.info("Current latency ($currentLatency) within 95% of max ($maxLatency), not adjusting") 140 | return currentRateLimiterValue 141 | } 142 | else { 143 | // increase a reasonable amount 144 | // should provide a very gentle increase when we get close to the right number 145 | val adjustmentFactor = (1 + cbrt(maxLatency.toDouble() - currentLatency) / maxLatency.toDouble()).coerceAtMost(maxIncrease) 146 | val newLimit = currentRateLimiterValue * adjustmentFactor 147 | log.info("Current limiter: $currentRateLimiterValue latency $currentLatency, max: $maxLatency adjustment factor: $adjustmentFactor, new limit: $newLimit") 148 | return newLimit 149 | } 150 | } 151 | 152 | fun getReadLatency() = 153 | metrics.selects.snapshot.get99thPercentile() * durationFactor 154 | 155 | 156 | fun getWriteLatency() = 157 | metrics.mutations.snapshot.get99thPercentile() * durationFactor 158 | 159 | fun getPopulateLatency() = 160 | metrics.populate.snapshot.get99thPercentile() * durationFactor 161 | 162 | fun reset() { 163 | isStepPhase = true 164 | rateLimiter.rate = stepValue 165 | } 166 | 167 | } -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit 84 | 85 | APP_NAME="Gradle" 86 | APP_BASE_NAME=${0##*/} 87 | 88 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 89 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 90 | 91 | # Use the maximum available, or set MAX_FD != -1 to use that value. 92 | MAX_FD=maximum 93 | 94 | warn () { 95 | echo "$*" 96 | } >&2 97 | 98 | die () { 99 | echo 100 | echo "$*" 101 | echo 102 | exit 1 103 | } >&2 104 | 105 | # OS specific support (must be 'true' or 'false'). 106 | cygwin=false 107 | msys=false 108 | darwin=false 109 | nonstop=false 110 | case "$( uname )" in #( 111 | CYGWIN* ) cygwin=true ;; #( 112 | Darwin* ) darwin=true ;; #( 113 | MSYS* | MINGW* ) msys=true ;; #( 114 | NONSTOP* ) nonstop=true ;; 115 | esac 116 | 117 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 118 | 119 | 120 | # Determine the Java command to use to start the JVM. 121 | if [ -n "$JAVA_HOME" ] ; then 122 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 123 | # IBM's JDK on AIX uses strange locations for the executables 124 | JAVACMD=$JAVA_HOME/jre/sh/java 125 | else 126 | JAVACMD=$JAVA_HOME/bin/java 127 | fi 128 | if [ ! -x "$JAVACMD" ] ; then 129 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 130 | 131 | Please set the JAVA_HOME variable in your environment to match the 132 | location of your Java installation." 133 | fi 134 | else 135 | JAVACMD=java 136 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | 142 | # Increase the maximum file descriptors if we can. 143 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 144 | case $MAX_FD in #( 145 | max*) 146 | MAX_FD=$( ulimit -H -n ) || 147 | warn "Could not query maximum file descriptor limit" 148 | esac 149 | case $MAX_FD in #( 150 | '' | soft) :;; #( 151 | *) 152 | ulimit -n "$MAX_FD" || 153 | warn "Could not set maximum file descriptor limit to $MAX_FD" 154 | esac 155 | fi 156 | 157 | # Collect all arguments for the java command, stacking in reverse order: 158 | # * args from the command line 159 | # * the main class name 160 | # * -classpath 161 | # * -D...appname settings 162 | # * --module-path (only if needed) 163 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 164 | 165 | # For Cygwin or MSYS, switch paths to Windows format before running java 166 | if "$cygwin" || "$msys" ; then 167 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 168 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 169 | 170 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 171 | 172 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 173 | for arg do 174 | if 175 | case $arg in #( 176 | -*) false ;; # don't mess with options #( 177 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 178 | [ -e "$t" ] ;; #( 179 | *) false ;; 180 | esac 181 | then 182 | arg=$( cygpath --path --ignore --mixed "$arg" ) 183 | fi 184 | # Roll the args list around exactly as many times as the number of 185 | # args, so each arg winds up back in the position where it started, but 186 | # possibly modified. 187 | # 188 | # NB: a `for` loop captures its iteration list before it begins, so 189 | # changing the positional parameters here affects neither the number of 190 | # iterations, nor the values presented in `arg`. 191 | shift # remove old arg 192 | set -- "$@" "$arg" # push replacement arg 193 | done 194 | fi 195 | 196 | # Collect all arguments for the java command; 197 | # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of 198 | # shell script including quotes and variable substitutions, so put them in 199 | # double quotes to make sure that they get re-expanded; and 200 | # * put everything else in single quotes, so that it's not re-expanded. 201 | 202 | set -- \ 203 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 204 | -classpath "$CLASSPATH" \ 205 | org.gradle.wrapper.GradleWrapperMain \ 206 | "$@" 207 | 208 | # Use "xargs" to parse quoted args. 209 | # 210 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 211 | # 212 | # In Bash we could simply go: 213 | # 214 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 215 | # set -- "${ARGS[@]}" "$@" 216 | # 217 | # but POSIX shell has neither arrays nor command substitution, so instead we 218 | # post-process each arg (as a line of input to sed) to backslash-escape any 219 | # character that might be a shell metacharacter, then use eval to reverse 220 | # that process (while maintaining the separation between arguments), and wrap 221 | # the whole thing up as a single "set" statement. 222 | # 223 | # This will of course break if any of these variables contains a newline or 224 | # an unmatched quote. 225 | # 226 | 227 | eval "set -- $( 228 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 229 | xargs -n1 | 230 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 231 | tr '\n' ' ' 232 | )" '"$@"' 233 | 234 | exec "$JAVACMD" "$@" 235 | -------------------------------------------------------------------------------- /src/main/kotlin/com/rustyrazorblade/easycassstress/SchemaBuilder.kt: -------------------------------------------------------------------------------- 1 | package com.rustyrazorblade.easycassstress 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude 4 | import com.fasterxml.jackson.core.FormatFeature 5 | import com.fasterxml.jackson.core.JsonParser 6 | import com.fasterxml.jackson.databind.ObjectMapper 7 | import org.apache.logging.log4j.kotlin.logger 8 | 9 | fun MutableMap.putInt(key: String, value: Int?) : MutableMap { 10 | if(value != null) { 11 | this[key] = value.toString() 12 | } 13 | return this 14 | } 15 | 16 | class SchemaBuilder(var baseStatement : String) { 17 | private var ttl: Long = 0 18 | private var compaction = "" 19 | private var compression = "" 20 | 21 | private var isCreateTable : Boolean 22 | 23 | var rowCache = "NONE" 24 | var keyCache = "ALL" 25 | 26 | var log = logger() 27 | 28 | val compactionShortcutRegex = """^(stcs|lcs|twcs|ucs)((?:,[0-9a-zA-Z]+)*)$""".toRegex() 29 | 30 | enum class WindowUnit(val s : String) { 31 | MINUTES("MINUTES"), 32 | HOURS("HOURS"), 33 | DAYS("DAYS"); 34 | 35 | companion object { 36 | fun get(s: String) : WindowUnit = when(s.toLowerCase()) { 37 | "minutes" -> MINUTES 38 | "hours" -> HOURS 39 | "days" -> DAYS 40 | else -> throw Exception("not a thing") 41 | } 42 | } 43 | 44 | 45 | } 46 | 47 | sealed class Compaction { 48 | 49 | val mapper = ObjectMapper() 50 | .setSerializationInclusion(JsonInclude.Include.NON_NULL) // no nulls 51 | .setSerializationInclusion(JsonInclude.Include.NON_EMPTY) // no empty fields 52 | .writerWithDefaultPrettyPrinter() 53 | 54 | open fun toCQL() = mapper.writeValueAsString(getOptions()).replace("\"", "'") 55 | 56 | abstract fun getOptions() : Map 57 | 58 | data class STCS(val min: Int?, 59 | val max: Int? = null) : Compaction() { 60 | override fun getOptions() = mutableMapOf( 61 | "class" to "SizeTieredCompactionStrategy" 62 | ).putInt( "max_threshold", max) 63 | .putInt( "min_threshold" ,min) 64 | } 65 | 66 | data class LCS(val sstableSizeInMb : Int? = null, 67 | val fanoutSize: Int? = null) : Compaction() { 68 | override fun getOptions() = 69 | mutableMapOf("class" to "LeveledCompactionStrategy") 70 | .putInt("sstable_size_in_mb", sstableSizeInMb) 71 | .putInt("fanout_size", fanoutSize) 72 | 73 | } 74 | 75 | data class TWCS(val windowSize: Int? = null, 76 | val windowUnit: WindowUnit? = null) : Compaction() { 77 | override fun getOptions() = 78 | mutableMapOf("class" to "TimeWindowCompactionStrategy", 79 | "compaction_window_unit" to (windowUnit?.s ?: "") ) 80 | .putInt("compaction_window_size", windowSize) 81 | } 82 | 83 | data class UCS(val scalingParameters: String): Compaction() { 84 | override fun getOptions() = 85 | mutableMapOf("class" to "UnifiedCompactionStrategy", 86 | "scaling_parameters" to scalingParameters) 87 | } 88 | 89 | data class Unknown(val raw: String) : Compaction() { 90 | override fun getOptions() = mapOf() 91 | override fun toCQL() = raw.trim().replace("\"", "'") 92 | } 93 | } 94 | 95 | init { 96 | 97 | val options = setOf(RegexOption.IGNORE_CASE, 98 | RegexOption.MULTILINE, 99 | RegexOption.DOT_MATCHES_ALL) 100 | 101 | val r = "^\\s?create\\s+table\\s.*".toRegex(options) 102 | 103 | isCreateTable = r.matches(baseStatement) 104 | log.debug("checking $baseStatement, isCreateTable=$isCreateTable") 105 | 106 | 107 | } 108 | 109 | companion object { 110 | fun create(baseStatement: String) : SchemaBuilder { 111 | return SchemaBuilder(baseStatement) 112 | } 113 | } 114 | 115 | fun withCompaction(compaction: String) : SchemaBuilder { 116 | 117 | this.compaction = parseCompaction(compaction).toCQL() 118 | return this 119 | } 120 | 121 | fun withCompression(compression: String) : SchemaBuilder { 122 | this.compression = compression.trim() 123 | return this 124 | } 125 | 126 | fun withDefaultTTL(ttl: Long) : SchemaBuilder { 127 | this.ttl = ttl 128 | return this 129 | } 130 | 131 | fun build() : String { 132 | val sb = StringBuilder(baseStatement) 133 | 134 | 135 | val parts = mutableListOf() 136 | 137 | // there's a whole bunch of flags we can only use in CREATE TABLE statements 138 | 139 | if(isCreateTable) { 140 | 141 | if(compaction.length > 0) 142 | parts.add("compaction = $compaction") 143 | if(compression.length > 0) 144 | parts.add("compression = $compression") 145 | 146 | parts.add("caching = {'keys': '$keyCache', 'rows_per_partition': '$rowCache'}") 147 | parts.add("default_time_to_live = $ttl") 148 | } 149 | 150 | val stuff = parts.joinToString(" AND ") 151 | 152 | if(stuff.length > 0 && !baseStatement.toLowerCase().contains("\\swith\\s".toRegex())) { 153 | sb.append(" WITH ") 154 | } else if(stuff.count() > 0) { 155 | sb.append(" AND ") 156 | } 157 | 158 | sb.append(stuff) 159 | 160 | val tmp = sb.toString() 161 | log.info(tmp) 162 | return tmp 163 | } 164 | 165 | fun withRowCache(rowCache: String): SchemaBuilder { 166 | this.rowCache = rowCache 167 | return this 168 | } 169 | 170 | fun withKeyCache(keyCache: String): SchemaBuilder { 171 | this.keyCache = keyCache 172 | return this 173 | } 174 | 175 | /** 176 | * Helper function for compaction shortcuts 177 | * If the functino parses, we return a 178 | * @see Issue 80 on Github 179 | */ 180 | fun parseCompaction(compaction: String) : Compaction { 181 | val parsed = compactionShortcutRegex.find(compaction) 182 | if(parsed == null) { 183 | log.error("Unknown compaction option: $compaction") 184 | return Compaction.Unknown(compaction) 185 | } 186 | val groups = parsed.groupValues 187 | val strategy = groups[1] 188 | val options = groups[2].removePrefix(",").split(",").filter{ it.length > 0} 189 | log.debug("Parsing $compaction: strategy: $strategy, options: $options / ${options.size}") 190 | 191 | return when(strategy) { 192 | "stcs" -> { 193 | when(options.size) { 194 | 0 -> Compaction.STCS(null, null) 195 | 2 -> Compaction.STCS(options[0].toInt(), options[1].toInt()) 196 | else -> Compaction.Unknown(compaction) 197 | } 198 | } 199 | /* 200 | lcs: leveled compaction, all defaults 201 | lcs,: leveled, override the default of 160 202 | lcs,,: leveled, override the default sstable size of 160 and fanout of 10 203 | */ 204 | "lcs" -> { 205 | when(options.size) { 206 | 0 -> Compaction.LCS() 207 | 1 -> Compaction.LCS(options.get(0).toInt()) 208 | 2 -> Compaction.LCS(options.get(0).toInt(), options.get(1).toInt()) 209 | else -> Compaction.Unknown(compaction) 210 | } 211 | } 212 | "twcs" -> { 213 | when(options.size) { 214 | 0 -> Compaction.TWCS() 215 | 2 -> Compaction.TWCS(options[0].toInt(), WindowUnit.get(options[1]) ) 216 | else -> Compaction.Unknown(compaction) 217 | } 218 | } 219 | "ucs" -> { 220 | /* 221 | Since we're using commas in the strategy's scaling parameters, 222 | we're just going to support the simple version, which is something like this: 223 | ucs,T10,T8 224 | 225 | If a user wants to use all the params, they're going to have to supply the entire strategy. 226 | 227 | ALTER TABLE your_table WITH compaction = { 'class': 'UnifiedCompactionStrategy', 228 | 'scaling_parameters': 'T8, T4, N, L4' }; 229 | target_sstable_size 230 | base_shard_count 231 | expired_sstable_check_frequency_seconds 232 | 233 | examples: 234 | * ucs,T10 235 | * ucs,T4,L20 236 | * ucs, 237 | */ 238 | 239 | Compaction.UCS( 240 | options.joinToString(",") 241 | ) 242 | } 243 | else -> Compaction.Unknown(compaction) 244 | } 245 | } 246 | 247 | 248 | 249 | } -------------------------------------------------------------------------------- /manual/MANUAL.adoc: -------------------------------------------------------------------------------- 1 | = easy-cass-stress 2 | Jon Haddad 3 | Anthony Grasso 4 | :toc: left 5 | :icon: font 6 | 7 | easy-cass-stress is a workload-centric stress tool for Apache Cassandra, written in Kotlin. 8 | Workloads are easy to write and because they are written in code. You 9 | have the ultimate flexibility to build whatever you want without having to learn and 10 | operate around the restrictions of a configuration DSL. Workloads can be tweaked via command line 11 | parameters to make them fit your environment more closely. 12 | 13 | One of the goals of easy-cass-stress is to provide enough pre-designed 14 | workloads _out of the box,_ so it’s unnecessary to code up a workload for 15 | most use cases. For instance, it’s very common to have a key value 16 | workload, and want to test that. easy-cass-stress allows you to customize a 17 | pre-configured key-value workload, using simple parameters to modify the 18 | workload to fit your needs. Several workloads are included, such as: 19 | 20 | * Time Series 21 | * Key / Value 22 | * Materialized Views 23 | * Collections (maps) 24 | * Counters 25 | 26 | The tool is flexible enough to design workloads which leverage multiple 27 | (thousands) of tables, hitting them as needed. Statistics are 28 | automatically captured by the Dropwizard metrics library. 29 | 30 | == Quickstart Example 31 | 32 | The goal of this project is to be testing common workloads (time series, key value) against a Cassandra cluster in 15 minutes or less. 33 | 34 | === Installation 35 | 36 | ==== Installing a Package 37 | 38 | *Note*: Installing from packages has not been migrated since I (Jon Haddad) began maintaining my own fork. I'll be updating this section soon. 39 | 40 | //// 41 | The easiest way to get started is to use your favorite package manager. 42 | 43 | The current version is {EASY_CASS_STRESS_VERSION}. 44 | 45 | ==== Deb Packages 46 | 47 | ``` 48 | $ echo "deb https://dl.bintray.com/rustyrazorblade/tlp-tools-deb weezy main" | sudo tee -a /etc/apt/sources.list 49 | $ sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 2895100917357435 50 | $ sudo apt update 51 | $ sudo apt install easy-cass-stress 52 | ``` 53 | 54 | ==== RPM Packages 55 | 56 | eYou'll need the bintray repo set up on your machine. Create this `/etc/yum.repos.d/tlp-tools.repo`: 57 | 58 | ``` 59 | [bintraybintray-rustyrazorblade-tlp-tools-rpm] 60 | name=bintray-rustyrazorblade-tlp-tools-rpm 61 | baseurl=https://dl.bintray.com/rustyrazorblade/tlp-tools-rpm 62 | gpgcheck=0 63 | repo_gpgcheck=0 64 | enabled=1 65 | ``` 66 | 67 | Then run the following to install: 68 | 69 | ``` 70 | $ yum install easy-cass-stress 71 | ``` 72 | 73 | 74 | Further information can be found on the https://bintray.com/beta/#/rustyrazorblade/tlp-tools-rpm?tab=packages[BinTray website]. 75 | 76 | ==== Tarball Install 77 | 78 | 79 | [subs="attributes"] 80 | ``` 81 | $ curl -L -O "https://dl.bintray.com/rustyrazorblade/tlp-tools-tarball/easy-cass-stress-{EASY_CASS_STRESS_VERSION}.tar 82 | $ tar -xzf easy-cass-stress-{EASY_CASS_STRESS_VERSION}.tar 83 | ``` 84 | 85 | //// 86 | 87 | === Building / Using the Stress Tool from Source 88 | 89 | This is currently the only way to use the latest version of easy-cass-stress. I'm working on getting the packages updated. 90 | 91 | //// 92 | This is advisable only if you're comfortable debugging the bash scripts, gradle, and Kotlin yourself and want to be either on the bleeding edge or add a feature. If not, that's OK, we recommend using one of the above packages instead. 93 | //// 94 | 95 | First you'll need to clone and build the repo. You can grab the source here and build via the included gradle script: 96 | 97 | 98 | ```bash 99 | $ git clone https://github.com/rustyrazorblade/easy-cass-stress.git 100 | $ cd easy-cass-stress 101 | $ ./gradlew shadowJar 102 | ``` 103 | 104 | You can now run the stress tool via the `bin/easy-cass-stress` script. This is not the same script you'll be running if you've installed from a package or the tarball. 105 | 106 | You can also create a zip, tar, or deb package by doing the following: 107 | 108 | ```bash 109 | $ ./gradlew distZip 110 | $ ./gradlew distTar 111 | $ ./gradlew deb 112 | ``` 113 | 114 | === Run Your First Stress Workload 115 | 116 | Assuming you have either a CCM cluster or are running a single node 117 | locally, you can run this quickstart. 118 | 119 | Either add the `bin` directory to your PATH or from within easy-cass-stress 120 | run the following command to execute 10,000 queries: 121 | 122 | ``` 123 | include::examples/easy-cass-stress-keyvalue.txt[lines=1] 124 | ``` 125 | 126 | You'll see the output of the keyspaces and tables that are created as well as some statistical information regarding the workload: 127 | 128 | [source,bash,options="nowrap"] 129 | ---- 130 | include::examples/easy-cass-stress-keyvalue.txt[lines=2..-1] 131 | ---- 132 | 133 | If you've made it this far, congrats! You've run your first workload. 134 | 135 | 136 | 137 | == Usage 138 | 139 | You'll probably want to do a bit more than simply run a few thousand queries against a KeyValue table with default settings. The nice part about easy-cass-stress is that it not only comes with a variety of workloads that you can run to test your cluster, but that it allows you to change many of the parameters. In the quickstart example we used the `-n` flag to change the total number of operations `easy-cass-stress` will execute against the database. There are many more options available, this section will cover some of them. 140 | 141 | === General Help 142 | 143 | `easy-cass-stress` will display the help if the `easy-cass-stress` command is run without any arguments or if the `--help` flag is passed: 144 | 145 | ```bash 146 | include::examples/easy-cass-stress-help.txt[] 147 | ``` 148 | 149 | === Listing All Workloads 150 | 151 | ``` 152 | include::examples/list-all.txt[] 153 | ``` 154 | 155 | === Getting information about a workload 156 | 157 | It's possible to get (some) information about a workload by using the info command. This area is a bit lacking at the moment. It currently only provides the schema and default read rate. 158 | 159 | ``` 160 | include::examples/info-key-value.txt[] 161 | ``` 162 | 163 | 164 | === Running a Customized Stress Workload 165 | 166 | Whenever possible we try to use human friendly numbers. Typing out `-n 1000000000` is error prone and hard to read, `-n 1B` is much easier. 167 | 168 | .Table Human Friendly Values 169 | |=== 170 | |Suffix|Implication|Example|Equivalent 171 | 172 | |k|Thousand|1k|1,000 173 | |m|Million|1m|1,000,000 174 | |b|Billion|1b|1,000,000,000 175 | 176 | |=== 177 | 178 | === Running a stress test for a given duration instead of a number of operations 179 | 180 | You might need to run a stress test for a given duration instead of providing a number of operations, especially in case of multithreaded stress runs. This is done by providing the duration in a human readable format with the `-d` argument. The minimum duration is 1 minute. 181 | For example, running a test for 1 hours and 30 minutes will be done as follows: 182 | 183 | ``` 184 | $ easy-cass-stress run KeyValue -d "1h 30m" 185 | ``` 186 | 187 | To run a test for 1 days, 3 hours and 15 minutes (why not?), run easy-cass-stress as follows: 188 | 189 | ``` 190 | $ easy-cass-stress run KeyValue -d "1d 3h 15m" 191 | ``` 192 | 193 | ==== Partition Keys 194 | 195 | A very useful feature is controlling how many partitions are read and written to for a given stress test. Doing a billion operations across a billion partitions is going to have a much different performance profile than writing to one hundred partitions, especially when mixed with different compaction settings. Using `-p` we can control how many partition keys a stress test will leverage. The keys are randomly chosen at the moment. 196 | 197 | 198 | ==== Read Rate 199 | 200 | It's possible to specify the read rate of a test as a double. For example, if you want to use 1% reads, you'd specify `-r .01`. The sum of the read rate and delete rate must be less than or equal to 1.0. 201 | 202 | ==== Delete Rate 203 | 204 | It's possible to specify the delete rate of a test as a double. For example, if you want to use 1% deletes, you'd specify `--deleterate .01`. The sum of the read rate and delete rate must be less than or equal to 1.0. 205 | 206 | ==== Compaction 207 | 208 | It's possible to change the compaction strategy used with the `--compaction flag`. At the moment this changes the compaction strategy of every table in the test. This will be addressed in the future to be more flexible. 209 | 210 | The `--compaction` flag can accept a raw string along these lines: 211 | 212 | ``` 213 | --compaction "{'class':'LeveledCompactionStrategy'}" 214 | ``` 215 | 216 | Alternatively, a shortcut format exists as of version 2.0: 217 | 218 | ``` 219 | --compaction lcs 220 | ``` 221 | 222 | The following shorthand formats are available: 223 | 224 | .Table Compaction ShortHand 225 | |=== 226 | |Syntax | Expansion 227 | |stcs 228 | a| 229 | `{'class':'SizeTieredCompactionStrategy'}` 230 | 231 | |stcs,4,32 232 | a| 233 | `{'class':'SizeTieredCompactionStrategy', 'min_threshold':4, 'max_threshold':32}` 234 | 235 | |lcs 236 | a| 237 | `{'class':'LeveledCompactionStrategy'}` 238 | 239 | |lcs,160 240 | a| 241 | `{'class':'LeveledCompactionStrategy', 'sstable_size_in_mb':'160'}` 242 | 243 | |lcs,160,10 244 | a| 245 | `{'class':'LeveledCompactionStrategy', 'sstable_size_in_mb':'160', 'fanout_size':10}` 246 | 247 | |twcs 248 | a| 249 | `{'class':'TimeWindowCompactionStrategy'}` 250 | 251 | |twcs,1,days 252 | a| 253 | `{'class':'TimeWindowCompactionStrategy', 'compaction_window_size':'1', 'compaction_window_unit':'DAYS'}` 254 | 255 | |=== 256 | 257 | 258 | ==== Compression 259 | 260 | It's possible to change the compression options used. At the moment this changes the compression options of every table in the test. This will be addressed in the future to be more flexible. 261 | 262 | 263 | ==== Customizing Fields 264 | 265 | To some extent, workloads can be customized by leveraging the `--fields` 266 | flag. For instance, if we look at the KeyValue workload, we have a table 267 | called `keyvalue` which has a `value` field. 268 | 269 | To customize the data we use for this field, we provide a generator at 270 | the command line. By default, the `value` field will use 100-200 271 | characters of random text. What if we’re storing blobs of text instead? 272 | Ideally we’d like to tweak this workload to be closer to our production 273 | use case. Let’s use random sections from various books: 274 | 275 | ``` 276 | $ easy-cass-stress run KeyValue --field.keyvalue.value='book(20,40)` 277 | ``` 278 | 279 | Instead of using random strings of garbage, the KeyValue workload will 280 | now use 20-40 words extracted from books. 281 | 282 | There are other generators available, such as names, gaussian numbers, 283 | and cities. Not every generator applies to every type. It’s up to the 284 | workload to specify which fields can be used this way. 285 | 286 | === Logging 287 | 288 | `easy-cass-stress` uses the https://logging.apache.org/[Log4J 2] logging framework. 289 | 290 | You can find the default log4j config inhttps://github.com/rustyrazorblade/easy-cass-stress/blob/main/src/main/resources/log4j2.yaml[`conf`, window="_blank"]. This should be suitable for most use cases. 291 | 292 | To use your own logging configuration, simply set the shell variable `EASY_CASS_STRESS_LOG4J` to the path of the new logging configuration before running `easy-cass-stress` to point to the config file of your choice. 293 | 294 | For more information on how to configure Log4J 2 please see the https://logging.apache.org/log4j/2.x/manual/configuration.html[configuration] documentation. 295 | 296 | 297 | ==== Debian Package 298 | 299 | The Debian package installs a basic configuration file to `/etc/easy-cass-stress/log4j2.yaml`. 300 | 301 | 302 | === Exporting Metrics 303 | 304 | easy-cass-stress automatically runs an HTTP server exporting metrics in Prometheus format on port 9501. 305 | 306 | === Workload Restrictions 307 | 308 | The `BasicTimeSeries` workload only supports Cassandra versions 3.0 and above. This is because range deletes are used by this workload during runtime. Range deletes are only support in Cassandra versions 3.0. An exception will is thrown if this workload is used and a Cassandra version less than 3.0 is detected during runtime. 309 | 310 | == Developer Docs 311 | 312 | === Building the documentation 313 | 314 | First generate the command examples with the following shell script: 315 | 316 | ```bash 317 | $ manual/generate_examples.sh 318 | ``` 319 | 320 | There’s a docker service to build the HTML manual: 321 | 322 | ``` 323 | $ docker-compose up docs 324 | ``` 325 | 326 | === Writing a Custom Workload 327 | 328 | `easy-cass-stress` is a work in progress. Writing a stress workload isn't documented yet as it is still changing. 329 | --------------------------------------------------------------------------------