├── .gitignore ├── .travis.yml ├── LICENCE ├── README.md ├── berserker-cassandra ├── README.md ├── pom.xml └── src │ └── main │ └── java │ └── io │ └── smartcat │ └── berserker │ └── cassandra │ ├── configuration │ ├── CassandraConfiguration.java │ └── PreparedStatement.java │ └── worker │ ├── CassandraWorker.java │ └── InvalidQueryMetadataException.java ├── berserker-commons ├── pom.xml └── src │ └── main │ └── java │ └── io │ └── smartcat │ └── berserker │ ├── api │ ├── AlreadyClosedException.java │ ├── DataSource.java │ ├── RateGenerator.java │ └── Worker.java │ └── configuration │ ├── BaseConfiguration.java │ ├── ConfigurationException.java │ ├── ConfigurationHelper.java │ ├── ConfigurationLoader.java │ ├── ConfigurationParseException.java │ ├── DataSourceConfiguration.java │ ├── GlobalConfiguration.java │ ├── LoadGeneratorConfiguration.java │ ├── MetricsReporterConfiguration.java │ ├── RateGeneratorConfiguration.java │ ├── WorkerConfiguration.java │ └── YamlConfigurationLoader.java ├── berserker-core ├── pom.xml └── src │ ├── main │ └── java │ │ └── io │ │ └── smartcat │ │ └── berserker │ │ ├── LoadGenerator.java │ │ ├── configuration │ │ ├── DefaultRateGeneratorConfiguration.java │ │ ├── JmxMetricsReporterConfiguration.java │ │ ├── RandomNumberDataSourceConfiguration.java │ │ ├── SimpleConsoleMetricsReporterConfiguration.java │ │ └── rategenerator │ │ │ ├── InvalidReferenceNameException.java │ │ │ ├── RateGeneratorConfigurationParser.java │ │ │ └── RateGeneratorExpressionParser.java │ │ ├── datasource │ │ ├── BufferedDataSource.java │ │ ├── RandomDoubleDataSource.java │ │ ├── RandomIntDataSource.java │ │ └── RandomLongDataSource.java │ │ ├── rategenerator │ │ ├── AdditionRateGenerator.java │ │ ├── ConstantRateGenerator.java │ │ ├── DivisionRateGenerator.java │ │ ├── MultiplicationRateGenerator.java │ │ ├── PeriodicRateGenerator.java │ │ ├── RateGeneratorProxy.java │ │ ├── SineRateGenerator.java │ │ ├── SquareRateGenerator.java │ │ ├── SubtractionRateGenerator.java │ │ └── TriangleRateGenerator.java │ │ ├── util │ │ └── LinkedEvictingBlockingQueue.java │ │ └── worker │ │ └── InternalWorker.java │ └── test │ ├── groovy │ └── io │ │ └── smartcat │ │ └── berserker │ │ └── configuration │ │ └── rategenerator │ │ └── RateGeneratorConfigurationParserSpec.groovy │ ├── java │ └── io │ │ └── smartcat │ │ └── berserker │ │ ├── LoadGeneratorTest.java │ │ ├── rategenerator │ │ └── PeriodicRateGeneratorTest.java │ │ ├── util │ │ └── LinkedEvictingBlockingQueueTest.java │ │ └── worker │ │ └── InternalWorkerTest.java │ └── resources │ ├── empty-csv.csv │ ├── invalid-csv.csv │ └── simple-csv.csv ├── berserker-http ├── README.md ├── pom.xml └── src │ └── main │ └── java │ └── io │ └── smartcat │ └── berserker │ └── http │ ├── configuration │ └── HttpConfiguration.java │ └── worker │ └── HttpWorker.java ├── berserker-kafka ├── README.md ├── pom.xml └── src │ └── main │ └── java │ └── io │ └── smartcat │ └── berserker │ └── kafka │ ├── configuration │ └── KafkaConfiguration.java │ └── worker │ └── KafkaWorker.java ├── berserker-mqtt ├── README.md ├── pom.xml └── src │ └── main │ └── java │ └── io │ └── smartcat │ └── berserker │ └── mqtt │ ├── configuration │ └── MqttConfiguration.java │ └── worker │ └── MqttWorker.java ├── berserker-rabbitmq ├── README.md ├── pom.xml └── src │ └── main │ └── java │ └── io │ └── smartcat │ └── berserker │ └── rabbitmq │ ├── configuration │ └── RabbitMqConfiguration.java │ └── worker │ └── RabbitMqWorker.java ├── berserker-ranger ├── pom.xml └── src │ └── main │ └── java │ └── io │ └── smartcat │ └── berserker │ └── ranger │ ├── configuration │ └── RangerConfiguration.java │ └── datasource │ └── RangerDataSource.java ├── berserker-runner ├── pom.xml └── src │ ├── example │ └── resources │ │ ├── ranger-cassandra.yml │ │ ├── ranger-http.yml │ │ ├── ranger-kafka.yml │ │ ├── ranger-mqtt.yml │ │ └── ranger-rabbitmq.yml │ └── main │ ├── java │ └── io │ │ └── smartcat │ │ └── berserker │ │ └── runner │ │ └── LoadGeneratorRunner.java │ └── resources │ └── logback.xml ├── checkstyle-suppressions.xml ├── checkstyle.properties ├── checkstyle.xml ├── images ├── architecture.png └── core-design.png ├── pom.xml ├── rate-generator-configuration.md └── release /.gitignore: -------------------------------------------------------------------------------- 1 | ## Common 2 | **/temp 3 | **/logs 4 | 5 | ## Java 6 | **/target 7 | *.settings 8 | *.classpath 9 | *.project 10 | *.java-version 11 | 12 | ## Intellij 13 | **/.idea 14 | *.iml 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | sudo: false 3 | jdk: 4 | - oraclejdk8 5 | matrix: 6 | include: 7 | - jdk: oraclejdk8 8 | script: travis_wait 30 mvn verify org.pitest:pitest-maven:mutationCoverage 9 | notifications: 10 | slack: 11 | secure: Vcrg6S2F40pTLir+D4eegQINQm0UKSRdwU5gzXwzfUX4pGLWPv1NzExZb6S8LXXTNU7j5Udop/XelgBps4iverYeDlEOSe+Abr8i7YTBvz1oyRzVj02HMtiVykJ76NbaVsjpdFXacW24FLlApbBjDvCp1E1xnvdVs4CH8qlY8IQE9vJW0YQNi0umOMVGyrQoIDbKha9NT/d+/HGkxSUB7X6t5G5ZAUoJryJQ9bvJrx9Vx0KQdKjYTCpLKzKbXKEv+hYsuUEozUXKwDZM+Xvuf2j0tSiRUqE1g550eoO/gUjfjza4vMG2LKX7B+IP29VNc8wDW3dt373hnk6qTeaBZOFjnf/XXB3vbQeFydFO2wmtF0XY2R2xqr2or+M71s1BjU+/AbbI5NEm5jBWy6x+P2Mb4NXbJ3wxVBH2vkdKJ2LIZL3xQCa86lLsdbdlItdY38ZSbLRTTlt+fatFErfeqj1LfOQFpwbQSyffB475m5WhOZw+iG8oiyuo7MoGoupmP7eQtJGO60mfEP7aYImmxHrX6wYtkyU55TNfYXx7plfpFVU6JUiDmbwqmWptywnY5YcXeIdvnRSfsl0YwpCbeCl+hi6fOV31vIwMhYTUVjA39SznO5vV34R2ep61pxv51lQyGxA4OyKyBGs1H6gOTSuPkwlr2l7H09FohahsFV4= 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Berserker 2 | 3 | Load generator with modular architecture. 4 | 5 | [![Build Status](https://travis-ci.org/smartcat-labs/berserker.svg?branch=master)](https://travis-ci.org/smartcat-labs/berserker) 6 | [ ![Download](https://api.bintray.com/packages/smartcat-labs/maven/berserker/images/download.svg) ](https://bintray.com/smartcat-labs/maven/berserker/_latestVersion) 7 | 8 | ## Introduction 9 | 10 | Berserker is designed to be modular from beginning as illustrated on the following diagram. 11 | 12 | ![Core Design](images/core-design.png) 13 | 14 | Rate generator controls the rate at which load generator operates, rate is expressed on per second basis, or better say, number of impulses which will be generated within one second. Each time impulse is generated load generator fetches data from data source and passes it to worker. Since those are simple interfaces, it is easy to add additional module implementing either data source, worker and even rate generator. 15 | Following diagram represents possible modules for Load Generator of which some are already implemented. 16 | 17 | ![Architecture](images/architecture.png) 18 | 19 | Berserker is designed as command line tool, but having modular architecture makes it easy to use it as Java library as well. 20 | 21 | ### Berserker Commons 22 | 23 | [Berserker Commons](berserker-commons) holds interface for core and configuration and it provides signature all the modules need to confront to be able to work together. 24 | 25 | ### Berserker Core 26 | 27 | [Berserker Core](berserker-core) contains load generator implementation, and common implementations of data source, rate generator and worker. 28 | 29 | ### Berserker Runner 30 | 31 | [Berserker Runner](berserker-runner) represents runnable jar where desired data source, rate generator and worker can be specified within YAML configuration. 32 | Following section illustrates YAML configuration example. 33 | 34 | ```yaml 35 | load-generator-configuration: 36 | data-source-configuration-name: Ranger 37 | rate-generator-configuration-name: default 38 | worker-configuration-name: Cassandra 39 | metrics-reporter-configuration-name: JMX 40 | thread-count: 10 41 | queue-capacity: 100000 42 | 43 | data-source-configuration: 44 | values: 45 | id: uuid() 46 | firstName: random(['Peter', 'Mike', 'Steven', 'Joshua', 'John', 'Brandon']) 47 | lastName: random(['Smith', 'Johnson', 'Williams', 'Davis', 'Jackson', 'White', 'Lewis', 'Clark']) 48 | age: random(20..45) 49 | email: string('{}@domain.com', randomLengthString(5)) 50 | statement: 51 | consistencyLevel: ONE 52 | query: string("INSERT INTO person (id, first_name, last_name, age, email) VALUES ({}, '{}', '{}', {}, '{}');", $id, $firstName, $lastName, $age, $email) 53 | output: $statement 54 | 55 | rate-generator-configuration: 56 | rates: 57 | r: 1000 58 | output: $r 59 | 60 | worker-configuration: 61 | connection-points: 0.0.0.0:32770 62 | keyspace: my_keyspace 63 | async: false 64 | bootstrap-commands: 65 | - "CREATE KEYSPACE IF NOT EXISTS my_keyspace WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};" 66 | - USE my_keyspace; 67 | - CREATE TABLE IF NOT EXISTS person (id uuid, first_name text, last_name text, age int, email text, primary key (id)); 68 | 69 | metrics-reporter-configuration: 70 | domain: berserker 71 | filter: 72 | ``` 73 | 74 | Main part of configuration is `load-generator-configuration` where concrete modules which will be used for data source, rate generator and worker need to be specified. After `load-generator-configuration` section, there should be exactly one section for data source, rate generator and worker. 75 | Each section is allowed to contain module specific configuration as configuration interpretation will be done by module itself. 76 | In order for berserker-runner to be able to find particular module, each module jar must be in classpath. 77 | 78 | #### Rate generator configuration 79 | 80 | Documentation on rate generator configuration can be found [here](rate-generator-configuration.md). 81 | 82 | ### Modules 83 | 84 | List of existing modules: 85 | 86 | #### Berserker Ranger 87 | 88 | [Berserker Ranger](berserker-ranger) is Ranger data source implementation. 89 | 90 | #### Berserker Kafka 91 | 92 | [Berserker Kafka](berserker-kafka) is worker implementation which sends messages to Kafka cluster. 93 | 94 | #### Berserker Cassandra 95 | [Berserker Cassandra](berserker-cassandra) is worker implementation which executes CQL statements on Cassandra cluster. 96 | 97 | #### Berserker HTTP 98 | [Berserker HTTP](berserker-http) is worker implementation which sends HTTP request on configured endpoint. 99 | 100 | #### Berserker RabbitMQ 101 | [Berserker RabbitMQ](berserker-rabbitmq) is worker implementation which sends AMQP messages to RabbitMQ. 102 | 103 | #### Berserker MQTT 104 | [Berserker MQTT](berserker-mqtt) is worker implementation which publishes messages to MQTT broker. 105 | 106 | ### Usage 107 | 108 | Berserker can be used either as a library or as a stand-alone command line tool. 109 | Configuration [examples](berserker-runner/src/example/resources) can be used as starting point. 110 | 111 | #### Library usage 112 | 113 | Artifact can be fetched from bintray. 114 | 115 | Add following `repository` element to your `` section in `pom.xml`: 116 | 117 | ```xml 118 | 119 | bintray-smartcat-labs-maven 120 | bintray 121 | https://dl.bintray.com/smartcat-labs/maven 122 | 123 | ``` 124 | 125 | Add the `dependency` element to your `` section in `pom.xml` depending which `artifact` and `version` you need: 126 | 127 | ```xml 128 | 129 | io.smartcat 130 | artifact 131 | version 132 | 133 | ``` 134 | 135 | #### Command line tool usage 136 | 137 | - Download latest [Berserker Runner](https://bintray.com/smartcat-labs/maven/berserker) version. 138 | - Create config file (example can be found [here](berserker-runner/src/example/resources/ranger-cassandra.yml)). 139 | - Run following command: `java -jar berserker-runner-.jar -c ` 140 | - If you need to specify logging options, you can run berserker this way: `java -jar -Dlogback.configurationFile= berserker-runner-.jar -c ` 141 | -------------------------------------------------------------------------------- /berserker-cassandra/README.md: -------------------------------------------------------------------------------- 1 | # Berserker Cassandra 2 | 3 | Worker implementation which executes CQL statements on Cassandra cluster. 4 | 5 | ## Configuration 6 | 7 | Example yaml configuration: 8 | 9 | ```yaml 10 | worker-configuration: 11 | connection-points: 127.0.0.1:9042 12 | # optional property which indicates whether SSL enabled connection should be used or not, can be 13 | # either true or false, if not set, defaults to false 14 | use-ssl: false 15 | keyspace: custom 16 | async: false 17 | bootstrap-commands: 18 | - "CREATE KEYSPACE IF NOT EXISTS custom WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};" 19 | - USE custom; 20 | - CREATE TABLE IF NOT EXISTS user (id bigint, firstName text, lastName text, year bigint, primary key (id)); 21 | prepared-statements: 22 | - id: st1 23 | query: INSERT INTO user (id, firstName, lastName, year) VALUES (?, ?, ?, ?); 24 | ``` 25 | -------------------------------------------------------------------------------- /berserker-cassandra/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | berserker 6 | io.smartcat 7 | 0.0.13-SNAPSHOT 8 | 9 | 10 | berserker-cassandra 11 | jar 12 | 13 | Berserker Cassandra 14 | Worker implementation which targets Cassandra cluster. 15 | 16 | 17 | UTF-8 18 | 3.3.0 19 | 20 | 21 | 22 | 23 | io.smartcat 24 | berserker-commons 25 | ${project.parent.version} 26 | 27 | 28 | com.datastax.cassandra 29 | cassandra-driver-core 30 | ${version.cassandra-driver} 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /berserker-cassandra/src/main/java/io/smartcat/berserker/cassandra/configuration/CassandraConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.cassandra.configuration; 2 | 3 | import java.net.InetSocketAddress; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | import java.util.Map; 7 | 8 | import com.fasterxml.jackson.core.type.TypeReference; 9 | import com.fasterxml.jackson.databind.ObjectMapper; 10 | 11 | import io.smartcat.berserker.api.Worker; 12 | import io.smartcat.berserker.cassandra.worker.CassandraWorker; 13 | import io.smartcat.berserker.configuration.ConfigurationParseException; 14 | import io.smartcat.berserker.configuration.WorkerConfiguration; 15 | 16 | import static io.smartcat.berserker.configuration.ConfigurationHelper.getMandatoryValue; 17 | import static io.smartcat.berserker.configuration.ConfigurationHelper.getOptionalValue; 18 | import static java.util.Collections.emptyList; 19 | 20 | /** 21 | * Configuration for Cassandra worker. Configuration map should contain following: 22 | *
    23 | *
  • connection-points - Comma separated values of hostname and port. (10.10.0.1:9042, host1:9042, host2:9043) 24 | * It cannot be null nor empty.
    • 25 | *
  • keyspace - Name of keyspace to use. Cannot be null nor empty.
    • 26 | *
  • async - Indicates whether statements will be executed synchronously or asynchronously. Can be either true 27 | * or false.
    • 28 | *
  • bootstrap-commands - List of CQL commands to execute only once after connection to Cassandra cluster is 29 | * established. Suitable for creating keyspaces, tables and populating some initial data if needed. It is optional.
    • 30 | *
  • prepared-statements - List of prepared statements to create. It is optional and each statement is a map 31 | * containing: 32 | *
      33 | *
    • id - Id of statement which can be later referenced.
      • 34 | *
    • query - Statement CQL query.
      • 35 | *
    36 | *
    • 37 | *
38 | */ 39 | public class CassandraConfiguration implements WorkerConfiguration { 40 | 41 | private static final String CONNECTION_POINTS = "connection-points"; 42 | private static final String USE_SSL = "use-ssl"; 43 | private static final String KEYSPACE = "keyspace"; 44 | private static final String ASYNC = "async"; 45 | private static final String BOOTSTRAP_COMMANDS = "bootstrap-commands"; 46 | private static final String PREPARED_STATEMENTS = "prepared-statements"; 47 | 48 | @Override 49 | public String getName() { 50 | return "Cassandra"; 51 | } 52 | 53 | @Override 54 | public Worker getWorker(Map configuration) throws ConfigurationParseException { 55 | List connectionPointsWithPorts = getConnectionPointsWithPorts(configuration); 56 | boolean useSSL = getOptionalValue(configuration, USE_SSL, false); 57 | String keyspace = getMandatoryValue(configuration, KEYSPACE); 58 | boolean async = getOptionalValue(configuration, ASYNC, false); 59 | List bootstrapDDLCommands = getOptionalValue(configuration, BOOTSTRAP_COMMANDS, new ArrayList<>(0)); 60 | List prepStatements = getPreparedStatements(configuration); 61 | return new CassandraWorker(connectionPointsWithPorts, useSSL, keyspace, async, bootstrapDDLCommands, 62 | prepStatements); 63 | } 64 | 65 | private List getConnectionPointsWithPorts(Map configuration) { 66 | String connectionPoints = (String) configuration.get(CONNECTION_POINTS); 67 | if (connectionPoints == null || configuration.isEmpty()) { 68 | throw new RuntimeException(CONNECTION_POINTS + " cannot be null nor empty."); 69 | } 70 | List result = new ArrayList<>(); 71 | for (String connectionPoint : connectionPoints.split(",")) { 72 | String[] hostnameAndPort = connectionPoint.trim().split(":"); 73 | InetSocketAddress address = new InetSocketAddress(hostnameAndPort[0], Integer.parseInt(hostnameAndPort[1])); 74 | result.add(address); 75 | } 76 | return result; 77 | } 78 | 79 | @SuppressWarnings("unchecked") 80 | private List getPreparedStatements(Map configuration) { 81 | List preparedStatements = (List) configuration.get(PREPARED_STATEMENTS); 82 | if (preparedStatements == null) { 83 | return emptyList(); 84 | } 85 | ObjectMapper objectMapper = new ObjectMapper(); 86 | return objectMapper.convertValue(preparedStatements, new TypeReference>() { 87 | }); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /berserker-cassandra/src/main/java/io/smartcat/berserker/cassandra/configuration/PreparedStatement.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.cassandra.configuration; 2 | 3 | /** 4 | * Prepared statement with id and query. 5 | */ 6 | public class PreparedStatement { 7 | 8 | private String id; 9 | private String query; 10 | 11 | /** 12 | * Constructs empty prepared statement. 13 | */ 14 | public PreparedStatement() { 15 | } 16 | 17 | /** 18 | * Constructs prepared statement with specified id and query. 19 | * 20 | * @param id Statement id. 21 | * @param query Statement query. 22 | */ 23 | public PreparedStatement(String id, String query) { 24 | this.id = id; 25 | this.query = query; 26 | } 27 | 28 | /** 29 | * Returns statement's id. 30 | * 31 | * @return Statement's id. 32 | */ 33 | public String getId() { 34 | return id; 35 | } 36 | 37 | /** 38 | * Sets statement's id. 39 | * 40 | * @param id Id to set. 41 | */ 42 | public void setId(String id) { 43 | this.id = id; 44 | } 45 | 46 | /** 47 | * Returns statement's query. 48 | * 49 | * @return Statement's query. 50 | */ 51 | public String getQuery() { 52 | return query; 53 | } 54 | 55 | /** 56 | * Sets statement's query. 57 | * 58 | * @param query Query to set. 59 | */ 60 | public void setQuery(String query) { 61 | this.query = query; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /berserker-cassandra/src/main/java/io/smartcat/berserker/cassandra/worker/CassandraWorker.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.cassandra.worker; 2 | 3 | import java.net.InetSocketAddress; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | 8 | import com.datastax.driver.core.*; 9 | import com.datastax.driver.core.Cluster.Builder; 10 | 11 | import com.google.common.util.concurrent.FutureCallback; 12 | import com.google.common.util.concurrent.Futures; 13 | import com.google.common.util.concurrent.MoreExecutors; 14 | import io.smartcat.berserker.api.Worker; 15 | import io.smartcat.berserker.cassandra.configuration.PreparedStatement; 16 | 17 | /** 18 | * Worker that executes CQL statements on provided Cassandra connection points. It uses DataStax's java driver 19 | * internally. 20 | */ 21 | public class CassandraWorker implements Worker> { 22 | 23 | private static final String QUERY = "query"; 24 | private static final String VALUES = "values"; 25 | private static final String PREPARED_STATEMENT_ID = "preparedStatementId"; 26 | private static final String CONSISTENCY_LEVEL = "consistencyLevel"; 27 | 28 | private final Cluster cluster; 29 | private final Session session; 30 | private final boolean async; 31 | private final Map preparedStatements; 32 | 33 | /** 34 | * Constructs Cassandra worker with specified properties. 35 | * 36 | * @param connectionPointsWithPorts List of connection points to use. 37 | * @param useSSL Enable ssl for connection. 38 | * @param keyspace Name of keyspace in database to use. 39 | * @param async Indicates whether statements will be executed synchronously or asynchronously. 40 | * @param bootstrapDDLCommands List of CQL commands to execute only once after connection to Cassandra cluster is 41 | * established. Suitable for creating keyspaces, tables and populating some initial data if needed. 42 | * @param prepStatements List of prepared statements to create. Each statement is defined with id and can be 43 | * referenced from {@link #accept(Map, Runnable, Runnable)} method. 44 | */ 45 | public CassandraWorker(List connectionPointsWithPorts, boolean useSSL, String keyspace, 46 | boolean async, List bootstrapDDLCommands, List prepStatements) { 47 | if (connectionPointsWithPorts == null || connectionPointsWithPorts.isEmpty()) { 48 | throw new IllegalArgumentException("List of connection points with ports cannot be null nor empty"); 49 | } 50 | if (keyspace == null || keyspace.isEmpty()) { 51 | throw new IllegalArgumentException("Keyspace cannot be null nor empty."); 52 | } 53 | 54 | Builder builder = Cluster.builder().addContactPointsWithPorts(connectionPointsWithPorts); 55 | 56 | if (useSSL) { 57 | builder = builder.withSSL(); 58 | } 59 | 60 | cluster = builder.build(); 61 | session = cluster.connect(); 62 | if (bootstrapDDLCommands != null) { 63 | for (String command : bootstrapDDLCommands) { 64 | session.execute(command); 65 | } 66 | } 67 | session.execute("USE " + keyspace + ";"); 68 | preparedStatements = new HashMap<>(); 69 | if (prepStatements != null && !prepStatements.isEmpty()) { 70 | for (PreparedStatement prepStatement : prepStatements) { 71 | RegularStatement toPrepare = new SimpleStatement(prepStatement.getQuery()); 72 | preparedStatements.put(prepStatement.getId(), session.prepare(toPrepare)); 73 | } 74 | } 75 | this.async = async; 76 | } 77 | 78 | /** 79 | * Accepts two possible combinations: 80 | *
    81 | *
  • Query, which contains: 82 | *
      83 | *
    • consistencyLevel - Consistency level of statement which will be executed. If not specified, ONE will be 84 | * used.
      • 85 | *
    • query - String representation of CQL statement which will be executed.
      • 86 | *
    87 | *
    • 88 | *
  • Prepared statement, which contains: 89 | *
      90 | *
    • consistencyLevel - Consistency level of statement which will be executed. If not specified, ONE will be 91 | * used.
      • 92 | *
    • preparedStatementId - Id of prepared statement to execute (defined within constructor).
      • 93 | *
    • values - List of values to bind to prepared statement's variables.
      • 94 | *
    95 | *
    • 96 | *
97 | * Depending on the map content, appropriate option will be executed (either query or prepared statement). 98 | */ 99 | @SuppressWarnings("unchecked") 100 | @Override 101 | public void accept(Map queryMetadata, Runnable commitSuccess, Runnable commitFailure) { 102 | ConsistencyLevel consistencyLevel = getConsistencyLevel(queryMetadata); 103 | String statement = (String) queryMetadata.get(QUERY); 104 | Statement toExecute; 105 | if (statement != null) { 106 | toExecute = new SimpleStatement(statement); 107 | } else { 108 | String statementId = (String) queryMetadata.get(PREPARED_STATEMENT_ID); 109 | if (statementId == null || statementId.isEmpty()) { 110 | throw new InvalidQueryMetadataException("Statement id cannot be null nor empty"); 111 | } 112 | List values = (List) queryMetadata.get(VALUES); 113 | if (values == null || values.isEmpty()) { 114 | throw new InvalidQueryMetadataException("Values cannot be null nor empty"); 115 | } 116 | com.datastax.driver.core.PreparedStatement preparedStatement = preparedStatements.get(statementId); 117 | if (preparedStatement == null) { 118 | throw new InvalidQueryMetadataException( 119 | "Prepared statement with id: " + statementId + " not defined within configuration."); 120 | } 121 | toExecute = preparedStatement.bind(values.toArray(new Object[values.size()])); 122 | } 123 | 124 | toExecute.setConsistencyLevel(consistencyLevel); 125 | ResultSetFuture future = session.executeAsync(toExecute); 126 | Futures.addCallback(future, new FutureCallback() { 127 | @Override 128 | public void onSuccess(ResultSet result) { 129 | commitSuccess.run(); 130 | } 131 | 132 | @Override 133 | public void onFailure(Throwable t) { 134 | commitFailure.run(); 135 | } 136 | }, MoreExecutors.directExecutor()); 137 | if (!async) { 138 | try { 139 | future.get(); 140 | } catch (Exception e) { 141 | throw new RuntimeException(e); 142 | } 143 | 144 | } 145 | } 146 | 147 | private ConsistencyLevel getConsistencyLevel(Map queryMetadata) { 148 | String consistency = (String) queryMetadata.get(CONSISTENCY_LEVEL); 149 | if (consistency == null || consistency.isEmpty()) { 150 | return ConsistencyLevel.ONE; 151 | } 152 | return ConsistencyLevel.valueOf(consistency); 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /berserker-cassandra/src/main/java/io/smartcat/berserker/cassandra/worker/InvalidQueryMetadataException.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.cassandra.worker; 2 | 3 | /** 4 | * Indicates invalid query metadata. 5 | */ 6 | public class InvalidQueryMetadataException extends RuntimeException { 7 | 8 | private static final long serialVersionUID = -5024863994836325282L; 9 | 10 | /** 11 | * Constructs an InvalidQueryMetadataException with no detail message. A detail message is a String that describes 12 | * this particular exception. 13 | */ 14 | public InvalidQueryMetadataException() { 15 | } 16 | 17 | /** 18 | * Constructs an InvalidQueryMetadataException with the specified detail message. A detail message is a String that 19 | * describes this particular exception. 20 | * 21 | * @param message The string that contains a detailed message. 22 | */ 23 | public InvalidQueryMetadataException(String message) { 24 | super(message); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /berserker-commons/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | berserker 6 | io.smartcat 7 | 0.0.13-SNAPSHOT 8 | 9 | 10 | berserker-commons 11 | jar 12 | 13 | Berserker Commons 14 | Holds interface for core and configuration and it provides signature all the modules need to confront to be able to work together. 15 | 16 | 17 | UTF-8 18 | 19 | 20 | 21 | 22 | org.slf4j 23 | slf4j-api 24 | 25 | 26 | ch.qos.logback 27 | logback-classic 28 | 29 | 30 | com.google.guava 31 | guava 32 | 33 | 34 | org.yaml 35 | snakeyaml 36 | 37 | 38 | com.fasterxml.jackson.core 39 | jackson-databind 40 | 41 | 42 | io.dropwizard.metrics 43 | metrics-core 44 | 45 | 46 | junit 47 | junit 48 | test 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /berserker-commons/src/main/java/io/smartcat/berserker/api/AlreadyClosedException.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.api; 2 | 3 | /** 4 | * Signals that method has been called on already closed object. 5 | */ 6 | public class AlreadyClosedException extends IllegalStateException { 7 | 8 | private static final long serialVersionUID = -2862469865231955799L; 9 | 10 | /** 11 | * Constructs an AlreadyClosedException with no detail message. A detail message is a String that describes this 12 | * particular exception. 13 | */ 14 | public AlreadyClosedException() { 15 | } 16 | 17 | /** 18 | * Constructs an AlreadyClosedException with the specified detail message. A detail message is a String that 19 | * describes this particular exception. 20 | * 21 | * @param message the String that contains a detailed message. 22 | */ 23 | public AlreadyClosedException(String message) { 24 | super(message); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /berserker-commons/src/main/java/io/smartcat/berserker/api/DataSource.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.api; 2 | 3 | /** 4 | * Interface providing API for source of data. 5 | * 6 | * @param Type of the data provided by data source implementation. 7 | */ 8 | public interface DataSource { 9 | 10 | /** 11 | * Returns true if data source can provide next value, otherwise false. 12 | * 13 | * @param time Relative time in nanoseconds from starting load generator. Time can be used for implementing time 14 | * dependent data source which can then return data with time stamps and/or data influenced by time in 15 | * any other way. 16 | * @return True if data source can provide next value, otherwise false. 17 | */ 18 | boolean hasNext(long time); 19 | 20 | /** 21 | * Returns next value from this data source. 22 | * 23 | * @param time Relative time in nanoseconds from starting load generator. Time can be used for implementing time 24 | * dependent data source which can then return data with time stamps and/or data influenced by time in 25 | * any other way. 26 | * @return Next value from this data source. 27 | */ 28 | T getNext(long time); 29 | } 30 | -------------------------------------------------------------------------------- /berserker-commons/src/main/java/io/smartcat/berserker/api/RateGenerator.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.api; 2 | 3 | /** 4 | * Interface providing API for rate generation. Generates rate per second. 5 | */ 6 | public interface RateGenerator { 7 | 8 | /** 9 | * Returns rate per second as a function of time. 10 | * 11 | * @param time Relative time in nanoseconds from starting load generator. 12 | * @return Rate per second as a function of time. 13 | */ 14 | double getRate(long time); 15 | } 16 | -------------------------------------------------------------------------------- /berserker-commons/src/main/java/io/smartcat/berserker/api/Worker.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.api; 2 | 3 | /** 4 | * Worker interface providing API for task execution. 5 | * 6 | * @param Type of data workerConfiguration accepts. 7 | */ 8 | public interface Worker { 9 | 10 | /** 11 | * Accepts message of type {@code } and processes it. 12 | * 13 | * @param message Message which will be processed. 14 | * @param commitSuccess Callback to be invoked when processing is successful. 15 | * @param commitFailure Callback to be invoked in case of a failure. 16 | */ 17 | void accept(T message, Runnable commitSuccess, Runnable commitFailure); 18 | } 19 | -------------------------------------------------------------------------------- /berserker-commons/src/main/java/io/smartcat/berserker/configuration/BaseConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration; 2 | 3 | /** 4 | * Marker interface for configurations. 5 | */ 6 | public interface BaseConfiguration { 7 | 8 | /** 9 | * Returns name of this configuration. 10 | * 11 | * @return name of this configuration. 12 | */ 13 | String getName(); 14 | } 15 | -------------------------------------------------------------------------------- /berserker-commons/src/main/java/io/smartcat/berserker/configuration/ConfigurationException.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration; 2 | 3 | /** 4 | * GlobalConfiguration exception. 5 | */ 6 | public class ConfigurationException extends Exception { 7 | 8 | private static final long serialVersionUID = -8589993260328531894L; 9 | 10 | /** 11 | * Default constructor. 12 | */ 13 | public ConfigurationException() { 14 | } 15 | 16 | /** 17 | * Constructor. 18 | * 19 | * @param message Error message. 20 | * @param cause Exception cause. 21 | * @param enableSuppression controls exception suppression. 22 | * @param writableStackTrace stack trace. 23 | */ 24 | public ConfigurationException(String message, Throwable cause, boolean enableSuppression, 25 | boolean writableStackTrace) { 26 | super(message, cause, enableSuppression, writableStackTrace); 27 | } 28 | 29 | /** 30 | * Constructor. 31 | * 32 | * @param message Error message. 33 | * @param cause Exception cause. 34 | */ 35 | public ConfigurationException(String message, Throwable cause) { 36 | super(message, cause); 37 | } 38 | 39 | /** 40 | * Constructor. 41 | * 42 | * @param message Error message. 43 | */ 44 | public ConfigurationException(String message) { 45 | super(message); 46 | } 47 | 48 | /** 49 | * Constructor. 50 | * 51 | * @param cause Exception cause. 52 | */ 53 | public ConfigurationException(Throwable cause) { 54 | super(cause); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /berserker-commons/src/main/java/io/smartcat/berserker/configuration/ConfigurationHelper.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.util.Map; 7 | 8 | /** 9 | * Helper class for configuration related operations. 10 | */ 11 | public class ConfigurationHelper { 12 | 13 | private static final Logger LOGGER = LoggerFactory.getLogger(ConfigurationHelper.class); 14 | 15 | private ConfigurationHelper() { 16 | } 17 | 18 | /** 19 | * Tries to get mandatory value, if value does not exist, exception is thrown. 20 | * 21 | * @param configuration Configuration from which to get value. 22 | * @param name Name of the value. 23 | * @param Type of the value. 24 | * @return Value if found, otherwise exception is thrown. 25 | */ 26 | public static T getMandatoryValue(Map configuration, String name) { 27 | T value = (T) configuration.get(name); 28 | if (value == null || (value instanceof String && ((String) value).isEmpty())) { 29 | throw new RuntimeException("'" + name + "' is mandatory."); 30 | } 31 | LOGGER.info("'" + name + "' set to value: " + value); 32 | return value; 33 | } 34 | 35 | /** 36 | * Tries to get optional value, if value does not exist, default value is returned. 37 | * 38 | * @param configuration Configuration from which to get value. 39 | * @param name Name of the value. 40 | * @param defaultValue Default value. 41 | * @param Type of the value. 42 | * @return Value if found, otherwise default value. 43 | */ 44 | public static T getOptionalValue(Map configuration, String name, T defaultValue) { 45 | T value = (T) configuration.get(name); 46 | if (value == null) { 47 | LOGGER.info("'" + name + "' not set, using default value: " + defaultValue); 48 | return defaultValue; 49 | } 50 | if (defaultValue != null && !defaultValue.getClass().isAssignableFrom(value.getClass())) { 51 | throw new RuntimeException("'" + name + "' set to wrong value. Value is of type: " 52 | + value.getClass().getName() + ", but should be of type: " + defaultValue.getClass().getName()); 53 | } 54 | LOGGER.info("'" + name + "' set to value: " + value); 55 | return value; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /berserker-commons/src/main/java/io/smartcat/berserker/configuration/ConfigurationLoader.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration; 2 | 3 | import java.net.URL; 4 | 5 | /** 6 | * Loader for {@link GlobalConfiguration}. 7 | */ 8 | public interface ConfigurationLoader { 9 | /** 10 | * Loads configuration from an implicit location. 11 | * 12 | * @return loaded configuration 13 | * @throws ConfigurationException in case the configuration cannot be loaded 14 | */ 15 | GlobalConfiguration loadConfig() throws ConfigurationException; 16 | 17 | /** 18 | * Loads configuration using an explicit location. 19 | * 20 | * @param url configuration location 21 | * @return loaded configuration 22 | * @throws ConfigurationException in case the configuration cannot be loaded 23 | */ 24 | GlobalConfiguration loadConfig(URL url) throws ConfigurationException; 25 | } 26 | -------------------------------------------------------------------------------- /berserker-commons/src/main/java/io/smartcat/berserker/configuration/ConfigurationParseException.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration; 2 | 3 | /** 4 | * Indicates error while parsing configuration. 5 | */ 6 | public class ConfigurationParseException extends ConfigurationException { 7 | 8 | private static final long serialVersionUID = -7101805420460054579L; 9 | 10 | /** 11 | * Default constructor. 12 | */ 13 | public ConfigurationParseException() { 14 | } 15 | 16 | /** 17 | * Constructor. 18 | * 19 | * @param message Error message. 20 | * @param cause Exception cause. 21 | * @param enableSuppression controls exception suppression. 22 | * @param writableStackTrace stack trace. 23 | */ 24 | public ConfigurationParseException(String message, Throwable cause, boolean enableSuppression, 25 | boolean writableStackTrace) { 26 | super(message, cause, enableSuppression, writableStackTrace); 27 | } 28 | 29 | /** 30 | * Constructor. 31 | * 32 | * @param message Error message. 33 | * @param cause Exception cause. 34 | */ 35 | public ConfigurationParseException(String message, Throwable cause) { 36 | super(message, cause); 37 | } 38 | 39 | /** 40 | * Constructor. 41 | * 42 | * @param message Error message. 43 | */ 44 | public ConfigurationParseException(String message) { 45 | super(message); 46 | } 47 | 48 | /** 49 | * Constructor. 50 | * 51 | * @param cause Exception cause. 52 | */ 53 | public ConfigurationParseException(Throwable cause) { 54 | super(cause); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /berserker-commons/src/main/java/io/smartcat/berserker/configuration/DataSourceConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration; 2 | 3 | import java.util.Map; 4 | 5 | import io.smartcat.berserker.api.DataSource; 6 | 7 | /** 8 | * Returns {@link DataSource} based on configuration parameters. Each {@link DataSource} implementation should go with 9 | * corresponding data source configuration implementation which would be used to construct that data source. 10 | */ 11 | public interface DataSourceConfiguration extends BaseConfiguration { 12 | 13 | /** 14 | * Returns data source based on configuration parameters. 15 | * 16 | * @param configuration Configuration specific to data source it should construct. 17 | * @return Instance of {@link DataSource}, never null. 18 | * 19 | * @throws ConfigurationParseException If there is an error during configuration parsing. 20 | */ 21 | DataSource getDataSource(Map configuration) throws ConfigurationParseException; 22 | } 23 | -------------------------------------------------------------------------------- /berserker-commons/src/main/java/io/smartcat/berserker/configuration/GlobalConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration; 2 | 3 | import java.util.Map; 4 | 5 | /** 6 | * Global configuration for Load Generator and its dependencies. 7 | */ 8 | public class GlobalConfiguration { 9 | 10 | /** 11 | * Configuration specific to {@link io.smartcat.berserker.api.DataSource DataSource}. 12 | */ 13 | public Map dataSourceConfiguration; 14 | 15 | /** 16 | * Configuration specific to {@link io.smartcat.berserker.api.Worker Worker}. 17 | */ 18 | public Map workerConfiguration; 19 | 20 | /** 21 | * Configuration specific to {@link io.smartcat.berserker.api.RateGenerator RateGenerator}. 22 | */ 23 | public Map rateGeneratorConfiguration; 24 | 25 | /** 26 | * Configuration specific to {@link com.codahale.metrics.Reporter Reporter}. 27 | */ 28 | public Map metricsReporterConfiguration; 29 | 30 | /** 31 | * Configuration explaining which {@link DataSourceConfiguration}, {@link WorkerConfiguration} and 32 | * {@link RateGeneratorConfiguration} implementations will be used. 33 | */ 34 | public LoadGeneratorConfiguration loadGeneratorConfiguration; 35 | } 36 | -------------------------------------------------------------------------------- /berserker-commons/src/main/java/io/smartcat/berserker/configuration/LoadGeneratorConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration; 2 | 3 | /** 4 | * Load Generator configuration explaining which configurations for data source, rate generator and worker should be 5 | * used. 6 | */ 7 | public class LoadGeneratorConfiguration { 8 | 9 | /** 10 | * Name of {@link io.smartcat.berserker.api.DataSource DataSource} configuration. 11 | */ 12 | public String dataSourceConfigurationName; 13 | 14 | /** 15 | * Name of {@link io.smartcat.berserker.api.RateGenerator RateGenerator} configuration. 16 | */ 17 | public String rateGeneratorConfigurationName; 18 | 19 | /** 20 | * Name of {@link io.smartcat.berserker.api.Worker Worker} configuration. 21 | */ 22 | public String workerConfigurationName; 23 | 24 | /** 25 | * Name of {@link com.codahale.metrics.Reporter Reporter} configuration. 26 | */ 27 | public String metricsReporterConfigurationName; 28 | 29 | /** 30 | * Prefix used for metrics. 31 | */ 32 | public String metricsPrefix; 33 | 34 | /** 35 | * Number of threads to be used by Load Generator. 36 | */ 37 | public int threadCount; 38 | 39 | /** 40 | * Capacity of queue to be used by Load Generator. 41 | */ 42 | public int queueCapacity; 43 | 44 | /** 45 | * Validates this configuration. 46 | * 47 | * @throws ConfigurationException If configuration is not correct. 48 | */ 49 | public void validate() throws ConfigurationException { 50 | validateProperty(dataSourceConfigurationName, "data-source-configuration-name"); 51 | validateProperty(rateGeneratorConfigurationName, "rate-generator-configuration-name"); 52 | validateProperty(workerConfigurationName, "worker-configuration-name"); 53 | validateProperty(threadCount, "thread-count"); 54 | validateProperty(queueCapacity, "queue-capacity"); 55 | } 56 | 57 | private void validateProperty(int property, String propertyName) throws ConfigurationException { 58 | if (property <= 0) { 59 | throw new ConfigurationException( 60 | "Property: '" + propertyName + "' is mandatory. It needs to be positive number."); 61 | } 62 | } 63 | private void validateProperty(String property, String propertyName) throws ConfigurationException { 64 | if (property == null || property.isEmpty()) { 65 | throw new ConfigurationException( 66 | "Property: '" + propertyName + "' is mandatory. It cannot be null nor empty."); 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /berserker-commons/src/main/java/io/smartcat/berserker/configuration/MetricsReporterConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration; 2 | 3 | import java.util.Map; 4 | 5 | import com.codahale.metrics.MetricRegistry; 6 | 7 | /** 8 | * Creates and starts {@link com.codahale.metrics.Reporter Reporter}. Each {@link com.codahale.metrics.Reporter 9 | * Reporter} should go with corresponding metrics reporter configuration implementation which would be used to 10 | * construct that metrics reporter. 11 | */ 12 | public interface MetricsReporterConfiguration extends BaseConfiguration { 13 | 14 | /** 15 | * Creates and starts reporter for specified metricRegistry based on provided configuration. 16 | * 17 | * @param metricRegistry Metric registry for which to create and start reporter. 18 | * @param configuration Configuration specific to metrics reporter it should construct. 19 | */ 20 | void createAndStartReporter(MetricRegistry metricRegistry, Map configuration); 21 | } 22 | -------------------------------------------------------------------------------- /berserker-commons/src/main/java/io/smartcat/berserker/configuration/RateGeneratorConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration; 2 | 3 | import java.util.Map; 4 | 5 | import io.smartcat.berserker.api.RateGenerator; 6 | 7 | /** 8 | * Returns {@link RateGenerator} based on configuration parameters. Each {@link RateGenerator} implementation should go 9 | * with corresponding rate generator configuration implementation which would be used to construct that rate generator. 10 | */ 11 | public interface RateGeneratorConfiguration extends BaseConfiguration { 12 | 13 | /** 14 | * Returns rate generator based on configuration parameters. 15 | * 16 | * @param configuration Configuration specific to rate generator it should construct. 17 | * @return Instance of {@link RateGenerator}, never null. 18 | * 19 | * @throws ConfigurationParseException If there is an error during configuration parsing. 20 | */ 21 | RateGenerator getRateGenerator(Map configuration) throws ConfigurationParseException; 22 | } 23 | -------------------------------------------------------------------------------- /berserker-commons/src/main/java/io/smartcat/berserker/configuration/WorkerConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration; 2 | 3 | import java.util.Map; 4 | 5 | import io.smartcat.berserker.api.Worker; 6 | 7 | /** 8 | * Returns {@link Worker} based on configuration parameters. Each {@link Worker} implementation should go with 9 | * corresponding worker configuration implementation which would be used to construct that worker. 10 | */ 11 | public interface WorkerConfiguration extends BaseConfiguration { 12 | 13 | /** 14 | * Returns worker based on configuration parameters. 15 | * 16 | * @param configuration Configuration specific to this worker. 17 | * @return Instance of {@link Worker}, never null. 18 | * 19 | * @throws ConfigurationParseException If there is an error during configuration parsing. 20 | */ 21 | Worker getWorker(Map configuration) throws ConfigurationParseException; 22 | } 23 | -------------------------------------------------------------------------------- /berserker-commons/src/main/java/io/smartcat/berserker/configuration/YamlConfigurationLoader.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration; 2 | 3 | import static com.google.common.base.CaseFormat.*; 4 | 5 | import java.beans.IntrospectionException; 6 | import java.io.File; 7 | import java.io.IOException; 8 | import java.io.InputStream; 9 | import java.net.URL; 10 | 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | import org.yaml.snakeyaml.Yaml; 14 | import org.yaml.snakeyaml.constructor.Constructor; 15 | import org.yaml.snakeyaml.error.YAMLException; 16 | import org.yaml.snakeyaml.introspector.Property; 17 | import org.yaml.snakeyaml.introspector.PropertyUtils; 18 | 19 | /** 20 | * YAML implementation of {@link ConfigurationLoader}. 21 | */ 22 | public class YamlConfigurationLoader implements ConfigurationLoader { 23 | 24 | private static final Logger LOGGER = LoggerFactory.getLogger(YamlConfigurationLoader.class); 25 | private static final String HYPHEN = "-"; 26 | private static final String CONFIGURATION_PROPERTY_NAME = "berserker.config"; 27 | 28 | /** 29 | * Default external configuration file name. 30 | */ 31 | private static final String DEFAULT_CONFIGURATION_URL = "berserker-default.yml"; 32 | 33 | /** 34 | * Determines and returns the external configuration URL. 35 | * 36 | * @return {@link URL} configuration file URL 37 | * @throws ConfigurationException in case of a bogus URL 38 | */ 39 | private URL getStorageConfigUrl() throws ConfigurationException { 40 | String configUrl = System.getProperty(CONFIGURATION_PROPERTY_NAME); 41 | if (configUrl == null) { 42 | configUrl = DEFAULT_CONFIGURATION_URL; 43 | LOGGER.info("Using default configuration " + DEFAULT_CONFIGURATION_URL); 44 | } 45 | 46 | URL url; 47 | try { 48 | url = new URL(configUrl); 49 | url.openStream().close(); // catches well-formed but bogus URLs 50 | } catch (Exception err) { 51 | ClassLoader loader = YamlConfigurationLoader.class.getClassLoader(); 52 | url = loader.getResource(configUrl); 53 | if (url == null) { 54 | String required = "file:" + File.separator + File.separator; 55 | if (!configUrl.startsWith(required)) { 56 | throw new ConfigurationException("Expecting URI in variable [" + CONFIGURATION_PROPERTY_NAME + "]. " 57 | + "Please prefix the file with " + required + File.separator + " for local files or " 58 | + required + "" + File.separator + " for remote files. Aborting."); 59 | } 60 | throw new ConfigurationException( 61 | "Cannot locate " + configUrl + ". If this is a local file, please confirm you've provided " 62 | + required + File.separator + " as a URI prefix."); 63 | } 64 | } 65 | return url; 66 | } 67 | 68 | @Override 69 | public GlobalConfiguration loadConfig() throws ConfigurationException { 70 | return loadConfig(getStorageConfigUrl()); 71 | } 72 | 73 | @Override 74 | public GlobalConfiguration loadConfig(URL url) throws ConfigurationException { 75 | try { 76 | LOGGER.info("Loading settings from {}", url); 77 | Constructor constructor = new Constructor(GlobalConfiguration.class); 78 | 79 | // treat dashed properties as camel case properties 80 | constructor.setPropertyUtils(new PropertyUtils() { 81 | @Override 82 | public Property getProperty(Class type, String name) throws IntrospectionException { 83 | if (name.contains(HYPHEN)) { 84 | name = LOWER_HYPHEN.to(LOWER_CAMEL, name); 85 | } 86 | return super.getProperty(type, name); 87 | } 88 | }); 89 | 90 | Yaml yaml = new Yaml(constructor); 91 | GlobalConfiguration result; 92 | try (InputStream is = url.openStream()) { 93 | result = yaml.loadAs(is, GlobalConfiguration.class); 94 | } catch (IOException e) { 95 | throw new AssertionError(e); 96 | } 97 | 98 | if (result == null) { 99 | throw new ConfigurationException("Invalid yaml"); 100 | } 101 | return result; 102 | } catch (YAMLException e) { 103 | throw new ConfigurationException("Invalid yaml", e); 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /berserker-core/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | berserker 6 | io.smartcat 7 | 0.0.13-SNAPSHOT 8 | 9 | 10 | berserker-core 11 | jar 12 | 13 | Berserker Core 14 | Contains load generator implementation, and common implementations of data source, rate generator and worker. 15 | 16 | 17 | UTF-8 18 | 1.1.8 19 | 1.1-groovy-2.4 20 | 3.2.0 21 | 2.4.12 22 | 23 | 24 | 25 | 26 | io.smartcat 27 | berserker-commons 28 | ${project.parent.version} 29 | 30 | 31 | org.parboiled 32 | parboiled-java 33 | ${version.parboiled-java} 34 | 35 | 36 | io.dropwizard.metrics 37 | metrics-jmx 38 | ${version.dropwizard.metrics} 39 | 40 | 41 | junit 42 | junit 43 | test 44 | 45 | 46 | org.spockframework 47 | spock-core 48 | ${version.spock-core} 49 | test 50 | 51 | 52 | cglib 53 | cglib-nodep 54 | ${version.cglib-nodep} 55 | test 56 | 57 | 58 | org.codehaus.groovy 59 | groovy-all 60 | ${version.groovy-all} 61 | test 62 | 63 | 64 | 65 | 66 | 67 | 68 | org.codehaus.gmavenplus 69 | gmavenplus-plugin 70 | 1.4 71 | 72 | 73 | 74 | compile 75 | testCompile 76 | 77 | 78 | 79 | 80 | 81 | maven-surefire-plugin 82 | 2.6 83 | 84 | false 85 | 86 | **/*Test.java 87 | **/*Spec.java 88 | 89 | 90 | 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/LoadGenerator.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker; 2 | 3 | import java.util.concurrent.TimeUnit; 4 | import java.util.concurrent.atomic.AtomicBoolean; 5 | import java.util.function.Consumer; 6 | 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import io.smartcat.berserker.api.DataSource; 11 | import io.smartcat.berserker.api.RateGenerator; 12 | 13 | /** 14 | * Load generator used to execute work tasks with data from provided data source. 15 | * 16 | * @param Type of data which will be used. 17 | */ 18 | public class LoadGenerator { 19 | 20 | private static final Logger LOGGER = LoggerFactory.getLogger(LoadGenerator.class); 21 | private static final long NANOS_IN_SECOND = TimeUnit.SECONDS.toNanos(1); 22 | private static final long TICK_PERIOD_IN_NANOS = 1000; 23 | 24 | private final DataSource dataSource; 25 | private final RateGenerator rateGenerator; 26 | private final Consumer worker; 27 | 28 | private AtomicBoolean terminate = new AtomicBoolean(false); 29 | 30 | /** 31 | * Constructs load generator with specified dataSource, rateGenerator and 32 | * worker. 33 | * 34 | * @param dataSource Data source from which load generator polls data. 35 | * @param rateGenerator Rate generator which generates rate based on time point. 36 | * @param worker Worker which accepts data polled from dataSource at rate provided by 37 | * rateGenerator. 38 | */ 39 | public LoadGenerator(DataSource dataSource, RateGenerator rateGenerator, Consumer worker) { 40 | this.dataSource = dataSource; 41 | this.rateGenerator = rateGenerator; 42 | this.worker = worker; 43 | } 44 | 45 | /** 46 | * Runs load generator. 47 | * 48 | * @throws IllegalStateException When run is attempted after load generator was terminated. 49 | */ 50 | public void run() { 51 | try { 52 | checkState(); 53 | LOGGER.info("Load generator started."); 54 | long beginning = System.nanoTime(); 55 | long previous = beginning; 56 | infiniteWhile: while (true) { 57 | if (terminate.get()) { 58 | LOGGER.info("Termination signal detected. Terminating load generator..."); 59 | break; 60 | } 61 | long now = System.nanoTime(); 62 | long fromBeginning = now - beginning; 63 | long elapsed = now - previous; 64 | double rate = rateGenerator.getRate(fromBeginning); 65 | long normalizedRate = normalizeRate(elapsed, rate); 66 | if (normalizedRate > 0) { 67 | previous += calculateConsumedTime(normalizedRate, rate); 68 | } 69 | for (int i = 0; i < normalizedRate; i++) { 70 | if (!dataSource.hasNext(fromBeginning)) { 71 | LOGGER.info("Reached end of data source. Terminating load generator..."); 72 | terminate.set(true); 73 | break infiniteWhile; 74 | } 75 | T data = dataSource.getNext(fromBeginning); 76 | worker.accept(data); 77 | } 78 | } 79 | LOGGER.info("Load generator terminated."); 80 | } catch (Exception e) { 81 | LOGGER.error("Terminating load generator due to error. Error: ", e); 82 | } 83 | } 84 | 85 | /** 86 | * Stops load generator. 87 | */ 88 | public void terminate() { 89 | terminate.set(true); 90 | LOGGER.info("Termination signal sent."); 91 | } 92 | 93 | private void checkState() { 94 | if (terminate.get()) { 95 | throw new IllegalStateException("Load generator is stopped and cannot be started again."); 96 | } 97 | } 98 | 99 | private long normalizeRate(long elapsed, double rate) { 100 | if (elapsed < TICK_PERIOD_IN_NANOS) { 101 | return 0; 102 | } 103 | return (long) (elapsed * rate / NANOS_IN_SECOND); 104 | } 105 | 106 | private long calculateConsumedTime(long normalizedRate, double rate) { 107 | return (long) (normalizedRate * NANOS_IN_SECOND / rate); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/configuration/DefaultRateGeneratorConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration; 2 | 3 | import java.util.Map; 4 | 5 | import io.smartcat.berserker.api.RateGenerator; 6 | import io.smartcat.berserker.configuration.rategenerator.RateGeneratorConfigurationParser; 7 | 8 | /** 9 | * Configuration to construct rate generator out of rate generator expressions. 10 | */ 11 | public class DefaultRateGeneratorConfiguration implements RateGeneratorConfiguration { 12 | 13 | @Override 14 | public String getName() { 15 | return "default"; 16 | } 17 | 18 | @Override 19 | public RateGenerator getRateGenerator(Map configuration) throws ConfigurationParseException { 20 | RateGeneratorConfigurationParser parser = new RateGeneratorConfigurationParser(configuration); 21 | return parser.build(); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/configuration/JmxMetricsReporterConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | import java.util.Map; 6 | import java.util.concurrent.TimeUnit; 7 | import java.util.stream.Collectors; 8 | 9 | import com.codahale.metrics.jmx.JmxReporter; 10 | import com.codahale.metrics.jmx.JmxReporter.Builder; 11 | import com.codahale.metrics.Metric; 12 | import com.codahale.metrics.MetricFilter; 13 | import com.codahale.metrics.MetricRegistry; 14 | 15 | /** 16 | * Configuration to construct {@link JmxReporter}. 17 | * Example of supported configuration: 18 | * 
19 |  * {@code
20 |  * metrics-reporter-configuration:
21 |  *   domain: berserkerMetrics
22 |  *   filter: io.smartcat.berserker.waitTime, io.smartcat.berserker.serviceTime}
23 |  *
24 | */ 25 | public class JmxMetricsReporterConfiguration implements MetricsReporterConfiguration { 26 | 27 | private static final String DOMAIN = "domain"; 28 | private static final String FILTER = "filter"; 29 | 30 | @Override 31 | public String getName() { 32 | return "JMX"; 33 | } 34 | 35 | @Override 36 | public void createAndStartReporter(MetricRegistry metricRegistry, Map configuration) { 37 | Builder builder = JmxReporter.forRegistry(metricRegistry).convertRatesTo(TimeUnit.DAYS); 38 | if (configuration.containsKey(DOMAIN)) { 39 | String domain = (String) configuration.get(DOMAIN); 40 | if (domain != null && !domain.isEmpty()) { 41 | builder.inDomain(domain); 42 | } 43 | } 44 | if (configuration.containsKey(FILTER)) { 45 | String filter = (String) configuration.get(FILTER); 46 | if (filter != null && !filter.isEmpty()) { 47 | List metrics = Arrays.asList(filter.split(",")).stream().map(x -> x.trim()) 48 | .collect(Collectors.toList()); 49 | MetricFilter metricFilter = new MetricFilter() { 50 | @Override 51 | public boolean matches(String name, Metric metric) { 52 | return metrics.contains(name); 53 | } 54 | }; 55 | builder.filter(metricFilter); 56 | } 57 | } 58 | JmxReporter reporter = builder.build(); 59 | reporter.start(); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/configuration/RandomNumberDataSourceConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration; 2 | 3 | import java.util.Map; 4 | 5 | import io.smartcat.berserker.api.DataSource; 6 | import io.smartcat.berserker.datasource.RandomDoubleDataSource; 7 | import io.smartcat.berserker.datasource.RandomIntDataSource; 8 | import io.smartcat.berserker.datasource.RandomLongDataSource; 9 | 10 | /** 11 | * Configuration to construct one of the following: {@link RandomIntDataSource}, {@link RandomLongDataSource} or 12 | * {@link RandomDoubleDataSource}. 13 | * 14 | * Map needs to contain key 'type' and have one of the following values: 'int', 15 | * 'long', 'double'. 16 | */ 17 | public class RandomNumberDataSourceConfiguration implements DataSourceConfiguration { 18 | 19 | private static final String TYPE = "type"; 20 | private static final String TYPE_INT = "int"; 21 | private static final String TYPE_LONG = "long"; 22 | private static final String TYPE_DOUBLE = "double"; 23 | 24 | @Override 25 | public String getName() { 26 | return "RandomDataSource"; 27 | } 28 | 29 | @Override 30 | public DataSource getDataSource(Map configuration) throws ConfigurationParseException { 31 | if (!configuration.containsKey(TYPE)) { 32 | throw new ConfigurationParseException("Property '" + TYPE + "' is mandatory."); 33 | } 34 | String type = (String) configuration.get(TYPE); 35 | if (TYPE_INT.equals(type)) { 36 | return new RandomIntDataSource(); 37 | } else if (TYPE_LONG.equals(type)) { 38 | return new RandomLongDataSource(); 39 | } else if (TYPE_DOUBLE.equals(type)) { 40 | return new RandomDoubleDataSource(); 41 | } 42 | throw new ConfigurationParseException("Value ' + type + ' is not supported for property '" + TYPE + "'."); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/configuration/SimpleConsoleMetricsReporterConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration; 2 | 3 | import java.util.Map; 4 | import java.util.concurrent.TimeUnit; 5 | 6 | import com.codahale.metrics.ConsoleReporter; 7 | import com.codahale.metrics.MetricRegistry; 8 | 9 | /** 10 | * Creates simple {@link ConsoleReporter} without any customization. 11 | */ 12 | public class SimpleConsoleMetricsReporterConfiguration implements MetricsReporterConfiguration { 13 | 14 | @Override 15 | public String getName() { 16 | return "SimpleConsoleReporter"; 17 | } 18 | 19 | @Override 20 | public void createAndStartReporter(MetricRegistry metricRegistry, Map configuration) { 21 | ConsoleReporter reporter = ConsoleReporter.forRegistry(metricRegistry).build(); 22 | reporter.start(10, TimeUnit.SECONDS); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/configuration/rategenerator/InvalidReferenceNameException.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration.rategenerator; 2 | 3 | /** 4 | * Signals that invalid reference name is used within configuration. 5 | */ 6 | public class InvalidReferenceNameException extends RuntimeException { 7 | 8 | private static final long serialVersionUID = -5468713809934047143L; 9 | 10 | /** 11 | * Constructs an {@link InvalidReferenceNameException} with {@code null} as its detail message. 12 | */ 13 | public InvalidReferenceNameException() { 14 | } 15 | 16 | /** 17 | * Constructs an {@link InvalidReferenceNameException} with message containing the invalid reference name. 18 | * 19 | * @param referenceName Name of the invalid reference name. 20 | */ 21 | public InvalidReferenceNameException(String referenceName) { 22 | super("Reference with name '" + referenceName + "' not defined within configuration."); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/configuration/rategenerator/RateGeneratorConfigurationParser.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration.rategenerator; 2 | 3 | import io.smartcat.berserker.api.RateGenerator; 4 | import io.smartcat.berserker.rategenerator.ConstantRateGenerator; 5 | import io.smartcat.berserker.rategenerator.RateGeneratorProxy; 6 | import org.parboiled.Parboiled; 7 | import org.parboiled.parserunners.ReportingParseRunner; 8 | import org.parboiled.support.ParsingResult; 9 | 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | /** 14 | * Constructs {@link RateGenerator} out of parsed configuration. 15 | */ 16 | public class RateGeneratorConfigurationParser { 17 | 18 | private static final String OFFSET = "offset"; 19 | private static final String RATES = "rates"; 20 | private static final String OUTPUT = "output"; 21 | 22 | private final Map rates; 23 | private final Object outputExpression; 24 | private Map proxyValues; 25 | private RateGeneratorExpressionParser parser; 26 | private ReportingParseRunner parseRunner; 27 | 28 | /** 29 | * Constructs rate generator configuration parser with specified configuration. 30 | * 31 | * @param config Configuration to parse. 32 | */ 33 | @SuppressWarnings("unchecked") 34 | public RateGeneratorConfigurationParser(Map config) { 35 | checkSectionExistence(config, RATES); 36 | checkSectionExistence(config, OUTPUT); 37 | this.rates = (Map) config.get(RATES); 38 | this.outputExpression = config.get(OUTPUT); 39 | } 40 | 41 | /** 42 | * Creates an instance of {@link RateGenerator} based on provided configuration. 43 | * 44 | * @return An instance of {@link RateGenerator}. 45 | */ 46 | @SuppressWarnings({ "unchecked" }) 47 | public RateGenerator build() { 48 | buildModel(); 49 | return parseRateGeneratorExpression(outputExpression); 50 | } 51 | 52 | private void buildModel() { 53 | this.proxyValues = new HashMap<>(); 54 | this.parser = Parboiled.createParser(RateGeneratorExpressionParser.class, proxyValues); 55 | this.parseRunner = new ReportingParseRunner<>(parser.rateGenerator()); 56 | if (rates != null) { 57 | createProxies(); 58 | parseRateGenerators(); 59 | } 60 | } 61 | 62 | private void checkSectionExistence(Map config, String name) { 63 | if (!config.containsKey(name)) { 64 | throw new RuntimeException("Configuration must contain '" + name + "' section."); 65 | } 66 | } 67 | 68 | private void createProxies() { 69 | for (Map.Entry entry : rates.entrySet()) { 70 | proxyValues.put(entry.getKey(), new RateGeneratorProxy()); 71 | } 72 | } 73 | 74 | private void parseRateGenerators() { 75 | for (Map.Entry entry : rates.entrySet()) { 76 | RateGenerator rateGenerator = parseRateGeneratorExpression(entry.getValue()); 77 | RateGeneratorProxy proxy = proxyValues.get(entry.getKey()); 78 | proxy.setDelegate(rateGenerator); 79 | entry.setValue(proxy); 80 | } 81 | } 82 | 83 | private RateGenerator parseRateGeneratorExpression(Object def) { 84 | // handle String as expression and all other types as primitives 85 | if (def instanceof String) { 86 | ParsingResult result = parseRunner.run((String) def); 87 | return result.valueStack.pop(); 88 | } else if (def instanceof Long) { 89 | return new ConstantRateGenerator((long) def); 90 | } else if (def instanceof Integer) { 91 | return new ConstantRateGenerator(((Number) def).longValue()); 92 | } else { 93 | throw new RuntimeException("Object type not supported: " + def.getClass().getName()); 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/datasource/BufferedDataSource.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.datasource; 2 | 3 | import java.util.Queue; 4 | import java.util.concurrent.LinkedBlockingQueue; 5 | import java.util.concurrent.atomic.AtomicBoolean; 6 | 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import io.smartcat.berserker.api.DataSource; 11 | 12 | /** 13 | * Data source buffer which polls from specified delegate data source from different thread. Blocks polling at the 14 | * beginning until buffer is filled to initialBufferFullness. 15 | *

16 | * Note:Only time independent {@link DataSource} implementations should be used with this implementation. 17 | * For more info read the next paragraph. 18 | *

19 | *

20 | * Buffered data source breaks {@link DataSource} interface since delegate data source is called 21 | * with time value of 0. This is due to buffer implementation which fills buffer from another thread. 22 | * Since delegate.getNext(...) is invoked from another thread, it does not have information on time 23 | * component, hence cannot invoke delegate.getNext(...) with proper value. If you are using 24 | * time dependent {@link DataSource} implementation, it will not work together with this implementation. 25 | *

26 | * 27 | * @param Type of the data provided by data source implementation. 28 | */ 29 | public class BufferedDataSource implements DataSource, AutoCloseable { 30 | 31 | private static final int DEFAULT_BUFFER_SIZE = 1024; 32 | private static final double DEFAULT_INITIAL_BUFFER_FULLNESS = 0.75; 33 | 34 | private static final Logger LOGGER = LoggerFactory.getLogger(BufferedDataSource.class); 35 | 36 | private final Object bufferFilled = new Object(); 37 | private final DataSource delegate; 38 | private final int bufferSize; 39 | private final Queue buffer; 40 | private final double initialBufferFullness; 41 | 42 | private AtomicBoolean terminateThread = new AtomicBoolean(false); 43 | private Thread thread; 44 | 45 | /** 46 | * Constructs a buffered data source with specified delegate data source. bufferSize is 47 | * set to 1024, initialBufferFullness is set to 75%. Blocks polling at the beginning until buffer is 48 | * filled to initialBufferFullness. 49 | * 50 | * @param delegate Data source which will be used to fill buffer. 51 | */ 52 | public BufferedDataSource(DataSource delegate) { 53 | this(delegate, DEFAULT_BUFFER_SIZE); 54 | } 55 | 56 | /** 57 | * Constructs a buffered data source with specified delegate data source and bufferSize. 58 | * initialBufferFullness is set to 75%. Blocks polling at the beginning until buffer is filled to 59 | * initialBufferFullness. 60 | * 61 | * @param delegate Data source which will be used to fill buffer. 62 | * @param bufferSize Size of the buffer. 63 | */ 64 | public BufferedDataSource(DataSource delegate, int bufferSize) { 65 | this(delegate, bufferSize, DEFAULT_INITIAL_BUFFER_FULLNESS); 66 | } 67 | 68 | /** 69 | * Constructs a buffered data source with specified delegate data source, bufferSize and 70 | * initialBufferFullness. Blocks polling at the beginning until buffer is filled to 71 | * initialBufferFullness. 72 | * 73 | * @param delegate Data source which will be used to fill buffer. 74 | * @param bufferSize Size of the buffer. 75 | * @param initialBufferFullness Initial fullness of buffer. 76 | */ 77 | public BufferedDataSource(DataSource delegate, int bufferSize, double initialBufferFullness) { 78 | this.delegate = delegate; 79 | this.bufferSize = bufferSize; 80 | this.buffer = new LinkedBlockingQueue<>(bufferSize); 81 | this.initialBufferFullness = initialBufferFullness; 82 | fillBuffer(); 83 | } 84 | 85 | @Override 86 | public boolean hasNext(long time) { 87 | return !buffer.isEmpty() || delegate.hasNext(time); 88 | } 89 | 90 | @Override 91 | public T getNext(long time) { 92 | try { 93 | bufferFilled.wait(); 94 | T result = buffer.poll(); 95 | LOGGER.trace("Polled value from buffer: {}", result); 96 | return result; 97 | } catch (InterruptedException e) { 98 | throw new RuntimeException(e); 99 | } 100 | } 101 | 102 | @Override 103 | public void close() { 104 | terminateThread.set(false); 105 | } 106 | 107 | private void fillBuffer() { 108 | thread = new Thread(() -> { 109 | LOGGER.trace("Filling buffer to initial buffer size..."); 110 | while (buffer.size() < bufferSize * initialBufferFullness) { 111 | buffer.offer(delegate.getNext(0)); 112 | } 113 | LOGGER.trace("Buffer filled to initial buffer size of {} elements", buffer.size()); 114 | bufferFilled.notify(); 115 | while (!terminateThread.get()) { 116 | buffer.offer(delegate.getNext(0)); 117 | } 118 | }); 119 | thread.start(); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/datasource/RandomDoubleDataSource.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.datasource; 2 | 3 | import java.util.Iterator; 4 | import java.util.SplittableRandom; 5 | 6 | import io.smartcat.berserker.api.DataSource; 7 | 8 | /** 9 | * Endless data source generating random Double values. 10 | */ 11 | public class RandomDoubleDataSource implements DataSource { 12 | 13 | private final Iterator it = new SplittableRandom().doubles().iterator(); 14 | 15 | @Override 16 | public boolean hasNext(long time) { 17 | return true; 18 | } 19 | 20 | @Override 21 | public Double getNext(long time) { 22 | return it.next(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/datasource/RandomIntDataSource.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.datasource; 2 | 3 | import java.util.Iterator; 4 | import java.util.SplittableRandom; 5 | 6 | import io.smartcat.berserker.api.DataSource; 7 | 8 | /** 9 | * Endless data source generating random Integer values. 10 | */ 11 | public class RandomIntDataSource implements DataSource { 12 | 13 | private final Iterator it = new SplittableRandom().ints().iterator(); 14 | 15 | @Override 16 | public boolean hasNext(long time) { 17 | return true; 18 | } 19 | 20 | @Override 21 | public Integer getNext(long time) { 22 | return it.next(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/datasource/RandomLongDataSource.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.datasource; 2 | 3 | import java.util.Iterator; 4 | import java.util.SplittableRandom; 5 | 6 | import io.smartcat.berserker.api.DataSource; 7 | 8 | /** 9 | * Endless data source generating random Long values. 10 | */ 11 | public class RandomLongDataSource implements DataSource { 12 | 13 | private final Iterator it = new SplittableRandom().longs().iterator(); 14 | 15 | @Override 16 | public boolean hasNext(long time) { 17 | return true; 18 | } 19 | 20 | @Override 21 | public Long getNext(long time) { 22 | return it.next(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/rategenerator/AdditionRateGenerator.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.rategenerator; 2 | 3 | import io.smartcat.berserker.api.RateGenerator; 4 | 5 | /** 6 | * Rate generator that adds up rates of two {@link RateGenerator}s. 7 | */ 8 | public class AdditionRateGenerator implements RateGenerator { 9 | 10 | private final RateGenerator summand1; 11 | private final RateGenerator summand2; 12 | 13 | /** 14 | * Constructs addition rate generator with two summands. 15 | * 16 | * @param summand1 Rate generator which will be used as summand 1 for this addition. 17 | * @param summand2 Rate generator which will be used as summand 2 for this addition. 18 | */ 19 | public AdditionRateGenerator(RateGenerator summand1, RateGenerator summand2) { 20 | if (summand1 == null) { 21 | throw new IllegalArgumentException("summand1 cannot be null."); 22 | } 23 | if (summand2 == null) { 24 | throw new IllegalArgumentException("summand2 cannot be null."); 25 | } 26 | this.summand1 = summand1; 27 | this.summand2 = summand2; 28 | } 29 | 30 | @Override 31 | public double getRate(long time) { 32 | return summand1.getRate(time) + summand2.getRate(time); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/rategenerator/ConstantRateGenerator.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.rategenerator; 2 | 3 | import io.smartcat.berserker.api.RateGenerator; 4 | 5 | /** 6 | * Rate generator which generates constant rate. 7 | */ 8 | public class ConstantRateGenerator implements RateGenerator { 9 | 10 | private final double perSecondRate; 11 | 12 | /** 13 | * Constructs rate generator with specified perSecondRate. 14 | * 15 | * @param perSecondRate Rate of the rate generator per second, must be positive number. 16 | */ 17 | public ConstantRateGenerator(double perSecondRate) { 18 | if (perSecondRate <= 0) { 19 | throw new IllegalArgumentException("Rate must be positive number."); 20 | } 21 | this.perSecondRate = perSecondRate; 22 | } 23 | 24 | @Override 25 | public double getRate(long time) { 26 | return perSecondRate; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/rategenerator/DivisionRateGenerator.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.rategenerator; 2 | 3 | import io.smartcat.berserker.api.RateGenerator; 4 | 5 | /** 6 | * Rate generator that divides rates of two {@link RateGenerator}s. 7 | */ 8 | public class DivisionRateGenerator implements RateGenerator { 9 | 10 | private final RateGenerator dividend; 11 | private final RateGenerator divisor; 12 | 13 | /** 14 | * Constructs division rate generator with dividend and divisor. 15 | * 16 | * @param dividend Rate generator which will be used as dividend for this division. 17 | * @param divisor Rate generator which will be used as divisor for this division. 18 | */ 19 | public DivisionRateGenerator(RateGenerator dividend, RateGenerator divisor) { 20 | if (dividend == null) { 21 | throw new IllegalArgumentException("dividend cannot be null."); 22 | } 23 | if (divisor == null) { 24 | throw new IllegalArgumentException("divisor cannot be null."); 25 | } 26 | this.dividend = dividend; 27 | this.divisor = divisor; 28 | } 29 | 30 | @Override 31 | public double getRate(long time) { 32 | return dividend.getRate(time) / divisor.getRate(time); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/rategenerator/MultiplicationRateGenerator.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.rategenerator; 2 | 3 | import io.smartcat.berserker.api.RateGenerator; 4 | 5 | /** 6 | * Rate generator that multiplies rates of two {@link RateGenerator}s. 7 | */ 8 | public class MultiplicationRateGenerator implements RateGenerator { 9 | 10 | private final RateGenerator factor1; 11 | private final RateGenerator factor2; 12 | 13 | /** 14 | * Constructs multiplication rate generator with two factors. 15 | * 16 | * @param factor1 Rate generator which will be used as factor 1 for this multiplication. 17 | * @param factor2 Rate generator which will be used as factor 2 for this multiplication. 18 | */ 19 | public MultiplicationRateGenerator(RateGenerator factor1, RateGenerator factor2) { 20 | if (factor1 == null) { 21 | throw new IllegalArgumentException("factor1 cannot be null."); 22 | } 23 | if (factor2 == null) { 24 | throw new IllegalArgumentException("factor2 cannot be null."); 25 | } 26 | this.factor1 = factor1; 27 | this.factor2 = factor2; 28 | } 29 | 30 | @Override 31 | public double getRate(long time) { 32 | return factor1.getRate(time) * factor2.getRate(time); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/rategenerator/PeriodicRateGenerator.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.rategenerator; 2 | 3 | import java.util.concurrent.TimeUnit; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import io.smartcat.berserker.api.RateGenerator; 9 | 10 | /** 11 | * Abstract class for implementing periodic rate generators. Handles period extraction and value normalization. 12 | */ 13 | public abstract class PeriodicRateGenerator implements RateGenerator { 14 | 15 | private static final Logger LOGGER = LoggerFactory.getLogger(PeriodicRateGenerator.class); 16 | 17 | /** 18 | * Duration of the period in nano seconds. 19 | */ 20 | protected final long periodInNanos; 21 | 22 | /** 23 | * Constructs rate generator with specified periodInSeconds. 24 | * 25 | * @param periodInSeconds Period of periodic function. Represented in seconds, must be positive number. 26 | */ 27 | public PeriodicRateGenerator(long periodInSeconds) { 28 | if (periodInSeconds <= 0) { 29 | throw new IllegalArgumentException("Period must be positive number."); 30 | } 31 | this.periodInNanos = TimeUnit.SECONDS.toNanos(periodInSeconds); 32 | } 33 | 34 | @Override 35 | public double getRate(long time) { 36 | double valueInPeriod = normalizeValue(time); 37 | double result = rateFunction(valueInPeriod); 38 | LOGGER.trace("rateFunction returned: {} for value: {}", result, valueInPeriod); 39 | return result < 0 ? 0d : result; 40 | } 41 | 42 | /** 43 | * Method implementing rate function for single period. 44 | * 45 | * @param value Value on x axis of periodic function representing single period. Value is guaranteed to be in range 46 | * [0,1). 47 | * @return Rate for this {@link PeriodicRateGenerator} implementation. 48 | */ 49 | protected abstract double rateFunction(double value); 50 | 51 | private double normalizeValue(long time) { 52 | long rangeValue = time % periodInNanos; 53 | return 1d * rangeValue / periodInNanos; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/rategenerator/RateGeneratorProxy.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.rategenerator; 2 | 3 | import io.smartcat.berserker.api.RateGenerator; 4 | 5 | /** 6 | * Proxy around rate generator. 7 | */ 8 | public class RateGeneratorProxy implements RateGenerator { 9 | 10 | private RateGenerator delegate; 11 | 12 | /** 13 | * Constructs proxy without delegate. 14 | */ 15 | public RateGeneratorProxy() { 16 | } 17 | 18 | /** 19 | * Constructs proxy with specified delegate. 20 | * 21 | * @param delegate Value which will be evaluated and cached. 22 | */ 23 | public RateGeneratorProxy(RateGenerator delegate) { 24 | setDelegate(delegate); 25 | } 26 | 27 | /** 28 | * Sets value to this proxy. 29 | * 30 | * @param delegate Value which will be evaluated and cached. 31 | */ 32 | public void setDelegate(RateGenerator delegate) { 33 | if (delegate == null) { 34 | throw new IllegalArgumentException("Delegate cannot be null."); 35 | } 36 | this.delegate = delegate; 37 | } 38 | 39 | @Override 40 | public double getRate(long time) { 41 | checkDelegate(); 42 | return delegate.getRate(time); 43 | } 44 | 45 | private void checkDelegate() { 46 | if (delegate == null) { 47 | throw new DelegateNotSetException(); 48 | } 49 | } 50 | 51 | /** 52 | * Signals that delegate is not set. 53 | */ 54 | public static class DelegateNotSetException extends RuntimeException { 55 | 56 | private static final long serialVersionUID = 6257779717961934851L; 57 | 58 | /** 59 | * Constructs {@link DelegateNotSetException} with default message. 60 | */ 61 | public DelegateNotSetException() { 62 | super("Delegate not set for ValueProxy."); 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/rategenerator/SineRateGenerator.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.rategenerator; 2 | 3 | /** 4 | * Rate generator which uses sine function to generate rate. 5 | */ 6 | public class SineRateGenerator extends PeriodicRateGenerator { 7 | 8 | private final double multiplier; 9 | private final double independentConstant; 10 | 11 | /** 12 | * Constructs rate generator with specified periodInSeconds, multiplier and 13 | * independentConstant. 14 | * 15 | * @param periodInSeconds Period in seconds, must be positive number. 16 | * @param multiplier Multiplies result of sine function. 17 | * @param independentConstant Adds to the result of multiplied sine function, must be positive number. 18 | */ 19 | public SineRateGenerator(long periodInSeconds, double multiplier, double independentConstant) { 20 | super(periodInSeconds); 21 | if (independentConstant <= 0) { 22 | throw new IllegalArgumentException("Independent constant must be positive number."); 23 | } 24 | this.multiplier = multiplier; 25 | this.independentConstant = independentConstant; 26 | } 27 | 28 | @Override 29 | protected double rateFunction(double value) { 30 | return Math.sin(value * 2 * Math.PI) * multiplier + independentConstant; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/rategenerator/SquareRateGenerator.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.rategenerator; 2 | 3 | /** 4 | * Rate generator which uses square wave to generate rate. 5 | */ 6 | public class SquareRateGenerator extends PeriodicRateGenerator { 7 | 8 | private final double leftSide; 9 | private final double lowerValue; 10 | private final double upperValue; 11 | 12 | /** 13 | * Constructs rate generator with specified periodInSeconds, leftSide, 14 | * lowerValue and upperValue. 15 | * 16 | * @param periodInSeconds Period in seconds, must be positive number. 17 | * @param leftSide Percentage of the period where function has value of lowerValue, must be in range 18 | * [0,1). 19 | * @param lowerValue Lower value of the square function, must be positive number. 20 | * @param upperValue Upper value of the square function, must be positive number. 21 | */ 22 | public SquareRateGenerator(long periodInSeconds, double leftSide, double lowerValue, double upperValue) { 23 | super(periodInSeconds); 24 | if (leftSide < 0 || leftSide >= 1) { 25 | throw new IllegalArgumentException("Left side must be in range [0,1)."); 26 | } 27 | if (lowerValue <= 0) { 28 | throw new IllegalArgumentException("Lower value must be positive number."); 29 | } 30 | if (upperValue <= 0) { 31 | throw new IllegalArgumentException("Upper value must be positive number."); 32 | } 33 | this.leftSide = leftSide; 34 | this.lowerValue = lowerValue; 35 | this.upperValue = upperValue; 36 | } 37 | 38 | @Override 39 | protected double rateFunction(double value) { 40 | return value < leftSide ? lowerValue : upperValue; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/rategenerator/SubtractionRateGenerator.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.rategenerator; 2 | 3 | import io.smartcat.berserker.api.RateGenerator; 4 | 5 | /** 6 | * Rate generator that subtracts rates of two {@link RateGenerator}s. 7 | */ 8 | public class SubtractionRateGenerator implements RateGenerator { 9 | 10 | private final RateGenerator minuend; 11 | private final RateGenerator subtrahend; 12 | 13 | /** 14 | * Constructs subtraction rate generator with minuend and subtrahend. 15 | * 16 | * @param minuend Rate generator which will be used as minuend for this subtraction. 17 | * @param subtrahend Rate generator which will be used as subtrahend for this subtraction. 18 | */ 19 | public SubtractionRateGenerator(RateGenerator minuend, RateGenerator subtrahend) { 20 | if (minuend == null) { 21 | throw new IllegalArgumentException("minuend cannot be null."); 22 | } 23 | if (subtrahend == null) { 24 | throw new IllegalArgumentException("subtrahend cannot be null."); 25 | } 26 | this.minuend = minuend; 27 | this.subtrahend = subtrahend; 28 | } 29 | 30 | @Override 31 | public double getRate(long time) { 32 | return minuend.getRate(time) - subtrahend.getRate(time); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/rategenerator/TriangleRateGenerator.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.rategenerator; 2 | 3 | /** 4 | * Rate generator which uses triangle wave to generate rate. 5 | */ 6 | public class TriangleRateGenerator extends PeriodicRateGenerator { 7 | 8 | private final double leftSide; 9 | private final double minValue; 10 | private final double maxValue; 11 | 12 | /** 13 | * Constructs rate generator with specified periodInSeconds, leftSide, 14 | * lowerValue and upperValue. 15 | * 16 | * @param periodInSeconds Period in seconds, must be positive number. 17 | * @param leftSide Percentage of the period where function is in ascending slope from minValue to 18 | * maxValue. Other part of period is in descending slope from maxValue to 19 | * minValue, must be in range [0,1). 20 | * @param minValue Minimum value of the triangle function, must be positive number. 21 | * @param maxValue Maximum value of the triangle function, must be positive number. 22 | */ 23 | public TriangleRateGenerator(long periodInSeconds, double leftSide, double minValue, double maxValue) { 24 | super(periodInSeconds); 25 | if (leftSide < 0 || leftSide >= 1) { 26 | throw new IllegalArgumentException("Left side must be in range [0, 1)."); 27 | } 28 | if (minValue <= 0) { 29 | throw new IllegalArgumentException("Min value must be positive number."); 30 | } 31 | if (maxValue <= 0) { 32 | throw new IllegalArgumentException("Max value must be positive number."); 33 | } 34 | this.leftSide = leftSide; 35 | this.minValue = minValue; 36 | this.maxValue = maxValue; 37 | } 38 | 39 | @Override 40 | protected double rateFunction(double value) { 41 | if (value < leftSide) { 42 | return ((maxValue - minValue) * value / leftSide) + minValue; 43 | } else { 44 | return ((maxValue - minValue) * (1 - value) / (1 - leftSide)) + minValue; 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /berserker-core/src/main/java/io/smartcat/berserker/util/LinkedEvictingBlockingQueue.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.util; 2 | 3 | import java.util.concurrent.locks.Condition; 4 | import java.util.concurrent.locks.ReentrantLock; 5 | 6 | /** 7 | * Blocking queue with eviction policy to evict elements if put is attempted when queue is full. Depending on 8 | * dropFromHead flag, can drop either from head or from tail. Implementation taken from 9 | * {@link java.util.concurrent.LinkedBlockingDeque} and adjusted. 10 | * 11 | * @param Type of elements held in this collection. 12 | */ 13 | public class LinkedEvictingBlockingQueue { 14 | 15 | private final boolean dropFromHead; 16 | private final int capacity; 17 | private final ReentrantLock lock = new ReentrantLock(); 18 | private final Condition notEmpty = lock.newCondition(); 19 | 20 | private transient int count; 21 | private transient Node head; 22 | private transient Node tail; 23 | 24 | /** 25 | * Constructs evicting blocking queue. dropFromHead is set to true and capacity is set to 26 | * {@code Integer.MAX_VALUE}. 27 | */ 28 | public LinkedEvictingBlockingQueue() { 29 | this(true); 30 | } 31 | 32 | /** 33 | * Constructs evicting blocking queue with specified dropFromHead flag. capacity is set to 34 | * {@code Integer.MAX_VALUE}. 35 | * 36 | * @param dropFromHead Flag to determine whether element from head or from tail will be dropped when put is invoked 37 | * on full queue. 38 | */ 39 | public LinkedEvictingBlockingQueue(boolean dropFromHead) { 40 | this(dropFromHead, Integer.MAX_VALUE); 41 | } 42 | 43 | /** 44 | * Constructs evicting blocking queue with specified dropFromHead flag and fixed capacity. 45 | * 46 | * @param dropFromHead Flag to determine whether element from head or from tail will be dropped when put is invoked 47 | * on full queue. 48 | * @param capacity Capacity of this queue, must be positive number. 49 | */ 50 | public LinkedEvictingBlockingQueue(boolean dropFromHead, int capacity) { 51 | if (capacity <= 0) { 52 | throw new IllegalArgumentException("Capacity must be positive number."); 53 | } 54 | this.dropFromHead = dropFromHead; 55 | this.capacity = capacity; 56 | } 57 | 58 | /** 59 | * Inserts the specified element at the tail of this queue. If queue if full, element is dropped and then specified 60 | * element is put. Depending on dropFromHead either element from head or from tail will be dropped from 61 | * the queue. 62 | * 63 | * @param e Element to put in this queue. 64 | * @return Element that was dropped, or null if no element was dropped. 65 | */ 66 | public T put(T e) { 67 | Node node = new Node(e); 68 | final ReentrantLock lock = this.lock; 69 | lock.lock(); 70 | try { 71 | T droppedNode = null; 72 | if (count >= capacity) { 73 | droppedNode = dropNode(); 74 | } 75 | linkToTail(node); 76 | return droppedNode; 77 | } finally { 78 | lock.unlock(); 79 | } 80 | } 81 | 82 | /** 83 | * Retrieves and removes the head of this queue, waiting if necessary until an element becomes available. 84 | * 85 | * @return The read of this queue. 86 | * @throws InterruptedException If interrupted while waiting. 87 | */ 88 | public T take() throws InterruptedException { 89 | final ReentrantLock lock = this.lock; 90 | lock.lock(); 91 | try { 92 | T x; 93 | while ((x = unlinkFromHead()) == null) 94 | notEmpty.await(); 95 | return x; 96 | } finally { 97 | lock.unlock(); 98 | } 99 | } 100 | 101 | /** 102 | * Returns the number of elements in this queue. 103 | * 104 | * @return the number of elements in this queue 105 | */ 106 | public int size() { 107 | final ReentrantLock lock = this.lock; 108 | lock.lock(); 109 | try { 110 | return count; 111 | } finally { 112 | lock.unlock(); 113 | } 114 | } 115 | 116 | private T dropNode() { 117 | if (dropFromHead) { 118 | return unlinkFromHead(); 119 | } else { 120 | return unlinkFromTail(); 121 | } 122 | } 123 | 124 | /** 125 | * Links node at tail, or returns false if full. 126 | */ 127 | private boolean linkToTail(Node node) { 128 | // assert lock.isHeldByCurrentThread(); 129 | if (count >= capacity) 130 | return false; 131 | Node l = tail; 132 | node.prev = l; 133 | tail = node; 134 | if (head == null) 135 | head = node; 136 | else 137 | l.next = node; 138 | ++count; 139 | notEmpty.signal(); 140 | return true; 141 | } 142 | 143 | /** 144 | * Removes and returns element from head, or null if empty. 145 | */ 146 | private T unlinkFromHead() { 147 | Node f = head; 148 | if (f == null) 149 | return null; 150 | Node n = f.next; 151 | T item = f.item; 152 | f.item = null; 153 | f.next = f; // help GC 154 | head = n; 155 | if (n == null) 156 | tail = null; 157 | else 158 | n.prev = null; 159 | --count; 160 | return item; 161 | } 162 | 163 | /** 164 | * Removes and returns element from tail, or null if empty. 165 | */ 166 | private T unlinkFromTail() { 167 | Node l = tail; 168 | if (l == null) 169 | return null; 170 | Node p = l.prev; 171 | T item = l.item; 172 | l.item = null; 173 | l.prev = l; // help GC 174 | tail = p; 175 | if (p == null) 176 | head = null; 177 | else 178 | p.next = null; 179 | --count; 180 | return item; 181 | } 182 | 183 | /** 184 | * Doubly-linked list node class. 185 | * 186 | * @param Type of node. 187 | */ 188 | private static class Node { 189 | 190 | E item; 191 | Node prev; 192 | Node next; 193 | 194 | Node(E x) { 195 | item = x; 196 | } 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /berserker-core/src/test/groovy/io/smartcat/berserker/configuration/rategenerator/RateGeneratorConfigurationParserSpec.groovy: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.configuration.rategenerator 2 | 3 | import io.smartcat.berserker.rategenerator.SineRateGenerator 4 | import io.smartcat.berserker.rategenerator.SquareRateGenerator 5 | import io.smartcat.berserker.rategenerator.TriangleRateGenerator 6 | import org.yaml.snakeyaml.Yaml 7 | import spock.lang.Specification 8 | 9 | 10 | class RateGeneratorConfigurationParserSpec extends Specification { 11 | 12 | private static final long SECONDS_IN_DAY = 86400L 13 | private static final long SECONDS_IN_HOUR = 3600L 14 | private static final long SECONDS_IN_MINUTE = 60L 15 | private static final long NANOS_IN_SECOND = 1_000_000_000L 16 | 17 | def "should parse number into constant rate generator"() { 18 | given: 19 | def config = ''' 20 | rates: 21 | firstRate: 4 22 | output: $firstRate 23 | ''' 24 | 25 | when: 26 | def rateGenerator = buildGenerator(config) 27 | 28 | then: 29 | rateGenerator.getRate(1) == 4 30 | } 31 | 32 | def "should parse simple addition rate generator"() { 33 | given: 34 | def config = ''' 35 | rates: 36 | r: 2.5 + 2 37 | output: $r 38 | ''' 39 | 40 | when: 41 | def rateGenerator = buildGenerator(config) 42 | 43 | then: 44 | rateGenerator.getRate(1) == 4.5 45 | } 46 | 47 | def "should parse addition and multiplication with parenthesis rate generator"() { 48 | given: 49 | def config = ''' 50 | rates: 51 | r: 2 * (3.5 + 4 ) 52 | output: $r 53 | ''' 54 | 55 | when: 56 | def rateGenerator = buildGenerator(config) 57 | 58 | then: 59 | rateGenerator.getRate(1) == 15 60 | } 61 | 62 | def "should parse addition and multiplication without parenthesis rate generator"() { 63 | given: 64 | def config = ''' 65 | rates: 66 | r: 2 * 3 + 4 67 | output: $r 68 | ''' 69 | 70 | when: 71 | def rateGenerator = buildGenerator(config) 72 | 73 | then: 74 | rateGenerator.getRate(1) == 10 75 | } 76 | 77 | def "should parse multiple complex operation rate generators"() { 78 | given: 79 | def config = ''' 80 | rates: 81 | a: 1 * 3 + 3 * 2 82 | b: (2 + 1)*5 83 | c: $b -$a + 2 * 2 84 | d: 100 / (5 + 5) 85 | e: 1 86 | output: $c - $d + $e 87 | ''' 88 | 89 | when: 90 | def rateGenerator = buildGenerator(config) 91 | 92 | then: 93 | rateGenerator.getRate(1) == 1 94 | } 95 | 96 | def "should parse triangle rate generator"() { 97 | given: 98 | def config = ''' 99 | rates: 100 | r: triangle(1d2h3m4s, 0.5, 1.1, 100) 101 | output: $r 102 | ''' 103 | 104 | when: 105 | def rateGenerator = buildGenerator(config) 106 | 107 | then: 108 | rateGenerator.delegate.class == TriangleRateGenerator 109 | rateGenerator.delegate.periodInNanos == (1 * SECONDS_IN_DAY + 2 * SECONDS_IN_HOUR + 3 * SECONDS_IN_MINUTE + 4) * NANOS_IN_SECOND 110 | rateGenerator.delegate.leftSide == 0.5 111 | rateGenerator.delegate.minValue == 1.1 112 | rateGenerator.delegate.maxValue == 100.0 113 | } 114 | 115 | def "should parse sine rate generator"() { 116 | given: 117 | def config = ''' 118 | rates: 119 | r: sin(2h10s, 10.4, 100) 120 | output: $r 121 | ''' 122 | 123 | when: 124 | def rateGenerator = buildGenerator(config) 125 | 126 | then: 127 | rateGenerator.delegate.class == SineRateGenerator 128 | rateGenerator.delegate.periodInNanos == (2 * SECONDS_IN_HOUR + 10) * NANOS_IN_SECOND 129 | rateGenerator.delegate.multiplier == 10.4 130 | rateGenerator.delegate.independentConstant == 100 131 | } 132 | 133 | def "should parse square rate generator"() { 134 | given: 135 | def config = ''' 136 | rates: 137 | r: square(10m, 0.7, 10, 1000) 138 | output: $r 139 | ''' 140 | 141 | when: 142 | def rateGenerator = buildGenerator(config) 143 | 144 | then: 145 | rateGenerator.delegate.class == SquareRateGenerator 146 | rateGenerator.delegate.periodInNanos == (10 * SECONDS_IN_MINUTE) * NANOS_IN_SECOND 147 | rateGenerator.delegate.leftSide == 0.7 148 | rateGenerator.delegate.lowerValue == 10 149 | rateGenerator.delegate.upperValue == 1000 150 | } 151 | 152 | def "should parse combination of several rate generators"() { 153 | given: 154 | def config = ''' 155 | rates: 156 | r: triangle(8s, 0.5, 1, 100) * 2 + square(10s, 0.2, 10, 200) + 120 * 4/2 157 | output: $r 158 | ''' 159 | 160 | when: 161 | def rateGenerator = buildGenerator(config) 162 | 163 | then: 164 | rateGenerator.getRate(0 * NANOS_IN_SECOND) == 252 165 | rateGenerator.getRate(4 * NANOS_IN_SECOND) == 640 166 | rateGenerator.getRate(8 * NANOS_IN_SECOND) == 442 167 | } 168 | 169 | def buildGenerator(config) { 170 | Yaml yaml = new Yaml() 171 | new RateGeneratorConfigurationParser(yaml.load(config)).build() 172 | } 173 | } -------------------------------------------------------------------------------- /berserker-core/src/test/java/io/smartcat/berserker/LoadGeneratorTest.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker; 2 | 3 | import java.util.concurrent.atomic.AtomicLong; 4 | 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | import io.smartcat.berserker.api.DataSource; 9 | import io.smartcat.berserker.datasource.RandomIntDataSource; 10 | import io.smartcat.berserker.rategenerator.ConstantRateGenerator; 11 | 12 | public class LoadGeneratorTest { 13 | 14 | private static final long DEFAULT_TEST_TIMEOUT = 3000; 15 | 16 | @Test(timeout = DEFAULT_TEST_TIMEOUT) 17 | public void loadGenerator_should_be_terminated_when_terminate_signal_is_sent() { 18 | // GIVEN 19 | LoadGenerator loadGenerator = new LoadGenerator<>(new RandomIntDataSource(), 20 | new ConstantRateGenerator(1000), (x) -> { }); 21 | runInBackground(() -> { 22 | // wait a bit for loadGenerator to run 23 | wait(1000); 24 | loadGenerator.terminate(); 25 | wait(500); 26 | }); 27 | 28 | // WHEN 29 | loadGenerator.run(); 30 | 31 | // THEN 32 | // test finishes without timeout 33 | } 34 | 35 | @Test(timeout = DEFAULT_TEST_TIMEOUT) 36 | public void loadGenerator_should_be_terminated_when_dataSource_has_no_next_element() { 37 | // GIVEN 38 | LoadGenerator loadGenerator = new LoadGenerator<>(new DataSource() { 39 | 40 | @Override 41 | public boolean hasNext(long time) { 42 | return false; 43 | } 44 | 45 | @Override 46 | public Integer getNext(long time) { 47 | return 123; 48 | } 49 | }, new ConstantRateGenerator(1000), (x) -> { }); 50 | 51 | // WHEN 52 | loadGenerator.run(); 53 | 54 | // THEN 55 | // test finishes without timeout 56 | } 57 | 58 | @Test 59 | public void worker_should_be_invoked_2000_times_when_loadGenerator_works_10_sec_with_rate_of_200() { 60 | testRateAndCountOfInvocation(10, 200); 61 | } 62 | 63 | @Test 64 | public void worker_should_be_invoked_200_000_times_when_loadGenerator_works_20_sec_with_rate_of_10_000() { 65 | testRateAndCountOfInvocation(20, 10_000); 66 | } 67 | 68 | @Test 69 | public void worker_should_be_invoked_3015_times_when_loadGenerator_works_30_sec_with_rate_of_100_point_5() { 70 | testRateAndCountOfInvocation(30, 100.5); 71 | } 72 | 73 | @Test 74 | public void worker_should_be_invoked_300_000_000_times_when_loadGenerator_works_30_sec_with_rate_of_10_000_000() { 75 | testRateAndCountOfInvocation(30, 10_000_000); 76 | } 77 | 78 | private void testRateAndCountOfInvocation(int numOfSeconds, double rate) { 79 | // GIVEN 80 | double tolerance = 0.005; 81 | final AtomicLong numOfInvoked = new AtomicLong(0); 82 | LoadGenerator loadGenerator = new LoadGenerator<>(new RandomIntDataSource(), 83 | new ConstantRateGenerator(rate), (x) -> numOfInvoked.incrementAndGet()); 84 | 85 | // WHEN 86 | runInBackground(() -> loadGenerator.run()); 87 | wait(numOfSeconds * 1_000); 88 | 89 | loadGenerator.terminate(); 90 | wait(500); 91 | 92 | // THEN 93 | Assert.assertEquals(1d * numOfSeconds * rate, numOfInvoked.get(), numOfSeconds * rate * tolerance); 94 | } 95 | 96 | private void runInBackground(Runnable target) { 97 | Thread t = new Thread(target); 98 | t.start(); 99 | } 100 | 101 | private void wait(int milliseconds) { 102 | try { 103 | Thread.sleep(milliseconds); 104 | } catch (InterruptedException e) { 105 | throw new RuntimeException(e); 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /berserker-core/src/test/java/io/smartcat/berserker/rategenerator/PeriodicRateGeneratorTest.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.rategenerator; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | import java.util.concurrent.TimeUnit; 7 | 8 | public class PeriodicRateGeneratorTest { 9 | 10 | private static final double DELTA = 0.001d; 11 | 12 | @Test 13 | public void rate_should_return_1_when_period_is_100_and_time_is_201() { 14 | // GIVEN 15 | ConcretePeriodicRateGenerator periodicRateGenerator = new ConcretePeriodicRateGenerator(100); 16 | 17 | // WHEN 18 | double rate = periodicRateGenerator.getRate(TimeUnit.SECONDS.toNanos(201)); 19 | 20 | // THEN 21 | Assert.assertEquals(1d, rate, DELTA); 22 | } 23 | 24 | @Test 25 | public void rate_should_return_0_when_period_is_50_and_time_is_150() { 26 | // GIVEN 27 | ConcretePeriodicRateGenerator periodicRateGenerator = new ConcretePeriodicRateGenerator(50); 28 | 29 | // WHEN 30 | double rate = periodicRateGenerator.getRate(TimeUnit.SECONDS.toNanos(150)); 31 | 32 | // THEN 33 | Assert.assertEquals(0d, rate, DELTA); 34 | } 35 | 36 | @Test 37 | public void rate_should_return_60_when_period_is_10_and_time_is_16() { 38 | // GIVEN 39 | ConcretePeriodicRateGenerator periodicRateGenerator = new ConcretePeriodicRateGenerator(10); 40 | 41 | // WHEN 42 | double rate = periodicRateGenerator.getRate(TimeUnit.SECONDS.toNanos(16)); 43 | 44 | // THEN 45 | Assert.assertEquals(60d, rate, DELTA); 46 | } 47 | 48 | @Test 49 | public void rate_should_return_99_when_period_is_100_and_time_is_399() { 50 | // GIVEN 51 | ConcretePeriodicRateGenerator periodicRateGenerator = new ConcretePeriodicRateGenerator(100); 52 | 53 | // WHEN 54 | double rate = periodicRateGenerator.getRate(TimeUnit.SECONDS.toNanos(399)); 55 | 56 | // THEN 57 | Assert.assertEquals(99d, rate, DELTA); 58 | } 59 | 60 | private class ConcretePeriodicRateGenerator extends PeriodicRateGenerator { 61 | 62 | public ConcretePeriodicRateGenerator(long periodInSeconds) { 63 | super(periodInSeconds); 64 | } 65 | 66 | @Override 67 | protected double rateFunction(double value) { 68 | return value * 100; 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /berserker-core/src/test/java/io/smartcat/berserker/util/LinkedEvictingBlockingQueueTest.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.util; 2 | 3 | import java.util.concurrent.CountDownLatch; 4 | 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | public class LinkedEvictingBlockingQueueTest { 9 | 10 | @Test(timeout = 500) 11 | public void take_should_block_thread_until_element_is_available_in_queue() throws InterruptedException { 12 | // GIVEN 13 | LinkedEvictingBlockingQueue queue = new LinkedEvictingBlockingQueue<>(); 14 | CountDownLatch countDownLatch = new CountDownLatch(1); 15 | newStartedThread(() -> { 16 | try { 17 | countDownLatch.await(); 18 | } catch (InterruptedException e) { 19 | } 20 | queue.put(2); 21 | }); 22 | 23 | // WHEN 24 | countDownLatch.countDown(); 25 | int val = queue.take(); 26 | 27 | // THEN 28 | Assert.assertEquals(2, val); 29 | } 30 | 31 | @Test(timeout = 2000) 32 | public void should_not_take_value_from_queue_when_value_is_not_put() throws InterruptedException { 33 | // GIVEN 34 | LinkedEvictingBlockingQueue queue = new LinkedEvictingBlockingQueue<>(); 35 | Thread takeFromQueueThread = newStartedThread(() -> { 36 | try { 37 | queue.take(); 38 | } catch (InterruptedException e) { 39 | } 40 | }); 41 | 42 | // THEN 43 | Thread.sleep(1000); 44 | Assert.assertTrue(takeFromQueueThread.isAlive()); 45 | takeFromQueueThread.interrupt(); 46 | } 47 | 48 | @Test(timeout = 500) 49 | public void put_should_drop_element_from_head_when_queue_is_full_and_dropFromHead_is_true() 50 | throws InterruptedException { 51 | // GIVEN 52 | LinkedEvictingBlockingQueue queue = new LinkedEvictingBlockingQueue<>(true, 2); 53 | queue.put(1); 54 | queue.put(2); 55 | 56 | // WHEN 57 | queue.put(3); 58 | 59 | // THEN 60 | Assert.assertEquals(new Integer(2), queue.take()); 61 | Assert.assertEquals(new Integer(3), queue.take()); 62 | } 63 | 64 | @Test(timeout = 500) 65 | public void put_should_drop_element_from_tail_when_queue_is_full_and_dropFromHead_is_false() 66 | throws InterruptedException { 67 | // GIVEN 68 | LinkedEvictingBlockingQueue queue = new LinkedEvictingBlockingQueue<>(false, 2); 69 | queue.put(1); 70 | queue.put(2); 71 | 72 | // WHEN 73 | queue.put(3); 74 | 75 | // THEN 76 | Assert.assertEquals(new Integer(1), queue.take()); 77 | Assert.assertEquals(new Integer(3), queue.take()); 78 | } 79 | 80 | private Thread newStartedThread(Runnable r) { 81 | Thread t = new Thread(r); 82 | t.setDaemon(true); 83 | t.start(); 84 | return t; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /berserker-core/src/test/java/io/smartcat/berserker/worker/InternalWorkerTest.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.worker; 2 | 3 | import java.util.concurrent.CountDownLatch; 4 | import java.util.concurrent.atomic.AtomicInteger; 5 | 6 | import org.junit.Assert; 7 | import org.junit.Test; 8 | 9 | import io.smartcat.berserker.LoadGenerator; 10 | import io.smartcat.berserker.api.DataSource; 11 | import io.smartcat.berserker.api.RateGenerator; 12 | import io.smartcat.berserker.api.Worker; 13 | import io.smartcat.berserker.rategenerator.ConstantRateGenerator; 14 | 15 | public class InternalWorkerTest { 16 | 17 | /** 18 | * InternalWorker has 4 threads and queue of 4 elements. Each thread is blocked until load generator depletes data 19 | * source. Data source has 10 values. At that moment, 4 threads have taken first four values and there is four more 20 | * values in queue to be processed. If load generator does not block, it means that InternalWorker is non-blocking. 21 | */ 22 | @Test(timeout = 3000) 23 | public void should_not_block_thread_when_delegate_of_asyncWorker_is_blocking() throws Exception { 24 | // GIVEN 25 | RateGenerator rg = new ConstantRateGenerator(10); 26 | Object lock = new Object(); 27 | AtomicInteger delegateInvokeCount = new AtomicInteger(); 28 | Worker delegate = (x, y, z) -> { 29 | synchronized (lock) { 30 | delegateInvokeCount.incrementAndGet(); 31 | } 32 | }; 33 | DataSource ds = new DataSource() { 34 | 35 | private int i = 0; 36 | 37 | @Override 38 | public boolean hasNext(long time) { 39 | return i < 10; 40 | } 41 | 42 | @Override 43 | public Integer getNext(long time) { 44 | return i++; 45 | } 46 | }; 47 | InternalWorker w = new InternalWorker<>(delegate, 4, true, null, 4); 48 | LoadGenerator lg = new LoadGenerator<>(ds, rg, w); 49 | 50 | // WHEN 51 | synchronized (lock) { 52 | lg.run(); 53 | } 54 | w.close(); 55 | 56 | // THEN 57 | // load generator was not blocked meaning that InternalWorker is non-blocking. 58 | } 59 | 60 | /** 61 | * InternalWorker has 3 threads and queue of 3 elements. Each thread is blocked until load generator depletes data 62 | * source. After threads have taken first 3 values and queue has taken next 3 values. Queue is full with values 4, 63 | * 5, 6. Since values are dropped from head and added to tail. After 3 droppings and 3 additions, queue has 7, 8, 64 | * and 9. 65 | */ 66 | @Test(timeout = 3000) 67 | public void should_drop_packet_from_head_of_the_queue_when_queue_is_full_and_dropFromHead_is_true() 68 | throws Exception { 69 | // GIVEN 70 | RateGenerator rg = new ConstantRateGenerator(10); 71 | CountDownLatch countDownLatch = new CountDownLatch(6); 72 | CountDownLatch dsCountDownLatch = new CountDownLatch(3); 73 | Object lock = new Object(); 74 | AtomicInteger delegateInvokeCount = new AtomicInteger(); 75 | Worker delegate = (x, y, z) -> { 76 | dsCountDownLatch.countDown(); 77 | synchronized (lock) { 78 | delegateInvokeCount.addAndGet(x); 79 | countDownLatch.countDown(); 80 | } 81 | }; 82 | DataSource ds = new DataSource() { 83 | 84 | private int i = 1; 85 | 86 | @Override 87 | public boolean hasNext(long time) { 88 | return i < 10; 89 | } 90 | 91 | @Override 92 | public Integer getNext(long time) { 93 | if (i == 4) { 94 | try { 95 | dsCountDownLatch.await(); 96 | } catch (InterruptedException ignored) { 97 | } 98 | } 99 | return i++; 100 | } 101 | }; 102 | InternalWorker w = new InternalWorker<>(delegate, 3, true, null, 3); 103 | LoadGenerator lg = new LoadGenerator<>(ds, rg, w); 104 | 105 | // WHEN 106 | synchronized (lock) { 107 | lg.run(); 108 | } 109 | countDownLatch.await(); 110 | w.close(); 111 | 112 | // THEN 113 | // Values that are processed should be: 1, 2, 3, 7, 8, 9. Total of 30. 114 | Assert.assertEquals(30, delegateInvokeCount.get()); 115 | } 116 | 117 | /** 118 | * InternalWorker has 3 threads and queue of 3 elements. Each thread is blocked until load generator depletes data 119 | * source. After threads have taken first 3 values and queue has taken next 3 values. Queue is full with values 4, 120 | * 5, 6. Since values are dropped from tail and added to tail. After 3 droppings and 3 additions, queue has 4, 5, 121 | * and 9. 122 | */ 123 | @Test(timeout = 3000) 124 | public void should_drop_packet_from_head_of_the_queue_when_queue_is_full_and_dropFromHead_is_false() 125 | throws Exception { 126 | // GIVEN 127 | RateGenerator rg = new ConstantRateGenerator(10); 128 | CountDownLatch countDownLatch = new CountDownLatch(6); 129 | CountDownLatch dsCountDownLatch = new CountDownLatch(3); 130 | Object lock = new Object(); 131 | AtomicInteger delegateInvokeCount = new AtomicInteger(); 132 | Worker delegate = (x, y, z) -> { 133 | dsCountDownLatch.countDown(); 134 | synchronized (lock) { 135 | delegateInvokeCount.addAndGet(x); 136 | countDownLatch.countDown(); 137 | } 138 | }; 139 | DataSource ds = new DataSource() { 140 | 141 | private int i = 1; 142 | 143 | @Override 144 | public boolean hasNext(long time) { 145 | return i < 10; 146 | } 147 | 148 | @Override 149 | public Integer getNext(long time) { 150 | if (i == 4) { 151 | try { 152 | dsCountDownLatch.await(); 153 | } catch (InterruptedException ignored) { 154 | } 155 | } 156 | return i++; 157 | } 158 | }; 159 | InternalWorker w = new InternalWorker<>(delegate, 3, false, null, 3); 160 | LoadGenerator lg = new LoadGenerator<>(ds, rg, w); 161 | 162 | // WHEN 163 | synchronized (lock) { 164 | lg.run(); 165 | } 166 | countDownLatch.await(); 167 | w.close(); 168 | 169 | // THEN 170 | // Values that are processed should be: 1, 2, 3, 4, 5, 9. Total of 24. 171 | Assert.assertEquals(24, delegateInvokeCount.get()); 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /berserker-core/src/test/resources/empty-csv.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartcat-labs/berserker/a80dab85f9cf8d608895935f097afee5017048e8/berserker-core/src/test/resources/empty-csv.csv -------------------------------------------------------------------------------- /berserker-core/src/test/resources/invalid-csv.csv: -------------------------------------------------------------------------------- 1 | Emma; Cooper, 3 -------------------------------------------------------------------------------- /berserker-core/src/test/resources/simple-csv.csv: -------------------------------------------------------------------------------- 1 | John, Doe, 31, 555-232-13143, true 2 | Jane, Doe, 24, 555-234-34843, false 3 | Michael, Smith, 45, 555-233-84932, true 4 | William, Johns, 22, 555-898-23421, true 5 | Emma, Cooper, 33, 555-849-23211, false -------------------------------------------------------------------------------- /berserker-http/README.md: -------------------------------------------------------------------------------- 1 | # Berserker HTTP 2 | 3 | Worker implementation which sends HTTP request on configured endpoint. 4 | 5 | Configuration can define following properties: 6 | 1. `async` - Can be `true` or `false`. Determines whether messages will be sent in asynchronous fashion or not. Optional, if not specified defaults to `false`. 7 | 2. `keep-alive` - Can be `true` or `false`. Determines whether HTTP keep-alive is enabled or disabled. Optional, defaults to `true`. 8 | 3. `max-connections` - The maximum number of connections a HTTP client can handle, or `-1` for no connection limit. Optional, if not specified defaults to `-1`. 9 | 4. `max-connections-per-host` - The maximum number of connections per host a HTTP client can handle, or `-1` for no connection limit. Optional, if not specified defaults to `-1`. 10 | 5. `connect-timeout` - The maximum time in millisecond a HTTP client can wait when connecting to a remote host. Optional, if not specified defaults to `5000`. 11 | 6. `read-timeout` - The maximum time in millisecond a HTTP client can stay idle. Optional, if not specified defaults to `60000`. 12 | 7. `pooled-connection-idle-timeout` - The maximum time in millisecond a HTTP client will keep connection in pool. Optional, if not specified defaults to `60000`. 13 | 8. `request-timeout` - The maximum time in millisecond a HTTP client waits until the response is completed. Optional, if not specified defaults to `60000`. 14 | 9. `follow-redirect` - Can be `true` or `false`. Determines whether HTTP redirect is enabled. Optional, if not specified defaults to `true`. 15 | 10. `max-redirects` - The maximum number of HTTP redirects. Optional, if not specified defaults to `5`. 16 | 11. `max-request-retry` - The number of time the library will retry when an error occurs by the remote server. Optional, if not specified defaults to `5`. 17 | 12. `connection-ttl` - The maximum time in millisecond a HTTP client will keep connection in the pool, or `-1` to keep connection while possible. Optional, if not specified defaults to `-1`. 18 | 13. `base-url` - Can be concatenated with request property `url-sufix` to construct URL. Optional, depending on whether `url` or `url-sufix` is specified. 19 | 14. `headers` - Contains headers in a form of name-value map which will be added to each request. Optional. 20 | 15. `error-codes` - List of HTTP codes that should be considered errors. Optional, defaults to all `4**` and `5**` codes. 21 | 22 | Worker `accept` method expects following properties: 23 | 1. `url` - Whole url to be used, it ignores `base-url`. Mutually exclusive with `url-sufix`. 24 | 2. `url-sufix` - Concatenates to `base-url` to construct url. Mutually exclusive with `url`. 25 | 3. `method-type` - Method type of the request. Mandatory. 26 | 4. `headers` - Header names with its values. It will override headers with same name defined in configuration. Optional. 27 | 5. `body` - Payload of the request. Available only when `POST` or `PUT` method types are used. Optional. 28 | 29 | ## Configuration 30 | 31 | Example yaml configuration: 32 | 33 | ```yaml 34 | worker-configuration: 35 | async: false 36 | keep-alive: true 37 | max-connections: -1 38 | max-connections-per-host: -1 39 | connect-timeout: 5000 40 | read-timeout: 60000 41 | pooled-connection-idle-timeout: 60000 42 | request-timeout: 60000 43 | follow-redirect: true 44 | max-redirects: 5 45 | max-request-retry: 5 46 | connection-ttl: -1 47 | base-url: http://localhost:8080/api/item 48 | headers: 49 | Content-Type: application/json 50 | X-Custom-Header-1: custom-value 51 | error-codes: 52 | - 400 53 | - 401 54 | - 403 55 | - 404 56 | ``` 57 | 58 | For whole configuration, take a look at [Ranger-HTTP example](../berserker-runner/src/example/resources/ranger-http.yml). 59 | -------------------------------------------------------------------------------- /berserker-http/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | berserker 6 | io.smartcat 7 | 0.0.13-SNAPSHOT 8 | 9 | 10 | berserker-http 11 | jar 12 | 13 | Berserker HTTP 14 | Worker implementation which sends HTTP requests. 15 | 16 | 17 | UTF-8 18 | 2.1.0-alpha26 19 | 20 | 21 | 22 | 23 | io.smartcat 24 | berserker-commons 25 | ${project.parent.version} 26 | 27 | 28 | org.asynchttpclient 29 | async-http-client 30 | ${version.async-http-client} 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /berserker-http/src/main/java/io/smartcat/berserker/http/configuration/HttpConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.http.configuration; 2 | 3 | import static io.smartcat.berserker.configuration.ConfigurationHelper.getOptionalValue; 4 | 5 | import java.util.Arrays; 6 | import java.util.HashMap; 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | import io.smartcat.berserker.api.Worker; 11 | import io.smartcat.berserker.configuration.WorkerConfiguration; 12 | import io.smartcat.berserker.http.worker.HttpWorker; 13 | 14 | /** 15 | * Configuration for HTTP worker. 16 | */ 17 | public class HttpConfiguration implements WorkerConfiguration { 18 | 19 | private static final String ASYNC = "async"; 20 | private static final String KEEP_ALIVE = "keep-alive"; 21 | private static final String MAX_CONNECTIONS = "max-connections"; 22 | private static final String MAX_CONNECTIONS_PER_HOST = "max-connections-per-host"; 23 | private static final String CONNECT_TIMEOUT = "connect-timeout"; 24 | private static final String READ_TIMEOUT = "read-timeout"; 25 | private static final String POOLED_CONNECTION_IDLE_TIMEOUT = "pooled-connection-idle-timeout"; 26 | private static final String REQUEST_TIMEOUT = "request-timeout"; 27 | private static final String FOLLOW_REDIRECT = "follow-redirect"; 28 | private static final String MAX_REDIRECTS = "max-redirects"; 29 | private static final String MAX_REQUEST_RETRY = "max-request-retry"; 30 | private static final String CONNECTION_TTL = "connection-ttl"; 31 | private static final String BASE_URL = "base-url"; 32 | private static final String HEADERS = "headers"; 33 | private static final String ERROR_CODES = "error-codes"; 34 | 35 | private static final List DEFAULT_ERROR_CODES = Arrays.asList(400, 401, 402, 403, 404, 405, 406, 407, 408, 36 | 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 421, 422, 423, 424, 426, 428, 429, 431, 451, 500, 501, 37 | 502, 503, 504, 505, 506, 507, 508, 510, 511); 38 | 39 | @Override 40 | public String getName() { 41 | return "HTTP"; 42 | } 43 | 44 | /** 45 | * Creates an instance of {@link HttpWorker} for given set of configuration properties. 46 | * Configuration map should contain following: 47 | *
    48 | *
  • async - Indicates whether request should be sent in async or sync fashion. Optional, 49 | * defaults to false.
    • 50 | *
  • keep-alive - Indicates whether HTTP keep-alive is enabled or disabled. Optional, 51 | * defaults to true.
    • 52 | *
  • max-connections - The maximum number of connections a HTTP client can handle, or 53 | * -1 for no connection limit. Optional, defaults to -1.
    • 54 | *
  • max-connections-per-host - The maximum number of connections per host a HTTP client can 55 | * handle, or -1 for no connection limit. Optional, defaults to -1.
    • 56 | *
  • connect-timeout - The maximum time in millisecond a HTTP client can wait when connecting 57 | * to a remote host. Optional, defaults to 5000.
    • 58 | *
  • read-timeout - The maximum time in millisecond a HTTP client can stay idle. Optional, 59 | * defaults to 60000.
    • 60 | *
  • pooled-connection-idle-timeout - The maximum time in millisecond a HTTP client will keep 61 | * connection in pool. Optional, defaults to 60000.
    • 62 | *
  • request-timeout - The maximum time in millisecond a HTTP client waits until the response 63 | * is completed. Optional, defaults to 60000.
    • 64 | *
  • follow-redirect - Indicates whether HTTP redirect is enabled. Optional, defaults to 65 | * true.
    • 66 | *
  • max-redirects - The maximum number of HTTP redirects. Optional, defaults to 67 | * 5.
    • 68 | *
  • max-request-retry - The number of time the library will retry when an error occurs by 69 | * the remote server. Optional, defaults to 5.
    • 70 | *
  • connection-ttl - The maximum time in millisecond a HTTP client will keep connection in 71 | * the pool, or `-1` to keep connection while possible. Optional, defaults to -1.
    • 72 | *
  • base-url - Base url to use, mandatory if worker will accept 73 | * url-sufix, unnecessary if worker will accept url.
    • 74 | *
  • headers - Contains headers in a form of name-value map which will be added to each 75 | * request. Optional.
    • 76 | *
  • error-codes - List of HTTP codes to be considered as errors. Optional, defaults to 77 | * all 4** and 5** codes.
    • 78 | *
79 | * @param configuration Configuration specific to this worker. 80 | * @return An instance of {@link HttpWorker}. 81 | */ 82 | @Override 83 | public Worker getWorker(Map configuration) { 84 | boolean async = getOptionalValue(configuration, ASYNC, false); 85 | boolean keepAlive = getOptionalValue(configuration, KEEP_ALIVE, true); 86 | int maxConnections = getOptionalValue(configuration, MAX_CONNECTIONS, -1); 87 | int maxConnectionsPerHost = getOptionalValue(configuration, MAX_CONNECTIONS_PER_HOST, -1); 88 | int connectTimeout = getOptionalValue(configuration, CONNECT_TIMEOUT, 5000); 89 | int readTimeout = getOptionalValue(configuration, READ_TIMEOUT, 60000); 90 | int pooledConnectionIdleTimeout = getOptionalValue(configuration, POOLED_CONNECTION_IDLE_TIMEOUT, 60000); 91 | int requestTimeout = getOptionalValue(configuration, REQUEST_TIMEOUT, 60000); 92 | boolean followRedirect = getOptionalValue(configuration, FOLLOW_REDIRECT, true); 93 | int maxRedirects = getOptionalValue(configuration, MAX_REDIRECTS, 5); 94 | int maxRequestRetry = getOptionalValue(configuration, MAX_REQUEST_RETRY, 5); 95 | int connectionTtl = getOptionalValue(configuration, CONNECTION_TTL, -1); 96 | List errorCodes = getOptionalValue(configuration, ERROR_CODES, DEFAULT_ERROR_CODES); 97 | String baseUrl = (String) configuration.get(BASE_URL); 98 | Map headers = getHeaders(configuration); 99 | 100 | return new HttpWorker(async, keepAlive, maxConnections, maxConnectionsPerHost, connectTimeout, readTimeout, 101 | pooledConnectionIdleTimeout, requestTimeout, followRedirect, maxRedirects, maxRequestRetry, 102 | connectionTtl, baseUrl, headers, errorCodes); 103 | } 104 | 105 | @SuppressWarnings("unchecked") 106 | private Map getHeaders(Map configuration) { 107 | Map result = new HashMap<>(); 108 | Map headers = (Map) configuration.get(HEADERS); 109 | if (headers == null) { 110 | return result; 111 | } 112 | for (Map.Entry header : headers.entrySet()) { 113 | if (!(header.getValue() instanceof String)) { 114 | throw new RuntimeException("All headers need to have string value. Header: " + header.getKey() 115 | + " has value: " + header.getValue() + " of type: " + header.getValue().getClass().getName()); 116 | } 117 | result.put(header.getKey(), (String) header.getValue()); 118 | } 119 | return result; 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /berserker-http/src/main/java/io/smartcat/berserker/http/worker/HttpWorker.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.http.worker; 2 | 3 | import java.util.Arrays; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | 8 | import static org.asynchttpclient.Dsl.*; 9 | 10 | import org.asynchttpclient.*; 11 | import io.smartcat.berserker.api.Worker; 12 | 13 | /** 14 | * Worker that sends HTTP requests to HTTP server. 15 | */ 16 | public class HttpWorker implements Worker> { 17 | 18 | private static final String URL = "url"; 19 | private static final String URL_SUFIX = "url-sufix"; 20 | private static final String HEADERS = "headers"; 21 | private static final String METHOD_TYPE = "method-type"; 22 | private static final String BODY = "body"; 23 | private static final String GET = "GET"; 24 | private static final String POST = "POST"; 25 | private static final String PUT = "PUT"; 26 | private static final String DELETE = "DELETE"; 27 | private static final String HEAD = "HEAD"; 28 | private static final List METHOD_TYPES = Arrays.asList(GET, POST, PUT, DELETE, HEAD); 29 | 30 | private final boolean async; 31 | private final String baseUrl; 32 | private final Map headers; 33 | private final List errorCodes; 34 | 35 | private AsyncHttpClient asyncHttpClient; 36 | 37 | /** 38 | * Constructs HTTP worker with specified properties. 39 | * 40 | * @param async Indicates whether HTTP worker should behave in asynchronous or synchronous manner. 41 | * @param keepAlive Indicates whether HTTP keep-alive is enabled or not. 42 | * @param maxConnections The maximum number of connections an {@link AsyncHttpClient} can handle. 43 | * @param maxConnectionsPerHost The maximum number of connections per host an AsyncHttpClient can handle. 44 | * @param connectTimeout The maximum time in millisecond an {@link AsyncHttpClient} can wait when connecting to a 45 | * remote host. 46 | * @param readTimeout The maximum time in millisecond an {@link AsyncHttpClient} can stay idle. 47 | * @param pooledConnectionIdleTimeout The maximum time in millisecond an {@link AsyncHttpClient} will keep 48 | * connection in pool. 49 | * @param requestTimeout The maximum time in millisecond an {@link AsyncHttpClient} waits until the response is 50 | * completed. 51 | * @param followRedirect Indicates whether HTTP redirect is enabled. 52 | * @param maxRedirects The maximum number of HTTP redirects. 53 | * @param maxRequestRetry The number of time the library will retry when an {@link java.io.IOException} is thrown by 54 | * the remote server. 55 | * @param connectionTtl The maximum time in millisecond an {@link AsyncHttpClient} will keep connection in the pool, 56 | * or -1 to keep connection while possible. 57 | * @param baseUrl Can be concatenated with request property url-sufix to constructs URL. 58 | * @param headers Map of headers to use for each request. 59 | * @param errorCodes List of codes to be considered errors. 60 | */ 61 | public HttpWorker(boolean async, boolean keepAlive, int maxConnections, int maxConnectionsPerHost, 62 | int connectTimeout, int readTimeout, int pooledConnectionIdleTimeout, int requestTimeout, 63 | boolean followRedirect, int maxRedirects, int maxRequestRetry, int connectionTtl, String baseUrl, 64 | Map headers, List errorCodes) { 65 | this.async = async; 66 | this.baseUrl = baseUrl; 67 | this.headers = headers; 68 | this.errorCodes = errorCodes; 69 | 70 | AsyncHttpClientConfig config = new DefaultAsyncHttpClientConfig.Builder().setKeepAlive(keepAlive) 71 | .setMaxConnections(maxConnections).setMaxConnectionsPerHost(maxConnectionsPerHost) 72 | .setConnectTimeout(connectTimeout).setReadTimeout(readTimeout) 73 | .setPooledConnectionIdleTimeout(pooledConnectionIdleTimeout).setRequestTimeout(requestTimeout) 74 | .setFollowRedirect(followRedirect).setMaxRedirects(maxRedirects).setMaxRequestRetry(maxRequestRetry) 75 | .setConnectionTtl(connectionTtl).build(); 76 | asyncHttpClient = new DefaultAsyncHttpClient(config); 77 | } 78 | 79 | /** 80 | * Accepts following arguments: 81 | *
    82 | *
  • url - Url to which request will be sent, base-url from configuration 83 | * is ignored in this case. Optional, but either url or url-sufix must 84 | * appear.
    • 85 | *
  • url-sufix - Url sufix to concatenate to base-url from configuration. 86 | * base-url needs to be present in this case, otherwise exception will be thrown. Optional, but 87 | * either url or url-sufix must appear.
    • 88 | *
  • headers - Key - value map of header names and header values. It will be merged with 89 | * headers from configuration and override same headers. Optional.
    • 90 | *
  • method-type - Method type to use for this request. Mandatory.
    • 91 | *
  • body - Body content, applicable only for POST and PUT method 92 | * types.
    • 93 | *
94 | */ 95 | @Override 96 | public void accept(Map requestMetadata, Runnable commitSuccess, Runnable commitFailure) { 97 | String url = (String) requestMetadata.get(URL); 98 | String urlSufix = (String) requestMetadata.get(URL_SUFIX); 99 | Map requestHeaders = getHeaders(requestMetadata); 100 | String methodType = getMethodType(requestMetadata); 101 | String body = getBodyIfNeeded(requestMetadata, methodType); 102 | 103 | String calculatedUrl = getCalculatedUrl(url, urlSufix); 104 | Map calculatedHeaders = getCalculatedHeaders(requestHeaders); 105 | 106 | ListenableFuture responseFuture = asyncHttpClient.executeRequest( 107 | createRequest(methodType, calculatedUrl, calculatedHeaders, body), 108 | new AsyncCompletionHandler() { 109 | @Override 110 | public Response onCompleted(Response response) throws Exception { 111 | if (errorCodes.contains(response.getStatusCode())) { 112 | commitFailure.run(); 113 | } else { 114 | commitSuccess.run(); 115 | } 116 | return response; 117 | } 118 | 119 | @Override 120 | public void onThrowable(Throwable t) { 121 | commitFailure.run(); 122 | } 123 | }); 124 | if (!async) { 125 | try { 126 | responseFuture.get(); 127 | } catch (Exception e) { 128 | throw new RuntimeException(e); 129 | } 130 | } 131 | } 132 | 133 | @SuppressWarnings("unchecked") 134 | private Map getHeaders(Map requestMetadata) { 135 | Map result = new HashMap<>(); 136 | Map headers = (Map) requestMetadata.get(HEADERS); 137 | if (headers == null) { 138 | return result; 139 | } 140 | for (Map.Entry header : headers.entrySet()) { 141 | if (!(header.getValue() instanceof String)) { 142 | throw new RuntimeException("All headers need to have string value. Header: " + header.getKey() 143 | + " has value: " + header.getValue() + " of type: " + header.getValue().getClass().getName()); 144 | } 145 | result.put(header.getKey(), (String) header.getValue()); 146 | } 147 | return result; 148 | } 149 | 150 | private String getMethodType(Map requestMetadata) { 151 | String methodType = (String) requestMetadata.get(METHOD_TYPE); 152 | if (methodType == null) { 153 | throw new RuntimeException("Method type is mandatory."); 154 | } 155 | if (!METHOD_TYPES.contains(methodType)) { 156 | throw new RuntimeException( 157 | "Expected any of supported method types: " + METHOD_TYPES + " but method type was: " + methodType); 158 | } 159 | return methodType; 160 | } 161 | 162 | private String getBodyIfNeeded(Map requestMetadata, String methodType) { 163 | if (methodType.equals(POST) || methodType.equals(PUT)) { 164 | return (String) requestMetadata.get(BODY); 165 | } 166 | return null; 167 | } 168 | 169 | private String getCalculatedUrl(String url, String urlSufix) { 170 | if (url == null && urlSufix == null) { 171 | throw new RuntimeException("One needs to be specified, either url or url-sufix."); 172 | } 173 | if (url != null && urlSufix != null) { 174 | throw new RuntimeException("Cannot have both url and url-sufix."); 175 | } 176 | String result = null; 177 | if (url != null) { 178 | result = url; 179 | } 180 | if (urlSufix != null && baseUrl == null) { 181 | throw new RuntimeException("base-url must be specified when url-sufix is used."); 182 | } 183 | if (urlSufix != null) { 184 | result = baseUrl + urlSufix; 185 | } 186 | return result; 187 | } 188 | 189 | private Map getCalculatedHeaders(Map requestHeaders) { 190 | Map result = new HashMap<>(); 191 | result.putAll(headers); 192 | result.putAll(requestHeaders); 193 | return result; 194 | } 195 | 196 | private Request createRequest(String methodType, String url, Map requestHeaders, String body) { 197 | final RequestBuilder request; 198 | 199 | switch (methodType) { 200 | case GET: 201 | request = get(url); 202 | break; 203 | case POST: 204 | request = post(url); 205 | if (body != null) { 206 | request.setBody(body.getBytes()); 207 | } 208 | break; 209 | case PUT: 210 | request = put(url); 211 | if (body != null) { 212 | request.setBody(body.getBytes()); 213 | } 214 | break; 215 | case DELETE: 216 | request = delete(url); 217 | break; 218 | case HEAD: 219 | request = head(url); 220 | break; 221 | default: 222 | throw new RuntimeException("Unsupported method type: " + methodType); 223 | } 224 | requestHeaders.forEach((name, value) -> request.setHeader(name, value)); 225 | return request.build(); 226 | } 227 | } 228 | -------------------------------------------------------------------------------- /berserker-kafka/README.md: -------------------------------------------------------------------------------- 1 | # Berserker Kafka 2 | 3 | Worker implementation which sends messages to Kafka cluster. 4 | 5 | Configuration can define following properties: 6 | 1. `async` - Can be `true` or `false`. Determines whether messages will be sent in asynchronous fashion or not. Optional, if not specified defaults to `false`. 7 | 2. `topic` - Name of the topic to which message will be sent to. Optional. Can be overridden by message level topic property. 8 | 3. `producer-configuration` - Below this placeholder Kafka specific properties should be defined. List of properties is defined within Kafka [documentation](https://kafka.apache.org/documentation/#producerconfigs). 9 | 10 | Worker `accept` method expects following properties: 11 | 1. `key` - (String) Key of Kafka message. Mandatory. 12 | 2. `value` - (String) Value of Kafka message. Mandatory. 13 | 3. `topic` - (String) Name of the topic to which message will be sent to. Optional. Overrides configuration level topic property. 14 | 15 | ## Configuration 16 | 17 | Example yaml configuration: 18 | 19 | ```yaml 20 | worker-configuration: 21 | async: true 22 | topic: topic1 23 | producer-configuration: 24 | bootstrap.servers: 192.168.0.5:32772 25 | ``` 26 | -------------------------------------------------------------------------------- /berserker-kafka/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | berserker 6 | io.smartcat 7 | 0.0.13-SNAPSHOT 8 | 9 | 10 | berserker-kafka 11 | jar 12 | 13 | Berserker Kafka 14 | Worker implementation which sends messages to Kafka cluster. 15 | 16 | 17 | UTF-8 18 | 0.10.2.0 19 | 20 | 21 | 22 | 23 | io.smartcat 24 | berserker-commons 25 | ${project.parent.version} 26 | 27 | 28 | org.apache.kafka 29 | connect-api 30 | ${version.kafka} 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /berserker-kafka/src/main/java/io/smartcat/berserker/kafka/configuration/KafkaConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.kafka.configuration; 2 | 3 | import java.util.Map; 4 | 5 | import io.smartcat.berserker.api.Worker; 6 | import io.smartcat.berserker.configuration.WorkerConfiguration; 7 | import io.smartcat.berserker.kafka.worker.KafkaWorker; 8 | 9 | import static io.smartcat.berserker.configuration.ConfigurationHelper.getOptionalValue; 10 | 11 | /** 12 | * Configuration for Kafka worker. 13 | */ 14 | public class KafkaConfiguration implements WorkerConfiguration { 15 | 16 | private static final String ASYNC = "async"; 17 | private static final String TOPIC = "topic"; 18 | private static final String PRODUCER_CONFIGURATION = "producer-configuration"; 19 | 20 | @Override 21 | public String getName() { 22 | return "Kafka"; 23 | } 24 | 25 | /** 26 | * Creates an instance of {@link KafkaWorker} for given set of configuration properties. 27 | * Configuration map should contain following: 28 | *
    29 | *
  • async - Indicates whether messages should be sent to broker in an async or sync 30 | * fashion. Optional, defaults to false.
    • 31 | *
  • topic - Name of the topic to which messages will be sent. Optional, defaults to 32 | * null. Can be overridden 33 | * by message level topic property.
    • 34 | *
  • producer-configuration - Set of producer properties as defined within 35 | * configuration properties.
    • 36 | *
37 | * @param configuration Configuration specific to this worker. 38 | * @return An instance of {@link KafkaWorker}. 39 | */ 40 | @Override 41 | public Worker getWorker(Map configuration) { 42 | boolean async = getOptionalValue(configuration, ASYNC, false); 43 | String topic = getOptionalValue(configuration, TOPIC, null); 44 | Map producerConfiguration = (Map) configuration.get(PRODUCER_CONFIGURATION); 45 | return new KafkaWorker(producerConfiguration, async, topic); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /berserker-kafka/src/main/java/io/smartcat/berserker/kafka/worker/KafkaWorker.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.kafka.worker; 2 | 3 | import java.util.Map; 4 | import java.util.concurrent.Future; 5 | 6 | import org.apache.kafka.clients.producer.*; 7 | import org.apache.kafka.common.serialization.StringSerializer; 8 | 9 | import io.smartcat.berserker.api.Worker; 10 | 11 | /** 12 | * Worker that publishes accepted message to Kafka cluster. 13 | */ 14 | public class KafkaWorker implements Worker>, AutoCloseable { 15 | 16 | private static final String KEY = "key"; 17 | private static final String VALUE = "value"; 18 | private static final String TOPIC = "topic"; 19 | 20 | private final Map configuration; 21 | private final boolean async; 22 | private final String topic; 23 | 24 | private Producer producer; 25 | 26 | /** 27 | * Constructs Kafka worker with specified properties. 28 | * 29 | * @param configuration Map containing configuration properties to be used by {@link KafkaProducer}. Map contains 30 | * common Kafka producer 31 | * configuration properties. 32 | * @param async Indicates whether messages should be sent asynchronously or synchronously. 33 | * @param topic Kafka topic to which to send messages. Optional. 34 | */ 35 | public KafkaWorker(Map configuration, boolean async, String topic) { 36 | this.configuration = configuration; 37 | this.async = async; 38 | this.topic = topic; 39 | init(); 40 | } 41 | 42 | /** 43 | * Accepts following arguments: 44 | *
    45 | *
  • key - Key of the message. Optional, if not specified, Kafka producer will calculate 46 | * partition based on message value.
    • 47 | *
  • value - Value of the message. Mandatory.
    • 48 | *
  • topic - Topic to which to send message. Mandatory if topic on configuration level is not 49 | * specified. If it is, this topic value will override it. If topic is not specified neither on configuration level 50 | * nor here, exception will be thrown.
    • 51 | *
52 | */ 53 | @Override 54 | public void accept(Map message, Runnable commitSuccess, Runnable commitFailure) { 55 | String key = (String) message.get(KEY); 56 | String value = (String) message.get(VALUE); 57 | if (value == null) { 58 | throw new RuntimeException("'value' is mandatory."); 59 | } 60 | String messageLevelTopic = (String) message.get(TOPIC); 61 | String calculatedTopic = getCalculatedTopic(messageLevelTopic); 62 | ProducerRecord record = new ProducerRecord<>(calculatedTopic, key, value); 63 | Future futureResponse = producer.send(record, (metadata, exception) -> { 64 | if (exception == null) { 65 | commitSuccess.run(); 66 | } else { 67 | commitFailure.run(); 68 | } 69 | }); 70 | if (!async) { 71 | try { 72 | futureResponse.get(); 73 | } catch (Exception e) { 74 | throw new RuntimeException(e); 75 | } 76 | } 77 | } 78 | 79 | @Override 80 | public void close() { 81 | producer.close(); 82 | } 83 | 84 | private void init() { 85 | StringSerializer keySerializer = getSerializer(configuration, true); 86 | StringSerializer valueSerializer = getSerializer(configuration, false); 87 | producer = new KafkaProducer<>(configuration, keySerializer, valueSerializer); 88 | } 89 | 90 | private StringSerializer getSerializer(Map configuration, boolean isKey) { 91 | StringSerializer serializer = new StringSerializer(); 92 | serializer.configure(configuration, isKey); 93 | return serializer; 94 | } 95 | 96 | private String getCalculatedTopic(String messageLevelTopic) { 97 | String calculatedTopic = messageLevelTopic != null ? messageLevelTopic : topic; 98 | if (calculatedTopic == null) { 99 | throw new RuntimeException("Topic must be present on either configuration or message level."); 100 | } 101 | return calculatedTopic; 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /berserker-mqtt/README.md: -------------------------------------------------------------------------------- 1 | # Berserker RabbitMQ 2 | 3 | Worker implementation which publishes messages to MQTT broker. 4 | 5 | Configuration defines several properties: 6 | 7 | 1. `async` - Can be `true` or `false`. Determines whether messages will be sent in asynchronous fashion or not. Optional, if not specified, defaults to `false`. 8 | 2. `broker-url`- URL of MQTT broker to which to connect to. Mandatory. 9 | 3. `client-id` - Client ID. Mandatory. 10 | 4. `max-inflight` - Maximum number of inflight messages. Should be increased when using asynchronous worker or QoS > 0. Value should follow the desired rate. Optional, if not specified, defaults to 10. 11 | 5. `clean-session` - Can be `true` or `false`. Determines whether client and server should remember state across restarts and reconnects. Optional, if not specified, defaults to `true`. 12 | 6. `connection-timeout` - Connection timeout in seconds. Optional, if not specified, defaults to `30` seconds. 13 | 7. `mqtt-version` - MQTT version to use. Possible values: `3.1.1` and `3.1`. Optional, if not specified, defaults to `3.1.1`, if that fails, tries `3.1`. 14 | 8. `username` - Username to connect to MQTT broker. Optional, if not specified, no username will be used. 15 | 9. `password` - Password to connect to MQTT broker. Optional, if not specified, no password will be used. 16 | 17 | Worker `accept` method expects following properties: 18 | 19 | 1. `topic` (String) - Topic to which message will be published. Mandatory. 20 | 2. `qos` (Integer) - Quality of Service. Possible values: 0 (At most once), 1 (At least once), 2 (Exactly once). Mandatory. 21 | 3. `payload` (String) - Payload of message to be published. Mandatory. 22 | 23 | ## Configuration 24 | 25 | Example yaml configuration: 26 | 27 | ```yaml 28 | worker-configuration: 29 | async: false 30 | broker-url: "tcp://localhost:1883" 31 | client-id: client-1 32 | clean-session: true 33 | connection-timeout: 20 34 | mqtt-version: 3.1.1 35 | ``` 36 | 37 | For whole configuration, take a look at [Ranger-MQTT example](../berserker-runner/src/example/resources/ranger-mqtt.yml). 38 | -------------------------------------------------------------------------------- /berserker-mqtt/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | berserker 6 | io.smartcat 7 | 0.0.13-SNAPSHOT 8 | 9 | 10 | berserker-mqtt 11 | jar 12 | 13 | Berserker MQTT 14 | Worker implementation which publishes messages MQTT topics. 15 | 16 | 17 | UTF-8 18 | 5.1.1 19 | 1.2.1 20 | 21 | 22 | 23 | 24 | io.smartcat 25 | berserker-commons 26 | ${project.parent.version} 27 | 28 | 29 | org.eclipse.paho 30 | org.eclipse.paho.client.mqttv3 31 | ${version.eclipse.paho} 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /berserker-mqtt/src/main/java/io/smartcat/berserker/mqtt/configuration/MqttConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.mqtt.configuration; 2 | 3 | import static io.smartcat.berserker.configuration.ConfigurationHelper.getMandatoryValue; 4 | import static io.smartcat.berserker.configuration.ConfigurationHelper.getOptionalValue; 5 | 6 | import java.util.Map; 7 | 8 | import org.eclipse.paho.client.mqttv3.MqttConnectOptions; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import io.smartcat.berserker.api.Worker; 13 | import io.smartcat.berserker.configuration.ConfigurationParseException; 14 | import io.smartcat.berserker.configuration.WorkerConfiguration; 15 | import io.smartcat.berserker.mqtt.worker.MqttWorker; 16 | 17 | /** 18 | * Configuration for MQTT worker. 19 | */ 20 | public class MqttConfiguration implements WorkerConfiguration { 21 | 22 | private static final Logger LOGGER = LoggerFactory.getLogger(MqttConfiguration.class); 23 | 24 | private static final String ASYNC = "async"; 25 | private static final String BROKER_URL = "broker-url"; 26 | private static final String CLIENT_ID = "client-id"; 27 | 28 | private static final String MAX_INFLIGHT = "max-inflight"; 29 | private static final String CLEAN_SESSION = "clean-session"; 30 | private static final String CONNECTION_TIMEOUT = "connection-timeout"; 31 | private static final String MQTT_VERSION = "mqtt-version"; 32 | private static final String USERNAME = "username"; 33 | private static final String PASSWORD = "password"; 34 | 35 | private static final String MQTT_VERSION_3_1_1 = "3.1.1"; 36 | private static final String MQTT_VERSION_3_1 = "3.1"; 37 | 38 | @Override 39 | public String getName() { 40 | return "MQTT"; 41 | } 42 | 43 | @Override 44 | public Worker getWorker(Map configuration) throws ConfigurationParseException { 45 | boolean async = getOptionalValue(configuration, ASYNC, false); 46 | String brokerUrl = getMandatoryValue(configuration, BROKER_URL); 47 | String clientId = getMandatoryValue(configuration, CLIENT_ID); 48 | int maxInflight = getOptionalValue(configuration, MAX_INFLIGHT, 10); 49 | boolean cleanSession = getOptionalValue(configuration, CLEAN_SESSION, true); 50 | int connectionTimeout = getOptionalValue(configuration, CONNECTION_TIMEOUT, 30); 51 | int mqttVersion = calculateMqttVersion((String) configuration.get(MQTT_VERSION)); 52 | String username = (String) configuration.get(USERNAME); 53 | String password = (String) configuration.get(PASSWORD); 54 | 55 | return new MqttWorker(async, brokerUrl, clientId, maxInflight, cleanSession, connectionTimeout, mqttVersion, 56 | username, password); 57 | } 58 | 59 | private int calculateMqttVersion(String mqttVersion) { 60 | if (mqttVersion == null) { 61 | LOGGER.info("'" + MQTT_VERSION + "' not set, using default version: 3.1.1 or 3.1 if that fails."); 62 | return 0; 63 | } else if (MQTT_VERSION_3_1_1.equals(mqttVersion)) { 64 | LOGGER.info("'" + MQTT_VERSION + "' set to value: " + MQTT_VERSION_3_1_1); 65 | return MqttConnectOptions.MQTT_VERSION_3_1_1; 66 | } else if (MQTT_VERSION_3_1.equals(mqttVersion)) { 67 | LOGGER.info("'" + MQTT_VERSION + "' set to value: " + MQTT_VERSION_3_1); 68 | return MqttConnectOptions.MQTT_VERSION_3_1; 69 | } else { 70 | throw new RuntimeException("'" + MQTT_VERSION + "' has invalid value: " + mqttVersion 71 | + " Only supported versions are: " + MQTT_VERSION_3_1 + " and " + MQTT_VERSION_3_1_1); 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /berserker-mqtt/src/main/java/io/smartcat/berserker/mqtt/worker/MqttWorker.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.mqtt.worker; 2 | 3 | import java.util.Map; 4 | 5 | import org.eclipse.paho.client.mqttv3.*; 6 | import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence; 7 | 8 | import io.smartcat.berserker.api.Worker; 9 | 10 | /** 11 | * MQTT worker which publishes messages to MQTT broker. 12 | */ 13 | public class MqttWorker implements Worker>, AutoCloseable { 14 | 15 | private static final String TOPIC = "topic"; 16 | private static final String QOS = "qos"; 17 | private static final String PAYLOAD = "payload"; 18 | 19 | private final boolean async; 20 | private final MqttAsyncClient client; 21 | private final MqttConnectOptions connectOptions; 22 | private volatile boolean connected = false; 23 | 24 | /** 25 | * Constructs MQTT asynchronous worker with specified client and connectOptions to be 26 | * used. 27 | * 28 | * @param async Indicates whether worker should behave in asynchronous fashion or not. True if worker is to behave 29 | * asynchronously, otherwise false. 30 | * @param brokerUrl Url to MQTT broker. 31 | * @param clientId ID of MQTT client. 32 | * @param maxInflight Maximum number of inflight messages. 33 | * @param cleanSession Indicates whether client and server should remember state across restarts and reconnects or 34 | * not. True is state is not to be remembered, otherwise false. 35 | * @param connectionTimeout Connection timeout in seconds. 36 | * @param mqttVersion Version of MQTT specification to use. 37 | * @param username Username to connect to MQTT broker. 38 | * @param password Password to connect to MQTT broker. 39 | */ 40 | public MqttWorker(boolean async, String brokerUrl, String clientId, int maxInflight, boolean cleanSession, 41 | int connectionTimeout, int mqttVersion, String username, String password) { 42 | try { 43 | this.async = async; 44 | this.client = new MqttAsyncClient(brokerUrl, clientId, new MemoryPersistence()); 45 | this.connectOptions = new MqttConnectOptions(); 46 | this.connectOptions.setMaxInflight(maxInflight); 47 | this.connectOptions.setCleanSession(cleanSession); 48 | this.connectOptions.setConnectionTimeout(connectionTimeout); 49 | this.connectOptions.setMqttVersion(mqttVersion); 50 | if (username != null) { 51 | this.connectOptions.setUserName(username); 52 | } 53 | if (password != null) { 54 | this.connectOptions.setPassword(password.toCharArray()); 55 | } 56 | } catch (Exception e) { 57 | throw new RuntimeException(e); 58 | } 59 | } 60 | 61 | @Override 62 | public void accept(Map message, Runnable commitSuccess, Runnable commitFailure) { 63 | ensureConnected(); 64 | String topic = (String) message.get(TOPIC); 65 | int qos = (Integer) message.get(QOS); 66 | String payload = (String) message.get(PAYLOAD); 67 | MqttMessage mqttMessage = new MqttMessage(payload.getBytes()); 68 | mqttMessage.setQos(qos); 69 | try { 70 | IMqttDeliveryToken token = client.publish(topic, mqttMessage, null, new IMqttActionListener() { 71 | @Override 72 | public void onSuccess(IMqttToken asyncActionToken) { 73 | commitSuccess.run(); 74 | } 75 | 76 | @Override 77 | public void onFailure(IMqttToken asyncActionToken, Throwable exception) { 78 | commitFailure.run(); 79 | } 80 | }); 81 | if (!async) { 82 | token.waitForCompletion(); 83 | } 84 | } catch (Exception e) { 85 | throw new RuntimeException(e); 86 | } 87 | } 88 | 89 | @Override 90 | public void close() throws Exception { 91 | try { 92 | client.close(); 93 | } catch (MqttException e) { 94 | throw new RuntimeException(e); 95 | } 96 | } 97 | 98 | private void ensureConnected() { 99 | if (!connected) { 100 | synchronized (this) { 101 | if (!connected) { 102 | try { 103 | client.connect(connectOptions).waitForCompletion(); 104 | connected = true; 105 | } catch (MqttException e) { 106 | throw new RuntimeException(e); 107 | } 108 | } 109 | } 110 | } 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /berserker-rabbitmq/README.md: -------------------------------------------------------------------------------- 1 | # Berserker RabbitMQ 2 | 3 | Worker implementation which sends AMQP messages to RabbitMQ. 4 | 5 | Configuration defines five properties in order for `ConnectionFactory` to be created: 6 | 7 | 1. `username` 8 | 2. `password` 9 | 3. `host` 10 | 4. `port` 11 | 5. `virtual-host` 12 | 13 | Worker `accept` method expects following properties: 14 | 15 | 1. `exchangeName` (String) - Mandatory 16 | 2. `routingKey` (String) - Mandatory 17 | 3. `messageContent` (String) - Mandatory 18 | 4. `contentType` - (String) - Optional 19 | 5. `contentEncoding` - (String) - Optional 20 | 6. `headers` - (Map) - Optional 21 | 7. `deliveryMode` - (Integer) - Optional 22 | 8. `priority` - (Integer) - Optional 23 | 9. `correlationId` - (String) - Optional 24 | 10. `replyTo` - (String) - Optional 25 | 11. `expiration` - (String) - Optional 26 | 12. `messageId` - (String) - Optional 27 | 13. `timestamp` - (Date) - Optional 28 | 14. `type` - (String) - Optional 29 | 15. `userId` - (String) - Optional 30 | 16. `appId` - (String) - Optional 31 | 17. `clusterId` - (String) - Optional 32 | 33 | ## Configuration 34 | 35 | Example yaml configuration: 36 | 37 | ```yaml 38 | worker-configuration: 39 | username: test 40 | password: test 41 | host: localhost 42 | port: 5672 43 | virtual-host: /rabbitmq_test 44 | ``` 45 | 46 | For whole configuration, take a look at [Ranger-RabbitMQ example](../berserker-runner/src/example/resources/ranger-rabbitmq.yml). -------------------------------------------------------------------------------- /berserker-rabbitmq/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | berserker 6 | io.smartcat 7 | 0.0.13-SNAPSHOT 8 | 9 | 10 | berserker-rabbitmq 11 | jar 12 | 13 | Berserker RabbitMQ 14 | Worker implementation which sends messages to RabbitMQ. 15 | 16 | 17 | UTF-8 18 | 5.7.3 19 | 20 | 21 | 22 | 23 | io.smartcat 24 | berserker-commons 25 | ${project.parent.version} 26 | 27 | 28 | com.rabbitmq 29 | amqp-client 30 | ${version.rabbitmq} 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /berserker-rabbitmq/src/main/java/io/smartcat/berserker/rabbitmq/configuration/RabbitMqConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.rabbitmq.configuration; 2 | 3 | import java.io.IOException; 4 | import java.util.Map; 5 | import java.util.concurrent.TimeoutException; 6 | 7 | import com.rabbitmq.client.Connection; 8 | import com.rabbitmq.client.ConnectionFactory; 9 | 10 | import io.smartcat.berserker.api.Worker; 11 | import io.smartcat.berserker.configuration.ConfigurationParseException; 12 | import io.smartcat.berserker.configuration.WorkerConfiguration; 13 | import io.smartcat.berserker.rabbitmq.worker.RabbitMqWorker; 14 | 15 | /** 16 | * Configuration for RabbitMQ worker. 17 | */ 18 | public class RabbitMqConfiguration implements WorkerConfiguration { 19 | 20 | private static final String USERNAME = "username"; 21 | private static final String PASSWORD = "password"; 22 | private static final String HOST = "host"; 23 | private static final String PORT = "port"; 24 | private static final String VIRTUAL_HOST = "virtual-host"; 25 | 26 | @Override 27 | public String getName() { 28 | return "RabbitMQ"; 29 | } 30 | 31 | @Override 32 | public Worker getWorker(Map configuration) throws ConfigurationParseException { 33 | String username = (String) configuration.get(USERNAME); 34 | String password = (String) configuration.get(PASSWORD); 35 | String host = (String) configuration.get(HOST); 36 | int port = (int) configuration.get(PORT); 37 | String virtualHost = (String) configuration.get(VIRTUAL_HOST); 38 | ConnectionFactory factory = new ConnectionFactory(); 39 | factory.setUsername(username); 40 | factory.setPassword(password); 41 | factory.setVirtualHost(virtualHost); 42 | factory.setHost(host); 43 | factory.setPort(port); 44 | Connection connection; 45 | try { 46 | connection = factory.newConnection(); 47 | } catch (IOException | TimeoutException e) { 48 | throw new ConfigurationParseException(e); 49 | } 50 | return new RabbitMqWorker(connection); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /berserker-rabbitmq/src/main/java/io/smartcat/berserker/rabbitmq/worker/RabbitMqWorker.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.rabbitmq.worker; 2 | 3 | import java.io.IOException; 4 | import java.util.Date; 5 | import java.util.Map; 6 | 7 | import com.rabbitmq.client.AMQP.BasicProperties; 8 | import com.rabbitmq.client.Channel; 9 | import com.rabbitmq.client.Connection; 10 | 11 | import io.smartcat.berserker.api.Worker; 12 | 13 | /** 14 | * Worker that publishes accepted message to RabbitMQ. Message must contain exchangeName, 15 | * routingKey and messageContent fields. 16 | */ 17 | public class RabbitMqWorker implements Worker>, AutoCloseable { 18 | 19 | private static final String EXCHANGE_NAME = "exchangeName"; 20 | private static final String ROUTING_KEY = "routingKey"; 21 | private static final String MESSAGE_CONTENT = "messageContent"; 22 | 23 | private static final String CONTENT_TYPE = "contentType"; 24 | private static final String CONTENT_ENCODING = "contentEncoding"; 25 | private static final String HEADERS = "headers"; 26 | private static final String DELIVERY_MODE = "deliveryMode"; 27 | private static final String PRIORITY = "priority"; 28 | private static final String CORRELATION_ID = "correlationId"; 29 | private static final String REPLY_TO = "replyTo"; 30 | private static final String EXPIRATION = "expiration"; 31 | private static final String MESSAGE_ID = "messageId"; 32 | private static final String TIMESTAMP = "timestamp"; 33 | private static final String TYPE = "type"; 34 | private static final String USER_ID = "userId"; 35 | private static final String APP_ID = "appId"; 36 | private static final String CLUSTER_ID = "clusterId"; 37 | 38 | private final Connection connection; 39 | private final Channel channel; 40 | 41 | /** 42 | * Constructs RabbitMQ worker with specified connection to be used. 43 | * 44 | * @param connection Connection object to RabbitMQ. 45 | */ 46 | public RabbitMqWorker(Connection connection) { 47 | this.connection = connection; 48 | try { 49 | this.channel = connection.createChannel(); 50 | } catch (IOException e) { 51 | throw new RuntimeException(e); 52 | } 53 | } 54 | 55 | @Override 56 | public void accept(Map message, Runnable commitSuccess, Runnable commitFailure) { 57 | String exchangeName = (String) message.get(EXCHANGE_NAME); 58 | String routingKey = (String) message.get(ROUTING_KEY); 59 | BasicProperties props = createProperties(message); 60 | String messageContent = (String) message.get(MESSAGE_CONTENT); 61 | try { 62 | channel.basicPublish(exchangeName, routingKey, props, messageContent.getBytes()); 63 | commitSuccess.run(); 64 | } catch (IOException e) { 65 | commitFailure.run(); 66 | throw new RuntimeException(e); 67 | } 68 | } 69 | 70 | @SuppressWarnings({ "rawtypes", "unchecked" }) 71 | private BasicProperties createProperties(Map message) { 72 | BasicProperties.Builder propsBuilder = new BasicProperties.Builder(); 73 | String contentType = (String) message.get(CONTENT_TYPE); 74 | if (!isEmpty(contentType)) { 75 | propsBuilder.contentType(contentType); 76 | } 77 | String contentEncoding = (String) message.get(CONTENT_ENCODING); 78 | if (!isEmpty(contentEncoding)) { 79 | propsBuilder.contentEncoding(contentEncoding); 80 | } 81 | Map headers = (Map) message.get(HEADERS); 82 | if (headers != null && !headers.isEmpty()) { 83 | propsBuilder.headers(headers); 84 | } 85 | Integer deliveryMode = (Integer) message.get(DELIVERY_MODE); 86 | if (deliveryMode != null) { 87 | propsBuilder.deliveryMode(deliveryMode); 88 | } 89 | Integer priority = (Integer) message.get(PRIORITY); 90 | if (priority != null) { 91 | propsBuilder.priority(priority); 92 | } 93 | String correlationId = (String) message.get(CORRELATION_ID); 94 | if (!isEmpty(correlationId)) { 95 | propsBuilder.correlationId(correlationId); 96 | } 97 | String replyTo = (String) message.get(REPLY_TO); 98 | if (!isEmpty(replyTo)) { 99 | propsBuilder.replyTo(replyTo); 100 | } 101 | String expiration = (String) message.get(EXPIRATION); 102 | if (!isEmpty(expiration)) { 103 | propsBuilder.expiration(expiration); 104 | } 105 | String messageId = (String) message.get(MESSAGE_ID); 106 | if (!isEmpty(messageId)) { 107 | propsBuilder.messageId(messageId); 108 | } 109 | Date timestamp = (Date) message.get(TIMESTAMP); 110 | if (timestamp != null) { 111 | propsBuilder.timestamp(timestamp); 112 | } 113 | String type = (String) message.get(TYPE); 114 | if (!isEmpty(type)) { 115 | propsBuilder.type(type); 116 | } 117 | String userId = (String) message.get(USER_ID); 118 | if (!isEmpty(userId)) { 119 | propsBuilder.userId(userId); 120 | } 121 | String appId = (String) message.get(APP_ID); 122 | if (!isEmpty(appId)) { 123 | propsBuilder.appId(appId); 124 | } 125 | String clusterId = (String) message.get(CLUSTER_ID); 126 | if (!isEmpty(clusterId)) { 127 | propsBuilder.clusterId(clusterId); 128 | } 129 | return propsBuilder.build(); 130 | } 131 | 132 | @Override 133 | public void close() throws Exception { 134 | channel.close(); 135 | connection.close(); 136 | } 137 | 138 | private boolean isEmpty(String s) { 139 | return s == null || s.isEmpty(); 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /berserker-ranger/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | berserker 6 | io.smartcat 7 | 0.0.13-SNAPSHOT 8 | 9 | 10 | berserker-ranger 11 | jar 12 | 13 | Berserker Ranger 14 | Ranger data source implementation. 15 | 16 | 17 | UTF-8 18 | 0.0.12 19 | 20 | 21 | 22 | 23 | io.smartcat 24 | berserker-core 25 | ${project.parent.version} 26 | 27 | 28 | io.smartcat 29 | ranger 30 | ${version.ranger} 31 | 32 | 33 | junit 34 | junit 35 | test 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /berserker-ranger/src/main/java/io/smartcat/berserker/ranger/configuration/RangerConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.ranger.configuration; 2 | 3 | import java.util.Map; 4 | 5 | import io.smartcat.berserker.api.DataSource; 6 | import io.smartcat.berserker.configuration.ConfigurationParseException; 7 | import io.smartcat.berserker.configuration.DataSourceConfiguration; 8 | import io.smartcat.berserker.ranger.datasource.RangerDataSource; 9 | import io.smartcat.ranger.ObjectGenerator; 10 | import io.smartcat.ranger.parser.ConfigurationParser; 11 | 12 | /** 13 | * Configuration to construct {@link RangerDataSource}. 14 | */ 15 | public class RangerConfiguration implements DataSourceConfiguration { 16 | 17 | @Override 18 | public String getName() { 19 | return "Ranger"; 20 | } 21 | 22 | @Override 23 | public DataSource getDataSource(Map configuration) throws ConfigurationParseException { 24 | ObjectGenerator> objectGenerator = new ConfigurationParser(configuration).build(); 25 | return new RangerDataSource(objectGenerator); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /berserker-ranger/src/main/java/io/smartcat/berserker/ranger/datasource/RangerDataSource.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.ranger.datasource; 2 | 3 | import java.util.Map; 4 | 5 | import io.smartcat.berserker.api.DataSource; 6 | import io.smartcat.ranger.ObjectGenerator; 7 | 8 | /** 9 | * Ranger data source implementation. 10 | */ 11 | public class RangerDataSource implements DataSource> { 12 | 13 | private final ObjectGenerator> objectGenerator; 14 | 15 | /** 16 | * Constructs ranger data source with specified aggregatedObjectGenerator. 17 | * 18 | * @param objectGenerator Generator which will be used to generate objects. 19 | */ 20 | public RangerDataSource(ObjectGenerator> objectGenerator) { 21 | this.objectGenerator = objectGenerator; 22 | } 23 | 24 | @Override 25 | public boolean hasNext(long time) { 26 | return true; 27 | } 28 | 29 | @Override 30 | public Map getNext(long time) { 31 | return (Map) objectGenerator.next(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /berserker-runner/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | berserker 6 | io.smartcat 7 | 0.0.13-SNAPSHOT 8 | 9 | 10 | berserker-runner 11 | jar 12 | 13 | Berserker Runner 14 | Represents runnable jar where desired data source, rate generator and worker can be specified within YAML configuration. 15 | 16 | 17 | UTF-8 18 | 0.9.11 19 | 20 | 21 | 22 | 23 | io.smartcat 24 | berserker-ranger 25 | ${project.parent.version} 26 | 27 | 28 | io.smartcat 29 | berserker-kafka 30 | ${project.parent.version} 31 | 32 | 33 | io.smartcat 34 | berserker-cassandra 35 | ${project.parent.version} 36 | 37 | 38 | io.smartcat 39 | berserker-http 40 | ${project.parent.version} 41 | 42 | 43 | io.smartcat 44 | berserker-rabbitmq 45 | ${project.parent.version} 46 | 47 | 48 | io.smartcat 49 | berserker-mqtt 50 | ${project.parent.version} 51 | 52 | 53 | commons-cli 54 | commons-cli 55 | 56 | 57 | org.reflections 58 | reflections 59 | ${version.reflections} 60 | 61 | 62 | junit 63 | junit 64 | test 65 | 66 | 67 | 68 | 69 | 70 | 71 | org.apache.maven.plugins 72 | maven-shade-plugin 73 | 74 | 75 | package 76 | 77 | shade 78 | 79 | 80 | false 81 | false 82 | 83 | 84 | *:* 85 | 86 | META-INF/*.SF 87 | META-INF/*.DSA 88 | META-INF/*.RSA 89 | 90 | 91 | 92 | 93 | 94 | classworlds:classworlds 95 | junit:junit 96 | org.apache.maven:lib:tests 97 | 98 | 99 | 100 | 102 | io.smartcat.berserker.runner.LoadGeneratorRunner 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | -------------------------------------------------------------------------------- /berserker-runner/src/example/resources/ranger-cassandra.yml: -------------------------------------------------------------------------------- 1 | load-generator-configuration: 2 | data-source-configuration-name: Ranger 3 | rate-generator-configuration-name: default 4 | worker-configuration-name: Cassandra 5 | metrics-reporter-configuration-name: JMX 6 | thread-count: 10 7 | queue-capacity: 100000 8 | 9 | data-source-configuration: 10 | values: 11 | id: uuid() 12 | firstName: random(['Peter', 'Mike', 'Steven', 'Joshua', 'John', 'Brandon']) 13 | lastName: random(['Smith', 'Johnson', 'Williams', 'Davis', 'Jackson', 'White', 'Lewis', 'Clark']) 14 | age: random(20..45) 15 | email: string('{}@domain.com', randomContentString(5)) 16 | statement: 17 | consistencyLevel: ONE 18 | query: string("INSERT INTO person (id, first_name, last_name, age, email) VALUES ({}, '{}', '{}', {}, '{}');", $id, $firstName, $lastName, $age, $email) 19 | output: $statement 20 | 21 | rate-generator-configuration: 22 | rates: 23 | r: 1000 24 | output: $r 25 | 26 | worker-configuration: 27 | connection-points: 0.0.0.0:32770 28 | keyspace: my_keyspace 29 | async: false 30 | bootstrap-commands: 31 | - "CREATE KEYSPACE IF NOT EXISTS my_keyspace WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};" 32 | - USE my_keyspace; 33 | - CREATE TABLE IF NOT EXISTS person (id uuid, first_name text, last_name text, age int, email text, primary key (id)); 34 | 35 | metrics-reporter-configuration: 36 | domain: berserker 37 | filter: -------------------------------------------------------------------------------- /berserker-runner/src/example/resources/ranger-http.yml: -------------------------------------------------------------------------------- 1 | load-generator-configuration: 2 | data-source-configuration-name: Ranger 3 | rate-generator-configuration-name: default 4 | worker-configuration-name: HTTP 5 | metrics-reporter-configuration-name: JMX 6 | thread-count: 10 7 | queue-capacity: 100000 8 | 9 | # This example configuration each time generates randomly 1 of 5 possible types of HTTP requests: 10 | # 1. GET on http://localhost:8080/api/item/count 11 | # 2. GET on http://localhost:8080/api/item/$id - where id is random number from 1 to 100 12 | # 3. POST on http://localhost:8080/api/item/new with content-type header and JSON payload 13 | # 4. PUT on http://localhost:8080/api/item/$id - with content-type header and JSON payload where id is random number from 201 to 300 14 | # 5. DELETE on http://localhost:8080/api/item/$id - where id is random number from 301 to 400 15 | # 16 | # Note that each request can either have url or url-sufix, they are mutually exclusive. url-sufix is possible only if worker configuration 17 | # contains base-url property defined. 18 | # Headers defined on configuration level will be overridden with headers defined on request level. 19 | 20 | data-source-configuration: 21 | values: 22 | getCountRequest: 23 | url-sufix: /count 24 | method-type: GET 25 | getItemRequest: 26 | url: string("http://localhost:8080/api/item/{}", random(1..100)) 27 | method-type: GET 28 | postItemRequestData: 29 | id: random(101..200) 30 | value: string("Val {}", $id) 31 | postItemRequest: 32 | url-sufix: /new 33 | method-type: POST 34 | body: json($postItemRequestData) 35 | headers: 36 | Content-Type: application/json 37 | X-Custom-Header-1: new-val-1 38 | X-Custom-Header-3: val-3 39 | putItemRequestData: 40 | id: random(201..300) 41 | value: string("Val {}", $id) 42 | putItemRequest: 43 | url-sufix: string("/{}", $putItemRequestData.id) 44 | method-type: PUT 45 | body: json($putItemRequestData) 46 | headers: 47 | Content-Type: application/json 48 | X-Custom-Header-1: new-val-1 49 | X-Custom-Header-3: val-3 50 | deleteItemRequest: 51 | url-sufix: string("/{}", random(301..400)) 52 | method-type: DELETE 53 | output: random([$getCountRequest, $getItemRequest, $postItemRequest, $putItemRequest, $deleteItemRequest]) 54 | 55 | rate-generator-configuration: 56 | rates: 57 | r: 50000 58 | output: $r 59 | 60 | worker-configuration: 61 | async: false 62 | keep-alive: true 63 | max-connections: -1 64 | max-connections-per-host: -1 65 | connect-timeout: 5000 66 | read-timeout: 60000 67 | pooled-connection-idle-timeout: 60000 68 | request-timeout: 60000 69 | follow-redirect: true 70 | max-redirects: 5 71 | max-request-retry: 5 72 | connection-ttl: -1 73 | base-url: http://localhost:8080/api/item 74 | headers: 75 | X-Custom-Header-1: val-1 76 | X-Custom-Header-2: val-2 77 | error-codes: 78 | - 400 79 | - 401 80 | - 403 81 | - 404 82 | 83 | 84 | metrics-reporter-configuration: 85 | domain: berserker 86 | filter: 87 | -------------------------------------------------------------------------------- /berserker-runner/src/example/resources/ranger-kafka.yml: -------------------------------------------------------------------------------- 1 | load-generator-configuration: 2 | data-source-configuration-name: Ranger 3 | rate-generator-configuration-name: default 4 | worker-configuration-name: Kafka 5 | metrics-reporter-configuration-name: SimpleConsoleReporter 6 | thread-count: 4 7 | queue-capacity: 1000 8 | 9 | data-source-configuration: 10 | values: 11 | values: circular([random(0.0..100),random(100..600),random(600..1000)]) 12 | tag1: 13 | host: circular(['192.168.0.1', '192.168.0.2', '192.168.0.3']) 14 | systemName: cassandra-cluster 15 | statementType: random(["SELECT", "UPDATE"]) 16 | measurement: 17 | name: random(['queryReport_count'], ['requestRate'] 18 | type: SIMPLE 19 | value: $values 20 | time: now() 21 | timeunit: MILLISECONDS 22 | tags: $tag1 23 | fields: 24 | dummy: string("dummy") 25 | kafkaMessage: 26 | key: randomContentString(10, ['A'..'Z', '0'..'9']) 27 | value: json($measurement) 28 | topic: random(['topic1', 'topic2', 'topic3']) 29 | 30 | output: $kafkaMessage 31 | 32 | rate-generator-configuration: 33 | rates: 34 | r: 100 35 | output: $r 36 | 37 | worker-configuration: 38 | async: true 39 | topic: topic1 40 | producer-configuration: 41 | bootstrap.servers: 192.168.0.1:9092,192.168.0.2:9092,192.168.0.3:9092 42 | 43 | metrics-reporter-configuration: 44 | -------------------------------------------------------------------------------- /berserker-runner/src/example/resources/ranger-mqtt.yml: -------------------------------------------------------------------------------- 1 | load-generator-configuration: 2 | data-source-configuration-name: Ranger 3 | rate-generator-configuration-name: default 4 | worker-configuration-name: MQTT 5 | metrics-reporter-configuration-name: SimpleConsoleReporter 6 | thread-count: 10 7 | queue-capacity: 1000 8 | 9 | data-source-configuration: 10 | values: 11 | genre: random(['horror', 'comedy', 'action', 'sci-fi', 'drama', 'thriller']) 12 | year: random(1980..2017) 13 | message: 14 | payload: string("{}-{}", $genre, $year) 15 | qos: 0 16 | topic: test 17 | output: $message 18 | 19 | rate-generator-configuration: 20 | rates: 21 | r: 100 22 | output: $r 23 | 24 | worker-configuration: 25 | async: true 26 | broker-url: "tcp://localhost:1883" 27 | client-id: berserker 28 | max-inflight: 200 29 | 30 | -------------------------------------------------------------------------------- /berserker-runner/src/example/resources/ranger-rabbitmq.yml: -------------------------------------------------------------------------------- 1 | load-generator-configuration: 2 | data-source-configuration-name: Ranger 3 | rate-generator-configuration-name: default 4 | worker-configuration-name: RabbitMQ 5 | metrics-reporter-configuration-name: JMX 6 | thread-count: 10 7 | queue-capacity: 100000 8 | 9 | # RabbitMQ worker expects object with following properties, all are RabbitMQ common properties: 10 | # 1. exchangeName (String) - Mandatory 11 | # 2. routingKey (String) - Mandatory 12 | # 3. messageContent (String) - Mandatory 13 | # 4. contentType - (String) - Optional 14 | # 5. contentEncoding - (String) - Optional 15 | # 6. headers - (Map) - Optional 16 | # 7. deliveryMode - (Integer) - Optional 17 | # 8. priority - (Integer) - Optional 18 | # 9. correlationId - (String) - Optional 19 | # 10. replyTo - (String) - Optional 20 | # 11. expiration - (String) - Optional 21 | # 12. messageId - (String) - Optional 22 | # 13. timestamp - (Date) - Optional 23 | # 14. type - (String) - Optional 24 | # 15. userId - (String) - Optional 25 | # 16. appId - (String) - Optional 26 | # 17. clusterId - (String) - Optional 27 | 28 | data-source-configuration: 29 | values: 30 | message: 31 | id: uuid() 32 | value: random(1..100) 33 | rabbitMessage: 34 | exchangeName: testExchange 35 | routingKey: test 36 | messageContent: json($message) 37 | priority: 0 38 | delivery_mode: 2 39 | headers: 40 | contentType: text/json 41 | routing-key: test 42 | contentType: text/json 43 | output: $rabbitMessage 44 | 45 | rate-generator-configuration: 46 | rates: 47 | r: 100 48 | output: $r 49 | 50 | # Configuration defines common properties necessary to create ConnectionFactory 51 | 52 | worker-configuration: 53 | username: test 54 | password: test 55 | host: localhost 56 | port: 5672 57 | virtual-host: /rabbitmq_test 58 | 59 | metrics-reporter-configuration: 60 | domain: berserker 61 | filter: -------------------------------------------------------------------------------- /berserker-runner/src/main/java/io/smartcat/berserker/runner/LoadGeneratorRunner.java: -------------------------------------------------------------------------------- 1 | package io.smartcat.berserker.runner; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.net.URL; 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | import java.util.Map; 9 | import java.util.Set; 10 | 11 | import io.smartcat.berserker.worker.InternalWorker; 12 | import org.apache.commons.cli.CommandLine; 13 | import org.apache.commons.cli.CommandLineParser; 14 | import org.apache.commons.cli.DefaultParser; 15 | import org.apache.commons.cli.HelpFormatter; 16 | import org.apache.commons.cli.Option; 17 | import org.apache.commons.cli.Options; 18 | import org.apache.commons.cli.ParseException; 19 | import org.reflections.Reflections; 20 | 21 | import com.codahale.metrics.MetricRegistry; 22 | 23 | import io.smartcat.berserker.LoadGenerator; 24 | import io.smartcat.berserker.api.DataSource; 25 | import io.smartcat.berserker.api.RateGenerator; 26 | import io.smartcat.berserker.api.Worker; 27 | import io.smartcat.berserker.configuration.BaseConfiguration; 28 | import io.smartcat.berserker.configuration.ConfigurationParseException; 29 | import io.smartcat.berserker.configuration.DataSourceConfiguration; 30 | import io.smartcat.berserker.configuration.GlobalConfiguration; 31 | import io.smartcat.berserker.configuration.LoadGeneratorConfiguration; 32 | import io.smartcat.berserker.configuration.MetricsReporterConfiguration; 33 | import io.smartcat.berserker.configuration.RateGeneratorConfiguration; 34 | import io.smartcat.berserker.configuration.WorkerConfiguration; 35 | import io.smartcat.berserker.configuration.YamlConfigurationLoader; 36 | 37 | /** 38 | * Runner which takes configuration file, constructs {@link LoadGenerator} with depending {@link DataSource}, 39 | * {@link RateGenerator} and {@link Worker} and runs load generator. 40 | */ 41 | public class LoadGeneratorRunner { 42 | 43 | private static final String BERSERKER_BASE_PACKAGE = "io.smartcat.berserker"; 44 | private static final String CONFIG_SHORT = "c"; 45 | private static final String CONFIG_LONG = "config"; 46 | 47 | private LoadGeneratorRunner() { 48 | } 49 | 50 | /** 51 | * Main method for starting load generator runner. 52 | * 53 | * @param args Command line arguments. 54 | */ 55 | public static void main(String[] args) { 56 | Options options = getOptions(); 57 | CommandLineParser parser = new DefaultParser(); 58 | CommandLine cmd = null; 59 | try { 60 | cmd = parser.parse(options, args); 61 | } catch (ParseException e) { 62 | HelpFormatter formatter = new HelpFormatter(); 63 | formatter.printHelp("java -jar berserker-runner.jar -c ", options); 64 | } 65 | if (cmd == null) { 66 | return; 67 | } 68 | try { 69 | generateLoad(cmd.getOptionValue(CONFIG_LONG)); 70 | } catch (Exception e) { 71 | throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); 72 | } 73 | } 74 | 75 | @SuppressWarnings({ "unchecked", "rawtypes" }) 76 | private static void generateLoad(String configFilePath) throws Exception { 77 | YamlConfigurationLoader configurationLoader = new YamlConfigurationLoader(); 78 | GlobalConfiguration configuration = configurationLoader.loadConfig(getURL(configFilePath)); 79 | LoadGeneratorConfiguration loadGeneratorConfiguration = configuration.loadGeneratorConfiguration; 80 | loadGeneratorConfiguration.validate(); 81 | DataSource dataSource = getDataSource(loadGeneratorConfiguration.dataSourceConfigurationName, 82 | configuration.dataSourceConfiguration); 83 | RateGenerator rateGenerator = getRateGenerator(loadGeneratorConfiguration.rateGeneratorConfigurationName, 84 | configuration.rateGeneratorConfiguration); 85 | Worker workerDelegate = getWorker(loadGeneratorConfiguration.workerConfigurationName, 86 | configuration.workerConfiguration); 87 | InternalWorker worker = wrapIntoInternalWorker(workerDelegate, loadGeneratorConfiguration.queueCapacity, 88 | loadGeneratorConfiguration.threadCount, loadGeneratorConfiguration.metricsPrefix); 89 | createAndStartReporter(worker.getMetricRegistry(), loadGeneratorConfiguration.metricsReporterConfigurationName, 90 | configuration.metricsReporterConfiguration); 91 | LoadGenerator loadGenerator = new LoadGenerator(dataSource, rateGenerator, worker); 92 | loadGenerator.run(); 93 | } 94 | 95 | private static Options getOptions() { 96 | Option configOption = new Option(CONFIG_SHORT, CONFIG_LONG, true, 97 | "Path to config YAML file containing runner configuration."); 98 | configOption.setRequired(true); 99 | 100 | Options options = new Options(); 101 | options.addOption(configOption); 102 | return options; 103 | } 104 | 105 | private static URL getURL(String path) throws IOException { 106 | return new File(path).getCanonicalFile().toURI().toURL(); 107 | } 108 | 109 | private static DataSource getDataSource(String name, Map configuration) 110 | throws InstantiationException, IllegalAccessException, ConfigurationParseException { 111 | DataSourceConfiguration dataSourceConfiguration = getConfigurationWithName(name, DataSourceConfiguration.class); 112 | return dataSourceConfiguration.getDataSource(configuration); 113 | } 114 | 115 | private static RateGenerator getRateGenerator(String name, Map configuration) 116 | throws ConfigurationParseException { 117 | RateGeneratorConfiguration rateGeneratorConfiguration = getConfigurationWithName(name, 118 | RateGeneratorConfiguration.class); 119 | return rateGeneratorConfiguration.getRateGenerator(configuration); 120 | } 121 | 122 | private static Worker getWorker(String name, Map configuration) 123 | throws ConfigurationParseException { 124 | WorkerConfiguration workerConfiguration = getConfigurationWithName(name, WorkerConfiguration.class); 125 | return workerConfiguration.getWorker(configuration); 126 | } 127 | 128 | @SuppressWarnings({ "unchecked", "rawtypes" }) 129 | private static InternalWorker wrapIntoInternalWorker(Worker workerDelegate, int queueCapacity, int threadCount, 130 | String metricsPrefix) { 131 | return new InternalWorker(workerDelegate, queueCapacity, true, metricsPrefix, threadCount); 132 | } 133 | 134 | private static void createAndStartReporter(MetricRegistry metricRegistry, String name, 135 | Map configuration) { 136 | MetricsReporterConfiguration metricsReporterConfiguration = getConfigurationWithName(name, 137 | MetricsReporterConfiguration.class); 138 | metricsReporterConfiguration.createAndStartReporter(metricRegistry, configuration); 139 | } 140 | 141 | private static T getConfigurationWithName(String name, Class clazz) { 142 | try { 143 | List configurations = new ArrayList<>(); 144 | List classNames = new ArrayList<>(); 145 | for (Class configurationClass : getSubTypesOf(clazz)) { 146 | T configuration = configurationClass.newInstance(); 147 | if (name.equals(configuration.getName())) { 148 | configurations.add(configuration); 149 | classNames.add(configurationClass.getCanonicalName()); 150 | } 151 | } 152 | if (configurations.isEmpty()) { 153 | throw new RuntimeException("Configuration with name: " + name + " not found."); 154 | } 155 | if (configurations.size() > 1) { 156 | throw new RuntimeException("Found " + configurations.size() + " configurations on classpath for name: " 157 | + name + ", but expected 1. Configuration classes found: " + classNames.toString()); 158 | } 159 | return configurations.get(0); 160 | } catch (Exception e) { 161 | throw new RuntimeException(e); 162 | } 163 | } 164 | 165 | private static Set> getSubTypesOf(Class clazz) { 166 | Reflections reflections = new Reflections(BERSERKER_BASE_PACKAGE); 167 | return reflections.getSubTypesOf(clazz); 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /berserker-runner/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{yyyy-MM-dd HH:mm:ss.SSS} %level [%thread] %logger{1} - %msg%n 8 | 9 | 10 | 11 | 12 | ${LOG_DIR}/berserker.log 13 | 14 | %d{yyyy-MM-dd HH:mm:ss.SSS} %level [%thread] %logger{1} - %msg%n 15 | 16 | 17 | 18 | ${LOG_DIR}/archived/berserker.%d{yyyy-MM-dd}.%i.log 19 | 20 | 10MB 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /checkstyle-suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /checkstyle.properties: -------------------------------------------------------------------------------- 1 | suppression_file=checkstyle-suppressions.xml 2 | -------------------------------------------------------------------------------- /checkstyle.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 70 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /images/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartcat-labs/berserker/a80dab85f9cf8d608895935f097afee5017048e8/images/architecture.png -------------------------------------------------------------------------------- /images/core-design.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartcat-labs/berserker/a80dab85f9cf8d608895935f097afee5017048e8/images/core-design.png -------------------------------------------------------------------------------- /rate-generator-configuration.md: -------------------------------------------------------------------------------- 1 | # Rate Generator Configuration 2 | 3 | Berserker supports configuration of rate generator through YAML file. Basic file structure is following: 4 | 5 | ```yaml 6 | rates: 7 | rateA: 10 8 | output: $rateA 9 | ``` 10 | There can be arbitrary number of rate generator definitions within `rates` property. Final output rate generator that will be used is references under `output` property. 11 | Rate generator is represented in number of operations per second, or to be more precise, number of times DataSource's `getNext()` method and Worker's `accept()` method will be invoked per second. 12 | 13 | # Primitive rate generators 14 | 15 | Rate generator can be represented as primitive number: 16 | 17 | ```yaml 18 | rates: 19 | rateA: 10 20 | rateB: 20.5 21 | ``` 22 | 23 | This will construct two rate generators `rateA` and `rateB` with rates of `10` and `20.5` respectively. 24 | 25 | # Reference rate generators 26 | 27 | Once defined, rate generator can be reused by its reference: 28 | 29 | ```yaml 30 | rates: 31 | rateA: 10 32 | rateB: $rateA 33 | ``` 34 | 35 | This example might not illustrate full potential of references or might not present right use case, but it is actually really useful when used with [combining rate generators](#combining-rate-generators). 36 | 37 | # Function rate generators 38 | 39 | Besides [primitive rate generators](#primitive-rate-generators), functions can be defined. Functions are defined as functions of time where result represents rate. 40 | 41 | ## Period literal 42 | 43 | Before functions are explained, lets take a look at period literal. 44 | Period literal is used as argument to define duration of period in all periodic functions, here are few examples with explanations: 45 | 46 | ``` 47 | 2d10m = 2 days and 10 minutes 48 | 2d5h1s = 2 days, 5 hours and 1 second 49 | 1d2h3m4s = 1 day, 2 hours, 3 minutes and 4 seconds 50 | 2h = 2 hours 51 | 10s = 10 seconds 52 | ``` 53 | 54 | Period literal does not need to contain all time units (days, hours, minutes and seconds), but it must contain at least one. However, time units must be in day-hour-minute-second order with possibility of omitting any of it. 55 | 56 | ## Triangle function 57 | 58 | Triangle function behaves as same-named period wave ([Triangle wave](https://en.wikipedia.org/wiki/Triangle_wave)) with generalization that it can represent also [Sawtooth wave](https://en.wikipedia.org/wiki/Sawtooth_wave). 59 | It is defined with four parameters as follows: 60 | 61 | ```yaml 62 | rates: 63 | rateA: triangle(2d10m, 0.2, 100, 15000) 64 | ``` 65 | 66 | First parameter represents period duration. Period duration is defined with [period literal](#period-literal). 67 | Second parameter represents percentage of the period where function is in ascending slope, or where maximum lies. Function starts from minimum and raising to maximum, and then again lowering to minimum, that is where period ends. 68 | Third parameter represents minimum value. 69 | Fourth parameter represents maximum value. 70 | 71 | Examples: 72 | 73 | ```yaml 74 | rates: 75 | rateA: triangle(10s, 0.3, 10, 20) 76 | rateB: triangle(20, 0.5, 1, 100) 77 | ``` 78 | 79 | Rate generator `rateA` starts from value `10` (minimum) it raises from `10` to `20` and at second 3 is at value `20` (maximum). Afterwards, rate is dropping from `20` to `10` for next 7 seconds. And then cycle starts again. 80 | Rate generator `rateB` starts from value `1` (minimum) it raises from `1` to `100` and at second 10 is at value `100` (maximum). Afterwards, rate is dropping from `100` to `1` for next 10 seconds. And then cycle starts again. 81 | 82 | ## Sine function 83 | 84 | Sine function behaves as sine wave ([Sine wave](https://en.wikipedia.org/wiki/Sine_wave)). 85 | It is defined with 3 parameters as follows: 86 | 87 | ```yaml 88 | rates: 89 | rateA: sin(10h, 200, 150) 90 | ``` 91 | 92 | First parameter represents period duration. Period duration is defined with [period literal](#period-literal). 93 | Second parameter represents multiplier by which sine function will be multiplied (instead of having values between [1, -1], it will have values between [multiplier, -multiplier]). But be aware that rate cannot be negative, so any number lower than 0 will be treated as 0. That's where third parameter comes into play. 94 | Third parameter represents independent constant which will be added to sine function value. This will allow for sine shape wave that can have positive number for minimum. 95 | 96 | Examples: 97 | 98 | ```yaml 99 | rates: 100 | rateA: sin(10s, 100, 101) 101 | rateB: sin(20s, 1000, 2000) 102 | ``` 103 | 104 | Rate generator `rateA` generates sine signal where minimum is at 1 and maximum is at 201 105 | Rate generator `rateB` generates sine signal where minimum is at 1000 and maximum is at 3000. 106 | 107 | ## Square function 108 | 109 | Square function behaves as square wave ([Square wave](https://en.wikipedia.org/wiki/Square_wave)). 110 | It is defined with 4 parameters as follows: 111 | 112 | ```yaml 113 | rates: 114 | rateA: square(20s, 0.4, 10, 100) 115 | ``` 116 | 117 | First parameter represents period duration. Period duration is defined with [period literal](#period-literal). 118 | Second parameter represents percentage of the period where function returns lower value. Function returns lower value until percentage is reached. When percentage is reached, function returns upper value until the end of period. 119 | Third parameter represents lower value. 120 | Fourth parameter represents upper value. 121 | 122 | Examples: 123 | 124 | ```yaml 125 | rates: 126 | rateA: square(20s, 0.2, 10, 20) 127 | rateB: square(10, 0.6, 5, 150) 128 | ``` 129 | 130 | Rate generator `rateA` returns value 10 for first 4 seconds, afterwards, it returns value 20 until the end of period (rest 16 seconds). 131 | Rate generator `rateB` returns value 5 for first 6 seconds, afterwards, it returns value 150 until the end of period (rest 4 seconds). 132 | 133 | # Combining rate generators 134 | 135 | While rate generators can be defined as primitive values or as functions, real power comes when they are combined. That can be achieved with simple mathematical operations. This can be best explained with examples: 136 | 137 | ```yaml 138 | rates: 139 | rateA: 4 + 10 140 | rateB: square(10s, 0.4, 100, 1000) + 2 * square(10s, 0.5, 10, 50) 141 | rateC: 2 * ($rateA + $rateB) 142 | rateD: ($rateC - 10.5) / 2 143 | ``` 144 | 145 | As shown in examples, primitive, function and reference rate generators can be combined with any of the following operations (+, -, *, /) taking into account operation precedence. Also precedence can be enforced using parentheses. -------------------------------------------------------------------------------- /release: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Verify 2 args, first is release version, second is next version 4 | if [ "$#" -ne 2 ]; then 5 | echo "Two parameters required, first is release version, second is next version" >&2 6 | exit 1 7 | fi 8 | 9 | # Verify that release version and next version are not the same 10 | if [ "$1" = "$2" ]; then 11 | echo "Release version cannot be the same as next version" >&2 12 | exit 1 13 | fi 14 | 15 | # Verify that release version is not already released 16 | if [ "$(git tag 2>/dev/null | grep "$1")" = "$1" ]; then 17 | echo "Release version $1 already released" >&2 18 | exit 1 19 | fi 20 | 21 | # Verify that next version is not already released 22 | if [ "$(git tag 2>/dev/null | grep "$2")" = "$2" ]; then 23 | echo "Next version $2 already released" >&2 24 | exit 1 25 | fi 26 | 27 | # Verify that there are no any local changes 28 | git diff-index --quiet HEAD -- 29 | if [ $? -ne 0 ]; then 30 | echo "Local branch must be clean, please stash, commit or revert local changes" 31 | exit 1 32 | fi 33 | 34 | # Checkout 'dev' branch 35 | echo "Checkout 'dev' branch" 36 | git checkout dev >/dev/null 2>&1 37 | if [ $? -ne 0 ]; then 38 | echo "Failed to checkout 'dev' branch" >&2 39 | exit 1 40 | fi 41 | 42 | # Pull latest changes 43 | echo "Pull latest changes from origin" 44 | git pull >/dev/null 2>&1 45 | if [ $? -ne 0 ]; then 46 | echo "Failed to pull latest changes from origin" >&2 47 | exit 1 48 | fi 49 | 50 | # Run mvn clean install 51 | echo "Run 'mvn clean install'" 52 | mvn clean install >/dev/null 2>&1 53 | if [ $? -ne 0 ]; then 54 | echo "'mvn clean install' finished with error" >&2 55 | exit 1 56 | fi 57 | 58 | # Run mvn clean install -P extras 59 | echo "Run 'mvn clean install -P extras'" 60 | mvn clean install -P extras >/dev/null 2>&1 61 | if [ $? -ne 0 ]; then 62 | echo "'mvn clean install -P extras' finished with error" >&2 63 | exit 1 64 | fi 65 | 66 | # Set release version 67 | echo "Run 'mvn versions:set -DnewVersion=$1'" 68 | mvn versions:set -DnewVersion="$1" >/dev/null 2>&1 69 | if [ $? -ne 0 ]; then 70 | echo "'mvn versions:set -DnewVersion=$1' finished with error" >&2 71 | exit 1 72 | fi 73 | 74 | # Commit release version 75 | echo "Run 'mvn versions:commit'" 76 | mvn versions:commit >/dev/null 2>&1 77 | if [ $? -ne 0 ]; then 78 | echo "'mvn versions:commit' finished with error" >&2 79 | exit 1 80 | fi 81 | 82 | # Add changes to git index 83 | echo "Add release version changes to git index" 84 | git add -u >/dev/null 2>&1 85 | if [ $? -ne 0 ]; then 86 | echo "Failed to add release version changes to git index" >&2 87 | exit 1 88 | fi 89 | 90 | # Commit changes to git 91 | echo "Commit release version changes" 92 | git commit -m "Released $1" >/dev/null 2>&1 93 | if [ $? -ne 0 ]; then 94 | echo "Failed to commit release version" >&2 95 | exit 1 96 | fi 97 | 98 | # Deploy new version to repository 99 | echo "Run 'mvn clean deploy -P extras'" 100 | mvn clean deploy -P extras >/dev/null 2>&1 101 | if [ $? -ne 0 ]; then 102 | echo "'mvn clean deploy -P extras' finished with error" >&2 103 | exit 1 104 | fi 105 | 106 | # Tag version 107 | echo "Tag version $1" 108 | git tag "$1" >/dev/null 2>&1 109 | if [ $? -ne 0 ]; then 110 | echo "Failed to tag version $1" >&2 111 | exit 1 112 | fi 113 | 114 | # Set next version 115 | echo "Run 'mvn versions:set -DnewVersion=$2'" 116 | mvn versions:set -DnewVersion="$2" >/dev/null 2>&1 117 | if [ $? -ne 0 ]; then 118 | echo "'mvn versions:set -DnewVersion=$2' finished with error" >&2 119 | exit 1 120 | fi 121 | 122 | # Commit next version 123 | echo "Run 'mvn versions:commit'" 124 | mvn versions:commit >/dev/null 2>&1 125 | if [ $? -ne 0 ]; then 126 | echo "'mvn versions:commit' finished with error" >&2 127 | exit 1 128 | fi 129 | 130 | # Add changes to git index 131 | echo "Add release version changes to git index" 132 | git add -u >/dev/null 2>&1 133 | if [ $? -ne 0 ]; then 134 | echo "Failed to add release version changes to git index" >&2 135 | exit 1 136 | fi 137 | 138 | # Commit changes to git 139 | echo "Commit next version changes" 140 | git commit -m "Set version to $2 for next iteration" >/dev/null 2>&1 141 | if [ $? -ne 0 ]; then 142 | echo "Failed to commit next version" >&2 143 | exit 1 144 | fi 145 | 146 | # Push 'dev' to origin 147 | echo "Push 'dev' changes to origin" 148 | git push origin dev >/dev/null 2>&1 149 | if [ $? -ne 0 ]; then 150 | echo "Failed to push 'dev' changes to origin" >&2 151 | exit 1 152 | fi 153 | 154 | # Checkout master branch 155 | echo "Checkout 'master' branch" 156 | git checkout master >/dev/null 2>&1 157 | if [ $? -ne 0 ]; then 158 | echo "Failed to checkout master branch" >&2 159 | exit 1 160 | fi 161 | 162 | # Merge 'dev' into 'master' 163 | echo "Merge 'dev' branch into 'master'" 164 | git merge dev >/dev/null 2>&1 165 | if [ $? -ne 0 ]; then 166 | echo "Failed to merge 'dev' branch into 'master'" >&2 167 | exit 1 168 | fi 169 | 170 | # Push changes to origin 171 | echo "Push 'master' changes to origin" 172 | git push origin master >/dev/null 2>&1 173 | if [ $? -ne 0 ]; then 174 | echo "Failed to push 'master' changes to origin" >&2 175 | exit 1 176 | fi 177 | 178 | # Push tags to origin 179 | echo "Push tags to origin" 180 | git push --tags >/dev/null 2>&1 181 | if [ $? -ne 0 ]; then 182 | echo "Failed to push tags to origin" >&2 183 | exit 1 184 | fi 185 | 186 | --------------------------------------------------------------------------------