├── .github └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── README.md ├── dcos ├── node.json └── seed.json ├── doc ├── getting-started-environment.jpg ├── khermes-architecture.png ├── khermes-console-seed.png ├── khermes-console.png └── kibana.png ├── docker ├── confluent-demo-compose.yml ├── docker-entrypoint-dcos.sh ├── docker-entrypoint-local.sh ├── full-demo-compose.yml ├── landoop-demo-compose.yml └── simple-env-compose.yml ├── pom.xml ├── project └── scalastyle_config.xml ├── scripts ├── kill ├── node ├── nodes ├── seed └── seed_debug └── src ├── main ├── resources │ ├── application.conf │ ├── locales │ │ ├── email │ │ │ └── EN.json │ │ ├── geo │ │ │ ├── EN.json │ │ │ └── ES.json │ │ ├── music │ │ │ ├── EN.json │ │ │ └── ES.json │ │ └── name │ │ │ ├── EN.json │ │ │ └── ES.json │ ├── logback.xml │ └── web │ │ ├── console.html │ │ ├── css │ │ └── jquery.terminal.css │ │ ├── index.html │ │ └── js │ │ ├── app.js │ │ ├── console.js │ │ ├── jquery-1.7.1.min.js │ │ ├── jquery.mousewheel-min.js │ │ └── jquery.terminal.min.js └── scala │ └── com │ └── stratio │ └── khermes │ ├── Khermes.scala │ ├── clients │ ├── http │ │ ├── flows │ │ │ └── WSFlow.scala │ │ └── protocols │ │ │ └── WSProtocolMessage.scala │ └── shell │ │ └── KhermesConsoleHelper.scala │ ├── cluster │ ├── collector │ │ └── CommandCollectorActor.scala │ └── supervisor │ │ ├── KhermesClientActor.scala │ │ └── NodeSupervisorActor.scala │ ├── commons │ ├── config │ │ └── AppConfig.scala │ ├── constants │ │ └── AppConstants.scala │ ├── exceptions │ │ └── KhermesException.scala │ └── implicits │ │ ├── AppImplicits.scala │ │ └── AppSerializer.scala │ ├── helpers │ ├── faker │ │ ├── Faker.scala │ │ └── generators │ │ │ ├── DatetimeGenerator.scala │ │ │ ├── EmailGenerator.scala │ │ │ ├── GeoGenerator.scala │ │ │ ├── MusicGenerator.scala │ │ │ ├── NameGenerator.scala │ │ │ └── NumberGenerator.scala │ └── twirl │ │ └── TwirlHelper.scala │ ├── metrics │ ├── KhermesMetrics.scala │ └── MetricsReporter.scala │ └── persistence │ ├── dao │ ├── BaseDAO.scala │ └── ZkDAO.scala │ └── kafka │ └── KafkaClient.scala └── test ├── resources ├── application.conf ├── locales │ ├── email │ │ ├── EN.json │ │ └── XX.json │ ├── geo │ │ ├── ES.json │ │ ├── US.json │ │ ├── XX.json │ │ ├── YY.json │ │ └── ZZ.json │ ├── music │ │ ├── EN.json │ │ ├── ES.json │ │ ├── XX.json │ │ ├── YY.json │ │ └── ZZ.json │ └── name │ │ ├── EN.json │ │ ├── ES.json │ │ └── XX.json ├── music.static.scala.html └── static.scala.html └── scala └── com └── stratio └── khermes ├── KhermesTestIT.scala ├── clients ├── http │ └── protocols │ │ └── WSProtocolMessageTest.scala └── shell │ └── KhermesConsoleHelperTest.scala ├── cluster ├── BaseActorTest.scala └── supervisor │ ├── KhermesClientActorTest.scala │ └── NodeSupervisorActorTest.scala ├── commons └── config │ └── AppConfigTest.scala ├── helpers ├── faker │ ├── FakerGeneratorTest.scala │ └── generators │ │ ├── DatetimeGeneratorTest.scala │ │ ├── EmailGeneratorTest.scala │ │ ├── GeoGeneratorTest.scala │ │ ├── MusicGeneratorTest.scala │ │ ├── NameGeneratorTest.scala │ │ └── NumberGeneratorTest.scala └── twirl │ └── TwirlHelperTest.scala ├── persistence ├── dao │ └── ZkDAOTest.scala └── kafka │ └── KafkaClientTest.scala └── utils └── EmbeddedServersUtils.scala /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | Add a description of your changes here. 4 | 5 | ### Testing 6 | - [ ] Unit, integration, acceptance tests as appropriate 7 | - [ ] Coverage (>90%) 8 | 9 | ### Documentation 10 | - [ ] Changelog update 11 | - [ ] Documentation entry 12 | 13 | ### Scalastyle 14 | Result of scalastyle execution: 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # use glob syntax. 2 | syntax: glob 3 | *.ser 4 | *.class 5 | *~ 6 | *.bak 7 | #*.off 8 | *.old 9 | 10 | # eclipse conf file 11 | .settings 12 | .classpath 13 | .project 14 | .manager 15 | .scala_dependencies 16 | 17 | # idea 18 | .idea 19 | *.iml 20 | 21 | # building 22 | target 23 | build 24 | null 25 | tmp 26 | temp 27 | test-output 28 | build.log 29 | 30 | # other scm 31 | .svn 32 | .CVS 33 | .hg* 34 | 35 | # switch to regexp syntax. 36 | # syntax: regexp 37 | # ^\.pc/ 38 | 39 | #SHITTY output not in target directory 40 | /dependency-reduced-pom.xml 41 | 42 | # Documentation autogenerated 43 | javadoc 44 | apidocs 45 | 46 | # Logs 47 | *.log 48 | esdata/ 49 | data/ 50 | 51 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | 3 | script: "mvn clean compile test" 4 | 5 | 6 | jdk: 7 | - oraclejdk8 -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 0.2.0 (upcoming) 4 | 5 | * Generate songs, including artist, album and genre 6 | * Email generator 7 | 8 | ## 0.1.0 (January 11, 2017) 9 | 10 | * Produce messages to kafka using Stratio Khermes with Akka. 11 | * Geolocation. 12 | * Khermes can now run in Docker. 13 | * Generate numbers. 14 | * Feature/khermes clustering. 15 | * Create kafka producer. 16 | * Implementation for random names. 17 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM qa.stratio.com/stratio/ubuntu-base:16.04 2 | 3 | MAINTAINER stratio 4 | 5 | ARG VERSION 6 | 7 | COPY target/khermes-${VERSION}-allinone.jar /khermes.jar 8 | COPY docker/docker-entrypoint-dcos.sh / 9 | COPY docker/docker-entrypoint-local.sh / 10 | COPY src/main/resources/application.conf / 11 | RUN touch /khermes.log 12 | 13 | EXPOSE 8080 14 | 15 | ENTRYPOINT ["/docker-entrypoint-local.sh"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | © 2017 Stratio Big Data Inc., Sucursal en España. 2 | 3 | This software is licensed under the Apache License, Version 2.0. 4 | This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 5 | without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 6 | See the terms of the License for more details. 7 | 8 | SPDX-License-Identifier: Apache-2.0. 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Stratio Khermes. [![Build Status](https://travis-ci.org/Stratio/khermes.svg?branch=master)](https://travis-ci.org/Stratio/khermes)[![Coverage Status](https://coveralls.io/repos/github/Stratio/khermes/badge.svg?branch=master)](https://coveralls.io/github/Stratio/khermes?branch=master) 2 | 3 | ## Overview. 4 | 5 | 6 | 7 | When you have a complex system architecture with a huge variety of services, the first question that arises is: "What happens when I start to generate tons of events and what is the behaviour of the system when it starts to process them?". For these reasons we are devoloping a high configurable and scalable tool called Khermes that can answer this "a priori simple" question. 8 | 9 | > "Khermes is a distributed fake data generator used in distributed environments that produces a high volume of events in a scalable way". 10 | 11 | ## Full Documentation 12 | 13 | See the [Wiki](https://github.com/Stratio/khermes/wiki/) for full documentation, examples, operational details and other information. 14 | 15 | ## Build 16 | To build the project, run the tests and generate the khermes jar artifact you should execute: 17 | ```sh 18 | $ mvn clean package 19 | ``` 20 | 21 | We have create both shell scripts and docker-compose files to make easier for you to start using khermes. For further info on how to get started please go to the [Getting Started](https://github.com/Stratio/khermes/wiki/Getting-started) section of our Wiki. 22 | 23 | ## Licenses. 24 | Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) 25 | 26 | ## Bugs and Feedback 27 | 28 | For bugs, questions and discussions please use the [GitHub Issues](https://github.com/Stratio/khermes/issues). 29 | 30 | ## Development. 31 | 32 | Want to contribute? Great! 33 | **Khermes is open source and we need you to keep growing.** 34 | If you want to contribute do not hesitate to create a Pull Request with your changes! 35 | -------------------------------------------------------------------------------- /dcos/node.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/khermes-node", 3 | "instances": 1, 4 | "cpus": 0.1, 5 | "mem": 128, 6 | "disk": 0, 7 | "gpus": 0, 8 | "backoffSeconds": 1, 9 | "backoffFactor": 1.15, 10 | "maxLaunchDelaySeconds": 3600, 11 | "container": { 12 | "type": "DOCKER", 13 | "docker": { 14 | "image": "ardlema/khermes:0.32", 15 | "privileged": false, 16 | "forcePullImage": false 17 | } 18 | }, 19 | "upgradeStrategy": { 20 | "minimumHealthCapacity": 1, 21 | "maximumOverCapacity": 1 22 | }, 23 | "unreachableStrategy": { 24 | "inactiveAfterSeconds": 300, 25 | "expungeAfterSeconds": 600 26 | }, 27 | "killSelection": "YOUNGEST_FIRST", 28 | "portDefinitions": [ 29 | { 30 | "port": 0, 31 | "protocol": "tcp", 32 | "name": "akka-tcp" 33 | } 34 | ], 35 | "requirePorts": true, 36 | "env": { 37 | "SEED": "false", 38 | "SEED_IP": "172.17.0.3", 39 | "SEED_PORT": "9743" 40 | } 41 | } -------------------------------------------------------------------------------- /dcos/seed.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "/khermes-seed-90", 3 | "instances": 1, 4 | "cpus": 0.1, 5 | "mem": 128, 6 | "disk": 0, 7 | "gpus": 0, 8 | "backoffSeconds": 1, 9 | "backoffFactor": 1.15, 10 | "maxLaunchDelaySeconds": 3600, 11 | "container": { 12 | "type": "DOCKER", 13 | "docker": { 14 | "image": "ardlema/khermes:0.90", 15 | "privileged": false, 16 | "forcePullImage": false 17 | } 18 | }, 19 | "upgradeStrategy": { 20 | "minimumHealthCapacity": 1, 21 | "maximumOverCapacity": 1 22 | }, 23 | "unreachableStrategy": { 24 | "inactiveAfterSeconds": 300, 25 | "expungeAfterSeconds": 600 26 | }, 27 | "killSelection": "YOUNGEST_FIRST", 28 | "portDefinitions": [ 29 | { 30 | "port": 0, 31 | "protocol": "tcp", 32 | "name": "akka-tcp" 33 | } 34 | ], 35 | "requirePorts": true, 36 | "env": { 37 | "SEED": "true" 38 | } 39 | } -------------------------------------------------------------------------------- /doc/getting-started-environment.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stratio/khermes/88a839693b4b3ad9aef27422d4daf9115e41d069/doc/getting-started-environment.jpg -------------------------------------------------------------------------------- /doc/khermes-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stratio/khermes/88a839693b4b3ad9aef27422d4daf9115e41d069/doc/khermes-architecture.png -------------------------------------------------------------------------------- /doc/khermes-console-seed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stratio/khermes/88a839693b4b3ad9aef27422d4daf9115e41d069/doc/khermes-console-seed.png -------------------------------------------------------------------------------- /doc/khermes-console.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stratio/khermes/88a839693b4b3ad9aef27422d4daf9115e41d069/doc/khermes-console.png -------------------------------------------------------------------------------- /doc/kibana.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stratio/khermes/88a839693b4b3ad9aef27422d4daf9115e41d069/doc/kibana.png -------------------------------------------------------------------------------- /docker/confluent-demo-compose.yml: -------------------------------------------------------------------------------- 1 | # 2 | # © 2017 Stratio Big Data Inc., Sucursal en España. 3 | # 4 | # This software is licensed under the Apache 2.0. 5 | # This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | # without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | # See the terms of the License for more details. 8 | # 9 | # SPDX-License-Identifier: Apache-2.0. 10 | # 11 | 12 | version: '2' 13 | services: 14 | zookeeper: 15 | image: confluentinc/cp-zookeeper:3.1.0 16 | network_mode: host 17 | environment: 18 | ZOOKEEPER_CLIENT_PORT: 2181 19 | ZOOKEEPER_TICK_TIME: 2000 20 | 21 | schema-registry: 22 | image: confluentinc/cp-schema-registry:3.1.0 23 | network_mode: host 24 | depends_on: 25 | - zookeeper 26 | environment: 27 | SCHEMA_REGISTRY_HOST_NAME: schema-registry 28 | SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: localhost:2181 29 | 30 | kafka: 31 | image: confluentinc/cp-kafka:3.1.0 32 | network_mode: host 33 | depends_on: 34 | - zookeeper 35 | environment: 36 | KAFKA_BROKER_ID: 1 37 | KAFKA_ZOOKEEPER_CONNECT: localhost:2181 38 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 39 | 40 | kafka-connect: 41 | image: confluentinc/cp-kafka-connect:3.1.0 42 | network_mode: host 43 | depends_on: 44 | - kafka 45 | environment: 46 | CONNECT_BOOTSTRAP_SERVERS: localhost:9092 47 | CONNECT_REST_PORT: 8082 48 | CONNECT_GROUP_ID: "default" 49 | CONNECT_CONFIG_STORAGE_TOPIC: "default.config" 50 | CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets" 51 | CONNECT_STATUS_STORAGE_TOPIC: "default.status" 52 | CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter" 53 | CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" 54 | CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://localhost:8081" 55 | CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://localhost:8081" 56 | CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" 57 | CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" 58 | CONNECT_REST_ADVERTISED_HOST_NAME: "default-config" 59 | CONNECT_ZOOKEEPER_CONNECT: "localhost:2181" 60 | 61 | elasticsearch: 62 | image: elasticsearch:2.4.1 63 | network_mode: host 64 | volumes: 65 | - $PWD/esdata:/usr/share/elasticsearch/data 66 | 67 | kibana: 68 | image: kibana:4.6.4 69 | network_mode: host 70 | depends_on: 71 | - elasticsearch 72 | environment: 73 | ELASTICSEARCH_URL: http://localhost:9200 74 | 75 | khermes: 76 | image: ardlema/khermes:0.102 77 | network_mode: host 78 | depends_on: 79 | - kafka-connect 80 | -------------------------------------------------------------------------------- /docker/docker-entrypoint-dcos.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | # 3 | # © 2017 Stratio Big Data Inc., Sucursal en España. 4 | # 5 | # This software is licensed under the Apache 2.0. 6 | # This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 7 | # without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 8 | # See the terms of the License for more details. 9 | # 10 | # SPDX-License-Identifier: Apache-2.0. 11 | # 12 | 13 | 14 | if [ $SEED = "true" ]; then 15 | HOSTNAME="$(hostname)" 16 | SEED_IP_ADDRESS="$(dig +short $HOSTNAME)" 17 | java -jar -Dkhermes.ws=true -Dakka.remote.netty.tcp.port=$PORT0 -Dakka.remote.netty.tcp.hostname=$SEED_IP_ADDRESS -Dakka.cluster.seed-nodes.0=akka.tcp://khermes@$SEED_IP_ADDRESS:$PORT0 -Dzookeeper.connection=master.mesos:2181 /khermes.jar 18 | else 19 | java -jar -Dkhermes.client=true -Dkhermes.ws=false -Dakka.remote.netty.tcp.port=$PORT0 -Dakka.cluster.seed-nodes.0=akka.tcp://khermes@$SEED_IP:$SEED_PORT -Dzookeeper.connection=master.mesos:2181 /khermes.jar 20 | fi 21 | 22 | tail -F /khermes.log -------------------------------------------------------------------------------- /docker/docker-entrypoint-local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | # 3 | # © 2017 Stratio Big Data Inc., Sucursal en España. 4 | # 5 | # This software is licensed under the Apache 2.0. 6 | # This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 7 | # without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 8 | # See the terms of the License for more details. 9 | # 10 | # SPDX-License-Identifier: Apache-2.0. 11 | # 12 | 13 | 14 | if [ $SEED = "true" ]; then 15 | java -jar -Dkhermes.ws=true -Dakka.remote.netty.tcp.port=$SEED_PORT -Dakka.remote.netty.tcp.hostname=localhost -Dakka.cluster.seed-nodes.0=akka.tcp://khermes@localhost:$SEED_PORT -Dmetrics.graphite.enabled=$METRICS_ENABLED -Dmetrics.graphite.name=$GRAPHITE_METRICS_NAME -Dzookeeper.connection=localhost:$ZK_PORT /khermes.jar 16 | else 17 | java -jar -Dkhermes.client=true -Dkhermes.ws=false -Dakka.remote.netty.tcp.port=$NODE_PORT -Dakka.cluster.seed-nodes.0=akka.tcp://khermes@localhost:$SEED_PORT -Dmetrics.graphite.enabled=$METRICS_ENABLED -Dmetrics.graphite.name=$GRAPHITE_METRICS_NAME -Dzookeeper.connection=localhost:$ZK_PORT /khermes.jar 18 | fi 19 | 20 | tail -F /khermes.log -------------------------------------------------------------------------------- /docker/full-demo-compose.yml: -------------------------------------------------------------------------------- 1 | # 2 | # © 2017 Stratio Big Data Inc., Sucursal en España. 3 | # 4 | # This software is licensed under the Apache 2.0. 5 | # This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | # without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | # See the terms of the License for more details. 8 | # 9 | # SPDX-License-Identifier: Apache-2.0. 10 | # 11 | 12 | version: '2' 13 | services: 14 | zookeeper: 15 | image: confluentinc/cp-zookeeper:3.1.0 16 | network_mode: host 17 | environment: 18 | ZOOKEEPER_CLIENT_PORT: 2181 19 | ZOOKEEPER_TICK_TIME: 2000 20 | 21 | schema-registry: 22 | image: confluentinc/cp-schema-registry:3.1.0 23 | network_mode: host 24 | depends_on: 25 | - zookeeper 26 | environment: 27 | SCHEMA_REGISTRY_HOST_NAME: schema-registry 28 | SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: localhost:2181 29 | 30 | kafka: 31 | image: confluentinc/cp-kafka:3.1.0 32 | network_mode: host 33 | depends_on: 34 | - zookeeper 35 | environment: 36 | KAFKA_BROKER_ID: 1 37 | KAFKA_ZOOKEEPER_CONNECT: localhost:2181 38 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 39 | 40 | kafka-connect: 41 | image: confluentinc/cp-kafka-connect:3.1.0 42 | network_mode: host 43 | depends_on: 44 | - kafka 45 | environment: 46 | CONNECT_BOOTSTRAP_SERVERS: localhost:9092 47 | CONNECT_REST_PORT: 8082 48 | CONNECT_GROUP_ID: "default" 49 | CONNECT_CONFIG_STORAGE_TOPIC: "default.config" 50 | CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets" 51 | CONNECT_STATUS_STORAGE_TOPIC: "default.status" 52 | CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter" 53 | CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" 54 | CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://localhost:8081" 55 | CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://localhost:8081" 56 | CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" 57 | CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" 58 | CONNECT_REST_ADVERTISED_HOST_NAME: "default-config" 59 | CONNECT_ZOOKEEPER_CONNECT: "localhost:2181" 60 | 61 | elasticsearch: 62 | image: elasticsearch:2.4.1 63 | network_mode: host 64 | volumes: 65 | - $PWD/esdata:/usr/share/elasticsearch/data 66 | 67 | kibana: 68 | image: kibana:4.6.4 69 | network_mode: host 70 | depends_on: 71 | - elasticsearch 72 | environment: 73 | ELASTICSEARCH_URL: http://localhost:9200 74 | 75 | khermes-seed: 76 | image: ardlema/khermes:latest 77 | network_mode: host 78 | depends_on: 79 | - kafka-connect 80 | environment: 81 | SEED: "true" 82 | SEED_PORT: 2552 83 | ZK_PORT: 2181 84 | 85 | khermes-node-1: 86 | image: ardlema/khermes:latest 87 | network_mode: host 88 | depends_on: 89 | - khermes-seed 90 | environment: 91 | SEED: "false" 92 | NODE_PORT: 2553 93 | SEED_PORT: 2552 94 | ZK_PORT: 2181 95 | 96 | khermes-node-2: 97 | image: ardlema/khermes:latest 98 | network_mode: host 99 | depends_on: 100 | - khermes-seed 101 | environment: 102 | SEED: "false" 103 | NODE_PORT: 2554 104 | SEED_PORT: 2552 105 | ZK_PORT: 2181 106 | 107 | khermes-node-3: 108 | image: ardlema/khermes:latest 109 | network_mode: host 110 | depends_on: 111 | - khermes-seed 112 | environment: 113 | SEED: "false" 114 | NODE_PORT: 2555 115 | SEED_PORT: 2552 116 | ZK_PORT: 2181 -------------------------------------------------------------------------------- /docker/landoop-demo-compose.yml: -------------------------------------------------------------------------------- 1 | # 2 | # © 2017 Stratio Big Data Inc., Sucursal en España. 3 | # 4 | # This software is licensed under the Apache 2.0. 5 | # This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | # without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | # See the terms of the License for more details. 8 | # 9 | # SPDX-License-Identifier: Apache-2.0. 10 | # 11 | 12 | version: '2' 13 | services: 14 | landoop-fast-data: 15 | image: landoop/fast-data-dev:cp3.1 16 | network_mode: host 17 | 18 | elasticsearch: 19 | image: elasticsearch:2.4.1 20 | network_mode: host 21 | volumes: 22 | - $PWD/esdata:/usr/share/elasticsearch/data 23 | 24 | kibana: 25 | image: kibana:4.6.4 26 | network_mode: host 27 | depends_on: 28 | - elasticsearch 29 | environment: 30 | ELASTICSEARCH_URL: http://localhost:9200 31 | 32 | khermes-seed: 33 | image: ardlema/khermes:latest 34 | network_mode: host 35 | depends_on: 36 | - landoop-fast-data 37 | environment: 38 | SEED: "true" 39 | SEED_PORT: 2552 40 | ZK_PORT: 2181 41 | METRICS_ENABLED: "false" 42 | GRAPHITE_METRICS_NAME: "khermes-seed" 43 | 44 | khermes-node-1: 45 | image: ardlema/khermes:latest 46 | network_mode: host 47 | depends_on: 48 | - khermes-seed 49 | environment: 50 | SEED: "false" 51 | NODE_PORT: 2553 52 | SEED_PORT: 2552 53 | ZK_PORT: 2181 54 | METRICS_ENABLED: "false" 55 | GRAPHITE_METRICS_NAME: "khermes-node1" 56 | 57 | khermes-node-2: 58 | image: ardlema/khermes:latest 59 | network_mode: host 60 | depends_on: 61 | - khermes-seed 62 | environment: 63 | SEED: "false" 64 | NODE_PORT: 2554 65 | SEED_PORT: 2552 66 | ZK_PORT: 2181 67 | METRICS_ENABLED: "false" 68 | GRAPHITE_METRICS_NAME: "khermes-node2" 69 | 70 | khermes-node-3: 71 | image: ardlema/khermes:latest 72 | network_mode: host 73 | depends_on: 74 | - khermes-seed 75 | environment: 76 | SEED: "false" 77 | NODE_PORT: 2555 78 | SEED_PORT: 2552 79 | ZK_PORT: 2181 80 | METRICS_ENABLED: "false" 81 | GRAPHITE_METRICS_NAME: "khermes-node3" -------------------------------------------------------------------------------- /docker/simple-env-compose.yml: -------------------------------------------------------------------------------- 1 | # 2 | # © 2017 Stratio Big Data Inc., Sucursal en España. 3 | # 4 | # This software is licensed under the Apache 2.0. 5 | # This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | # without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | # See the terms of the License for more details. 8 | # 9 | # SPDX-License-Identifier: Apache-2.0. 10 | # 11 | 12 | version: '2' 13 | services: 14 | zookeeper: 15 | image: confluentinc/cp-zookeeper:3.1.0 16 | network_mode: host 17 | environment: 18 | ZOOKEEPER_CLIENT_PORT: 2181 19 | ZOOKEEPER_TICK_TIME: 2000 20 | 21 | kafka: 22 | image: confluentinc/cp-kafka:3.1.0 23 | network_mode: host 24 | depends_on: 25 | - zookeeper 26 | environment: 27 | KAFKA_BROKER_ID: 1 28 | KAFKA_ZOOKEEPER_CONNECT: localhost:2181 29 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 -------------------------------------------------------------------------------- /project/scalastyle_config.xml: -------------------------------------------------------------------------------- 1 | 2 | Scalastyle standard configuration 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | -------------------------------------------------------------------------------- /scripts/kill: -------------------------------------------------------------------------------- 1 | ps aux | grep khermes | grep -v grep | awk '{print $2}' | xargs kill -9 2 | -------------------------------------------------------------------------------- /scripts/node: -------------------------------------------------------------------------------- 1 | java -jar -Dkhermes.client=false -Dkhermes.ws=false -Dakka.remote.netty.tcp.port=2554 -Dakka.cluster.seed-nodes.0=akka.tcp://khermes@localhost:2552 -Dmetrics.graphite.name="khermes-node1" ../target/khermes-0.2.0-SNAPSHOT-allinone.jar 2 | -------------------------------------------------------------------------------- /scripts/nodes: -------------------------------------------------------------------------------- 1 | java -jar -Dkhermes.client=false -Dkhermes.ws=false -Dakka.remote.netty.tcp.port=2553 -Dakka.cluster.seed-nodes.0=akka.tcp://khermes@localhost:2552 -Dmetrics.graphite.name="khermes-node1" ../target/khermes-0.2.0-SNAPSHOT-allinone.jar & 2 | java -jar -Dkhermes.client=false -Dkhermes.ws=false -Dakka.remote.netty.tcp.port=2554 -Dakka.cluster.seed-nodes.0=akka.tcp://khermes@localhost:2552 -Dmetrics.graphite.name="khermes-node2" ../target/khermes-0.2.0-SNAPSHOT-allinone.jar & 3 | java -jar -Dkhermes.client=false -Dkhermes.ws=false -Dakka.remote.netty.tcp.port=2555 -Dakka.cluster.seed-nodes.0=akka.tcp://khermes@localhost:2552 -Dmetrics.graphite.name="khermes-node3" ../target/khermes-0.2.0-SNAPSHOT-allinone.jar & 4 | java -jar -Dkhermes.client=false -Dkhermes.ws=false -Dakka.remote.netty.tcp.port=2556 -Dakka.cluster.seed-nodes.0=akka.tcp://khermes@localhost:2552 -Dmetrics.graphite.name="khermes-node4" ../target/khermes-0.2.0-SNAPSHOT-allinone.jar 5 | -------------------------------------------------------------------------------- /scripts/seed: -------------------------------------------------------------------------------- 1 | java -jar -Dkhermes.ws=true -Dakka.remote.netty.tcp.port=2552 -Dakka.remote.netty.tcp.hostname=localhost -Dakka.cluster.seed-nodes.0=akka.tcp://khermes@localhost:2552 -Dmetrics.graphite.name="khermes-seed" ../target/khermes-0.2.0-SNAPSHOT-allinone.jar 2 | -------------------------------------------------------------------------------- /scripts/seed_debug: -------------------------------------------------------------------------------- 1 | java -jar -Dkhermes.client=true -Dakka.remote.netty.tcp.port=2552 -Dakka.remote.netty.tcp.hostname=localhost -Dakka.cluster.seed-nodes.0=akka.tcp://khermes@localhost:2552 -Dmetrics.graphite.name="khermes-seed" -agentlib:jdwp=transport=dt_socket,address=9999,server=y,suspend=y ../target/khermes-0.2.0-SNAPSHOT-allinone.jar 2 | -------------------------------------------------------------------------------- /src/main/resources/application.conf: -------------------------------------------------------------------------------- 1 | khermes { 2 | ws { 3 | hostname = 0.0.0.0 4 | port = 8080 5 | } 6 | templates-path = "/tmp/khermes/templates" 7 | client = false 8 | ws = true 9 | } 10 | 11 | akka { 12 | loglevel = "error" 13 | actor { 14 | provider = "akka.cluster.ClusterActorRefProvider" 15 | debug { 16 | receive = on 17 | lifecycle = on 18 | } 19 | } 20 | remote { 21 | log-remote-lifecycle-events = off 22 | netty.tcp { 23 | hostname = localhost 24 | port = 2553 25 | } 26 | } 27 | cluster { 28 | roles = [backend] 29 | seed-nodes = [${?VALUE}] 30 | auto-down-unreachable-after = 10s 31 | } 32 | 33 | http.server.idle-timeout = 1000s 34 | } 35 | 36 | zookeeper { 37 | connection = "localhost:2181" 38 | connectionTimeout = 15000 39 | sessionTimeout = 60000 40 | retryAttempts = 50 41 | retryInterval = 10000 42 | } 43 | 44 | metrics { 45 | logger { 46 | enabled = false 47 | name = "khermes" 48 | } 49 | 50 | graphite { 51 | enabled = false 52 | name = "khermes" 53 | host = 0.0.0.0 54 | port = 2003 55 | } 56 | } -------------------------------------------------------------------------------- /src/main/resources/locales/email/EN.json: -------------------------------------------------------------------------------- 1 | [ 2 | "gmail.com", 3 | "hotmail.com", 4 | "yahoo.com", 5 | "msn.com", 6 | "outlook.com", 7 | "aol.com" 8 | ] 9 | -------------------------------------------------------------------------------- /src/main/resources/locales/geo/ES.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"latitude":28.452717,"longitude":-13.863761,"city":"Fuerteventura"}, 3 | {"latitude":27.814847,"longitude":-17.887056,"city":"Hierro"}, 4 | {"latitude":28.626478,"longitude":-17.755611,"city":"Santa Cruz De La Palma"}, 5 | {"latitude":27.931886,"longitude":-15.386586,"city":"Gran Canaria"}, 6 | {"latitude":28.945464,"longitude":-13.605225,"city":"Las Palmas"}, 7 | {"latitude":28.044475,"longitude":-16.572489,"city":"Tenerife"}, 8 | {"latitude":28.482653,"longitude":-16.341536,"city":"Tenerife"}, 9 | {"latitude":35.279817,"longitude":-2.956256,"city":"Melilla"}, 10 | {"latitude":38.948528,"longitude":-1.863517,"city":"Albacete"}, 11 | {"latitude":38.282169,"longitude":-0.558156,"city":"Alicante"}, 12 | {"latitude":36.843936,"longitude":-2.370097,"city":"Almeria"}, 13 | {"latitude":43.563567,"longitude":-6.034622,"city":"Aviles"}, 14 | {"latitude":37.842006,"longitude":-4.848878,"city":"Cordoba"}, 15 | {"latitude":43.301097,"longitude":-2.910608,"city":"Bilbao"}, 16 | {"latitude":41.297078,"longitude":2.078464,"city":"Barcelona"}, 17 | {"latitude":38.89125,"longitude":-6.821333,"city":"Badajoz"}, 18 | {"latitude":43.302061,"longitude":-8.377256,"city":"La Coruna"}, 19 | {"latitude":37.133222,"longitude":-3.635694,"city":"Granada"}, 20 | {"latitude":41.900969,"longitude":2.760547,"city":"Gerona"}, 21 | {"latitude":37.188731,"longitude":-3.777356,"city":"Granada"}, 22 | {"latitude":40.294139,"longitude":-3.723833,"city":"Madrid"}, 23 | {"latitude":38.872858,"longitude":1.373117,"city":"Ibiza"}, 24 | {"latitude":36.744622,"longitude":-6.060111,"city":"Jerez"}, 25 | {"latitude":37.774972,"longitude":-0.812389,"city":"Murcia"}, 26 | {"latitude":40.493556,"longitude":-3.566764,"city":"Madrid"}, 27 | {"latitude":36.6749,"longitude":-4.499106,"city":"Malaga"}, 28 | {"latitude":39.862597,"longitude":4.218647,"city":"Menorca"}, 29 | {"latitude":37.174917,"longitude":-5.615944,"city":"Sevilla"}, 30 | {"latitude":39.9375,"longitude":-3.503333,"city":"Ocana"}, 31 | {"latitude":42.770039,"longitude":-1.646331,"city":"Pamplona"}, 32 | {"latitude":37.951111,"longitude":-1.230319,"city":"Murcia"}, 33 | {"latitude":41.147392,"longitude":1.167172,"city":"Reus"}, 34 | {"latitude":36.645211,"longitude":-6.349458,"city":"Rota"}, 35 | {"latitude":40.952117,"longitude":-5.501986,"city":"Salamanca"}, 36 | {"latitude":39.598889,"longitude":2.702778,"city":"Son Bonet"}, 37 | {"latitude":39.551675,"longitude":2.738808,"city":"Palma De Mallorca"}, 38 | {"latitude":39.862222,"longitude":4.258333,"city":"San Luis"}, 39 | {"latitude":43.356519,"longitude":-1.790611,"city":"San Sebastian"}, 40 | {"latitude":42.896333,"longitude":-8.415144,"city":"Santiago"}, 41 | {"latitude":42.338611,"longitude":1.409167,"city":"Seo De Urgel"}, 42 | {"latitude":40.496747,"longitude":-3.445872,"city":"Madrid"}, 43 | {"latitude":39.489314,"longitude":-0.481625,"city":"Valencia"}, 44 | {"latitude":41.706111,"longitude":-4.851944,"city":"Valladolid"}, 45 | {"latitude":40.370678,"longitude":-3.785144,"city":"Madrid"}, 46 | {"latitude":42.882836,"longitude":-2.724469,"city":"Vitoria"}, 47 | {"latitude":42.2318,"longitude":-8.626775,"city":"Vigo"}, 48 | {"latitude":43.427064,"longitude":-3.820006,"city":"Santander"}, 49 | {"latitude":41.666242,"longitude":-1.041553,"city":"Zaragoza"}, 50 | {"latitude":37.4180,"longitude":-5.893106,"city":"Sevilla"}, 51 | {"latitude":41.5209,"longitude":2.10508,"city":"Sabadell"}, 52 | {"latitude":39.55361,"longitude":2.727778,"city":"Palma de Mallorca"}, 53 | {"latitude":42.4542,"longitude":-2.32083,"city":"Logroño-Agoncillo"}, 54 | {"latitude":35.8969,"longitude":-5.29908,"city":"Ceuta"}, 55 | {"latitude":28.0296,"longitude":-17.2146,"city":"La Gomera"}, 56 | {"latitude":36.75,"longitude":-5.166667,"city":"Ronda"}, 57 | {"latitude":42.357628,"longitude":-3.620764,"city":"Burgos"}, 58 | {"latitude":42.5890,"longitude":-5.655556,"city":"Leon"}, 59 | {"latitude":41.727778,"longitude":0.535833,"city":"Lleida"}, 60 | {"latitude":42.080833,"longitude":-0.323333,"city":"Huesca"}, 61 | {"latitude":38.856389,"longitude":-3.97,"city":"Ciudad Real"}, 62 | {"latitude":41.358901,"longitude":2.178447,"city":"Barcelona"}, 63 | {"latitude":28.967298,"longitude":-13.527528,"city":"Arrecife Lanzarote"}, 64 | {"latitude":28.470068,"longitude":-16.242471,"city":"Santa Cruz de Tenerife"}, 65 | {"latitude":36.702989,"longitude":-4.41395,"city":"Malaga"}, 66 | {"latitude":42.241537,"longitude":-8.728799,"city":"Vigo"}, 67 | {"latitude":36.534821,"longitude":-6.290649,"city":"Cadiz"}, 68 | {"latitude":36.070781,"longitude":-5.602764,"city":"Tarifa"}, 69 | {"latitude":36.128889,"longitude":-5.441111,"city":"Algeciras"}, 70 | {"latitude":40.889233,"longitude":-4.239478,"city":"Segovia"} 71 | ] -------------------------------------------------------------------------------- /src/main/resources/locales/music/EN.json: -------------------------------------------------------------------------------- 1 | [{"song": "Shape of You", "artist": "Ed Sheeran", "album": "Shape of You", "genre": "Pop"}, 2 | {"song": "I Don’t Wanna Live Forever (Fifty Shades Darker) - From 'Fifty Shades Darker (Original Motion Picture Soundtrack)'", "artist": "ZAYN, Taylor Swift", "album": "I Don’t Wanna Live Forever (Fifty Shades Darker)", "genre": "Pop"}, 3 | {"song": "It Ain't Me (with Selena Gomez)", "artist": "Kygo, Selena Gomez", "album": "It Ain't Me (with Selena Gomez)", "genre": "Dance"}, 4 | {"song": "Paris", "artist": "The Chainsmokers", "album": "Paris", "genre": "Dance"}, 5 | {"song": "Despacito (Featuring Daddy Yankee)", "artist": "Luis Fonsi, Daddy Yankee", "album": "Despacito (Featuring Daddy Yankee)", "genre": "Latin"}, 6 | {"song": "Scared To Be Lonely", "artist": "Martin Garrix, Dua Lipa", "album": "Scared To Be Lonely", "genre": "Pop"}, 7 | {"song": "Castle on the Hill", "artist": "Ed Sheeran", "album": "Castle on the Hill", "genre": "Pop"}, 8 | {"song": "Chained To The Rhythm", "artist": "Katy Perry, Skip Marley", "album": "Chained To The Rhythm", "genre": "Dance"}, 9 | {"song": "Rockabye (feat. Sean Paul & Anne-Marie)", "artist": "Clean Bandit, Anne-Marie, Sean Paul", "album": "Rockabye (feat. Sean Paul & Anne-Marie)", "genre": "Dance"}, 10 | {"song": "I Feel It Coming", "artist": "The Weeknd, Daft Punk", "album": "Starboy", "genre": "Dance"}, 11 | {"song": "Say You Won't Let Go", "artist": "James Arthur", "album": "Back from the Edge", "genre": "Pop"}, 12 | {"song": "Issues", "artist": "Julia Michaels", "album": "Issues", "genre": "Pop"}, 13 | {"song": "Starboy", "artist": "The Weeknd, Daft Punk", "album": "Starboy", "genre": "Dance"}, 14 | {"song": "Closer", "artist": "The Chainsmokers, Halsey", "album": "Closer", "genre": "Pop"}, 15 | {"song": "Call on Me - Ryan Riback Remix", "artist": "Starley, Ryan Riback", "album": "Call on Me (Ryan Riback Remix)", "genre": "Dance"}, 16 | {"song": "Bad and Boujee (feat. Lil Uzi Vert)", "artist": "Migos, Lil Uzi Vert", "album": "Culture", "genre": "Hip Hop"}, 17 | {"song": "Run Up (feat. PARTYNEXTDOOR & Nicki Minaj)", "artist": "Major Lazer, PARTYNEXTDOOR, Nicki Minaj", "album": "Run Up (feat. PARTYNEXTDOOR & Nicki Minaj)", "genre": "Hip Hop"}, 18 | {"song": "How Far I'll Go - From 'Vaiana'", "artist": "Alessia Cara", "album": "How Far I'll Go (From 'Vaiana')", "genre": "Pop"}, 19 | {"song": "That's What I Like", "artist": "Bruno Mars", "album": "24K Magic", "genre": "Pop"}, 20 | {"song": "Bad Things - With Camila Cabello", "artist": "Machine Gun Kelly, Camila Cabello", "album": "Bad Things", "genre": "Hip Hop"}, 21 | {"song": "Chantaje", "artist": "Shakira, Maluma", "album": "Chantaje", "genre": "Latin"}, 22 | {"song": "Just Hold On", "artist": "Steve Aoki, Louis Tomlinson", "album": "Just Hold On", "genre": "Dance"}, 23 | {"song": "Believer", "artist": "Imagine Dragons", "album": "Believer", "genre": "Rock"}, 24 | {"song": "Don't Wanna Know", "artist": "Maroon 5, Kendrick Lamar", "album": "Don't Wanna Know", "genre": "Pop"}, 25 | {"song": "Weak", "artist": "AJR", "album": "What Everyone's Thinking - EP", "genre": "Pop"}, 26 | {"song": "Fake Love", "artist": "Drake", "album": "Fake Love", "genre": "Hip Hop"}, 27 | {"song": "Solo Dance", "artist": "Martin Jensen", "album": "Solo Dance", "genre": "Dance"}, 28 | {"song": "Let Me Love You", "artist": "DJ Snake, Justin Bieber", "album": "Encore", "genre": "Dance"}, 29 | {"song": "All Night", "artist": "The Vamps, Matoma", "album": "All Night", "genre": "Dance"}, 30 | {"song": "All Time Low", "artist": "Jon Bellion", "album": "The Human Condition", "genre": "Hip Hop"}, 31 | {"song": "24K Magic", "artist": "Bruno Mars", "album": "24K Magic", "genre": "Pop"}, 32 | {"song": "Cold", "artist": "Maroon 5, Future", "album": "Cold", "genre": "Pop"}, 33 | {"song": "Bounce Back", "artist": "Big Sean", "album": "I Decided.", "genre": "Hip Hop"}, 34 | {"song": "Hear Me Now", "artist": "Alok, Bruno Martini, Zeeba", "album": "Hear Me Now", "genre": "Dance"}, 35 | {"song": "How Would You Feel (Paean)", "artist": "Ed Sheeran", "album": "How Would You Feel (Paean)", "genre": "Love"}, 36 | {"song": "One Dance", "artist": "Drake, WizKid, Kyla", "album": "Views", "genre": "Dance"}, 37 | {"song": "iSpy (feat. Lil Yachty)", "artist": "Kyle, Lil Yachty", "album": "iSpy (feat. Lil Yachty)", "genre": "Hip Hop"}, 38 | {"song": "El Amante", "artist": "Nicky Jam", "album": "Fénix", "genre": "Latin"}, 39 | {"song": "Reggaetón Lento (Bailemos)", "artist": "CNCO", "album": "Primera Cita", "genre": "Latin"}, 40 | {"song": "Don't Leave", "artist": "Snakehips, MØ", "album": "Don't Leave", "genre": "Rap"}, 41 | {"song": "Shed A Light", "artist": "Robin Schulz, David Guetta, Cheat Codes", "album": "Shed A Light", "genre": "Dance"}, 42 | {"song": "Cold Water (feat. Justin Bieber & MØ)", "artist": "Major Lazer, MØ, Justin Bieber", "album": "Cold Water (feat. Justin Bieber & MØ)", "genre": "Dance"}, 43 | {"song": "Congratulations", "artist": "Post Malone, Quavo", "album": "Stoney", "genre": "Rap"}, 44 | {"song": "In the Name of Love", "artist": "Martin Garrix, Bebe Rexha", "album": "In the Name of Love", "genre": "Pop"}, 45 | {"song": "Now and Later", "artist": "Sage The Gemini", "album": "Now and Later", "genre": "Rap"}, 46 | {"song": "Side To Side", "artist": "Ariana Grande, Nicki Minaj", "album": "Dangerous Woman", "genre": "Dance"}, 47 | {"song": "I Would Like", "artist": "Zara Larsson", "album": "I Would Like", "genre": "Pop"}, 48 | {"song": "My Way", "artist": "Calvin Harris", "album": "My Way", "genre": "Dance"}, 49 | {"song": "Black Beatles", "artist": "Rae Sremmurd, Gucci Mane", "album": "SremmLife 2 (Deluxe)", "genre": "Rap"}, 50 | {"song": "T-Shirt", "artist": "Migos", "album": "Culture", "genre": "Rap"}] -------------------------------------------------------------------------------- /src/main/resources/locales/music/ES.json: -------------------------------------------------------------------------------- 1 | [{"song": "Despacito (Featuring Daddy Yankee)", "artist": "Luis Fonsi, Daddy Yankee", "album": "Despacito (Featuring Daddy Yankee)", "genre": "Latin"}, 2 | {"song": "Shape of You", "artist": "Ed Sheeran", "album": "Shape of You", "genre": "Pop"}, 3 | {"song": "El Amante", "artist": "Nicky Jam", "album": "Fénix", "genre": "Latin"}, 4 | {"song": "Reggaetón Lento (Bailemos)", "artist": "CNCO", "album": "Primera Cita", "genre": "Latin"}, 5 | {"song": "Chantaje", "artist": "Shakira, Maluma", "album": "Chantaje", "genre": "Latin"}, 6 | {"song": "Rockabye (feat. Sean Paul & Anne-Marie)", "artist": "Clean Bandit, Anne-Marie, Sean Paul", "album": "Rockabye (feat. Sean Paul & Anne-Marie)", "genre": "Pop"}, 7 | {"song": "I Don’t Wanna Live Forever (Fifty Shades Darker) - From 'Fifty Shades Darker (Original Motion Picture Soundtrack)'", "artist": "ZAYN, Taylor Swift", "album": "I Don’t Wanna Live Forever (Fifty Shades Darker)", "genre": "Pop"}, 8 | {"song": "Lumbra", "artist": "Cali Y El Dandee, Shaggy", "album": "Lumbra", "genre": "Latin"}, 9 | {"song": "La Rompe Corazones", "artist": "Daddy Yankee, Ozuna", "album": "La Rompe Corazones", "genre": "Latin"}, 10 | {"song": "Safari", "artist": "J Balvin, Pharrell Williams, BIA, Sky", "album": "Energía", "genre": "Latin"}, 11 | {"song": "Otra Vez (feat. J Balvin)", "artist": "Zion & Lennox, J Balvin", "album": "Motivan2", "genre": "Latin"}, 12 | {"song": "Paris", "artist": "The Chainsmokers", "album": "Paris", "genre": "Dance"}, 13 | {"song": "Gyal You A Party Animal - Remix", "artist": "Charly Black, Daddy Yankee", "album": "Gyal You A Party Animal (Remix)", "genre": "Latin"}, 14 | {"song": "Castle on the Hill", "artist": "Ed Sheeran", "album": "Castle on the Hill", "genre": "Pop"}, 15 | {"song": "Sola (Remix) (feat. Daddy Yankee, Wisin, Farruko, Zion & Lennox)", "artist": "Anuel Aa, Daddy Yankee, Wisin, Farruko, Zion & Lennox", "album": "Sola (Remix) (feat. Daddy Yankee, Wisin, Farruko, Zion & Lennox)", "genre": "Latin"}, 16 | {"song": "Scared To Be Lonely", "artist": "Martin Garrix, Dua Lipa", "album": "Scared To Be Lonely", "genre": "Pop"}, 17 | {"song": "Te Quiero Pa´Mi", "artist": "Don Omar, Zion & Lennox", "album": "King Of Kings 10th Anniversary (Remastered)", "genre": "Latin"}, 18 | {"song": "Vacaciones", "artist": "Wisin", "album": "Vacaciones", "genre": "Latin"}, 19 | {"song": "Ando buscando (feat. Piso 21)", "artist": "Carlos Baute, Piso 21", "album": "Ando buscando (feat. Piso 21)", "genre": "Latin"}, 20 | {"song": "It Ain't Me (with Selena Gomez)", "artist": "Kygo, Selena Gomez", "album": "It Ain't Me (with Selena Gomez)", "genre": "Dance"}, 21 | {"song": "Me Llamas (feat. Maluma) - Remix", "artist": "Piso 21, Maluma", "album": "Me Llamas (feat. Maluma) (Remix)", "genre": "Latin"}, 22 | {"song": "Vente Pa' Ca", "artist": "Ricky Martin, Maluma", "album": "Vente Pa' Ca", "genre": "Latin"}, 23 | {"song": "Chained To The Rhythm", "artist": "Katy Perry, Skip Marley", "album": "Chained To The Rhythm", "genre": "Pop"}, 24 | {"song": "Traicionera", "artist": "Sebastian Yatra", "album": "Traicionera", "genre": "Latin"}, 25 | {"song": "Manicomio", "artist": "Cosculluela", "album": "Blanco Perla", "genre": "Latin"}, 26 | {"song": "Ay MI Dios (feat. Pitbull, Yandel & Chacal)", "artist": "IAmChino, Pitbull, Yandel, El Chacal", "album": "Ay MI Dios (feat. Pitbull, Yandel & Chacal)", "genre": "Latin"}, 27 | {"song": "I Feel It Coming", "artist": "The Weeknd, Daft Punk", "album": "Starboy", "genre": "Dance"}, 28 | {"song": "24K Magic", "artist": "Bruno Mars", "album": "24K Magic", "genre": "Pop"}, 29 | {"song": "Hasta Que Se Seque el Malecón - Remix", "artist": "Jacob Forever, Farruko", "album": "Hasta Que Se Seque el Malecón (Remix)", "genre": "Latin"}, 30 | {"song": "Don't Wanna Know", "artist": "Maroon 5, Kendrick Lamar", "album": "Don't Wanna Know", "genre": "Pop"}, 31 | {"song": "Dile Que Tu Me Quieres", "artist": "Ozuna", "album": "Dile Que Tu Me Quieres", "genre": "Latin"}, 32 | {"song": "Let Me Love You", "artist": "DJ Snake, Justin Bieber", "album": "Encore", "genre": "Dance"}, 33 | {"song": "Say You Won't Let Go", "artist": "James Arthur", "album": "Back from the Edge", "genre": "Pop"}, 34 | {"song": "La Bicicleta", "artist": "Carlos Vives, Shakira", "album": "La Bicicleta", "genre": "Latin"}, 35 | {"song": "Starboy", "artist": "The Weeknd, Daft Punk", "album": "Starboy", "genre": "Dance"}, 36 | {"song": "Deja Que Te Bese", "artist": "Alejandro Sanz, Marc Anthony", "album": "Deja Que Te Bese", "genre": "Latin"}, 37 | {"song": "Closer", "artist": "The Chainsmokers, Halsey", "album": "Closer", "genre": "Pop"}, 38 | {"song": "Bailame Despacio", "artist": "Xantos, Dynell", "album": "Revolucionario", "genre": "Latin"}, 39 | {"song": "Fuego", "artist": "Juanes", "album": "Fuego", "genre": "Latin"}, 40 | {"song": "Desde que Estamos Juntos", "artist": "Melendi", "album": "Quítate las Gafas", "genre": "Latin"}, 41 | {"song": "Andas En Mi Cabeza", "artist": "Chino & Nacho, Daddy Yankee", "album": "Andas En Mi Cabeza", "genre": "Latin"}, 42 | {"song": "Quiero Que Sepas", "artist": "Juan Magán", "album": "Quiero Que Sepas", "genre": "Latin"}, 43 | {"song": "Cómo Te Atreves", "artist": "Morat", "album": "Sobre El Amor Y Sus Efectos Secundarios", "genre": "Pop"}, 44 | {"song": "In the Name of Love", "artist": "Martin Garrix, Bebe Rexha", "album": "In the Name of Love", "genre": "Dance"}, 45 | {"song": "One Dance", "artist": "Drake, WizKid, Kyla", "album": "Views", "genre": "Dance"}, 46 | {"song": "My Way", "artist": "Calvin Harris", "album": "My Way", "genre": "Dance"}, 47 | {"song": "Borro Cassette", "artist": "Maluma", "album": "Pretty Boy, Dirty Boy", "genre": "Latin"}, 48 | {"song": "DUELE EL CORAZON", "artist": "Enrique Iglesias, Wisin", "album": "DUELE EL CORAZON", "genre": "Latin"}, 49 | {"song": "El Perdedor - The Remix", "artist": "Maluma, Yandel", "album": "El Perdedor (The Remix)", "genre": "Latin"}, 50 | {"song": "Cold Water (feat. Justin Bieber & MØ)", "artist": "Major Lazer, MØ, Justin Bieber", "album": "Cold Water (feat. Justin Bieber & MØ)", "genre": "Dance"}] -------------------------------------------------------------------------------- /src/main/resources/locales/name/EN.json: -------------------------------------------------------------------------------- 1 | { 2 | "firstNames": [ 3 | "Amy", "Anna", "Rebecca", "Virginia", "Kathleen", "Martha", "Debra", "Amanda", "Stephanie", "Carolyn", "Christine", 4 | "Marie", "Janet", "Catherine", "Frances", "Ann", "Joyce", "Diane", "Alice", "Julie", "Heather", "Doris", "Evelyn", 5 | "Jean", "Cheryl", "James", "John", "Robert", "Michael", "William", "Richard", "Charles", "Joseph", "Thomas", 6 | "Christopher", "Paul", "Mark", "Donald", "George", "Kenneth", "Steven", "Edward", "Brian", "Ronald", "Anthony", 7 | "Kevin", "Jason", "Matthew", "Gary", "Timothy" 8 | ], 9 | "lastNames": [ 10 | "Smith", "Johnson", "Williams", "Jones", "Brown", "Davis", "Miller", "Wilson", "Moore", "Taylor", "Anderson", 11 | "Thomas", "Jackson", "White", "Harris", "Martin", "Thompson", "Robinson", "Clark", "Lewis", "Walker", "Hall", 12 | "Allen", "Young", "King", "Wright", "Hill", "Scott", "Green", "Adams", "Baker", "Nelson", "Carter", "Mitchell", 13 | "Roberts", "Turner", "Phillips", "Campbell", "Parker", "Evans", "Edwards", "Collins", "Stewart", "Morris", "Rogers", 14 | "Reed", "Cook", "Morgan", "Bell", "Murphy" 15 | ] 16 | } -------------------------------------------------------------------------------- /src/main/resources/locales/name/ES.json: -------------------------------------------------------------------------------- 1 | { 2 | "firstNames": [ 3 | "Gonzalo", "Moisés", "Sebastián", "Oscar", "Ernesto", "Petronila", "Adón", "Marcos", "José", "Elisa", "Iván", 4 | "Ángel", "Jonathan", "Samuel", "Fabiola", "Aurelia", "Ireneo", "Toribio", "Santiago", "Aristides", "Agustín", 5 | "Jaume", "Celso", "Carolina", "Sergio", "Columbano", "Luis", "Zacarías", "Antonia", "Leandro", "Fabián", "Humberto", 6 | "Paula", "Julia", "Ariadna", "Carmen", "Víctor", "Rocío", "Martín", "Baltasar", "Poncio", "Mario", "Aurelio", 7 | "Probo", "Fidel", "Manuela", "Sonia", "Constancio", "Matilde", "Vladimiro" 8 | ], 9 | "lastNames": [ 10 | "Jiménez", "Martínez", "Gallardo", "Fuentes", "Caballero", "Álvarez", "Ortega", "Vázquez", "Méndez", "Castillo", 11 | "Ruiz", "Serrano", "Cano", "Marín", "Garrido", "Vicente", "Hernández", "Gutiérrez", "Benítez", "Cortés", "Pérez", 12 | "Muñoz", "Calvo", "Lozano", "Lorenzo", "Castro", "León", "Soto", "Martín", "Moya", "Bravo", "Román", "Peña", 13 | "Flores", "Díaz", "Pastor", "Crespo", "Parra", "Ferrer", "Molina", "Montero", "Santos", "Rubio", "Cruz", "Gallego", 14 | "Sáez", "Aguilar", "Soler", "Herrero", "Alonso" 15 | ] 16 | } -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 14 | 15 | 19 | 26 | 27 | ./khermes.log 28 | true 29 | 30 | %-5relative %-5level %logger{35} - %msg%n 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /src/main/resources/web/console.html: -------------------------------------------------------------------------------- 1 | 11 | 12 | 13 | 14 | 15 | 16 | Khermes Terminal 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/main/resources/web/index.html: -------------------------------------------------------------------------------- 1 | 11 | 12 | 13 | 14 | 15 | 16 | Bootstrap, from Twitter 17 | 18 | 19 | 20 | 21 | 22 | 23 | 28 | 29 | 30 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 |
44 |
45 |
46 | 51 | Khermes 52 |
53 |
54 |
55 | 56 |
57 |
58 | 59 |
60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /src/main/resources/web/js/app.js: -------------------------------------------------------------------------------- 1 | /* 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | setupWebSocket(wsURL("input"), "input"); 12 | setupWebSocket(wsURL("output"), "output"); 13 | 14 | function setupWebSocket(endpoint, name) { 15 | if(window[name]) { 16 | window[name].close(); 17 | } 18 | 19 | var ws = new WebSocket(endpoint) 20 | 21 | ws.onopen = function(event) { 22 | console.info("Connected to the server") 23 | }; 24 | 25 | ws.onmessage = function(event) { 26 | console.log(event); 27 | createEventDiv(event); 28 | }; 29 | 30 | ws.onclose = function() { 31 | console.info("Disconnected to the server"); 32 | setupWebSocket(this.url) 33 | }; 34 | 35 | window[name] = ws; 36 | } 37 | 38 | function wsURL(path) { 39 | var protocol = (location.protocol === 'https:') ? 'wss://' : 'ws://'; 40 | var url = protocol + location.host; 41 | if(location.hostname === 'localhost') { 42 | url += '/' + location.pathname.split('/')[1]; 43 | } else { 44 | url += '/'; 45 | } 46 | return url + path; 47 | }; 48 | 49 | function createEventDiv(obj) { 50 | var newDiv = ""; 51 | newDiv += '"; 54 | $(".feedback" ).remove(); 55 | $("div#content").append(newDiv); 56 | }; 57 | 58 | function send() { 59 | var messageToSend = document.getElementById('command').value; 60 | document.getElementById('command').value = ''; 61 | window["input"].send(messageToSend); 62 | }; -------------------------------------------------------------------------------- /src/main/resources/web/js/jquery.mousewheel-min.js: -------------------------------------------------------------------------------- 1 | (function(c){function g(a){var b=a||window.event,i=[].slice.call(arguments,1),e=0,h=0,f=0;a=c.event.fix(b);a.type="mousewheel";if(b.wheelDelta)e=b.wheelDelta/120;if(b.detail)e=-b.detail/3;f=e;if(b.axis!==undefined&&b.axis===b.HORIZONTAL_AXIS){f=0;h=-1*e}if(b.wheelDeltaY!==undefined)f=b.wheelDeltaY/120;if(b.wheelDeltaX!==undefined)h=-1*b.wheelDeltaX/120;i.unshift(a,e,h,f);return(c.event.dispatch||c.event.handle).apply(this,i)}var d=["DOMMouseScroll","mousewheel"];if(c.event.fixHooks)for(var j=d.length;j;)c.event.fixHooks[d[--j]]= 2 | c.event.mouseHooks;c.event.special.mousewheel={setup:function(){if(this.addEventListener)for(var a=d.length;a;)this.addEventListener(d[--a],g,false);else this.onmousewheel=g},teardown:function(){if(this.removeEventListener)for(var a=d.length;a;)this.removeEventListener(d[--a],g,false);else this.onmousewheel=null}};c.fn.extend({mousewheel:function(a){return a?this.bind("mousewheel",a):this.trigger("mousewheel")},unmousewheel:function(a){return this.unbind("mousewheel",a)}})})(jQuery); 3 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/Khermes.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes 12 | 13 | import java.io.File 14 | import java.net.InetAddress 15 | import java.util.Date 16 | 17 | import akka.actor.{ActorRef, ActorSystem, Props} 18 | import akka.http.scaladsl.Http 19 | import akka.http.scaladsl.server.Directives._ 20 | import com.stratio.khermes.clients.http.flows.WSFlow 21 | import com.stratio.khermes.cluster.collector.CommandCollectorActor 22 | import com.stratio.khermes.cluster.supervisor.{KhermesClientActor, NodeSupervisorActor} 23 | import com.stratio.khermes.commons.constants.AppConstants 24 | import com.stratio.khermes.commons.implicits.AppImplicits 25 | import com.stratio.khermes.metrics.MetricsReporter 26 | import com.typesafe.config.Config 27 | import com.typesafe.scalalogging.LazyLogging 28 | 29 | import scala.concurrent.ExecutionContextExecutor 30 | import scala.util.{Failure, Success, Try} 31 | 32 | /** 33 | * Entry point of the application. 34 | */ 35 | object Khermes extends App with LazyLogging { 36 | 37 | import AppImplicits._ 38 | welcome 39 | createPaths 40 | MetricsReporter.start 41 | 42 | val khermesSupervisor = workerSupervisor 43 | 44 | if(config.getString("khermes.client") == "true") { 45 | clientActor(khermesSupervisor) 46 | } 47 | 48 | if(config.getString("khermes.ws") == "true") { 49 | wsHttp() 50 | } 51 | 52 | /** 53 | * Prints a welcome message with some information about the system. 54 | * @param system 55 | */ 56 | def welcome(implicit system: ActorSystem, config: Config): Unit = { 57 | logger.info( 58 | s""" 59 | |╦╔═┬ ┬┌─┐┬─┐┌┬┐┌─┐┌─┐ 60 | |╠╩╗├─┤├┤ ├┬┘│││├┤ └─┐ 61 | |╩ ╩┴ ┴└─┘┴└─┴ ┴└─┘└─┘ Powered by Stratio (www.stratio.com) 62 | | 63 | |> System Name : ${system.name} 64 | |> Start time : ${new Date(system.startTime)} 65 | |> Number of CPUs: ${Runtime.getRuntime.availableProcessors} 66 | |> Total memory : ${Runtime.getRuntime.totalMemory} 67 | |> Free memory : ${Runtime.getRuntime.freeMemory} 68 | """.stripMargin) 69 | } 70 | 71 | /** 72 | * Creates necessary paths used mainly to generate and compile Twirl templates. 73 | * @param config with all necessary configuration. 74 | */ 75 | def createPaths(implicit config: Config): Unit = { 76 | val templatesFile = new File(config.getString("khermes.templates-path")) 77 | if(!templatesFile.exists()) { 78 | logger.info(s"Creating templates path: ${templatesFile.getAbsolutePath}") 79 | templatesFile.mkdirs() 80 | } 81 | } 82 | 83 | 84 | def workerSupervisor(implicit config: Config, 85 | system: ActorSystem, 86 | executionContext: ExecutionContextExecutor): ActorRef = 87 | system.actorOf(Props(new NodeSupervisorActor()), "khermes-supervisor") 88 | 89 | def clientActor(khermesSupervisor: ActorRef)(implicit config: Config, 90 | system: ActorSystem, 91 | executionContext: ExecutionContextExecutor): Unit = { 92 | 93 | val clientActor = system.actorOf(Props(new KhermesClientActor()), "khermes-client") 94 | clientActor ! KhermesClientActor.Start 95 | } 96 | 97 | def wsHttp()(implicit config: Config, 98 | system: ActorSystem, 99 | executionContext: ExecutionContextExecutor): Unit = { 100 | val commandCollector = system.actorOf(CommandCollectorActor.props) 101 | 102 | val routes = 103 | get { 104 | pathPrefix("css") { 105 | getFromResourceDirectory("web/css") 106 | } ~ 107 | pathPrefix("js") { 108 | getFromResourceDirectory("web/js") 109 | } ~ 110 | pathSingleSlash { 111 | getFromResource("web/index.html") 112 | } ~ 113 | path("console") { 114 | getFromResource("web/console.html") 115 | } ~ 116 | path("input") { 117 | handleWebSocketMessages(WSFlow.inputFlow(commandCollector)) 118 | } ~ 119 | path("output") { 120 | handleWebSocketMessages(WSFlow.outputFlow) 121 | } 122 | } 123 | 124 | val host = Try(config.getString("khermes.ws.host")).getOrElse({ 125 | logger.info("khermes.ws.host is not defined. Setting default: localhost") 126 | AppConstants.DefaultWSHost 127 | }) 128 | 129 | val port = Try(config.getInt("khermes.ws.port")).getOrElse({ 130 | logger.info("khermes.ws.port is not defined. Setting default: 8080") 131 | AppConstants.DefaultWSPort 132 | }) 133 | 134 | logger.info("Binding routes......") 135 | val binding = Http().bindAndHandle(routes, host, port) 136 | 137 | binding.onComplete { 138 | case Success(b) ⇒ logger.info(s"Started WebSocket Command Server online at ${b.localAddress}") 139 | case Failure(t) ⇒ logger.error("Failed to start HTTP server") 140 | } 141 | 142 | while(true){} 143 | binding.flatMap(_.unbind()).onComplete(_ ⇒ system.terminate()) 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/clients/http/flows/WSFlow.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.clients.http.flows 12 | 13 | import akka.NotUsed 14 | import akka.actor.ActorRef 15 | import akka.http.scaladsl.model.ws.TextMessage.Strict 16 | import akka.http.scaladsl.model.ws.{Message, TextMessage} 17 | import akka.stream.scaladsl.{Flow, Sink, Source} 18 | import com.stratio.khermes.clients.http.protocols.{WSProtocolMessage, WsProtocolCommand} 19 | import com.stratio.khermes.cluster.collector 20 | import com.stratio.khermes.cluster.collector.CommandCollectorActor 21 | import com.typesafe.scalalogging.LazyLogging 22 | import org.json4s.Formats 23 | import org.json4s.native.Serialization.{read, write} 24 | 25 | import scala.util.Try 26 | 27 | /** 28 | * All flows that will be used in the communication with the WebSocket should be implemented here. 29 | */ 30 | case object WSFlow extends LazyLogging { 31 | 32 | val source = Source.actorPublisher[CommandCollectorActor.Result](collector.CommandCollectorActor.props) 33 | 34 | /** 35 | * Defines what to do when the CommandCollector produces an output. In first instance only it needs to write 36 | * the result in the websocket. 37 | * @param formats needed for serialization and deserialization. 38 | * @return a flow needed to run the server. 39 | */ 40 | def outputFlow()(implicit formats: Formats): Flow[Any, Strict, NotUsed] = 41 | Flow.fromSinkAndSource(Sink.ignore, source.map(x => { 42 | TextMessage.Strict(write(x)) 43 | })) 44 | 45 | /** 46 | * Defines how to parse messages that have been sent from the websocket. It converts the message to a ProtocolMessage 47 | * and sends messages to the collector. 48 | * @param commandCollector is an actorref that will receive orders to execute in the cluster. 49 | * @param formats needed for serialization and deserialization. 50 | * @return a flow needed to run the server. 51 | */ 52 | def inputFlow(commandCollector: ActorRef)(implicit formats: Formats): Flow[Message, Message, Any] = 53 | Flow[Any].mapConcat { 54 | case tm: TextMessage ⇒ 55 | val message = tm.getStrictText 56 | Try(WsProtocolCommand.parseTextBlock(message)).toOption.map(commandCollector ! _) 57 | .getOrElse(logger.error(s"Imposible to serialize message: $message")) 58 | Nil 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/clients/http/protocols/WSProtocolMessage.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.clients.http.protocols 12 | 13 | import com.stratio.khermes.clients.http.protocols.WsProtocolCommand.WsProtocolCommandValue 14 | 15 | case class WSProtocolMessage(command: WsProtocolCommandValue, args: Map[String, String]) 16 | 17 | case object WsProtocolCommand extends Enumeration { 18 | 19 | type WsProtocolCommandValue = Value 20 | 21 | val Ls = Value("ls") 22 | val Start = Value("start") 23 | val Stop = Value("stop") 24 | val CreateTwirlTemplate = Value("create twirl-template") 25 | val CreateKafkaConfig = Value("create kafka-config") 26 | val CreateGeneratorConfig = Value("create generator-config") 27 | val CreateAvroConfig = Value("create avro-config") 28 | 29 | val ShowTwirlTemplate = Value("show twirl-template") 30 | val ShowKafkaConfig = Value("show kafka-config") 31 | val ShowGeneratorConfig = Value("show generator-config") 32 | val ShowAvroConfig = Value("show avro-config") 33 | 34 | val ArgsName = "name" 35 | val ArgsContent = "content" 36 | val ArgsTwirlTemplate = "twirl-template" 37 | val ArgsKafkaConfig = "kafka-config" 38 | val ArgsGeneratorConfig = "generator-config" 39 | val ArgsAvroConfig = "avro-config" 40 | val ArgsNodeIds = "node-ids" 41 | 42 | //scalastyle:off 43 | def parseTextBlock(block: String): WSProtocolMessage = { 44 | def parseLines(lines: Seq[String], 45 | commandOption: Option[String], 46 | argOption: Option[String], 47 | args: Map[String, String], 48 | isCommand: Boolean, 49 | isArg: Boolean) : (Option[String], Map[String, String]) = { 50 | lines.headOption match { 51 | case None => (commandOption, args) 52 | case Some(line) if line.trim == "" => 53 | parseLines(lines.tail, commandOption, argOption, args, isCommand, isArg) 54 | case Some(line) if line.toLowerCase == "[command]" => 55 | parseLines(lines.tail, commandOption, argOption, args, true, false) 56 | case Some(line) if line.toLowerCase().startsWith("[") => 57 | parseLines(lines.tail, commandOption, Option(line.replace("[", "").replace("]", "")), args, false, true) 58 | case Some(line) if isCommand => 59 | parseLines(lines.tail, Option(line), argOption, args, true, false) 60 | case Some(line) if isArg => 61 | val arg = argOption.getOrElse( 62 | throw new IllegalStateException("Something was wrong taking the name of the arg (it is None)")) 63 | val value = (args.get(arg).toSeq ++ Seq(line)).mkString("\n") 64 | val newArgs = args + (arg -> value) 65 | parseLines(lines.tail, commandOption, argOption, newArgs, false, true) 66 | case _ => 67 | parseLines(lines.tail, commandOption, argOption, args, isCommand, isArg) 68 | } 69 | } 70 | 71 | val result = parseLines(block.split("\n"), None, None, Map.empty, false, false) 72 | WSProtocolMessage( 73 | WsProtocolCommand.withName(result._1.getOrElse( 74 | throw new IllegalStateException("Impossible to parse command"))), result._2) 75 | } 76 | //scalastyle:on 77 | } 78 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/clients/shell/KhermesConsoleHelper.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.clients.shell 12 | 13 | import com.stratio.khermes.cluster.supervisor.KhermesClientActor 14 | import com.stratio.khermes.commons.config.AppConfig 15 | import com.stratio.khermes.commons.constants.AppConstants._ 16 | import com.stratio.khermes.commons.implicits.AppImplicits.configDAO 17 | import jline.console.ConsoleReader 18 | import jline.console.completer.{ArgumentCompleter, NullCompleter, StringsCompleter} 19 | 20 | import scala.util.Try 21 | 22 | // TODO (Alvaro Nistal): This should be refactored. 23 | case class KhermesConsoleHelper(client: KhermesClientActor) { 24 | 25 | lazy val reader = createDefaultReader() 26 | 27 | //scalastyle:off 28 | def parseLines: Unit = { 29 | reader.readLine.trim match { 30 | case value if value.startsWith("save") => 31 | KhermesConsoleHelper.save(KhermesConsoleHelper.commandArgumentsAndValues(value), this.setConfiguration()) 32 | parseLines 33 | 34 | case value if value.startsWith("show") => 35 | println(KhermesConsoleHelper.show(KhermesConsoleHelper.commandArgumentsAndValues(value))) 36 | parseLines 37 | 38 | case value if value.startsWith("start") => 39 | start(KhermesConsoleHelper.commandArgumentsAndValues(value)) 40 | parseLines 41 | 42 | case value if value.startsWith("stop") => 43 | stop(KhermesConsoleHelper.commandArgumentsAndValues(value)) 44 | parseLines 45 | 46 | case "ls" => 47 | ls() 48 | parseLines 49 | 50 | case "help" => 51 | println(HelpMessage) 52 | parseLines 53 | 54 | case "clear" => 55 | clearScreen 56 | parseLines 57 | 58 | case "exit" | "quit" | "bye" => 59 | System.exit(0) 60 | 61 | case "" => 62 | parseLines 63 | 64 | case _ => 65 | println(CommandNotFoundMessage) 66 | parseLines 67 | } 68 | reader.setPrompt("\u001B[33mkhermes> \u001B[0m") 69 | } 70 | 71 | def setConfiguration(): Option[String] = { 72 | println("Press Control + D to finish") 73 | val parsedBlock = Option(parseBlock()) 74 | reader.setPrompt("\u001B[33mkhermes> \u001B[0m") 75 | parsedBlock 76 | } 77 | 78 | def start(args: Map[String, String]): Unit = { 79 | Try { 80 | val khermes = args("generator-config") 81 | val kafka = args("kafka-config") 82 | val template = args("twirl-template") 83 | val avro = Try(args("avro-template")) 84 | val ids = Try(args("ids").split(" ").toSeq) 85 | val khermesConfig = AppConfig( 86 | configDAO.read(s"$GeneratorConfigPath/$khermes"), 87 | configDAO.read(s"$KafkaConfigPath/$kafka"), 88 | configDAO.read(s"$TwirlTemplatePath/$template"), 89 | Try(configDAO.read(s"$AvroConfigPath/$avro")).toOption 90 | ) 91 | client.start(khermesConfig, ids.getOrElse(Seq.empty)) 92 | }.getOrElse(println(s"Bad arguments.")) 93 | } 94 | 95 | def stop(args: Map[String, String]): Unit = { 96 | val ids = Try(args("ids").split(" ").toSeq) 97 | ids.toOption.map(id => println(s"Sending $id stop message")) 98 | client.stop(ids.getOrElse(Seq.empty)) 99 | } 100 | 101 | def ls(): Unit = { 102 | println("Node Id Status") 103 | println("------------------------------------ ------") 104 | client.ls 105 | Thread.sleep(KhermesConsoleHelper.TimeoutWhenLsMessage) 106 | } 107 | 108 | def clearScreen: Unit = { 109 | reader.clearScreen() 110 | } 111 | 112 | //scalastyle:on 113 | 114 | def parseBlock(result: String = ""): String = { 115 | reader.setPrompt("") 116 | Option(reader.readLine()).map(currentLine => parseBlock(s"$result\n$currentLine")).getOrElse(result) 117 | } 118 | 119 | protected[this] def createDefaultReader(): ConsoleReader = { 120 | val reader = new ConsoleReader() 121 | reader.setHandleUserInterrupt(false) 122 | reader.setExpandEvents(false) 123 | val completer = new StringsCompleter("bye", "clear", "exit", "help", "ls", "save", "show", "start", "stop", "quit") 124 | reader.addCompleter(completer) 125 | val argumentShowCompleter = new ArgumentCompleter( 126 | new StringsCompleter("show"), 127 | new StringsCompleter("--kafka-config", "--twirl-template", "--generator-config", "--avro-template"), 128 | new NullCompleter()) 129 | val argumentSaveCompleter = new ArgumentCompleter( 130 | new StringsCompleter("save"), 131 | new StringsCompleter("--kafka-config", "--twirl-template", "--generator-config", "--avro-template"), 132 | new NullCompleter()) 133 | val argumentStartCompleter = new ArgumentCompleter( 134 | new StringsCompleter("start"), 135 | new StringsCompleter("--kafka-config", "--twirl-template", "--generator-config", "--avro-template", "--ids"), 136 | new NullCompleter()) 137 | val argumentStopCompleter = new ArgumentCompleter(new StringsCompleter("stop"), 138 | new StringsCompleter("--ids"), 139 | new NullCompleter()) 140 | reader.addCompleter(argumentShowCompleter) 141 | reader.addCompleter(argumentSaveCompleter) 142 | reader.addCompleter(argumentStartCompleter) 143 | reader.addCompleter(argumentStopCompleter) 144 | reader.setPrompt("\u001B[33mkhermes> \u001B[0m") 145 | reader 146 | } 147 | } 148 | 149 | object KhermesConsoleHelper { 150 | 151 | val TimeoutWhenLsMessage = 200L 152 | 153 | def show(args: Map[String, String]): String = { 154 | val a = args.map { 155 | case ("generator-config", value) => Try(configDAO.read(s"$GeneratorConfigPath/$value")) 156 | .getOrElse(s"Khermes $value config is empty") 157 | case ("kafka-config", value) => Try(configDAO.read(s"$KafkaConfigPath/$value")) 158 | .getOrElse(s"Khermes $value config is empty") 159 | case ("twirl-template", value) => Try(configDAO.read(s"$TwirlTemplatePath/$value")) 160 | .getOrElse(s"Khermes $value config is empty") 161 | case ("avro-template", value) => Try(configDAO.read(s"$AvroConfigPath/$value")) 162 | .getOrElse(s"Khermes $value config is empty") 163 | case value => s"Arg ${value._1} is not correct." 164 | } 165 | a.mkString 166 | } 167 | 168 | def save(args: Map[String, String], config: Option[String]): Unit = { 169 | Try { 170 | require(args.nonEmpty) 171 | args.foreach { 172 | case ("generator-config", value) => configDAO.create(s"$GeneratorConfigPath/$value", config.get) 173 | case ("kafka-config", value) => configDAO.create(s"$KafkaConfigPath/$value", config.get) 174 | case ("twirl-template", value) => configDAO.create(s"$TwirlTemplatePath/$value", config.get) 175 | case ("avro-template", value) => configDAO.create(s"$AvroConfigPath/$value", config.get) 176 | case value => println(s"Arg ${value._1} is not correct.") 177 | } 178 | }.getOrElse(println("You must provide arguments.")) 179 | } 180 | 181 | /** 182 | * Help to obtain arguments and their values. 183 | * 184 | * @param line 185 | * @return Map with arguments and their values. 186 | */ 187 | def commandArgumentsAndValues(line: String): Map[String, String] = { 188 | val splitWords = line.split("--").filter(_ != "") 189 | val filterFirstWord = splitWords.drop(1).map(_.trim) 190 | filterFirstWord 191 | .map(_.split(" ", 2)) 192 | .map(c => c.head -> c.tail.mkString) 193 | .toMap 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/cluster/collector/CommandCollectorActor.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.cluster.collector 12 | 13 | import java.util.UUID 14 | 15 | import akka.actor.{ActorLogging, Props} 16 | import akka.cluster.MemberStatus 17 | import akka.cluster.pubsub.DistributedPubSub 18 | import akka.cluster.pubsub.DistributedPubSubMediator.Publish 19 | import akka.stream.actor.ActorPublisher 20 | import com.stratio.khermes.clients.http.protocols.WsProtocolCommand.WsProtocolCommandValue 21 | import com.stratio.khermes.clients.http.protocols.{WSProtocolMessage, WsProtocolCommand} 22 | import com.stratio.khermes.cluster.collector.CommandCollectorActor.CheckCommandHasEnd 23 | import com.stratio.khermes.cluster.supervisor.NodeSupervisorActor 24 | import com.stratio.khermes.cluster.supervisor.NodeSupervisorActor.Result 25 | import com.stratio.khermes.commons.config.AppConfig 26 | import com.stratio.khermes.commons.constants.AppConstants 27 | import com.stratio.khermes.commons.implicits.AppImplicits._ 28 | 29 | import scala.concurrent.duration._ 30 | import scala.util.{Failure, Success, Try} 31 | 32 | class CommandCollectorActor extends ActorPublisher[CommandCollectorActor.Result] with ActorLogging { 33 | 34 | val MaxCommandTimeout = 10L 35 | val CheckCommandStateTimeout = 100 milliseconds 36 | val mediator = DistributedPubSub(context.system).mediator 37 | 38 | var commands = scala.collection.mutable.HashMap.empty[String, (Long, List[Result])] 39 | 40 | override def preStart: Unit = { 41 | implicit val executionContext = context.system.dispatcher 42 | context.system.eventStream.subscribe(self, classOf[CommandCollectorActor.Result]) 43 | context.system.scheduler.schedule(0 milliseconds, CheckCommandStateTimeout, self, CheckCommandHasEnd) 44 | } 45 | 46 | //scalastyle:off 47 | override def receive: Receive = { 48 | case WSProtocolMessage(WsProtocolCommand.Ls, _) => 49 | ls 50 | 51 | case WSProtocolMessage(WsProtocolCommand.Start, args) => 52 | start(args) 53 | 54 | case WSProtocolMessage(WsProtocolCommand.Stop, args) => 55 | stop(args) 56 | 57 | case WSProtocolMessage(WsProtocolCommand.CreateTwirlTemplate, args) => 58 | createConfig(args, WsProtocolCommand.CreateTwirlTemplate, AppConstants.TwirlTemplatePath) 59 | 60 | case WSProtocolMessage(WsProtocolCommand.CreateGeneratorConfig, args) => 61 | createConfig(args, WsProtocolCommand.CreateGeneratorConfig, AppConstants.GeneratorConfigPath) 62 | 63 | case WSProtocolMessage(WsProtocolCommand.CreateKafkaConfig, args) => 64 | createConfig(args, WsProtocolCommand.CreateKafkaConfig, AppConstants.KafkaConfigPath) 65 | 66 | case WSProtocolMessage(WsProtocolCommand.CreateAvroConfig, args) => 67 | createConfig(args, WsProtocolCommand.CreateAvroConfig, AppConstants.AvroConfigPath) 68 | 69 | case WSProtocolMessage(WsProtocolCommand.ShowTwirlTemplate, args) => 70 | showConfig(args, WsProtocolCommand.ShowTwirlTemplate, AppConstants.TwirlTemplatePath) 71 | 72 | case WSProtocolMessage(WsProtocolCommand.ShowGeneratorConfig, args) => 73 | showConfig(args, WsProtocolCommand.ShowGeneratorConfig, AppConstants.GeneratorConfigPath) 74 | 75 | case WSProtocolMessage(WsProtocolCommand.ShowKafkaConfig, args) => 76 | showConfig(args, WsProtocolCommand.ShowKafkaConfig, AppConstants.KafkaConfigPath) 77 | 78 | case WSProtocolMessage(WsProtocolCommand.ShowAvroConfig, args) => 79 | showConfig(args, WsProtocolCommand.ShowAvroConfig, AppConstants.AvroConfigPath) 80 | 81 | case result: NodeSupervisorActor.Result => 82 | collectResult(result) 83 | 84 | case CheckCommandHasEnd => 85 | checkCommandHasEnd 86 | 87 | case message: CommandCollectorActor.Result => 88 | performOnNext(message) 89 | } 90 | 91 | def collectResult(result: NodeSupervisorActor.Result): Unit = { 92 | Try(commands(result.commandId)).toOption 93 | .map(x => commands += (result.commandId -> (x._1, (x._2 ::: List(result))))) 94 | .getOrElse(commands += result.commandId -> (System.currentTimeMillis(), List(result))) 95 | } 96 | 97 | def ls(): Unit = { 98 | val commandId = UUID.randomUUID().toString 99 | mediator ! Publish("content", NodeSupervisorActor.List(Seq.empty, commandId)) 100 | } 101 | 102 | def start(args: Map[String, String]): Unit = { 103 | val argsTwirlTemplate = args.get(WsProtocolCommand.ArgsTwirlTemplate).getOrElse( 104 | throw new IllegalArgumentException("a twirl-template must be supplied when you send a Start signal")) 105 | val argsKafkaConfig = args.get(WsProtocolCommand.ArgsKafkaConfig).getOrElse( 106 | throw new IllegalArgumentException("a kafka-config must be supplied when you send a Start signal")) 107 | val argsGeneratorConfig = args.get(WsProtocolCommand.ArgsGeneratorConfig).getOrElse( 108 | throw new IllegalArgumentException("a generator-config must be supplied when you send a Start signal")) 109 | val argsAvroConfigOption = args.get(WsProtocolCommand.ArgsAvroConfig) 110 | val nodeIds = args.get(WsProtocolCommand.ArgsNodeIds).map(value => value.split(" ")).toSeq.flatten 111 | 112 | val twirlTemplate = configDAO.read(s"${AppConstants.TwirlTemplatePath}/$argsTwirlTemplate") 113 | val kafkaConfig = configDAO.read(s"${AppConstants.KafkaConfigPath}/$argsKafkaConfig") 114 | val generatorConfig = configDAO.read(s"${AppConstants.GeneratorConfigPath}/$argsGeneratorConfig") 115 | val avroConfig = argsAvroConfigOption.map( 116 | argsAvroConfig => configDAO.read(s"${AppConstants.AvroConfigPath}/$argsAvroConfig")) 117 | 118 | mediator ! Publish("content", 119 | NodeSupervisorActor.Start(nodeIds, AppConfig(generatorConfig, kafkaConfig, twirlTemplate, avroConfig))) 120 | self ! Result("OK", s"Sending Start signal to nodes ${nodeIds.mkString(" ")}") 121 | } 122 | 123 | def stop(args: Map[String, String]): Unit = { 124 | val nodeIds = args.get(WsProtocolCommand.ArgsNodeIds).map(value => value.split(" ")).toSeq.flatten 125 | mediator ! Publish("content", NodeSupervisorActor.Stop(nodeIds)) 126 | self ! Result("OK", s"Sending Stop signal to nodes ${nodeIds.mkString(" ")}") 127 | } 128 | 129 | def createConfig(args: Map[String, String], protocolCommand: WsProtocolCommandValue, basePath: String): Unit = { 130 | val name = args.get(WsProtocolCommand.ArgsName).getOrElse( 131 | throw new IllegalArgumentException(s"Not found name for ${protocolCommand.toString}")) 132 | val content = args.get(WsProtocolCommand.ArgsContent).getOrElse( 133 | throw new IllegalArgumentException(s"not found content for ${protocolCommand.toString}")) 134 | 135 | configDAO.create(s"$basePath/$name", content) 136 | self ! Result("OK", s"Created node in ZK: $basePath/$name") 137 | } 138 | 139 | def showConfig(args: Map[String, String], protocolCommand: WsProtocolCommandValue, basePath: String): Unit = { 140 | val name = args.getOrElse(WsProtocolCommand.ArgsName, "") 141 | if (name == "") { 142 | val list = Try(configDAO.list(s"$basePath")) match { 143 | case Success(list) => s"OK \n$list" 144 | case Failure(e) => s"There is none $basePath stored." 145 | } 146 | self ! Result(s"$list", s"Show config of $basePath") 147 | } else { 148 | val read = Try(configDAO.read(s"$basePath/$name")) match { 149 | case Success(config) => s"OK \n$config" 150 | case Failure(e) => s"$name config does not exist." 151 | } 152 | self ! Result(s"$read", s"Show config of $basePath/$name") 153 | } 154 | } 155 | 156 | def checkCommandHasEnd(): Unit = { 157 | val currentMembersInCluster = membersInCluster 158 | commands.filter(element => { 159 | (currentMembersInCluster == element._2._2.size || System.currentTimeMillis() - element._2._1 > MaxCommandTimeout) 160 | }).map(element => { 161 | val result = element._2._2.map(_.value).mkString("\n") 162 | commands.remove(element._1) 163 | context.system.eventStream.publish(CommandCollectorActor.Result(result)) 164 | }) 165 | } 166 | 167 | def performOnNext(message: CommandCollectorActor.Result): Unit = { 168 | if (totalDemand > 0 && isActive) { 169 | onNext(message) 170 | } 171 | } 172 | 173 | def membersInCluster: Int = 174 | akka.cluster.Cluster(context.system).state.members.filter(_.status == MemberStatus.Up).size 175 | } 176 | 177 | object CommandCollectorActor { 178 | 179 | case object CheckCommandHasEnd 180 | 181 | case class Result(value: String) 182 | 183 | def props: Props = Props[CommandCollectorActor] 184 | } 185 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/cluster/supervisor/KhermesClientActor.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.cluster.supervisor 12 | 13 | import java.util.UUID 14 | 15 | import akka.actor.{ActorLogging, Props} 16 | import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} 17 | import akka.stream.actor.ActorPublisher 18 | import com.stratio.khermes.clients.shell.KhermesConsoleHelper 19 | import com.stratio.khermes.commons.config.AppConfig 20 | 21 | import scala.concurrent.Future 22 | 23 | /** 24 | * This actor starts a client with an interactive shell to throw commands through the cluster. 25 | * TODO: (Alvaro Nistal) This class must be deleted and refactored to another one that will use WebSockets with 26 | * the command collector. 27 | */ 28 | class KhermesClientActor extends ActorPublisher[String] with ActorLogging { 29 | 30 | import DistributedPubSubMediator.Publish 31 | val mediator = DistributedPubSub(context.system).mediator 32 | 33 | override def preStart: Unit = { 34 | context.system.eventStream.subscribe(self, classOf[String]) 35 | } 36 | 37 | override def receive: Receive = { 38 | case KhermesClientActor.Start => 39 | import scala.concurrent.ExecutionContext.Implicits.global 40 | Future(new KhermesConsoleHelper(this).parseLines) 41 | 42 | case result: NodeSupervisorActor.Result => 43 | //scalastyle:off 44 | println(result.value) 45 | //scalastyle:on 46 | } 47 | 48 | /** 49 | * Sends to the cluster a list message. 50 | */ 51 | def ls: Unit = { 52 | mediator ! Publish("content", NodeSupervisorActor.List(Seq.empty, UUID.randomUUID().toString)) 53 | } 54 | 55 | /** 56 | * Starts event generation in N nodes. 57 | * @param khermesConfig with Khermes' configuration. 58 | * @param nodeIds with the ids that should be start the generation. 59 | * If this Seq is empty it will try to start all of them. 60 | */ 61 | def start(khermesConfig: AppConfig, 62 | nodeIds: Seq[String]): Unit = { 63 | mediator ! Publish("content", 64 | NodeSupervisorActor.Start(nodeIds, khermesConfig)) 65 | } 66 | 67 | 68 | /** 69 | * Stops event generation in N nodes. 70 | * @param nodeIds with the ids that should be stop the generation. 71 | * If this Seq is empty it will try to start all of them. 72 | */ 73 | def stop(nodeIds: Seq[String]): Unit = 74 | mediator ! Publish("content", NodeSupervisorActor.Stop(nodeIds)) 75 | } 76 | 77 | 78 | object KhermesClientActor { 79 | 80 | case object Start 81 | 82 | def props: Props = Props(new KhermesClientActor()) 83 | 84 | // TODO (Alvaro Nistal) this method should be refactored. 85 | def messageFeedback(khermesConfigOption: Option[String], 86 | kafkaConfigOption: Option[String], 87 | templateOption: Option[String]): String = { 88 | var m = List[String]() 89 | if (khermesConfigOption.isEmpty) m = "khermes" :: m 90 | if (kafkaConfigOption.isEmpty) m = "kafka" :: m 91 | if (templateOption.isEmpty) m = "template" :: m 92 | if (m.isEmpty) "Your configuration is OK" else s"Error: To start nodes is necessary to set ${m.mkString(" and ")} configuration." 93 | } 94 | } 95 | 96 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/cluster/supervisor/NodeSupervisorActor.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.cluster.supervisor 12 | 13 | import java.util.UUID 14 | 15 | import akka.actor.{Actor, ActorLogging} 16 | import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} 17 | import com.stratio.khermes.cluster.supervisor.NodeSupervisorActor.Result 18 | import com.stratio.khermes.commons.config.AppConfig 19 | import com.stratio.khermes.helpers.faker.Faker 20 | import com.stratio.khermes.helpers.twirl.TwirlHelper 21 | import com.stratio.khermes.metrics.KhermesMetrics 22 | import com.stratio.khermes.persistence.kafka.KafkaClient 23 | import com.typesafe.config.Config 24 | import com.typesafe.scalalogging.LazyLogging 25 | import org.apache.avro.Schema.Parser 26 | import play.twirl.api.Txt 27 | import tech.allegro.schema.json2avro.converter.JsonAvroConverter 28 | 29 | import scala.annotation.tailrec 30 | import scala.util.Try 31 | 32 | /** 33 | * Supervisor that will manage a thread that will generate data along the cluster. 34 | * @param config with all needed configuration. 35 | */ 36 | class NodeSupervisorActor(implicit config: Config) extends Actor with ActorLogging with KhermesMetrics { 37 | 38 | import DistributedPubSubMediator.Subscribe 39 | 40 | val mediator = DistributedPubSub(context.system).mediator 41 | mediator ! Subscribe("content", self) 42 | 43 | var khermesExecutor: Option[NodeExecutorThread] = None 44 | val id = UUID.randomUUID.toString 45 | 46 | val khermes = Faker(Try(config.getString("khermes.i18n")).toOption.getOrElse("EN"), 47 | Try(config.getString("khermes.strategy")).toOption) 48 | 49 | override def receive: Receive = { 50 | case NodeSupervisorActor.Start(ids, hc) => 51 | log.debug("Received start message") 52 | 53 | execute(ids, () => { 54 | if (khermesExecutor.isEmpty) { 55 | khermesExecutor = Option(new NodeExecutorThread(hc)) 56 | } else { 57 | khermesExecutor.foreach(_.stopExecutor) 58 | khermesExecutor = Option(new NodeExecutorThread(hc)) 59 | } 60 | khermesExecutor.foreach(_.start()) 61 | }) 62 | 63 | case NodeSupervisorActor.Stop(ids) => 64 | log.debug("Received stop message") 65 | execute(ids, () => { 66 | khermesExecutor.foreach(_.stopExecutor) 67 | khermesExecutor = None 68 | }) 69 | 70 | case NodeSupervisorActor.List(ids, commandId) => 71 | log.debug("Received list message") 72 | execute(ids, () => { 73 | val status = khermesExecutor.map(_.status).getOrElse(false) 74 | sender ! Result(s"$id | $status", commandId) 75 | //context.system.eventStream.publish(s"$id | $status") 76 | }) 77 | } 78 | 79 | protected[this] def isAMessageForMe(ids: Seq[String]): Boolean = ids.exists(x => id == x) 80 | 81 | protected[this] def execute(ids: Seq[String], callback: () => Unit): Unit = 82 | if (ids.nonEmpty && !isAMessageForMe(ids)) log.debug("Is not a message for me!") else callback() 83 | } 84 | 85 | /** 86 | * Thread that will generate data and it is controlled by the supervisor thanks to stopExecutor method. 87 | * @param hc with the Hermes' configuration. 88 | * @param config with general configuration. 89 | */ 90 | class NodeExecutorThread(hc: AppConfig)(implicit config: Config) extends NodeExecutable 91 | with LazyLogging with KhermesMetrics { 92 | 93 | val converter = new JsonAvroConverter() 94 | var running: Boolean = false 95 | val messageSentCounterMetric = getAvailableCounterMetrics("khermes-messages-count") 96 | val messageSentMeterMetric = getAvailableMeterMetrics("khermes-messages-meter") 97 | /** 98 | * Starts the thread. 99 | * @param hc with all configuration needed to start the thread. 100 | */ 101 | override def start(hc: AppConfig): Unit = run() 102 | 103 | override def stopExecutor: Unit = running = false 104 | 105 | def status: Boolean = running 106 | 107 | //scalastyle:off 108 | override def run(): Unit = { 109 | running = true 110 | val kafkaClient = new KafkaClient[Object](hc.kafkaConfig) 111 | val template = TwirlHelper.template[(Faker) => Txt](hc.templateContent, hc.templateName) 112 | val khermes = Faker(hc.khermesI18n, hc.strategy) 113 | 114 | val parserOption = hc.avroSchema.map(new Parser().parse(_)) 115 | 116 | val timeoutNumberOfEventsOption = hc.timeoutNumberOfEventsOption 117 | val timeoutNumberOfEventsDurationOption = hc.timeoutNumberOfEventsDurationOption 118 | val stopNumberOfEventsOption = hc.stopNumberOfEventsOption 119 | 120 | /** 121 | * If you are defining the following example configuration: 122 | * timeout-rules { 123 | * number-of-events: 1000 124 | * duration: 2 seconds 125 | * } 126 | * Then when the node produces 1000 events, it will wait 2 seconds to start producing again. 127 | * @param numberOfEvents with the current number of events generated. 128 | */ 129 | def performTimeout(numberOfEvents: Int): Unit = 130 | for { 131 | timeoutNumberOfEvents <- timeoutNumberOfEventsOption 132 | timeoutNumberOfEventsDuration <- timeoutNumberOfEventsDurationOption 133 | if (numberOfEvents % timeoutNumberOfEvents == 0) 134 | } yield ({ 135 | logger.debug(s"Sleeping executor thread $timeoutNumberOfEventsDuration") 136 | Thread.sleep(timeoutNumberOfEventsDuration.toMillis) 137 | }) 138 | 139 | 140 | /** 141 | * Starts to generate events in a recursive way. 142 | * Note that this generation only will stop in two cases: 143 | * 1. The user sends an stop event to the supervisor actor; the supervisor change the state of running to false, 144 | * stopping the execution. 145 | * 2. The user defines the following Hermes' configuration: 146 | * stop-rules { 147 | * number-of-events: 5000 148 | * } 149 | * In this case only it will generate 5000 events, then automatically the thread puts its state of running to 150 | * false stopping the event generation. 151 | * @param numberOfEvents with the current number of events generated. 152 | */ 153 | @tailrec 154 | def recursiveGeneration(numberOfEvents: Int): Unit = 155 | if (running) { 156 | logger.debug(s"$numberOfEvents") 157 | val json = template.static(khermes).toString() 158 | parserOption match { 159 | case None => { 160 | kafkaClient.send(hc.topic, json) 161 | increaseCounterMetric(messageSentCounterMetric) 162 | markMeterMetric(messageSentMeterMetric) 163 | performTimeout(numberOfEvents) 164 | } 165 | 166 | case Some(value) => { 167 | val record = converter.convertToGenericDataRecord(json.getBytes("UTF-8"), value) 168 | kafkaClient.send(hc.topic, record) 169 | increaseCounterMetric(messageSentCounterMetric) 170 | markMeterMetric(messageSentMeterMetric) 171 | performTimeout(numberOfEvents) 172 | } 173 | } 174 | if (stopNumberOfEventsOption.filter(_ == numberOfEvents).map(_ => stopExecutor).isEmpty) 175 | recursiveGeneration(numberOfEvents + 1) 176 | } 177 | 178 | recursiveGeneration(0) 179 | } 180 | } 181 | 182 | trait NodeExecutable extends Thread { 183 | 184 | def start(hc: AppConfig) 185 | 186 | def stopExecutor 187 | 188 | def status: Boolean 189 | } 190 | 191 | object NodeSupervisorActor { 192 | 193 | case class Start(workerIds: Seq[String], khermesConfig: AppConfig) 194 | 195 | case class Stop(workerIds: Seq[String]) 196 | 197 | case class List(workerIds: Seq[String], commandId: String) 198 | 199 | case class Result(value: String, commandId: String) 200 | 201 | object WorkerStatus extends Enumeration { 202 | val Started, Stopped = Value 203 | } 204 | 205 | } 206 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/commons/config/AppConfig.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.commons.config 12 | 13 | import java.time.Duration 14 | 15 | import com.stratio.khermes.commons.constants.AppConstants 16 | import com.typesafe.config.ConfigFactory 17 | 18 | import scala.util.Try 19 | import com.stratio.khermes.commons.config.AppConfig._ 20 | 21 | /** 22 | * Class used to load and parse configuration that will be used by the application. 23 | * Remember that it should be serializable because will be part as an Akka message. 24 | * 25 | * @param khermesConfigContent with configuration about khermes generator. 26 | * @param kafkaConfigContent with kafka configuration. 27 | * @param template to generate. 28 | * @param avroSchema in the case that you are using avro serialization. 29 | */ 30 | case class AppConfig(khermesConfigContent: String, 31 | kafkaConfigContent: String, 32 | template: String, 33 | avroSchema: Option[String] = None) { 34 | 35 | val khermesConfig = ConfigFactory.parseString(khermesConfigContent) 36 | val kafkaConfig = ConfigFactory.parseString(kafkaConfigContent) 37 | 38 | assertCorrectConfig() 39 | 40 | /** 41 | * Tries to parse the configuration and checks that the KhermesConfig object has all required fields. 42 | */ 43 | protected[this] def assertCorrectConfig(): Unit = { 44 | def buildErrors(mandatoryFields: Seq[String]): Seq[String] = 45 | for { 46 | mandatoryField <- mandatoryFields 47 | if Try(khermesConfig.getAnyRef(mandatoryField)).isFailure && Try(kafkaConfig.getAnyRef(mandatoryField)).isFailure 48 | } yield(s"$mandatoryField not found in the config.") 49 | 50 | val errors = buildErrors(MandatoryFields) ++ (if(configType == ConfigType.Avro) buildErrors(AvroMandatoryFields) else Seq.empty) 51 | assert(errors.isEmpty, errors.mkString("\n")) 52 | } 53 | 54 | def configType(): ConfigType.Value = 55 | if(kafkaConfig.getString("kafka.key.serializer") == AppConstants.KafkaAvroSerializer) { 56 | ConfigType.Avro 57 | } else { 58 | ConfigType.Json 59 | } 60 | 61 | def topic: String = khermesConfig.getString("khermes.topic") 62 | 63 | def templateName: String = khermesConfig.getString("khermes.template-name") 64 | 65 | def templateContent: String = template 66 | 67 | def khermesI18n: String = khermesConfig.getString("khermes.i18n") 68 | 69 | def strategy: Option[String] = Try(khermesConfig.getString("khermes.strategy")).toOption 70 | 71 | def timeoutNumberOfEventsOption: Option[Int] = Try(khermesConfig.getInt("khermes.timeout-rules.number-of-events")).toOption 72 | 73 | def timeoutNumberOfEventsDurationOption: Option[Duration] = Try(khermesConfig.getDuration("khermes.timeout-rules.duration")).toOption 74 | 75 | def stopNumberOfEventsOption: Option[Int] = Try(khermesConfig.getInt("khermes.stop-rules.number-of-events")).toOption 76 | 77 | } 78 | 79 | object AppConfig { 80 | 81 | val MandatoryFields = Seq( 82 | "khermes.topic", 83 | "khermes.template-name", 84 | "khermes.i18n", 85 | "kafka.key.serializer" 86 | ) 87 | 88 | val AvroMandatoryFields = Seq( 89 | "kafka.schema.registry.url" 90 | ) 91 | 92 | object ConfigType extends Enumeration { 93 | val Avro, Json = Value 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/commons/constants/AppConstants.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.commons.constants 12 | 13 | /** 14 | * Global constants used in the application. 15 | */ 16 | object AppConstants { 17 | 18 | val DefaultLocale = "ALL" 19 | val AkkaClusterName = "khermes" 20 | val DecimalValue = 10 21 | val SupervisorStart = 5 22 | val SupervisorStop = 30 23 | 24 | val GeneratedTemplatesPrefix = "generated-templates" 25 | val GeneratedClassesPrefix = "generated-classes" 26 | val KafkaAvroSerializer = "io.confluent.kafka.serializers.KafkaAvroSerializer" 27 | 28 | val ZookeeperParentPath = "/stratio/khermes" 29 | val ZookeeperConnection = "zookeeper.connection" 30 | val ZookeeperConnectionDefault = "master.mesos:2181" 31 | val ZookeeperConnectionTimeout = "zookeeper.connectionTimeout" 32 | val ZookeeperSessionTimeout = "zookeeper.sessionTimeout" 33 | val ZookeeperRetryAttempts = "zookeeper.retryAttempts" 34 | val ZookeeperRetryInterval = "zookeeper.retryInterval" 35 | 36 | val KafkaConfigPath = "kafka-config" 37 | val GeneratorConfigPath = "generator-config" 38 | val TwirlTemplatePath = "twirl-template" 39 | val AvroConfigPath = "avro-config" 40 | 41 | val CommandNotFoundMessage = "Command not found. Type help to list available commands." 42 | val HelpMessage = 43 | s"""Khermes client provides the next commands to manage your Khermes cluster: 44 | | Usage: COMMAND [args...] 45 | | 46 | | Commands: 47 | | start [command options] : Starts event generation in N nodes. 48 | | --generator-config : Khermes configuration 49 | | --kafka-config : Kafka configuration 50 | | --twirl-template : Template to generate data 51 | | --avro-template : Avro configuration 52 | | --ids : Node id where start khermes 53 | | stop [command options] : Stop event generation in N nodes. 54 | | --ids : Node id where start khermes 55 | | ls : List the nodes with their current status 56 | | save [command options] : Save your configuration in zookeeper 57 | | --generator-config : Khermes configuration 58 | | --kafka-config : Kafka configuration 59 | | --twirl-template : Template to generate data 60 | | --avro-template : Avro configuration 61 | | show [command options] : Show your configuration 62 | | --generator-config : Khermes configuration 63 | | --kafka-config : Kafka configuration 64 | | --twirl-template : Template to generate data 65 | | --avro-template : Avro configuration 66 | | clear : Clean the screen. 67 | | help : Print this usage. 68 | | exit | quit | bye : Exit of Khermes Cli. """.stripMargin 69 | 70 | val DefaultWSHost = "0.0.0.0" 71 | val DefaultWSPort = 8080 72 | 73 | val DefaultStrategy = Option("default") 74 | 75 | val LoggerEnabled = "metrics.logger.enabled" 76 | val LoggerEnabledDefault = false 77 | val GraphiteEnabled = "metrics.graphite.enabled" 78 | val GraphiteEnabledDefault = false 79 | val LoggerReporterName = "metrics.logger.name" 80 | val LoggerReporterNameDefault = "khermes" 81 | val GraphiteReporterName = "metrics.graphite.name" 82 | val GraphiteReporterNameDefault = "khermes" 83 | val GraphiteReporterHost = "metrics.graphite.host" 84 | val GraphiteReporterHostDefault = "localhost" 85 | val GraphiteReporterPort = "metrics.graphite.port" 86 | val GraphiteReporterPortDefault = 2003 87 | 88 | } 89 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/commons/exceptions/KhermesException.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.commons.exceptions 12 | 13 | /** 14 | * Generic exception used when Khermes fails. 15 | * TODO (Alvaro Nistal) Take a look if really it is necessary. 16 | * @param message to set. 17 | */ 18 | class KhermesException(message: String) extends Exception(message) 19 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/commons/implicits/AppImplicits.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.commons.implicits 12 | 13 | import akka.actor.ActorSystem 14 | import akka.stream.ActorMaterializer 15 | import com.stratio.khermes.commons.constants.AppConstants 16 | import com.stratio.khermes.persistence.dao.ZkDAO 17 | import com.typesafe.config.{Config, ConfigFactory, ConfigResolveOptions} 18 | import com.typesafe.scalalogging.LazyLogging 19 | 20 | /** 21 | * General implicits used in the application. 22 | */ 23 | object AppImplicits extends AppSerializer with LazyLogging { 24 | 25 | lazy implicit val executionContext = scala.concurrent.ExecutionContext.Implicits.global 26 | lazy implicit val config: Config = ConfigFactory 27 | .load(getClass.getClassLoader, 28 | ConfigResolveOptions.defaults.setAllowUnresolved(true)) 29 | .resolve 30 | 31 | lazy implicit val system: ActorSystem = ActorSystem(AppConstants.AkkaClusterName, config) 32 | lazy implicit val configDAO: ZkDAO = new ZkDAO 33 | lazy implicit val materializer = ActorMaterializer() 34 | } 35 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/commons/implicits/AppSerializer.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.commons.implicits 12 | 13 | import com.stratio.khermes.clients.http.protocols.WsProtocolCommand 14 | import org.json4s.ext.EnumNameSerializer 15 | import org.json4s.{DefaultFormats, Formats} 16 | 17 | /** 18 | * Defines how json is serialized / unserialized. 19 | * Remember that all custom serializations such as enums should be here. 20 | */ 21 | trait AppSerializer { 22 | 23 | implicit val json4sFormats: Formats = DefaultFormats + new EnumNameSerializer(WsProtocolCommand) 24 | } 25 | 26 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/helpers/faker/Faker.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.helpers.faker 12 | 13 | import java.io.{File, InputStream} 14 | 15 | import com.stratio.khermes.commons.constants.AppConstants 16 | import com.stratio.khermes.commons.exceptions.KhermesException 17 | import com.stratio.khermes.commons.implicits.AppSerializer 18 | import com.stratio.khermes.helpers.faker.generators._ 19 | import org.json4s.native.Serialization._ 20 | 21 | import scala.util.{Failure, Random, Success, Try} 22 | 23 | /** 24 | * Khermes util used for to generate random values. 25 | */ 26 | case class Faker(locale: String = AppConstants.DefaultLocale, strategy: Option[String] = None) extends AppSerializer { 27 | 28 | object Name extends NameGenerator(locale, strategy) 29 | 30 | object Number extends NumberGenerator 31 | 32 | object Geo extends GeoGenerator(locale) 33 | 34 | object Datetime extends DatetimeGenerator 35 | 36 | object Music extends MusicGenerator(locale) 37 | 38 | object Email extends EmailGenerator(locale) 39 | 40 | } 41 | 42 | trait FakerGenerator extends AppSerializer { 43 | 44 | def name: String 45 | 46 | /** 47 | * Returns a random element from a list. 48 | * 49 | * @param list initial list 50 | * @tparam T with the type of the list 51 | * @return a random element of the list or None if the list is empty. 52 | */ 53 | def randomElementFromAList[T](list: Seq[T]): Option[T] = 54 | if (list.nonEmpty) Option(list(Random.nextInt((list.size - 1) + 1))) else None 55 | 56 | //TODO: We should provide more strategies. 57 | def listWithStrategy[T](list: Seq[T], strategy: String): Seq[(T, Double)] = { 58 | strategy match { 59 | case "default" => listStrategyApply(list, 0.8) 60 | case _ => listStrategyApply(list, 0.5) 61 | } 62 | } 63 | 64 | def listStrategyApply[T](list: Seq[T], p: Double): Seq[(T, Double)] = { 65 | val first: (T, Double) = list.head -> p 66 | val tail: Seq[(T, Double)] = list.tail.map(x => 67 | x -> ((1 - p) / (list.size - 1)) 68 | ) 69 | tail :+ first 70 | } 71 | 72 | 73 | def repeatElementsInList[T](list: Seq[(T, Double)]): Seq[T] = { 74 | list.flatMap(x => 75 | Seq.fill((x._2 * 1000).toInt)(x._1) 76 | ) 77 | } 78 | 79 | def getResources(name: String): Seq[String] = Try( 80 | new File(getClass.getResource(s"/locales/$name").getFile)) match { 81 | case Success(resources) => resources.list().toSeq 82 | case Failure(_) => throw new KhermesException(s"Error loading invalid name /locales/$name") 83 | } 84 | 85 | def parse[T](unitName: String, locale: String)(implicit m: Manifest[T]): Either[String, T] = Try( 86 | read[T](getResource(unitName, locale))) match { 87 | case Success(model) => Right(model) 88 | case Failure(e) => Left(s"${e.getMessage}") 89 | } 90 | 91 | def parseErrors[T](maybeResources: Seq[Either[String, T]]): Seq[String] = { 92 | maybeResources.filter(_.isLeft).map(_.left.get) 93 | } 94 | 95 | def getResource(name: String, file: String): InputStream = Option( 96 | getClass.getResourceAsStream(s"/locales/$name/$file")).getOrElse( 97 | throw new KhermesException(s"Error loading invalid resource /locales/$name/$file")) 98 | } 99 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/helpers/faker/generators/DatetimeGenerator.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.helpers.faker.generators 12 | 13 | import java.security.InvalidParameterException 14 | 15 | import com.stratio.khermes.commons.exceptions.KhermesException 16 | import com.stratio.khermes.commons.implicits.AppSerializer 17 | import org.joda.time.format.DateTimeFormat 18 | import org.joda.time.{DateTime, LocalTime, Seconds} 19 | 20 | import scala.util.{Random, Try} 21 | 22 | /** 23 | * Generates random dates. 24 | */ 25 | class DatetimeGenerator extends AppSerializer { 26 | /** 27 | * Example: "dateTime("1970-1-12" ,"2017-1-1") -> 2005-03-01T20:34:30.000+01:00". 28 | * @return a random dateTime. 29 | */ 30 | def datetime(from: DateTime, to: DateTime, format: Option[String] = None): String = { 31 | assert(to.getMillis > from.getMillis, throw new InvalidParameterException(s"$to must be greater than $from")) 32 | val diff = Seconds.secondsBetween(from, to).getSeconds 33 | val randomDate = new Random(System.nanoTime) 34 | val date: DateTime = from.plusSeconds(randomDate.nextInt(diff.toInt)) 35 | format match { 36 | case Some(stringFormat) => Try(DateTimeFormat.forPattern(stringFormat).print(date)).getOrElse( 37 | throw new KhermesException(s"Invalid DateTimeFormat")) 38 | case None => date.toString() 39 | } 40 | } 41 | 42 | /** 43 | * Example: time() -> 15:30:00.000+01:00 44 | * @return a random time from 00:00:00.000 to 23:59:59.999 (ISO8601) 45 | */ 46 | def time(): String = { 47 | //scalastyle:off 48 | new LocalTime(Random.nextInt(24), Random.nextInt(60), Random.nextInt(60), Random.nextInt(1000)).toString() 49 | //scalastyle:on 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/helpers/faker/generators/EmailGenerator.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.helpers.faker.generators 12 | 13 | import com.stratio.khermes.commons.constants.AppConstants 14 | import com.stratio.khermes.commons.exceptions.KhermesException 15 | import com.stratio.khermes.commons.implicits.AppSerializer 16 | import com.stratio.khermes.helpers.faker.FakerGenerator 17 | import com.typesafe.scalalogging.LazyLogging 18 | 19 | import scala.util.Try 20 | 21 | case class EmailGenerator(locale: String) extends FakerGenerator 22 | with AppSerializer 23 | with LazyLogging { 24 | 25 | override def name: String = "email" 26 | 27 | lazy val emailModel = locale match { 28 | case AppConstants.DefaultLocale => 29 | val resources = getResources(name) 30 | .map(parse[Seq[String]](name, _)) 31 | val maybeErrors = parseErrors[Seq[String]](resources) 32 | if (maybeErrors.nonEmpty) logger.warn(s"$maybeErrors") 33 | resources 34 | case localeValue => 35 | val resource = Seq(parse[Seq[String]](name, s"$localeValue.json")) 36 | val maybeErrors = parseErrors[Seq[String]](resource) 37 | if (maybeErrors.nonEmpty) logger.warn(s"$maybeErrors") 38 | resource 39 | } 40 | 41 | def domains(emailModel: Seq[Either[String, Seq[String]]]): Seq[String] = 42 | emailModel 43 | .filter(_.isRight) 44 | .flatMap(_.right.get) 45 | 46 | /** 47 | * Returns an email address using a fullname and a random domain 48 | * @param fullname Name and surname 49 | * @return A valid email address, as string, concatenating the first letter from the name and the whole surname, 50 | * and finally a random domain 51 | */ 52 | def address(fullname: String): String = { 53 | val domain = randomElementFromAList[String](domains(emailModel)).getOrElse( 54 | throw new KhermesException(s"Error loading locate /locales/$name/$locale.json")) 55 | s"${getInitial(fullname)}${getSurname(fullname)}@$domain" 56 | } 57 | 58 | private def getInitial(fullname: String) = { 59 | Try(getName(fullname).charAt(0)).getOrElse(throw new KhermesException(s"Error parsing a no valid name")) 60 | } 61 | 62 | def getName(fullName: String): String = 63 | Try(fullName.trim.split(" ")(0)).getOrElse( 64 | throw new KhermesException(s"Error extracting the name value")).toLowerCase 65 | 66 | def getSurname(fullName: String): String = 67 | Try(fullName.trim.split(" ")(1)).getOrElse( 68 | throw new KhermesException(s"Error extracting the surname value")).toLowerCase 69 | } 70 | 71 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/helpers/faker/generators/GeoGenerator.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.helpers.faker.generators 12 | 13 | import com.stratio.khermes.commons.constants.AppConstants 14 | import com.stratio.khermes.commons.exceptions.KhermesException 15 | import com.stratio.khermes.commons.implicits.AppSerializer 16 | import com.stratio.khermes.helpers.faker.FakerGenerator 17 | import com.typesafe.scalalogging.LazyLogging 18 | 19 | /** 20 | * Generates random locations. 21 | */ 22 | case class GeoGenerator(locale: String) extends FakerGenerator 23 | with LazyLogging 24 | with AppSerializer { 25 | 26 | override def name: String = "geo" 27 | 28 | lazy val geoModel = locale match { 29 | case AppConstants.DefaultLocale => { 30 | val resources = getResources(name) 31 | .map(parse[Seq[GeoModel]](name, _)) 32 | if (parseErrors[Seq[GeoModel]](resources).nonEmpty) logger.warn(s"${parseErrors[Seq[GeoModel]](resources)}") 33 | resources 34 | } 35 | case localeMatch => Seq(parse[Seq[GeoModel]](name, s"$localeMatch.json")) 36 | } 37 | 38 | /** 39 | * Example: "geolocation() -> 40.493556, -3.566764, Madrid". 40 | * @return a random geolocation. 41 | */ 42 | def geolocation(): GeoModel = { 43 | 44 | randomElementFromAList[GeoModel](geoModelList(geoModel)) 45 | .getOrElse(throw new KhermesException(s"Error loading locate /locales/$name/$locale.json")) 46 | } 47 | 48 | /** 49 | * Example: "geolocationWithoutCity() -> 28.452717, -13.863761". 50 | * @return a random geolocation. 51 | */ 52 | def geolocationWithoutCity(): (Double, Double) = { 53 | val geo = geolocation 54 | (geo.longitude, geo.latitude) 55 | } 56 | 57 | /** 58 | * Example: "city() -> Tenerife". 59 | * @return a random city name. 60 | */ 61 | def city(): String = { 62 | geolocation.city 63 | } 64 | 65 | def geoModelList(l: Seq[Either[String, Seq[GeoModel]]]): Seq[GeoModel] = { 66 | l.filter(_.isRight).flatMap(_.right.get) 67 | } 68 | 69 | def geoWithoutCityList(l: Seq[Either[String, Seq[GeoModel]]]): Seq[(Double, Double)] = { 70 | geoModelList(l).map(geomodel => (geomodel.longitude, geomodel.latitude)) 71 | } 72 | 73 | def cityList(l: Seq[Either[String, Seq[GeoModel]]]): Seq[String] = { 74 | geoModelList(l).map(geomodel => geomodel.city) 75 | } 76 | 77 | /** 78 | * Return the locale code of input data 79 | * @return locale code (e.g. ES, EN) 80 | */ 81 | def country: String = locale 82 | 83 | } 84 | 85 | case class GeoModel(latitude: Double, longitude: Double, city: String) 86 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/helpers/faker/generators/MusicGenerator.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.helpers.faker.generators 12 | 13 | import com.stratio.khermes.commons.constants.AppConstants 14 | import com.stratio.khermes.commons.exceptions.KhermesException 15 | import com.stratio.khermes.commons.implicits.AppSerializer 16 | import com.stratio.khermes.helpers.faker.FakerGenerator 17 | import com.typesafe.scalalogging.LazyLogging 18 | 19 | case class MusicGenerator(locale: String) extends FakerGenerator 20 | with AppSerializer 21 | with LazyLogging { 22 | 23 | override def name: String = "music" 24 | 25 | lazy val musicModel = locale match { 26 | case AppConstants.DefaultLocale => { 27 | val resources = getResources(name) 28 | .map(parse[Seq[MusicModel]](name, _)) 29 | if (parseErrors[Seq[MusicModel]](resources).nonEmpty) logger.warn(s"${parseErrors[Seq[MusicModel]](resources)}") 30 | resources 31 | } 32 | case localeValue => Seq(parse[Seq[MusicModel]](name, s"$localeValue.json")) 33 | } 34 | 35 | 36 | def getMusic(maybeResources: Seq[Either[String, Seq[MusicModel]]]): Seq[MusicModel] = 37 | maybeResources.filter(_.isRight).flatMap(_.right.get) 38 | 39 | def playedSong: MusicModel = 40 | randomElementFromAList[MusicModel](getMusic(musicModel)).getOrElse( 41 | throw new KhermesException(s"Error loading locate /locales/$name/$locale.json")) 42 | } 43 | 44 | case class MusicModel(song: String, artist: String, album: String, genre: String) 45 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/helpers/faker/generators/NameGenerator.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.helpers.faker.generators 12 | 13 | import com.stratio.khermes.commons.constants.AppConstants 14 | import com.stratio.khermes.commons.implicits.AppSerializer 15 | import com.stratio.khermes.helpers.faker.FakerGenerator 16 | import com.typesafe.scalalogging.LazyLogging 17 | 18 | /** 19 | * Generates random names. 20 | */ 21 | case class NameGenerator(locale: String, strategy: Option[String]) extends FakerGenerator 22 | with AppSerializer 23 | with LazyLogging { 24 | 25 | override def name: String = "name" 26 | 27 | lazy val nameModel: Seq[Either[String, NameModel]] = locale match { 28 | case AppConstants.DefaultLocale => 29 | val resources = getResources(name) 30 | if (strategy.isDefined) { 31 | val res = resources.map(parse[NameModel](name, _) 32 | .right.map(elem => 33 | NameModel( 34 | repeatElementsInList(listWithStrategy(elem.firstNames, strategy.getOrElse(throw new IllegalArgumentException("Bad strategy definition")))) 35 | , repeatElementsInList(listWithStrategy(elem.lastNames, strategy.getOrElse(throw new IllegalArgumentException("Bad strategy definition"))))) 36 | )) 37 | if (parseErrors[NameModel](res).nonEmpty) logger.warn(s"${parseErrors[NameModel](res)}") 38 | res 39 | } else { 40 | val res = resources.map(parse[NameModel](name, _)) 41 | if (parseErrors[NameModel](res).nonEmpty) logger.warn(s"${parseErrors[NameModel](res)}") 42 | res 43 | } 44 | case localeValue => 45 | if (strategy.isDefined) { 46 | Seq(parse[NameModel](name, s"$localeValue.json") 47 | .right.map(elem => 48 | NameModel(repeatElementsInList(listWithStrategy(elem.firstNames, strategy.getOrElse(throw new IllegalArgumentException("Bad strategy definition")))), 49 | repeatElementsInList(listWithStrategy(elem.lastNames, strategy.getOrElse(throw new IllegalArgumentException("Bad strategy definition")))))) 50 | ) 51 | } else { 52 | Seq(parse[NameModel](name, s"$localeValue.json")) 53 | } 54 | } 55 | 56 | 57 | /** 58 | * Example: "Bruce Wayne". 59 | * @return a full name. 60 | */ 61 | def fullName(): String = s"$firstName $lastName" 62 | 63 | /** 64 | * Example: "Bruce Lex". 65 | * @return a middle name. 66 | */ 67 | def middleName(): String = s"$firstName $firstName" 68 | 69 | /** 70 | * Example: "Bruce". 71 | * @return a first name. 72 | */ 73 | def firstName(): String = 74 | randomElementFromAList[String](firstNames(nameModel)).getOrElse(throw new NoSuchElementException) 75 | 76 | 77 | /** 78 | * Example: "Wayne". 79 | * @return a last name. 80 | */ 81 | def lastName(): String = 82 | randomElementFromAList[String](lastNames(nameModel)).getOrElse(throw new NoSuchElementException) 83 | 84 | def lastNames(resources: Seq[Either[String, NameModel]]): Seq[String] = { 85 | getName(resources: Seq[Either[String, NameModel]]).flatMap(_.firstNames) 86 | } 87 | 88 | private def getName(resources: Seq[Either[String, NameModel]]): Seq[NameModel] = { 89 | resources.filter(_.isRight).map(_.right.get) 90 | } 91 | 92 | def firstNames(resources: Seq[Either[String, NameModel]]): Seq[String] = { 93 | getName(resources).flatMap(_.firstNames) 94 | } 95 | } 96 | 97 | case class NameModel(firstNames: Seq[String], lastNames: Seq[String]) 98 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/helpers/faker/generators/NumberGenerator.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.helpers.faker.generators 12 | 13 | import java.security.InvalidParameterException 14 | 15 | import com.stratio.khermes.commons.constants.AppConstants 16 | import com.stratio.khermes.commons.exceptions.KhermesException 17 | import com.stratio.khermes.commons.implicits.AppSerializer 18 | 19 | import scala.math.BigDecimal.RoundingMode 20 | import scala.util.Random 21 | 22 | /** 23 | * Generates random numbers. 24 | */ 25 | class NumberGenerator extends AppSerializer { 26 | 27 | /** 28 | * Example: "number(3) 123". 29 | * @param n integer size. 30 | * @return a random integer. 31 | */ 32 | def number(n: Int): Int = { 33 | assert(n >= 0 && n <= 9, throw new InvalidParameterException(s"$n must be between 0 and 9")) 34 | if (n == 0) { 35 | 0 36 | } else { 37 | val first = Random.nextInt(AppConstants.DecimalValue - 1) + 1 38 | val randSeq = first +: (1 until n).map { _ => Random.nextInt(AppConstants.DecimalValue) } 39 | BigInt(randSeq.mkString).toInt * randSign 40 | } 41 | } 42 | 43 | /** 44 | * Example: "number(3) 123". 45 | * @param n integer size. 46 | * @return a random string that contain the decimal part of a number. 47 | */ 48 | def numberDec(n: Int): String = { 49 | assert(n >= 0 && n <= 9, throw new InvalidParameterException(s"$n must be between 0 and 9")) 50 | if (n == 0) { 51 | "0" 52 | } else { 53 | val nonZero = Random.nextInt(AppConstants.DecimalValue - 1) + 1 54 | val randSeq = (1 until n).map { _ => Random.nextInt(AppConstants.DecimalValue) } :+ nonZero 55 | randSeq.mkString 56 | } 57 | } 58 | 59 | /** 60 | * Example: "number(3,Sign.-) -123". 61 | * @return an Integer positive or negative depending of Sign parameter. 62 | */ 63 | def number(n: Int, sign: NumberSign): Int = sign match { 64 | case Positive => Math.abs(number(n)) 65 | case Negative => Math.abs(number(n)) * -1 66 | case _ => throw new KhermesException(s"The sign must be Positive or Negative") 67 | } 68 | 69 | /** 70 | * @return a random 1 or -1. 71 | */ 72 | def randSign: Int = if (Random.nextBoolean) 1 else -1 73 | 74 | /** 75 | * Example: "number(3) -> 123.456". 76 | * @param n decimal part size. 77 | * @return a random double with same integer and decimal part and random sign. 78 | */ 79 | def decimal(n: Int): BigDecimal = setScale(number(n).toString + "." + numberDec(n), n) 80 | 81 | /** 82 | * Example: "decimal(3,Sign.-) -> -123.456". 83 | * @param n decimal part size. 84 | * @return a random double with same integer and decimal part with defined sign. 85 | */ 86 | def decimal(n: Int, sign: NumberSign): BigDecimal = setScale(number(n, sign).toString + "." + numberDec(n), n) 87 | 88 | /** 89 | * Example: "decimal(3,1) -> 123.4". 90 | * @param m integer part size. 91 | * @param n decimal part size. 92 | * @return a random double with random sign. 93 | */ 94 | def decimal(m: Int, n: Int): BigDecimal = setScale(number(m).toString + "." + numberDec(n), n) 95 | 96 | /** 97 | * Example: "decimal(3,2,Sign.-) -> -123.45". 98 | * @param m integer part size. 99 | * @param n decimal part size. 100 | * @param sign sign positive or negative. 101 | * @return a random double with defined sign. 102 | */ 103 | def decimal(m: Int, n: Int, sign: NumberSign): BigDecimal = setScale(number(m, sign).toString + "." + numberDec(n), n) 104 | 105 | /** 106 | * Example: "numberInRange(3,5) 4". 107 | * @param m one end of range. 108 | * @param n the other end of range. 109 | * @return a random BigDecimal. 110 | */ 111 | def numberInRange(m: Int, n: Int): Int = { 112 | Random.nextInt((Math.max(m, n) - Math.min(n, m)) + 1) + Math.min(n, m) 113 | } 114 | 115 | /** 116 | * Example: "decimalInRange(3,4) 3.1446350167374337". 117 | * @param m one end of range. 118 | * @param n the other end of range. 119 | * @return a random BigDecimal. 120 | */ 121 | def decimalInRange(m: Int, n: Int): BigDecimal = { 122 | BigDecimal(Random.nextDouble() * (Math.max(m, n) - Math.min(n, m)) + Math.min(n, m)) 123 | } 124 | 125 | def setScale(s: String, n: Int): BigDecimal = { 126 | if (n == 0) BigDecimal.valueOf(s.toDouble).setScale(1, RoundingMode.HALF_UP) else BigDecimal.valueOf( 127 | s.toDouble).setScale(n, RoundingMode.HALF_UP) 128 | } 129 | 130 | /** 131 | * Example rating(5) 0-4 132 | * @param n max integer rating (exclude) 133 | * @return random integer rating [0-n) 134 | */ 135 | def rating(n: Int): Int = { 136 | Random.nextInt(n) 137 | } 138 | } 139 | 140 | sealed trait NumberSign 141 | 142 | case object Positive extends NumberSign 143 | 144 | case object Negative extends NumberSign 145 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/helpers/twirl/TwirlHelper.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.helpers.twirl 12 | 13 | import java.io.File 14 | import java.lang.reflect.Method 15 | import java.net._ 16 | 17 | import com.stratio.khermes.commons.constants.AppConstants 18 | import com.typesafe.config.Config 19 | import com.typesafe.scalalogging.LazyLogging 20 | import play.twirl.compiler.{GeneratedSource, TwirlCompiler} 21 | 22 | import scala.collection.mutable 23 | import scala.reflect.internal.util.Position 24 | import scala.tools.nsc.reporters.ConsoleReporter 25 | import scala.tools.nsc.{Global, Settings} 26 | 27 | /** 28 | * Helper used to parse and compile templates using Twirl. 29 | */ 30 | object TwirlHelper extends LazyLogging { 31 | 32 | /** 33 | * Compiles and executes a template with Twirl. It follows the next steps: 34 | * Step 1) A string that represents the template is saved in Khermes' templates path. 35 | * Step 2) The engine generates a scala files to be compiled. 36 | * Step 3) The engine compiles the scala files generated in the previous step. 37 | * Step 4) Finally it executes the compiled files interpolating values with the template. 38 | * @param template a string with the template. 39 | * @param templateName the name of the file that will contain the content of the template. 40 | * @param config with Khermes' configuration. 41 | * @tparam T with the type of object to inject in the template. 42 | * @return a compiled and executed template. 43 | */ 44 | def template[T](template: String, templateName: String)(implicit config: Config): CompiledTemplate[T] = { 45 | val templatesPath = config.getString("khermes.templates-path") 46 | val templatePath = s"$templatesPath/$templateName.scala.html" 47 | scala.tools.nsc.io.File(templatePath).writeAll(template) 48 | 49 | val sourceDir = new File(templatesPath) 50 | val generatedDir = new File(s"$templatesPath/${AppConstants.GeneratedTemplatesPrefix}") 51 | val generatedClasses = new File(s"$templatesPath/${AppConstants.GeneratedClassesPrefix}") 52 | 53 | deleteRecursively(generatedDir) 54 | deleteRecursively(generatedClasses) 55 | generatedClasses.mkdirs() 56 | generatedDir.mkdirs() 57 | 58 | val helper = new CompilerHelper(sourceDir, generatedDir, generatedClasses) 59 | helper.compile[T](s"$templateName.scala.html", s"html.$templateName", Seq("com.stratio.khermes.helpers.faker.Faker")) 60 | } 61 | 62 | /** 63 | * If the template is wrong this exception informs about the mistake. 64 | * @param message with information about the error. 65 | * @param line that contains the error. 66 | * @param column that contains the error. 67 | */ 68 | case class CompilationError(message: String, line: Int, column: Int) extends RuntimeException(message) 69 | 70 | /** 71 | * Deletes all content in a path. 72 | * @param dir a file that represents the path to delete. 73 | */ 74 | protected[this] def deleteRecursively(dir: File) { 75 | if(dir.isDirectory) dir.listFiles().foreach(deleteRecursively) 76 | dir.delete() 77 | } 78 | 79 | /** 80 | * Helper used to compile templates internally. 81 | * @param sourceDir that contains original templates. 82 | * @param generatedDir that contains scala files from the templates. 83 | * @param generatedClasses that contains class files with the result of the compilation. 84 | */ 85 | protected[this] class CompilerHelper(sourceDir: File, generatedDir: File, generatedClasses: File) { 86 | 87 | val twirlCompiler = TwirlCompiler 88 | val classloader = new URLClassLoader(Array(generatedClasses.toURI.toURL), 89 | Class.forName("play.twirl.compiler.TwirlCompiler").getClassLoader) 90 | val compileErrors = new mutable.ListBuffer[CompilationError] 91 | 92 | val compiler = { 93 | def additionalClassPathEntry: Option[String] = Some( 94 | Class.forName("play.twirl.compiler.TwirlCompiler") 95 | .getClassLoader.asInstanceOf[URLClassLoader] 96 | .getURLs.map(url => new File(url.toURI)).mkString(":")) 97 | 98 | val settings = new Settings 99 | val scalaObjectSource = Class.forName("scala.Option").getProtectionDomain.getCodeSource 100 | 101 | val compilerPath = Class.forName("scala.tools.nsc.Interpreter").getProtectionDomain.getCodeSource.getLocation 102 | val libPath = scalaObjectSource.getLocation 103 | val pathList = List(compilerPath, libPath) 104 | val originalBootClasspath = settings.bootclasspath.value 105 | settings.bootclasspath.value = 106 | ((originalBootClasspath :: pathList) ::: additionalClassPathEntry.toList) mkString File.pathSeparator 107 | settings.outdir.value = generatedClasses.getAbsolutePath 108 | 109 | new Global(settings, new ConsoleReporter(settings) { 110 | override def printMessage(pos: Position, msg: String) = { 111 | compileErrors.append(CompilationError(msg, pos.line, pos.point)) 112 | } 113 | }) 114 | } 115 | 116 | def compile[T](templateName: String, className: String, additionalImports: Seq[String] = Nil): CompiledTemplate[T] = { 117 | val templateFile = new File(sourceDir, templateName) 118 | val Some(generated) = twirlCompiler.compile(templateFile, sourceDir, generatedDir, "play.twirl.api.TxtFormat", 119 | additionalImports = TwirlCompiler.DefaultImports ++ additionalImports) 120 | val mapper = GeneratedSource(generated) 121 | val run = new compiler.Run 122 | compileErrors.clear() 123 | run.compile(List(generated.getAbsolutePath)) 124 | 125 | compileErrors.headOption.foreach { 126 | case CompilationError(msg, line, column) => 127 | compileErrors.clear() 128 | throw CompilationError(msg, mapper.mapLine(line), mapper.mapPosition(column)) 129 | } 130 | new CompiledTemplate[T](className, classloader) 131 | } 132 | } 133 | 134 | /** 135 | * From a classname and a classloader it returns a result of a compiled and executed template. 136 | * @param className with the classname. 137 | * @param classloader with the classloader. 138 | * @tparam T with the type of object to inject in the template. 139 | */ 140 | class CompiledTemplate[T](className: String, classloader: URLClassLoader) { 141 | var method: Option[Method] = None 142 | var declaredField: Option[AnyRef] = None 143 | 144 | private def getF(template: Any) = { 145 | if(method.isEmpty) { 146 | method = Option(template.getClass.getMethod("f")) 147 | method.get.invoke(template).asInstanceOf[T] 148 | } else { 149 | method.get.invoke(template).asInstanceOf[T] 150 | } 151 | } 152 | /** 153 | * @return the result of a compiled and executed template. 154 | */ 155 | //scalastyle:off 156 | def static: T = { 157 | if(declaredField.isEmpty) { 158 | declaredField = Option(classloader.loadClass(className + "$").getDeclaredField("MODULE$").get(null)) 159 | getF(declaredField.get) 160 | } else { 161 | getF(declaredField.get) 162 | } 163 | } 164 | //scalastyle:on 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/metrics/KhermesMetrics.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.metrics 12 | 13 | import com.stratio.khermes.commons.constants.AppConstants 14 | import com.codahale.metrics.{Counter, Meter, SharedMetricRegistries, Timer} 15 | 16 | import scala.util.Try 17 | 18 | trait KhermesMetrics { 19 | 20 | val khermesConfig = com.stratio.khermes.commons.implicits.AppImplicits.config 21 | lazy val logReporterEnabled = Try(khermesConfig.getBoolean(AppConstants.LoggerEnabled)).getOrElse( 22 | AppConstants.LoggerEnabledDefault) 23 | lazy val graphiteReporterEnabled = Try(khermesConfig.getBoolean(AppConstants.GraphiteEnabled)).getOrElse( 24 | AppConstants.GraphiteEnabledDefault) 25 | val loggerReporterName = Try(khermesConfig.getString(AppConstants.LoggerReporterName)).getOrElse( 26 | AppConstants.LoggerReporterNameDefault) 27 | val graphiteReporterName = Try(khermesConfig.getString(AppConstants.GraphiteReporterName)).getOrElse( 28 | AppConstants.GraphiteReporterNameDefault) 29 | val availableMetrics = List(logReporter, graphiteReporter).flatten 30 | 31 | def getAvailableCounterMetrics(counterName: String): List[Counter] = 32 | for (availableMetric <- availableMetrics) yield 33 | SharedMetricRegistries.getOrCreate(availableMetric).counter(counterName) 34 | 35 | def getAvailableMeterMetrics(counterName: String): List[Meter] = 36 | for (availableMetric <- availableMetrics) yield 37 | SharedMetricRegistries.getOrCreate(availableMetric).meter(counterName) 38 | 39 | def increaseCounterMetric[T](metrics: List[Counter]): Unit = 40 | for (metric <- metrics) metric.inc() 41 | 42 | def markMeterMetric[T](metrics: List[Meter]): Unit = 43 | for (metric <- metrics) metric.mark() 44 | 45 | private def logReporter() = 46 | if (logReporterEnabled) { 47 | Some(loggerReporterName) 48 | } 49 | else { 50 | None 51 | } 52 | 53 | private def graphiteReporter() = 54 | if (graphiteReporterEnabled) { 55 | Some(graphiteReporterName) 56 | } 57 | else { 58 | None 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/metrics/MetricsReporter.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.metrics 12 | 13 | import java.net.InetSocketAddress 14 | import java.util.concurrent.TimeUnit 15 | 16 | import com.codahale.metrics.graphite.{Graphite, GraphiteReporter} 17 | import com.codahale.metrics.jvm.{ClassLoadingGaugeSet, GarbageCollectorMetricSet, MemoryUsageGaugeSet, ThreadStatesGaugeSet} 18 | import com.codahale.metrics.{MetricFilter, MetricRegistry, SharedMetricRegistries, Slf4jReporter} 19 | import com.stratio.khermes.commons.constants.AppConstants 20 | import com.typesafe.config.Config 21 | import com.typesafe.scalalogging.LazyLogging 22 | import org.slf4j.LoggerFactory 23 | 24 | import scala.util.Try 25 | 26 | object MetricsReporter extends LazyLogging { 27 | 28 | val config: Config = com.stratio.khermes.commons.implicits.AppImplicits.config 29 | val loggerReporterEnabled: Boolean = Try(config.getBoolean(AppConstants.LoggerEnabled)).getOrElse( 30 | AppConstants.LoggerEnabledDefault) 31 | val graphiteReporterEnabled: Boolean = Try(config.getBoolean(AppConstants.GraphiteEnabled)).getOrElse( 32 | AppConstants.GraphiteEnabledDefault) 33 | val loggerReporterName = Try(config.getString(AppConstants.LoggerReporterName)).getOrElse( 34 | AppConstants.LoggerReporterNameDefault) 35 | val graphiteReporterName = Try(config.getString(AppConstants.GraphiteReporterName)).getOrElse( 36 | AppConstants.GraphiteReporterNameDefault) 37 | val graphiteReporterHost = Try(config.getString(AppConstants.GraphiteReporterHost)).getOrElse( 38 | AppConstants.GraphiteReporterHostDefault) 39 | val graphiteReporterPort = Try(config.getInt(AppConstants.GraphiteReporterPort)).getOrElse( 40 | AppConstants.GraphiteReporterPortDefault) 41 | val metricsFrequencyInSeconds = 1 42 | 43 | def start(): Unit = { 44 | if (loggerReporterEnabled) { 45 | logger.info("Starting the logger metrics reporter...") 46 | startSlf4jReporter 47 | } 48 | 49 | if (graphiteReporterEnabled) { 50 | logger.info("Starting the graphite metrics reporter...") 51 | startGraphiteReporter 52 | } 53 | } 54 | 55 | private def startSlf4jReporter = { 56 | val logReporter = Slf4jReporter.forRegistry( 57 | SharedMetricRegistries.getOrCreate(loggerReporterName)) 58 | .outputTo(LoggerFactory.getLogger("metrics")) 59 | .convertRatesTo(TimeUnit.SECONDS) 60 | .convertDurationsTo(TimeUnit.MILLISECONDS) 61 | .build 62 | logReporter.start(metricsFrequencyInSeconds, TimeUnit.SECONDS) 63 | } 64 | 65 | private def startGraphiteReporter = { 66 | logger.info(s"Writing graphite data to: $graphiteReporterHost:$graphiteReporterPort") 67 | logger.error(s"WITH THE FOLLOWING PREFIX: $graphiteReporterName") 68 | val graphite = new Graphite(new InetSocketAddress(graphiteReporterHost, graphiteReporterPort)) 69 | val graphiteRegistry = SharedMetricRegistries.getOrCreate(graphiteReporterName) 70 | val graphiteReporter = GraphiteReporter 71 | .forRegistry( 72 | graphiteRegistry) 73 | .prefixedWith(graphiteReporterName) 74 | .convertRatesTo(TimeUnit.SECONDS) 75 | .convertDurationsTo(TimeUnit.MILLISECONDS) 76 | .filter(MetricFilter.ALL) 77 | .build(graphite) 78 | graphiteReporter.start(metricsFrequencyInSeconds, TimeUnit.SECONDS) 79 | setUpJVMInstrumentation(graphiteRegistry) 80 | } 81 | 82 | private def setUpJVMInstrumentation(graphiteRegistry: MetricRegistry) = { 83 | setUpClassLoadingMetricsGauge(graphiteRegistry) 84 | setUpMemoryUsageMetricsGauge(graphiteRegistry) 85 | setUpGarbageCollectorGauge(graphiteRegistry) 86 | setUpThreadsStatusGauge(graphiteRegistry) 87 | } 88 | 89 | private def setUpClassLoadingMetricsGauge(registry: MetricRegistry) = 90 | registry.register("ClassLoading", new ClassLoadingGaugeSet()) 91 | 92 | private def setUpMemoryUsageMetricsGauge(registry: MetricRegistry) = 93 | registry.register("MemoryUsage", new MemoryUsageGaugeSet()) 94 | 95 | private def setUpGarbageCollectorGauge(registry: MetricRegistry) = 96 | registry.register("GarbageCollector", new GarbageCollectorMetricSet()) 97 | 98 | private def setUpThreadsStatusGauge(registry: MetricRegistry) = 99 | registry.register("ThreadStates", new ThreadStatesGaugeSet()) 100 | } 101 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/persistence/dao/BaseDAO.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.persistence.dao 12 | 13 | trait BaseDAO[T] { 14 | 15 | def create(entity: T, data: T) 16 | 17 | def read(entity: T): T 18 | 19 | def update(entity: T, data: T) 20 | 21 | def delete(entity: T) 22 | 23 | def exists(entity: T): Boolean 24 | 25 | def list(entity: T): T 26 | } 27 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/persistence/dao/ZkDAO.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.persistence.dao 12 | 13 | import com.stratio.khermes.commons.constants.AppConstants 14 | import com.stratio.khermes.commons.exceptions.KhermesException 15 | import com.typesafe.scalalogging.LazyLogging 16 | import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory} 17 | import org.apache.curator.retry.ExponentialBackoffRetry 18 | 19 | import scala.util.{Failure, Success, Try} 20 | 21 | class ZkDAO(connectionServer: Option[String] = None) extends BaseDAO[String] with LazyLogging { 22 | val config = com.stratio.khermes.commons.implicits.AppImplicits.config 23 | lazy val curatorFramework: CuratorFramework = buildCurator(connectionServer.getOrElse(connectionString)) 24 | lazy val connectionString = Try(config.getString(AppConstants.ZookeeperConnection)).getOrElse( 25 | AppConstants.ZookeeperConnectionDefault) 26 | lazy val connectionTimeout = config.getInt(AppConstants.ZookeeperConnectionTimeout) 27 | lazy val sessionTimeout = config.getInt(AppConstants.ZookeeperSessionTimeout) 28 | lazy val retryAttempts = config.getInt(AppConstants.ZookeeperRetryAttempts) 29 | lazy val retryInterval = config.getInt(AppConstants.ZookeeperRetryInterval) 30 | 31 | override def create(path: String, config: String): Unit = Try( 32 | if (exists(path)) { 33 | update(path, config) 34 | } 35 | else { 36 | curatorFramework.create().creatingParentsIfNeeded().forPath(s"${AppConstants.ZookeeperParentPath}/$path") 37 | curatorFramework.setData().forPath(s"${AppConstants.ZookeeperParentPath}/$path", config.getBytes()) 38 | } 39 | ) match { 40 | case Success(ids) => ids.toString 41 | case Failure(e) => throw new KhermesException(e.getMessage) 42 | } 43 | 44 | override def read(path: String): String = Try( 45 | new String(curatorFramework.getData.forPath(s"${AppConstants.ZookeeperParentPath}/$path")) 46 | ) 47 | match { 48 | case Success(ids) => ids 49 | case Failure(e) => throw new KhermesException(e.getMessage) 50 | } 51 | 52 | override def exists(path: String): Boolean = { 53 | //scalastyle:off 54 | curatorFramework.checkExists().forPath(s"${AppConstants.ZookeeperParentPath}/$path") != null 55 | //scalastyle:on 56 | } 57 | 58 | override def delete(path: String): Unit = { 59 | curatorFramework.delete().deletingChildrenIfNeeded().forPath(s"/$path") 60 | } 61 | 62 | override def update(path: String, config: String): Unit = { 63 | curatorFramework.setData().forPath(s"${AppConstants.ZookeeperParentPath}/$path", config.getBytes()) 64 | } 65 | 66 | override def list(path: String): String = { 67 | val children = curatorFramework.getChildren.forPath(s"${AppConstants.ZookeeperParentPath}/$path") 68 | import collection.JavaConverters._ 69 | children.asScala.mkString("\n") 70 | } 71 | 72 | 73 | def buildCurator(connectionServer: String): CuratorFramework = { 74 | Try { 75 | val cf = CuratorFrameworkFactory.builder() 76 | .connectString(connectionServer) 77 | .connectionTimeoutMs(connectionTimeout) 78 | .sessionTimeoutMs(sessionTimeout) 79 | .retryPolicy(new ExponentialBackoffRetry(retryAttempts, retryInterval)) 80 | .build() 81 | cf.start() 82 | logger.info(s"Zookeeper connection to ${connectionServer} was STARTED.") 83 | cf 84 | }.getOrElse { 85 | logger.error("Impossible to start Zookeeper connection") 86 | throw new KhermesException("Impossible to start Zookeeper connection") 87 | } 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/main/scala/com/stratio/khermes/persistence/kafka/KafkaClient.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.persistence.kafka 12 | 13 | import java.util.Properties 14 | import java.util.concurrent.Future 15 | 16 | import com.typesafe.config.Config 17 | import com.typesafe.scalalogging.LazyLogging 18 | import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord, RecordMetadata} 19 | 20 | /** 21 | * Simple client used to send messages to a Kafka broker. 22 | * @param config with all Kafka configuration. 23 | */ 24 | class KafkaClient[K](config: Config) extends LazyLogging { 25 | 26 | lazy val producer: KafkaProducer[String, K] = new KafkaProducer(parseProperties()) 27 | 28 | /** 29 | * Parses Kafka's configuration to a properties object. 30 | * @param path that could be the configuration of a kafkaProducer or a kafkaConsumer. (kafkaProducer by default). 31 | * @return a parsed properties object. 32 | */ 33 | def parseProperties(path: String = "kafka"): Properties = { 34 | assert(config.hasPath(path), s"Not existing $path path in application.conf") 35 | import scala.collection.JavaConversions._ 36 | val props = new Properties() 37 | val map: Map[String, Object] = config.getConfig(path).entrySet().map({ entry => 38 | entry.getKey -> entry.getValue.unwrapped() 39 | })(collection.breakOut) 40 | props.putAll(map) 41 | props 42 | } 43 | 44 | /** 45 | * Sends a message to a topic. 46 | * @param topic with the Kafka's topic. 47 | * @param message with the message to send. 48 | * @return a future with the result of the operation. 49 | */ 50 | def send(topic: String, message: K): Future[RecordMetadata] = { 51 | val a = new ProducerRecord[String, K](topic, message) 52 | producer.send(a) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/test/resources/application.conf: -------------------------------------------------------------------------------- 1 | khermes { 2 | ws { 3 | hostname = 0.0.0.0 4 | port = 8081 5 | } 6 | templates-path = "/tmp/khermes/templates" 7 | client = false 8 | ws = false 9 | } 10 | 11 | akka { 12 | loglevel = "error" 13 | actor { 14 | provider = "akka.cluster.ClusterActorRefProvider" 15 | debug { 16 | receive = on 17 | lifecycle = on 18 | } 19 | } 20 | remote { 21 | log-remote-lifecycle-events = off 22 | netty.tcp { 23 | hostname = localhost 24 | port = 0 25 | } 26 | } 27 | cluster { 28 | roles = [backend] 29 | seed-nodes = [${?VALUE}] 30 | auto-down-unreachable-after = 10s 31 | } 32 | 33 | http.server.idle-timeout = 1000s 34 | } 35 | 36 | zookeeper { 37 | connection = "localhost:2181" 38 | connectionTimeout = 15000 39 | sessionTimeout = 60000 40 | retryAttempts = 5 41 | retryInterval = 10000 42 | } -------------------------------------------------------------------------------- /src/test/resources/locales/email/EN.json: -------------------------------------------------------------------------------- 1 | [ 2 | "gmail.com", 3 | "hotmail.com", 4 | "yahoo.com" 5 | ] 6 | -------------------------------------------------------------------------------- /src/test/resources/locales/email/XX.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /src/test/resources/locales/geo/ES.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"latitude":28.452717,"longitude":-13.863761,"city":"Fuerteventura"}, 3 | {"latitude":27.814847,"longitude":-17.887056,"city":"Hierro"}, 4 | {"latitude":28.626478,"longitude":-17.755611,"city":"Santa Cruz De La Palma"}, 5 | {"latitude":27.931886,"longitude":-15.386586,"city":"Gran Canaria"}, 6 | {"latitude":28.945464,"longitude":-13.605225,"city":"Las Palmas"}, 7 | {"latitude":28.044475,"longitude":-16.572489,"city":"Tenerife"} 8 | ] -------------------------------------------------------------------------------- /src/test/resources/locales/geo/US.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"longitude":39.6335556,"latitude":-86.8138056,"city":"Greencastle"}, 3 | {"longitude":41.9929342,"latitude":-86.1280125,"city":"Dowagiac"}, 4 | {"longitude":39.9750278,"latitude":-81.5775833,"city":"Cambridge"}, 5 | {"longitude":44.8436667,"latitude":-87.4215556,"city":"Sturgeon Bay"}, 6 | {"longitude":39.7948244,"latitude":-76.6471914,"city":"Stewartstown"}, 7 | {"longitude":45.6950000,"latitude":-118.841389,"city":"Pendleton"}, 8 | ] -------------------------------------------------------------------------------- /src/test/resources/locales/geo/XX.json: -------------------------------------------------------------------------------- 1 | [] -------------------------------------------------------------------------------- /src/test/resources/locales/geo/YY.json: -------------------------------------------------------------------------------- 1 | [{"latitude": "pepe", "longitude": -86.8138056}] -------------------------------------------------------------------------------- /src/test/resources/locales/geo/ZZ.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"latitude": 39.6335556, "longitude": -86.8138056}, 3 | {"latitude": 41.9929342, "longitude": -86.1280125}, 4 | {"latitude": 39.9750278, "longitude": -81.5775833}, 5 | {"latitude": 44.8436667, "longitud": -87.4215556}, 6 | {"latitud": 39.7948244, "longitude": -76.6471914} 7 | ] -------------------------------------------------------------------------------- /src/test/resources/locales/music/EN.json: -------------------------------------------------------------------------------- 1 | [{"song": "Shape of You", "artist": "Ed Sheeran", "album": "Shape of You", "genre": "Pop"}, 2 | {"song": "I Don’t Wanna Live Forever (Fifty Shades Darker) - From 'Fifty Shades Darker (Original Motion Picture Soundtrack)'", "artist": "ZAYN, Taylor Swift", "album": "I Don’t Wanna Live Forever (Fifty Shades Darker)", "genre": "Pop"}, 3 | {"song": "It Ain't Me (with Selena Gomez)", "artist": "Kygo, Selena Gomez", "album": "It Ain't Me (with Selena Gomez)", "genre": "Dance"}, 4 | {"song": "Paris", "artist": "The Chainsmokers", "album": "Paris", "genre": "Dance"}, 5 | {"song": "Despacito (Featuring Daddy Yankee)", "artist": "Luis Fonsi, Daddy Yankee", "album": "Despacito (Featuring Daddy Yankee)", "genre": "Latin"}, 6 | {"song": "Scared To Be Lonely", "artist": "Martin Garrix, Dua Lipa", "album": "Scared To Be Lonely", "genre": "Pop"}, 7 | {"song": "Castle on the Hill", "artist": "Ed Sheeran", "album": "Castle on the Hill", "genre": "Pop"}, 8 | {"song": "Chained To The Rhythm", "artist": "Katy Perry, Skip Marley", "album": "Chained To The Rhythm", "genre": "Dance"}, 9 | {"song": "Rockabye (feat. Sean Paul & Anne-Marie)", "artist": "Clean Bandit, Anne-Marie, Sean Paul", "album": "Rockabye (feat. Sean Paul & Anne-Marie)", "genre": "Dance"}, 10 | {"song": "I Feel It Coming", "artist": "The Weeknd, Daft Punk", "album": "Starboy", "genre": "Dance"}] -------------------------------------------------------------------------------- /src/test/resources/locales/music/ES.json: -------------------------------------------------------------------------------- 1 | [{"song": "Despacito (Featuring Daddy Yankee)", "artist": "Luis Fonsi, Daddy Yankee", "album": "Despacito (Featuring Daddy Yankee)", "genre": "Latin"}, 2 | {"song": "Shape of You", "artist": "Ed Sheeran", "album": "Shape of You", "genre": "Pop"}, 3 | {"song": "El Amante", "artist": "Nicky Jam", "album": "Fénix", "genre": "Latin"}, 4 | {"song": "Reggaetón Lento (Bailemos)", "artist": "CNCO", "album": "Primera Cita", "genre": "Latin"}, 5 | {"song": "Chantaje", "artist": "Shakira, Maluma", "album": "Chantaje", "genre": "Latin"}, 6 | {"song": "Rockabye (feat. Sean Paul & Anne-Marie)", "artist": "Clean Bandit, Anne-Marie, Sean Paul", "album": "Rockabye (feat. Sean Paul & Anne-Marie)", "genre": "Pop"}, 7 | {"song": "I Don’t Wanna Live Forever (Fifty Shades Darker) - From 'Fifty Shades Darker (Original Motion Picture Soundtrack)'", "artist": "ZAYN, Taylor Swift", "album": "I Don’t Wanna Live Forever (Fifty Shades Darker)", "genre": "Pop"}, 8 | {"song": "Lumbra", "artist": "Cali Y El Dandee, Shaggy", "album": "Lumbra", "genre": "Latin"}, 9 | {"song": "La Rompe Corazones", "artist": "Daddy Yankee, Ozuna", "album": "La Rompe Corazones", "genre": "Latin"}, 10 | {"song": "Safari", "artist": "J Balvin, Pharrell Williams, BIA, Sky", "album": "Energía", "genre": "Latin"}] -------------------------------------------------------------------------------- /src/test/resources/locales/music/XX.json: -------------------------------------------------------------------------------- 1 | [] -------------------------------------------------------------------------------- /src/test/resources/locales/music/YY.json: -------------------------------------------------------------------------------- 1 | [{"artist": "Luis Fonsi, Daddy Yankee", "album": "Despacito (Featuring Daddy Yankee)", "genre": "Latin"}] -------------------------------------------------------------------------------- /src/test/resources/locales/music/ZZ.json: -------------------------------------------------------------------------------- 1 | [{"cancion": "fake", "artist": "Luis Fonsi, Daddy Yankee", "album": "Despacito (Featuring Daddy Yankee)", "genre": "Latin"}] -------------------------------------------------------------------------------- /src/test/resources/locales/name/EN.json: -------------------------------------------------------------------------------- 1 | { 2 | "firstNames": ["Bruce", "Lex", "Clark", "Peter"], 3 | "lastNames": ["Wayne", "Luthor", "Kent", "Parker"] 4 | } -------------------------------------------------------------------------------- /src/test/resources/locales/name/ES.json: -------------------------------------------------------------------------------- 1 | { 2 | "firstNames": ["Alicia", "Juan", "Alberto", "Enrique"], 3 | "lastNames": ["Doblas", "Garcia", "Rodriguez", "Ruiz"] 4 | } -------------------------------------------------------------------------------- /src/test/resources/locales/name/XX.json: -------------------------------------------------------------------------------- 1 | { 2 | "firstNames": [], 3 | "lastNames": [] 4 | } -------------------------------------------------------------------------------- /src/test/resources/music.static.scala.html: -------------------------------------------------------------------------------- 1 | 13 | @import com.stratio.khermes.helpers.faker.Faker 14 | @import com.stratio.khermes.helpers.faker.generators.Positive 15 | @import scala.util.Random 16 | @import org.joda.time.DateTime 17 | 18 | @(faker: Faker) 19 | @defining(faker.Music.playedSong, faker.Name.fullName, faker.Geo.geolocation) { case (randomSong, fullname, geo) => 20 | 21 | { 22 | "song": "@(randomSong.song)", 23 | "artist": "@(randomSong.artist)", 24 | "genre": "@(randomSong.genre)", 25 | "album": "@(randomSong.album)", 26 | "playDuration": @(faker.Number.number(3,Positive)), 27 | "rating": @(faker.Number.rating(5)), 28 | "user": "@(fullname)", 29 | "email": "@(faker.Email.address(fullname))", 30 | "userType": "@(Seq("free", "membership")(Random.nextInt(2)))", 31 | "country": "@(faker.Geo.country)", 32 | "latitude": @(geo.latitude), 33 | "longitude": @(geo.longitude), 34 | "startTime": "@(s"${Random.nextInt(24)}:${Random.nextInt(60)}:${Random.nextInt(60)}.${Random.nextInt(1000)}")", 35 | "artistBirthday" : @(faker.Datetime.datetime(new DateTime("1970-1-1"),new DateTime("1985-1-1"))) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/test/resources/static.scala.html: -------------------------------------------------------------------------------- 1 | 13 | @import com.stratio.khermes.helpers.faker.Faker 14 | @import com.stratio.khermes.helpers.faker.generators.Positive 15 | 16 | @(faker: Faker) 17 | { 18 | "customerId": @(faker.Number.number(3,Positive)), 19 | "customerName": "@(faker.Name.fullName)", 20 | "latitude": @(faker.Geo.geolocation.latitude), 21 | "longitude": @(faker.Geo.geolocation.longitude), 22 | "productIds": [@((1 to 5).map(x => faker.Number.number(1, Positive)).mkString(","))] 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/KhermesTestIT.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes 12 | 13 | import com.stratio.khermes.cluster.BaseActorTest 14 | import org.junit.runner.RunWith 15 | import org.scalatest.junit.JUnitRunner 16 | 17 | @RunWith(classOf[JUnitRunner]) 18 | class KhermesTestIT extends BaseActorTest { 19 | 20 | "An KhermesRunner" should { 21 | "run main without any kind of error" in { 22 | Khermes.main(Array.empty) 23 | } 24 | 25 | "print a welcome message and start the akka system without errors" in { 26 | Khermes.welcome 27 | } 28 | 29 | "start a worker supervisor without errors" in { 30 | Khermes.workerSupervisor 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/clients/http/protocols/WSProtocolMessageTest.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.clients.http.protocols 12 | 13 | import org.junit.runner.RunWith 14 | import org.scalatest.junit.JUnitRunner 15 | import org.scalatest.{FlatSpec, Matchers} 16 | 17 | @RunWith(classOf[JUnitRunner]) 18 | class WSProtocolMessageTest extends FlatSpec with Matchers { 19 | 20 | "A WSProtocolCommand" should "parse a configuration" in { 21 | val block = 22 | """ 23 | |[command] 24 | |ls 25 | |[arg1] 26 | |one line content 27 | |[arg2] 28 | |multiline line 1 content 29 | |multiline line 2 content 30 | """.stripMargin 31 | 32 | val result = WsProtocolCommand.parseTextBlock(block) 33 | result should be(WSProtocolMessage(WsProtocolCommand.Ls, Map( 34 | "arg1" -> "one line content", 35 | "arg2" -> "multiline line 1 content\nmultiline line 2 content" 36 | ))) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/clients/shell/KhermesConsoleHelperTest.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.clients.shell 12 | 13 | import org.junit.runner.RunWith 14 | import org.scalatest.junit.JUnitRunner 15 | import org.scalatest.{FlatSpec, Matchers} 16 | 17 | @RunWith(classOf[JUnitRunner]) 18 | class KhermesConsoleHelperTest extends FlatSpec 19 | with Matchers { 20 | 21 | val khermesConsoleHelper = KhermesConsoleHelper 22 | 23 | it should "give a map with arg as key and a list of the values" in { 24 | val line = "start --kafka k1 --template t1 --khermes k1 --ids 112312-32123213 3123-12343-453534-5345" 25 | khermesConsoleHelper.commandArgumentsAndValues(line) shouldBe Map("kafka" -> "k1", "template" -> "t1", "khermes" -> "k1", "ids" -> "112312-32123213 3123-12343-453534-5345") 26 | } 27 | 28 | it should "when an arg do not have value raise an exception" in { 29 | val line = "start --kafka --template t1 --khermes k1 --ids 1123-123212-3213 31231-2343453-5345345" 30 | khermesConsoleHelper.commandArgumentsAndValues(line) shouldBe Map("kafka" -> "", "template" -> "t1", "khermes" -> "k1", "ids" -> "1123-123212-3213 31231-2343453-5345345") 31 | } 32 | 33 | it should "give an empty map when there are not args" in { 34 | val line = "save" 35 | khermesConsoleHelper.commandArgumentsAndValues(line) shouldBe Map() 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/cluster/BaseActorTest.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.cluster 12 | 13 | import akka.actor.ActorSystem 14 | import akka.testkit.{DefaultTimeout, ImplicitSender, TestKit} 15 | import com.stratio.khermes.commons.implicits.AppImplicits 16 | import com.typesafe.config.{Config, ConfigFactory} 17 | import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} 18 | import scala.concurrent.duration._ 19 | import akka.util.Timeout 20 | 21 | /** 22 | * Generic class used to test actors. All tests that uses akka should extend this class. 23 | */ 24 | abstract class BaseActorTest extends TestKit(ActorSystem("ActorTest", ConfigFactory.load())) 25 | with DefaultTimeout 26 | with ImplicitSender 27 | with WordSpecLike 28 | with Matchers 29 | with BeforeAndAfterAll { 30 | 31 | lazy implicit val config: Config = AppImplicits.config 32 | lazy implicit val executionContext = AppImplicits.executionContext 33 | override implicit val timeout: Timeout = Timeout(10 seconds) 34 | 35 | override def afterAll { 36 | system.terminate() 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/cluster/supervisor/KhermesClientActorTest.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.cluster.supervisor 12 | 13 | import org.junit.runner.RunWith 14 | import org.scalatest.junit.JUnitRunner 15 | import org.scalatest.{FlatSpec, Matchers} 16 | 17 | @RunWith(classOf[JUnitRunner]) 18 | class KhermesClientActorTest extends FlatSpec 19 | with Matchers{ 20 | it should "give a message with configuration empties" in { 21 | val khermes = KhermesClientActor 22 | khermes.messageFeedback(None,None,None) shouldBe "Error: To start nodes is necessary to set template and kafka and khermes configuration." 23 | } 24 | it should "give a message with kafka and template configuration" in { 25 | val khermes = KhermesClientActor 26 | khermes.messageFeedback(Option("khermes"),None,None) shouldBe "Error: To start nodes is necessary to set template and kafka configuration." 27 | } 28 | it should "give a message with template configuration" in { 29 | val khermes = KhermesClientActor 30 | khermes.messageFeedback(Option("khermes"),Option("kafka"),None) shouldBe "Error: To start nodes is necessary to set template configuration." 31 | } 32 | it should "do not give a message because the configurations are OK" in { 33 | val khermes = KhermesClientActor 34 | khermes.messageFeedback(Option("khermes"),Option("kafka"), Option("template")) shouldBe "Your configuration is OK" 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/cluster/supervisor/NodeSupervisorActorTest.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.cluster.supervisor 12 | 13 | import akka.actor.Props 14 | import com.stratio.khermes.cluster.BaseActorTest 15 | import com.stratio.khermes.cluster.supervisor.NodeSupervisorActor.{Start, WorkerStatus} 16 | import com.stratio.khermes.commons.config.AppConfig 17 | import org.junit.runner.RunWith 18 | import org.scalatest.junit.JUnitRunner 19 | 20 | import scala.concurrent.duration._ 21 | 22 | //@RunWith(classOf[JUnitRunner]) 23 | class NodeSupervisorActorTest extends BaseActorTest { 24 | 25 | val nodeSupervisor = system.actorOf(Props(new NodeSupervisorActor()), "node-supervisor") 26 | 27 | val kafkaConfigContent = 28 | """ 29 | |kafka { 30 | | bootstrap.servers = "localhost:9092" 31 | | acks = "-1" 32 | | key.serializer = "org.apache.kafka.common.serialization.StringSerializer" 33 | | value.serializer = "org.apache.kafka.common.serialization.StringSerializer" 34 | |} 35 | """.stripMargin 36 | 37 | val khermesConfigContent = 38 | """ 39 | |khermes { 40 | | templates-path = "/tmp/khermes/templates" 41 | | topic = "chustas" 42 | | template-name = "chustasTemplate" 43 | | i18n = "ES" 44 | |} 45 | """.stripMargin 46 | 47 | val templateContent = 48 | """ 49 | |@import com.stratio.khermes.helpers.faker.Faker 50 | | 51 | |@(faker: Faker) 52 | |{ 53 | | "name" : "@(faker.Name.firstName)" 54 | |} 55 | """.stripMargin 56 | 57 | val avroContent = 58 | """ 59 | |{ 60 | | "type": "record", 61 | | "name": "myrecord", 62 | | "fields": 63 | | [ 64 | | {"name": "name", "type":"int"} 65 | | ] 66 | |} 67 | """.stripMargin 68 | 69 | "An WorkerSupervisorActor" should { 70 | "Start n threads of working kafka producers" in { 71 | within(10 seconds) { 72 | nodeSupervisor ! Start(Seq.empty, AppConfig(khermesConfigContent, kafkaConfigContent, templateContent)) 73 | expectMsg(WorkerStatus.Started) 74 | } 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/commons/config/AppConfigTest.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.commons.config 12 | 13 | import com.typesafe.scalalogging.LazyLogging 14 | import org.junit.runner.RunWith 15 | import org.scalatest.junit.JUnitRunner 16 | import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} 17 | 18 | @RunWith(classOf[JUnitRunner]) 19 | class AppConfigTest extends FlatSpec 20 | with Matchers 21 | with BeforeAndAfter 22 | with LazyLogging { 23 | 24 | val khermesConfig = 25 | """ 26 | |khermes { 27 | | templates-path = "/some/test/path" 28 | | template-name = "someTemplate" 29 | | topic = "someTopic" 30 | | i18n = "EN" 31 | |} 32 | """.stripMargin 33 | 34 | val jsonKafkaConfig = 35 | """ 36 | |kafka { 37 | | bootstrap.servers = "localhost:9092" 38 | | acks = "-1" 39 | | key.serializer = "org.apache.kafka.common.serialization.StringSerializer" 40 | | value.serializer = "org.apache.kafka.common.serialization.StringSerializer" 41 | |} 42 | """.stripMargin 43 | 44 | val avroKafkaConfig = 45 | """ 46 | |kafka { 47 | | bootstrap.servers = "localhost:9092" 48 | | acks = "-1" 49 | | key.serializer = "io.confluent.kafka.serializers.KafkaAvroSerializer" 50 | | value.serializer = "io.confluent.kafka.serializers.KafkaAvroSerializer" 51 | | schema.registry.url = "http://localhost:16803" 52 | |} 53 | """.stripMargin 54 | 55 | val wrongKhermesConfig = 56 | """ 57 | |khermes { 58 | | templates-path = "/some/test/path" 59 | | template-name = "someTemplate" 60 | | i18n = "EN" 61 | |} 62 | """.stripMargin 63 | 64 | val wrongKafkaConfig = 65 | """ 66 | |kafka { 67 | | bootstrap.servers = "localhost:9092" 68 | | acks = "-1" 69 | | key.serializer = "io.confluent.kafka.serializers.KafkaAvroSerializer" 70 | | value.serializer = "io.confluent.kafka.serializers.KafkaAvroSerializer" 71 | |} 72 | """.stripMargin 73 | 74 | val template = 75 | """ 76 | |@import com.stratio.khermes.helpers.generator.Khermes 77 | | 78 | |@(khermes: Khermes) 79 | |{ 80 | | "name" : "@(khermes.Name.firstName)" 81 | |} 82 | """.stripMargin 83 | 84 | val avroSchema = 85 | """ 86 | |{ 87 | | "type": "record", 88 | | "name": "myrecord", 89 | | "fields": 90 | | [ 91 | | {"name": "name", "type":"int"} 92 | | ] 93 | |} 94 | """.stripMargin 95 | 96 | val khermesConfigWithoutTimeOutRules = 97 | """ 98 | |khermes { 99 | | templates-path = "/some/test/path" 100 | | template-name = "someTemplate" 101 | | topic = "someTopic" 102 | | i18n = "EN" 103 | |} 104 | """.stripMargin 105 | 106 | val khermesConfigWithNumberOfEvents = 107 | """ 108 | |khermes { 109 | | templates-path = "/some/test/path" 110 | | template-name = "someTemplate" 111 | | topic = "someTopic" 112 | | i18n = "EN" 113 | | timeout-rules { 114 | | number-of-events: 1000 115 | | } 116 | |} 117 | """.stripMargin 118 | 119 | val khermesConfigWithDuration = 120 | """ 121 | |khermes { 122 | | templates-path = "/some/test/path" 123 | | template-name = "someTemplate" 124 | | topic = "someTopic" 125 | | i18n = "EN" 126 | | timeout-rules { 127 | | duration: 2 seconds 128 | | } 129 | |} 130 | """.stripMargin 131 | 132 | val khermesConfigWithStopRules = 133 | """ 134 | |khermes { 135 | | templates-path = "/some/test/path" 136 | | template-name = "someTemplate" 137 | | topic = "someTopic" 138 | | i18n = "EN" 139 | | stop-rules { 140 | | number-of-events: 500 141 | | } 142 | |} 143 | """.stripMargin 144 | 145 | "An KhermesConfig" should "parse a correct config when serializer is Json" in { 146 | val hc = AppConfig(khermesConfig, jsonKafkaConfig, template) 147 | checkCommonFields(hc) 148 | hc.configType should be(AppConfig.ConfigType.Json) 149 | hc.avroSchema should be(None) 150 | } 151 | 152 | it should "parse a correct config when the serializer is Avro" in { 153 | val hc = AppConfig(khermesConfig, avroKafkaConfig, template, Option(avroSchema)) 154 | checkCommonFields(hc) 155 | hc.configType should be(AppConfig.ConfigType.Avro) 156 | hc.avroSchema should be(Option(avroSchema)) 157 | } 158 | 159 | it should "throw an error when a mandatory field is not supplied when the serializer is JSON" in { 160 | a[AssertionError] should be thrownBy { 161 | AppConfig(wrongKhermesConfig, jsonKafkaConfig, template) 162 | } 163 | } 164 | 165 | it should "throw an error when a mandatory field is not supplied when the serializer is Avro" in { 166 | a[AssertionError] should be thrownBy { 167 | AppConfig(khermesConfig, wrongKafkaConfig, template) 168 | } 169 | } 170 | 171 | it should "return None when the parameter timeout-rules/number of events DOES NOT exist" in { 172 | val khermesConfig = AppConfig(khermesConfigWithoutTimeOutRules, avroKafkaConfig, template, Option(avroSchema)) 173 | val timeOutNumberOfEvents = khermesConfig.timeoutNumberOfEventsOption 174 | 175 | timeOutNumberOfEvents.isDefined should be(false) 176 | } 177 | 178 | it should "return Some when the parameter timeout-rules/number of events DOES exist" in { 179 | val khermesConfig = AppConfig(khermesConfigWithNumberOfEvents, avroKafkaConfig, template, Option(avroSchema)) 180 | val timeOutNumberOfEvents = khermesConfig.timeoutNumberOfEventsOption 181 | 182 | timeOutNumberOfEvents.isDefined should be(true) 183 | } 184 | 185 | it should "return None when the parameter timeout-rules/duration DOES NOT exist" in { 186 | val khermesConfig = AppConfig(khermesConfigWithoutTimeOutRules, avroKafkaConfig, template, Option(avroSchema)) 187 | val timeOutDuration = khermesConfig.timeoutNumberOfEventsDurationOption 188 | 189 | timeOutDuration.isDefined should be(false) 190 | } 191 | 192 | it should "return Some when the parameter timeout-rules/duration DOES exist" in { 193 | val khermesConfig = AppConfig(khermesConfigWithDuration, avroKafkaConfig, template, Option(avroSchema)) 194 | val timeOutDuration = khermesConfig.timeoutNumberOfEventsDurationOption 195 | 196 | timeOutDuration.isDefined should be(true) 197 | } 198 | 199 | it should "return None when the parameter stop-rules/number-of-events DOES NOT exist" in { 200 | val khermesConfig = AppConfig(khermesConfigWithoutTimeOutRules, avroKafkaConfig, template, Option(avroSchema)) 201 | val stopRulesNumberOfEvents = khermesConfig.stopNumberOfEventsOption 202 | 203 | stopRulesNumberOfEvents.isDefined should be(false) 204 | } 205 | 206 | it should "return Some when the parameter stop-rules/number-of-events DOES exist" in { 207 | val khermesConfig = AppConfig(khermesConfigWithStopRules, avroKafkaConfig, template, Option(avroSchema)) 208 | val stopRulesNumberOfEvents = khermesConfig.stopNumberOfEventsOption 209 | 210 | stopRulesNumberOfEvents.isDefined should be(true) 211 | } 212 | 213 | private[this] def checkCommonFields(hc: AppConfig): Unit 214 | 215 | = 216 | { 217 | hc.khermesI18n should be("EN") 218 | hc.templateContent should be(template) 219 | hc.templateName should be("someTemplate") 220 | hc.template should be(template) 221 | hc.topic should be("someTopic") 222 | } 223 | } 224 | -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/helpers/faker/FakerGeneratorTest.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.helpers 12 | 13 | import java.io.InputStream 14 | 15 | import com.stratio.khermes.commons.exceptions.KhermesException 16 | import com.stratio.khermes.helpers.faker.generators.MusicModel 17 | import com.stratio.khermes.helpers.faker.{Faker, FakerGenerator} 18 | import org.junit.runner.RunWith 19 | import org.scalatest.junit.JUnitRunner 20 | import org.scalatest.{EitherValues, FlatSpec, Matchers} 21 | 22 | @RunWith(classOf[JUnitRunner]) 23 | class FakerGeneratorTest extends FlatSpec with FakerGenerator with Matchers with EitherValues { 24 | 25 | override def name: String = "test" 26 | 27 | "FakeGenerator" should "return Left when a error occurs during parse process" in { 28 | parse("no-valid-name", "ES").left.value should be("Error loading invalid resource /locales/no-valid-name/ES") 29 | } 30 | 31 | it should "return Right when no errors occur during parse process" in { 32 | parse[Seq[MusicModel]]("music", "EN.json") should be('right) 33 | parse[Seq[MusicModel]]("music", "EN.json").right.value shouldBe a[Seq[_]] 34 | } 35 | 36 | it should "raise an exception when it gets a song that is corrupted" in { 37 | val khermesYY = Faker("YY") 38 | parseErrors(khermesYY.Music.musicModel).length should be(1) 39 | an[KhermesException] should be thrownBy khermesYY.Music.playedSong 40 | } 41 | 42 | it should "raise an exception when it gets a file with at least one song corrupted" in { 43 | val khermes = Faker() 44 | parseErrors(khermes.Music.musicModel).length should be(2) 45 | } 46 | 47 | it should "generates an element from a generic list" in { 48 | randomElementFromAList(List(1, 2, 3)) should contain oneOf(1, 2, 3) 49 | randomElementFromAList(List("a", "b", "c")) should contain oneOf("a", "b", "c") 50 | } 51 | 52 | it should "return nothing when a generic list is empty" in { 53 | randomElementFromAList(List.empty) should be(None) 54 | } 55 | 56 | it should "return a valid seq of files when path exists" in { 57 | getResources("music") should not be empty 58 | } 59 | 60 | it should "return an exception when path does not exist" in { 61 | intercept[KhermesException] { 62 | getResources("no-valid") 63 | } 64 | } 65 | 66 | it should "return a valid inputStream " in { 67 | getResource("music", "EN.json") shouldBe a[InputStream] 68 | } 69 | 70 | it should "return an exception when the resource does not exist" in { 71 | intercept[KhermesException] { 72 | getResource("no-valid-name", "EN.json") 73 | } 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/helpers/faker/generators/DatetimeGeneratorTest.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.helpers.faker.generators 12 | 13 | import java.security.InvalidParameterException 14 | 15 | import com.stratio.khermes.commons.exceptions.KhermesException 16 | import com.stratio.khermes.helpers.faker.Faker 17 | import org.joda.time.DateTime 18 | import org.joda.time.format.DateTimeFormat 19 | import org.junit.runner.RunWith 20 | import org.scalatest.junit.JUnitRunner 21 | import org.scalatest.{FlatSpec, Matchers} 22 | 23 | @RunWith(classOf[JUnitRunner]) 24 | class DatetimeGeneratorTest extends FlatSpec 25 | with Matchers { 26 | 27 | it should "generate a random date between two dates" in { 28 | val khermes = Faker() 29 | val startDate = new DateTime("1970-1-1") 30 | val endDate = new DateTime("2017-1-1") 31 | val randomDateString = khermes.Datetime.datetime(startDate, endDate) 32 | val randomDate = DateTime.parse(randomDateString) 33 | assert(startDate.compareTo(randomDate) * randomDate.compareTo(endDate) > 0) 34 | } 35 | 36 | it should "raise an exception if start Date is greater than end Date" in { 37 | val khermes = Faker() 38 | val startDate = new DateTime("2017-1-1") 39 | val endDate = new DateTime("1985-1-1") 40 | an[InvalidParameterException] should be thrownBy khermes.Datetime.datetime(startDate, endDate, None) 41 | } 42 | 43 | it should "generate a random date in a custom format" in { 44 | val khermes = Faker() 45 | val startDate = new DateTime("1970-1-1") 46 | val endDate = new DateTime("1985-1-1") 47 | val randomDateString = khermes.Datetime.datetime(startDate, endDate, Option("yyyy-MM-dd")) 48 | val randomDate = DateTime.parse(randomDateString) 49 | randomDateString shouldBe DateTimeFormat.forPattern(randomDateString.format("yyyy-MM-dd")).print(randomDate) 50 | } 51 | 52 | it should "generate a random date in a complex format" in { 53 | val khermes = Faker() 54 | val startDate = new DateTime("1970-1-1") 55 | val endDate = new DateTime("1985-1-1") 56 | val randomDateString = khermes.Datetime.datetime(startDate, endDate, Option("yyyy-MM-dd'T'HH:mm:ss.SSS")) 57 | val randomDate = DateTime.parse(randomDateString) 58 | randomDateString shouldBe DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS").print(randomDate) 59 | } 60 | 61 | it should "generate a random date with a bad format" in { 62 | val khermes = Faker() 63 | val startDate = new DateTime("1970-1-1") 64 | val endDate = new DateTime("1985-1-1") 65 | //scalastyle:off 66 | an[KhermesException] should be thrownBy khermes.Datetime.datetime(startDate, endDate, Option("Invalid format")) 67 | //scalastyle:on 68 | } 69 | 70 | "time" should "return a valid hour (0:0:0.000 - 23:59:59.999)" in { 71 | val khermes = Faker() 72 | val time = khermes.Datetime.time 73 | time shouldBe a[String] 74 | val timeAsString = time.split(":").toList 75 | timeAsString(0).toInt should (be >= 0 and be < 24) 76 | timeAsString(1).toInt should (be >= 0 and be < 60) 77 | timeAsString(2).split("\\.")(0).toInt should (be >= 0 and be < 60) 78 | timeAsString(2).split("\\.")(1).toInt should (be >= 0 and be < 1000) 79 | 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/helpers/faker/generators/EmailGeneratorTest.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.helpers.faker.generators 12 | 13 | import com.stratio.khermes.commons.exceptions.KhermesException 14 | import com.stratio.khermes.helpers.faker.Faker 15 | import org.junit.runner.RunWith 16 | import org.scalatest.junit.JUnitRunner 17 | import org.scalatest.{FlatSpec, Matchers} 18 | 19 | @RunWith(classOf[JUnitRunner]) 20 | class EmailGeneratorTest extends FlatSpec 21 | with Matchers { 22 | 23 | "Khermes" should "generate valid email address from a full name" in { 24 | val khermes = Faker("EN") 25 | val address = khermes.Email.address(" John Doe") 26 | val domain: String = address.split("@")(1) 27 | 28 | address should startWith("jdoe") 29 | khermes.Email.domains(khermes.Email.emailModel) should contain(domain) 30 | 31 | } 32 | 33 | 34 | "Khermes" should "fail when no domain exists" in { 35 | val khermes = Faker("XX") 36 | an[KhermesException] should be thrownBy khermes.Email.address("sample name") 37 | } 38 | 39 | "Khermes" should "fail when name is invalid" in { 40 | val khermes = Faker("EN") 41 | an[KhermesException] should be thrownBy khermes.Email.address(" ") 42 | } 43 | 44 | "Khermes" should "generate valid email using all locates" in { 45 | val khermes = Faker() 46 | val address = khermes.Email.address(" John Doe") 47 | val domain: String = address.split("@")(1) 48 | 49 | address should startWith("jdoe") 50 | khermes.Email.domains(khermes.Email.emailModel) should contain(domain) 51 | } 52 | 53 | 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/helpers/faker/generators/GeoGeneratorTest.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.helpers.faker.generators 12 | 13 | import com.stratio.khermes.commons.exceptions.KhermesException 14 | import com.stratio.khermes.helpers.faker.{FakerGenerator, Faker} 15 | import org.junit.runner.RunWith 16 | import org.scalatest.junit.JUnitRunner 17 | import org.scalatest.{FlatSpec, Matchers} 18 | 19 | @RunWith(classOf[JUnitRunner]) 20 | class GeoGeneratorTest extends FlatSpec with FakerGenerator with Matchers { 21 | 22 | override def name: String = "test" 23 | 24 | it should "generate valid locations: ES and US locales" in { 25 | 26 | val khermesES = Faker("ES") 27 | khermesES.Geo.geoModelList(khermesES.Geo.geoModel) should contain(khermesES.Geo.geolocation) 28 | 29 | val khermesUS = Faker("US") 30 | khermesUS.Geo.geoModelList(khermesUS.Geo.geoModel) should contain(khermesUS.Geo.geolocation) 31 | } 32 | 33 | it should "raise a NoSuchElementException when the locale is empty" in { 34 | val khermes = Faker("XX") 35 | an[KhermesException] should be thrownBy khermes.Geo.geolocation 36 | } 37 | 38 | it should "when you do not specify the locale try to use all the locales" in { 39 | val khermes = Faker() 40 | khermes.Geo.geoModelList(khermes.Geo.geoModel) should contain(khermes.Geo.geolocation) 41 | } 42 | 43 | it should "raise an exception when it gets a geolocation that not exists" in { 44 | val khermesFR = Faker("FR") 45 | an[KhermesException] should be thrownBy khermesFR.Geo.geolocation 46 | an[KhermesException] should be thrownBy khermesFR.Geo.city() 47 | an[KhermesException] should be thrownBy khermesFR.Geo.geolocationWithoutCity() 48 | } 49 | 50 | it should "generate a random city" in { 51 | val khermes = Faker() 52 | khermes.Geo.cityList(khermes.Geo.geoModel) should contain(khermes.Geo.city) 53 | } 54 | 55 | it should "generate a random geolocation without city" in { 56 | val khermes = Faker() 57 | khermes.Geo.geoWithoutCityList(khermes.Geo.geoModel) should contain(khermes.Geo.geolocationWithoutCity()) 58 | } 59 | 60 | it should "raise an exception when it gets a geolocation that is corrupted" in { 61 | val khermesYY = Faker("YY") 62 | parseErrors(khermesYY.Geo.geoModel).length should be(1) 63 | an[KhermesException] should be thrownBy khermesYY.Geo.geolocation 64 | } 65 | 66 | it should "raise an exception when it gets a file with at least one record corrupted" in { 67 | val khermes = Faker() 68 | parseErrors(khermes.Geo.geoModel).length should be(2) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/helpers/faker/generators/MusicGeneratorTest.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.helpers.faker.generators 12 | 13 | import com.stratio.khermes.commons.exceptions.KhermesException 14 | import com.stratio.khermes.helpers.faker.Faker 15 | import org.junit.runner.RunWith 16 | import org.scalatest.junit.JUnitRunner 17 | import org.scalatest.{FlatSpec, Matchers} 18 | 19 | @RunWith(classOf[JUnitRunner]) 20 | class MusicGeneratorTest extends FlatSpec 21 | with Matchers { 22 | 23 | 24 | "A Khermes" should "should generate valid music: with EN and ES locales" in { 25 | val khermesEN = Faker("EN") 26 | khermesEN.Music.getMusic(khermesEN.Music.musicModel) should contain(khermesEN.Music.playedSong) 27 | 28 | val khermesES = Faker("ES") 29 | khermesES.Music.getMusic(khermesES.Music.musicModel) should contain(khermesES.Music.playedSong) 30 | } 31 | 32 | it should "raise a NoSuchElementException when the music locale is empty" in { 33 | val khermes = Faker("XX") 34 | an[KhermesException] should be thrownBy khermes.Music.playedSong 35 | } 36 | 37 | it should "when you do not specify any locale try to use all the locales" in { 38 | val khermes = Faker() 39 | khermes.Music.getMusic(khermes.Music.musicModel) should contain(khermes.Music.playedSong) 40 | } 41 | 42 | it should "raise an exception when it gets a song that not exists" in { 43 | val khermesFR = Faker("FR") 44 | an[KhermesException] should be thrownBy khermesFR.Music.playedSong 45 | } 46 | 47 | "getMusic" should "return a seq with one music model" in { 48 | val generator = MusicGenerator("EN") 49 | generator.getMusic(Seq(Left("error"), Right(Seq(MusicModel("song", "artist", "album", "genre"))))) should be( 50 | Seq(MusicModel("song", "artist", "album", "genre"))) 51 | } 52 | 53 | "getMusic" should "return empty seq when no music model exists" in { 54 | val generator = MusicGenerator("EN") 55 | generator.getMusic(Seq(Left("error"))) should be(Seq()) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/helpers/faker/generators/NameGeneratorTest.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.helpers.faker.generators 12 | 13 | import java.util.NoSuchElementException 14 | 15 | import com.stratio.khermes.helpers.faker.Faker 16 | import org.junit.runner.RunWith 17 | import org.scalatest.junit.JUnitRunner 18 | import org.scalatest.{FlatSpec, Matchers} 19 | 20 | @RunWith(classOf[JUnitRunner]) 21 | class NameGeneratorTest extends FlatSpec 22 | with Matchers { 23 | 24 | 25 | 26 | 27 | it should "generate valid names: firstName lastName with EN and ES locales" in { 28 | val khermesEN = Faker() 29 | val fullNameEN = khermesEN.Name.fullName 30 | fullNameEN should fullyMatch regex """[a-zA-Z]+ [a-zA-Z]+""" 31 | khermesEN.Name.firstNames(khermesEN.Name.nameModel) should contain(fullNameEN.split(" ")(0)) 32 | khermesEN.Name.lastNames(khermesEN.Name.nameModel) should contain(fullNameEN.split(" ")(1)) 33 | 34 | val khermesES = Faker("ES") 35 | val fullNameES = khermesES.Name.fullName 36 | fullNameES should fullyMatch regex """[a-zA-Z]+ [a-zA-Z]+""" 37 | khermesES.Name.firstNames(khermesEN.Name.nameModel) should contain(fullNameES.split(" ")(0)) 38 | khermesES.Name.lastNames(khermesEN.Name.nameModel) should contain(fullNameES.split(" ")(1)) 39 | } 40 | 41 | it should "generate valid middle names: firstName firstName with EN and ES locales" in { 42 | val khermesEN = Faker() 43 | val middleNameEN = khermesEN.Name.middleName 44 | middleNameEN should fullyMatch regex """[a-zA-Z]+ [a-zA-Z]+""" 45 | khermesEN.Name.firstNames(khermesEN.Name.nameModel) should contain(middleNameEN.split(" ")(0)) 46 | khermesEN.Name.firstNames(khermesEN.Name.nameModel) should contain(middleNameEN.split(" ")(1)) 47 | 48 | val khermesES = Faker("ES") 49 | val middleNameES = khermesES.Name.middleName 50 | middleNameES should fullyMatch regex """[a-zA-Z]+ [a-zA-Z]+""" 51 | khermesES.Name.firstNames(khermesEN.Name.nameModel) should contain(middleNameES.split(" ")(0)) 52 | khermesES.Name.firstNames(khermesEN.Name.nameModel) should contain(middleNameES.split(" ")(1)) 53 | } 54 | 55 | it should "raise an exception when it gets a firstName/lastName and firstNames/lastNames are empty in the locale" in { 56 | val khermes = Faker("XX") 57 | //scalastyle:off 58 | an[NoSuchElementException] should be thrownBy khermes.Name.firstName() 59 | an[NoSuchElementException] should be thrownBy khermes.Name.lastName() 60 | //scalastyle:on 61 | } 62 | 63 | it should "raise an exception when it tries to load a locale that don't exist" in { 64 | //scalastyle:off 65 | val model = Faker("XY").Name.nameModel 66 | //scalastyle:on 67 | model.map(_.left.get should equal(s"Error loading invalid resource /locales/name/XY.json")) 68 | } 69 | 70 | "A Khermes" should "generates random firstNames and lastNames with EN and ES locales" in { 71 | val khermesEN = Faker() 72 | khermesEN.Name.firstNames(khermesEN.Name.nameModel) should contain(khermesEN.Name.firstName) 73 | khermesEN.Name.lastNames(khermesEN.Name.nameModel) should contain(khermesEN.Name.lastName) 74 | 75 | val khermesES = Faker("ES") 76 | khermesES.Name.firstNames(khermesEN.Name.nameModel) should contain(khermesES.Name.firstName) 77 | khermesES.Name.lastNames(khermesEN.Name.nameModel) should contain(khermesES.Name.lastName) 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/helpers/faker/generators/NumberGeneratorTest.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.helpers.faker.generators 12 | 13 | import java.security.InvalidParameterException 14 | 15 | import com.stratio.khermes.commons.exceptions.KhermesException 16 | import com.stratio.khermes.helpers.faker.Faker 17 | import org.junit.runner.RunWith 18 | import org.scalacheck.Prop.forAll 19 | import org.scalatest.junit.JUnitRunner 20 | import org.scalatest.{FlatSpec, Matchers} 21 | 22 | @RunWith(classOf[JUnitRunner]) 23 | class NumberGeneratorTest extends FlatSpec 24 | with Matchers { 25 | 26 | it should "generate a random integer of 0 digit give it 0" in { 27 | val khermesNum = Faker("") 28 | khermesNum.Number.number(0) shouldBe 0 29 | } 30 | 31 | it should "generate a random integer when it passed the number of digit" in { 32 | val khermesNum = Faker("") 33 | forAll { (n: Int) => 34 | numberOfDigitsFromANumber(khermesNum.Number.number(n)) == n 35 | } 36 | } 37 | 38 | it should "generate a random integer when it passed the number of digit and the sign" in { 39 | val khermesNum = Faker("") 40 | //scalastyle:off 41 | forAll { (n: Int) => 42 | khermesNum.Number.number(n, Positive) > 0 43 | } 44 | forAll { (n: Int) => 45 | khermesNum.Number.number(n, Negative) < 0 46 | } 47 | //scalastyle:on 48 | } 49 | 50 | it should "throw an InvalidParameterException when a negative digit is passed or greater than the VAL_MAX" in { 51 | val khermesNum = Faker("") 52 | //scalastyle:off 53 | an[InvalidParameterException] should be thrownBy khermesNum.Number.number(-2) 54 | an[InvalidParameterException] should be thrownBy khermesNum.Number.number(500) 55 | an[InvalidParameterException] should be thrownBy khermesNum.Number.decimal(-2) 56 | an[InvalidParameterException] should be thrownBy khermesNum.Number.decimal(2, -2) 57 | an[InvalidParameterException] should be thrownBy khermesNum.Number.decimal(2, 11) 58 | //scalastyle:on 59 | } 60 | it should "generate a random decimal of 0 digit give it 0.0" in { 61 | val khermesNum = Faker("") 62 | khermesNum.Number.decimal(0) shouldBe 0.0 63 | khermesNum.Number.decimal(0, 0) shouldBe 0.0 64 | } 65 | 66 | it should "generate a random decimal when it passed the number of digit" in { 67 | val khermesNum = Faker("") 68 | //scalastyle:off 69 | numberOfDigitsFromANumber(khermesNum.Number.decimal(2)) shouldBe 4 70 | numberOfDigitsFromANumber(khermesNum.Number.decimal(2, 4)) shouldBe 6 71 | numberOfDigitsFromANumber(khermesNum.Number.decimal(0, 2)) shouldBe 3 72 | numberOfDigitsFromANumber(khermesNum.Number.decimal(9, 9)) shouldBe 18 73 | numberOfDigitsFromANumber(khermesNum.Number.decimal(9, 0)) shouldBe 10 74 | numberOfDigitsFromANumber(khermesNum.Number.decimal(8, Positive)) shouldBe 16 75 | numberOfDigitsFromANumber(khermesNum.Number.decimal(7, Negative)) shouldBe 14 76 | numberOfDigitsFromANumber(khermesNum.Number.decimal(9, 7, Positive)) shouldBe 16 77 | numberOfDigitsFromANumber(khermesNum.Number.decimal(2, 1, Negative)) shouldBe 3 78 | //scalastyle:on 79 | } 80 | 81 | it should "throw an InvalidParameterException when pass an sign that is null" in { 82 | val khermesNum = Faker("") 83 | //scalastyle:off 84 | an[KhermesException] should be thrownBy khermesNum.Number.number(2, null) 85 | //scalastyle:on 86 | } 87 | 88 | it should "generate a random integer when it passed the range " in { 89 | val khermesNum = Faker("") 90 | forAll { (n: Int, m: Int) => 91 | khermesNum.Number.numberInRange(n, m) <= m && khermesNum.Number.numberInRange(n, m) >= n 92 | } 93 | //scalastyle:off 94 | numberOfDigitsFromANumber(khermesNum.Number.numberInRange(1, 9)) shouldBe 1 95 | numberOfDigitsFromANumber(khermesNum.Number.numberInRange(99, 10)) shouldBe 2 96 | numberOfDigitsFromANumber(khermesNum.Number.numberInRange(100, 999)) shouldBe 3 97 | //scalastyle:on 98 | } 99 | 100 | it should "generate a random decimal between the range" in { 101 | val khermesNum = Faker("") 102 | val decimalInRange = khermesNum.Number.decimalInRange(1, 2).toInt 103 | 104 | decimalInRange should be (1) 105 | } 106 | 107 | it should "generate a random decimal when it passed the range " in { 108 | val khermesNum = Faker("") 109 | forAll { (n: Int, m: Int) => 110 | khermesNum.Number.decimalInRange(n, m) <= m && khermesNum.Number.decimalInRange(n, m) >= n 111 | } 112 | } 113 | 114 | it should "generate a random rating number less than the number passed as parameter" in { 115 | val khermesNum = Faker("") 116 | 117 | val rating = khermesNum.Number.rating(1) 118 | rating shouldBe 0 119 | } 120 | 121 | it should "generate a random rating for all n's" in { 122 | val khermesNum = Faker("") 123 | 124 | forAll { (n: Int) => 125 | khermesNum.Number.rating(n) < n 126 | } 127 | } 128 | 129 | /** 130 | * Returns length of a Integer element. 131 | * @param n number to calculate length. 132 | * @return size of the integer. 133 | */ 134 | def numberOfDigitsFromANumber(n: Int): Int = if (n == 0) 1 else math.log10(math.abs(n)).toInt + 1 135 | 136 | /** 137 | * Returns length of a BigDecimal element. 138 | * @param n number to calculate length. 139 | * @return size of the BigDecimal. 140 | */ 141 | def numberOfDigitsFromANumber(n: BigDecimal): Int = n.abs.toString.length - 1 142 | } 143 | -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/helpers/twirl/TwirlHelperTest.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.helpers.twirl 12 | 13 | import com.stratio.khermes.Khermes 14 | import com.stratio.khermes.helpers.faker.Faker 15 | import com.stratio.khermes.helpers.twirl.TwirlHelper.CompilationError 16 | import com.typesafe.scalalogging.LazyLogging 17 | import org.junit.runner.RunWith 18 | import org.scalatest.junit.JUnitRunner 19 | import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} 20 | import play.twirl.api.Txt 21 | 22 | @RunWith(classOf[JUnitRunner]) 23 | class TwirlHelperTest extends FlatSpec 24 | with Matchers 25 | with BeforeAndAfter 26 | with LazyLogging { 27 | 28 | implicit val config = com.stratio.khermes.commons.implicits.AppImplicits.config 29 | 30 | before { 31 | Khermes.createPaths 32 | } 33 | 34 | "A TwirlHelper" should "compile a simple template without object injection" in { 35 | val template = "Hello world" 36 | val result = cleanContent(TwirlHelper.template[() => Txt](template, "templateTest").static().toString()) 37 | result should be("Hello world") 38 | } 39 | 40 | it should "compile a template and inject a string" in { 41 | val template = 42 | """ 43 | |@(name: String) 44 | |Hello @name 45 | """.stripMargin 46 | val result = cleanContent(TwirlHelper.template[(String) => Txt](template, "templateTest").static("Neo").toString()) 47 | result should be("Hello Neo") 48 | } 49 | 50 | it should "compile a template and inject an khermes helper" in { 51 | val template = 52 | """ 53 | |@(faker: Faker) 54 | |Hello @(faker.Name.firstName) 55 | """.stripMargin 56 | 57 | val khermes = new Faker("EN") 58 | 59 | val result = cleanContent( 60 | TwirlHelper.template[(Faker) => Txt](template, "templateTest").static(khermes).toString()) 61 | result should fullyMatch regex """Hello [a-zA-Z]+""" 62 | } 63 | 64 | it should "throw an error when the template is wrong" in { 65 | val template = "Hello @(error)" 66 | //scalastyle:off 67 | the[CompilationError] thrownBy (TwirlHelper.template[() => Txt] 68 | (template, "templateTest").static().toString()) should have('line (1), 'column (8)) 69 | //scalastyle:on 70 | } 71 | 72 | /** 73 | * Cleans the content deleting return carriages an not necessary spaces. 74 | * @param content with the original content. 75 | * @return a sanitized content. 76 | */ 77 | def cleanContent(content: String): String = content.replace("\n", "").replaceAll(" +", "") 78 | } 79 | -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/persistence/dao/ZkDAOTest.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.persistence.dao 12 | 13 | import com.stratio.khermes.commons.exceptions.KhermesException 14 | import com.stratio.khermes.utils.EmbeddedServersUtils 15 | import org.junit.runner.RunWith 16 | import org.scalatest.junit.JUnitRunner 17 | import org.scalatest.{FlatSpec, Matchers} 18 | 19 | @RunWith(classOf[JUnitRunner]) 20 | class ZkDAOTest extends FlatSpec 21 | with Matchers 22 | with EmbeddedServersUtils{ 23 | withEmbeddedZookeeper() { zookeeperServer => 24 | val khermesConfigDAO = new ZkDAO(Option(zookeeperServer.getConnectString)) 25 | 26 | "An KhermesConfig" should "be save in zookeeper path and could be loaded" in { 27 | khermesConfigDAO.create("testZkPath", "myConfig") 28 | val data = khermesConfigDAO.read("testZkPath") 29 | khermesConfigDAO.delete("stratio") 30 | data shouldBe "myConfig" 31 | } 32 | "An KhermesConfig" should "be updated when we save over an existing config" in { 33 | khermesConfigDAO.create("testZkPath", "myConfig") 34 | khermesConfigDAO.create("testZkPath", "myConfig2") 35 | val data = khermesConfigDAO.read("testZkPath") 36 | khermesConfigDAO.delete("stratio") 37 | data shouldBe "myConfig2" 38 | } 39 | "An KhermesConfig" should "have a way to obtain the list of configs" in { 40 | khermesConfigDAO.create("a/b/c","config1") 41 | khermesConfigDAO.create("a/b/d","config2") 42 | khermesConfigDAO.list("a/b") shouldBe "c\nd" 43 | } 44 | 45 | it should "raise an exception when it save or load a config in a path that does not exists" in { 46 | an[KhermesException] should be thrownBy khermesConfigDAO.read("") 47 | an[KhermesException] should be thrownBy khermesConfigDAO.create("", "config") 48 | } 49 | } 50 | } -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/persistence/kafka/KafkaClientTest.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.persistence.kafka 12 | 13 | import java.util.UUID 14 | 15 | import com.stratio.khermes.utils.EmbeddedServersUtils 16 | import com.typesafe.scalalogging.LazyLogging 17 | import org.junit.runner.RunWith 18 | import org.scalatest.junit.JUnitRunner 19 | import org.scalatest.{FlatSpec, Matchers} 20 | 21 | @RunWith(classOf[JUnitRunner]) 22 | class KafkaClientTest extends FlatSpec 23 | with Matchers 24 | with LazyLogging 25 | with EmbeddedServersUtils{ 26 | 27 | val Message = "testMessage" 28 | val Topic = s"topic-${UUID.randomUUID().toString}" 29 | val NumberOfMessages = 3 30 | 31 | "A KafkaClient" should "parse the configuration" in { 32 | withEmbeddedKafkaServer(Seq(Topic)) { kafkaServer => 33 | withKafkaClient[Object](kafkaServer) { kafkaClient => 34 | Option(kafkaClient.parseProperties().getProperty("bootstrap.servers")) should not be (None) 35 | } 36 | } 37 | } 38 | 39 | it should "produce messages in a topic" in { 40 | withEmbeddedKafkaServer(Seq(Topic)) { kafkaServer => 41 | withKafkaClient[Object](kafkaServer) { kafkaClient => 42 | (1 to NumberOfMessages).foreach(_ => { 43 | kafkaClient.send(Topic, Message) 44 | }) 45 | kafkaClient.producer.flush() 46 | kafkaClient.producer.close() 47 | } 48 | } 49 | } 50 | } -------------------------------------------------------------------------------- /src/test/scala/com/stratio/khermes/utils/EmbeddedServersUtils.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * © 2017 Stratio Big Data Inc., Sucursal en España. 3 | * 4 | * This software is licensed under the Apache 2.0. 5 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 6 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 7 | * See the terms of the License for more details. 8 | * 9 | * SPDX-License-Identifier: Apache-2.0. 10 | */ 11 | package com.stratio.khermes.utils 12 | 13 | import java.io.File 14 | import java.net.ServerSocket 15 | import java.util.Properties 16 | 17 | import com.stratio.khermes.persistence.kafka.KafkaClient 18 | import com.typesafe.config.ConfigFactory 19 | import com.typesafe.scalalogging.LazyLogging 20 | import kafka.server.{KafkaConfig, KafkaServer} 21 | import kafka.utils.{SystemTime, TestUtils} 22 | import org.apache.curator.test.TestingServer 23 | import org.apache.kafka.clients.producer.KafkaProducer 24 | import org.apache.kafka.common.protocol.SecurityProtocol 25 | import org.junit.rules.TemporaryFolder 26 | 27 | import scala.util.Try 28 | 29 | trait EmbeddedServersUtils extends LazyLogging { 30 | type TopicName = String 31 | val zookeeperConnectString = "127.0.0.1:2181" 32 | val tmpFolder = new TemporaryFolder() 33 | tmpFolder.create() 34 | val logDir = tmpFolder.newFolder("kafkatest") 35 | val loggingEnabled = true 36 | 37 | def withEmbeddedKafkaServer(topicsToBeCreated: Seq[TopicName])(function: KafkaServer => Any): Unit = { 38 | withEmbeddedZookeeper() { zookeeperServer => 39 | zookeeperServer.start 40 | val kafkaConfig = new KafkaConfig(kafkaConfiguration(logDir, zookeeperServer.getConnectString), loggingEnabled) 41 | 42 | logger.debug("Starting embedded Kafka broker (with log.dirs={} and ZK ensemble at {}) ...", 43 | logDir, zookeeperConnectString) 44 | 45 | val kafkaServer = TestUtils.createServer(kafkaConfig, SystemTime) 46 | Try { 47 | kafkaServer.startup 48 | val brokerList = 49 | s"""${kafkaServer.config.hostName}:${ 50 | Integer.toString(kafkaServer.boundPort(SecurityProtocol.PLAINTEXT)) 51 | }""" 52 | 53 | logger.debug("Startup of embedded Kafka broker at {} completed (with ZK ensemble at {}) ...", 54 | brokerList, zookeeperConnectString) 55 | 56 | function(kafkaServer) 57 | } 58 | kafkaServer.shutdown 59 | zookeeperServer.stop 60 | } 61 | } 62 | 63 | 64 | def withEmbeddedZookeeper()(function: TestingServer => Any): Unit = { 65 | function(new TestingServer(-1)) 66 | } 67 | 68 | def withKafkaProducer[V](kafkaServer: KafkaServer)(testFunction: KafkaProducer[String, V] => Any): Unit = { 69 | val props = kafkaServer.config.originals 70 | val producer: KafkaProducer[String, V] = new KafkaProducer(props) 71 | testFunction(producer) 72 | } 73 | 74 | def withKafkaClient[V](kafkaServer: KafkaServer)(function: KafkaClient[V] => Any): Unit = { 75 | val kafkaClient = new KafkaClient[V](ConfigFactory.parseMap(kafkaServer.config.originals)) 76 | function(kafkaClient) 77 | } 78 | 79 | //TODO: Accept initial config parameter (specific traits) 80 | private def kafkaConfiguration(logDir: File, zkConnectString: String) = { 81 | val kafkaConfig = new Properties() 82 | val randomPort = getRandomPort.toString 83 | kafkaConfig.put(KafkaConfig.ZkConnectProp, zkConnectString) 84 | kafkaConfig.put(KafkaConfig.BrokerIdProp, "0") 85 | kafkaConfig.put(KafkaConfig.HostNameProp, "127.0.0.1") 86 | kafkaConfig.put(KafkaConfig.PortProp, randomPort) 87 | kafkaConfig.put(KafkaConfig.NumPartitionsProp, "1") 88 | kafkaConfig.put(KafkaConfig.AutoCreateTopicsEnableProp, "true") 89 | kafkaConfig.put(KafkaConfig.MessageMaxBytesProp, "1000000") 90 | kafkaConfig.put(KafkaConfig.ControlledShutdownEnableProp, "true") 91 | kafkaConfig.put("kafka.bootstrap.servers", "127.0.0.1:" + randomPort) 92 | kafkaConfig.put("kafka.key.serializer", "org.apache.kafka.common.serialization.StringSerializer") 93 | kafkaConfig.put("kafka.value.serializer", "org.apache.kafka.common.serialization.StringSerializer") 94 | kafkaConfig.setProperty(KafkaConfig.LogDirProp, logDir.getAbsolutePath) 95 | //effectiveConfig.putAll(initialConfig); 96 | kafkaConfig 97 | } 98 | 99 | private def openSocket: ServerSocket = Try { 100 | new ServerSocket(0) 101 | }.recoverWith { case _: Throwable => Try(openSocket) }.get 102 | 103 | private def closeSocket(socket: ServerSocket): Unit = Try { 104 | socket.close() 105 | }.recoverWith { case _: Throwable => Try(closeSocket(socket)) } 106 | 107 | private def getRandomPort: Int = { 108 | val socket = openSocket 109 | val port = socket.getLocalPort 110 | closeSocket(socket) 111 | port 112 | } 113 | 114 | } 115 | --------------------------------------------------------------------------------