├── .gitignore ├── .scalafmt.conf ├── README.md ├── build.sbt ├── ddata ├── scripts │ └── create-nodes-gcp.sh └── src │ └── main │ ├── resources │ └── application.conf │ └── scala │ └── com │ └── lightbend │ └── akka │ └── bench │ └── ddata │ ├── DDataBenchmarkCoordinator.scala │ ├── DDataHost.scala │ ├── DistributedDataBenchmark.scala │ └── HttpApi.scala ├── infra ├── .gitignore ├── .ruby-version ├── Berksfile ├── Berksfile.lock ├── Gemfile ├── Gemfile.lock ├── README.md ├── benchmark-run-ddata.sh ├── benchmark-run-sharding-init-cassandra.sh ├── data_bags │ └── .gitkeep ├── environments │ └── .gitkeep ├── etc-hosts-akka-nodes-print.sh ├── etc-hosts-aws-re.sh ├── gcloud-clone-akka-sharding-nodes.sh ├── gcloud-new-akka-node.sh ├── gcloud-new-cassandra-nodes.sh ├── internal-ips.tmp ├── jdk-9.0.1_linux-x64_bin.tar.gz ├── littlechef.cfg-example ├── make-node-files-from-gcloud.sh ├── make-re-node-files-from-aws.sh ├── nodes │ ├── akka-cassandra-01.json │ ├── akka-cassandra-02.json │ ├── akka-cassandra-03.json │ ├── akka-cassandra-04.json │ ├── akka-cassandra-05.json │ ├── akka-cassandra-06.json │ ├── akka-cassandra-07.json │ ├── akka-cassandra-08.json │ ├── akka-cassandra-09.json │ ├── akka-cassandra-10.json │ ├── akka-cassandra-11.json │ ├── akka-cassandra-12.json │ ├── akka-cassandra-13.json │ ├── akka-cassandra-14.json │ ├── akka-cassandra-15.json │ ├── akka-cassandra-16.json │ ├── akka-cassandra-17.json │ ├── akka-cassandra-18.json │ ├── akka-cassandra-19.json │ ├── akka-cassandra-20.json │ ├── akka-cassandra.json.template │ ├── akka-node-002.json │ ├── akka-node-003.json │ ├── akka-node-010.json │ ├── akka-node-011.json │ ├── akka-node-012.json │ ├── akka-node-013.json │ ├── akka-node-014.json │ ├── akka-node-015.json │ ├── akka-node.json.template │ ├── akka-sharding-001.json │ ├── example-aws-node.json │ ├── moxie-a0.json │ ├── moxie-a1.json │ ├── moxie-a2.json │ ├── moxie-a3.json │ ├── moxie-a4.json │ ├── moxie-a5.json │ ├── moxie-a6.json │ ├── moxie-a7.json │ ├── moxie-a8.json │ ├── re-akka-eucentral-1a.json │ ├── re-akka-eucentral-1b.json │ ├── re-akka-eucentral-1c.json │ ├── re-akka-euwest-1a.json │ ├── re-akka-euwest-1b.json │ ├── re-akka-euwest-1c.json │ ├── re-akka.json.template │ ├── re-cassandra-eucentral-1a.json │ ├── re-cassandra-eucentral-1b.json │ ├── re-cassandra-eucentral-1c.json │ ├── re-cassandra-euwest-1a.json │ ├── re-cassandra-euwest-1b.json │ ├── re-cassandra-euwest-1c.json │ └── re-cassandra.json.template ├── prepare-all.sh ├── roles │ ├── akka-re.json │ ├── akka_team.json │ ├── benchmark-ddata.json │ ├── benchmark-sharding.json │ ├── benchmark.json │ ├── cassandra-re.json │ ├── cassandra.json │ ├── gce-tools.json │ ├── jenkins_master.json │ └── jenkins_worker.json ├── site-cookbooks │ ├── ab │ │ ├── .gitignore │ │ ├── .kitchen.yml │ │ ├── Berksfile │ │ ├── CHANGELOG.md │ │ ├── Gemfile │ │ ├── LICENSE │ │ ├── README.md │ │ ├── Thorfile │ │ ├── Vagrantfile │ │ ├── chefignore │ │ ├── metadata.json │ │ ├── metadata.rb │ │ └── recipes │ │ │ └── default.rb │ ├── akka-linux-tweaks │ │ ├── CHANGELOG.md │ │ ├── README.md │ │ ├── files │ │ │ └── default │ │ │ │ └── startup-script-akka-tweaks.sh │ │ ├── metadata.json │ │ ├── metadata.rb │ │ ├── recipes │ │ │ └── default.rb │ │ └── templates │ │ │ └── default │ │ │ └── akka.conf.rb │ ├── akka-multi-dc │ │ ├── .gitignore │ │ ├── CHANGELOG.md │ │ ├── README.md │ │ ├── attributes │ │ │ └── default.rb │ │ ├── metadata.json │ │ ├── metadata.rb │ │ ├── recipes │ │ │ └── default.rb │ │ └── templates │ │ │ └── default │ │ │ └── application.conf.erb │ ├── akka-user │ │ ├── .gitignore │ │ ├── CHANGELOG.md │ │ ├── README.md │ │ ├── metadata.json │ │ ├── metadata.rb │ │ └── recipes │ │ │ └── default.rb │ ├── cassandra-linux-tweaks │ │ ├── CHANGELOG.md │ │ ├── README.md │ │ ├── files │ │ │ └── default │ │ │ │ └── startup-script-cassandra-tweaks.sh │ │ ├── metadata.json │ │ ├── metadata.rb │ │ ├── recipes │ │ │ └── default.rb │ │ └── templates │ │ │ └── default │ │ │ └── security-cassandra.conf │ ├── cassandra-manual │ │ ├── CHANGELOG.md │ │ ├── README.md │ │ ├── attributes │ │ │ └── default.rb │ │ ├── metadata.json │ │ ├── metadata.rb │ │ ├── recipes │ │ │ └── default.rb │ │ └── templates │ │ │ └── default │ │ │ └── cassandra.yaml.rb │ ├── clone-akka-repos │ │ ├── .gitignore │ │ ├── CHANGELOG.md │ │ ├── README.md │ │ ├── metadata.json │ │ ├── metadata.rb │ │ └── recipes │ │ │ └── default.rb │ ├── docker_simple │ │ ├── .gitignore │ │ ├── .kitchen.yml │ │ ├── Berksfile │ │ ├── CHANGELOG.md │ │ ├── Gemfile │ │ ├── LICENSE │ │ ├── README.md │ │ ├── Thorfile │ │ ├── Vagrantfile │ │ ├── chefignore │ │ ├── metadata.json │ │ ├── metadata.rb │ │ └── recipes │ │ │ └── default.rb │ ├── gce-logging-agent │ │ ├── CHANGELOG.md │ │ ├── README.md │ │ ├── metadata.json │ │ ├── metadata.rb │ │ ├── recipes │ │ │ └── default.rb │ │ └── templates │ │ │ └── default │ │ │ └── akka.conf.rb │ ├── java-manual │ │ ├── CHANGELOG.md │ │ ├── README.md │ │ ├── metadata.json │ │ ├── metadata.rb │ │ └── recipes │ │ │ └── default.rb │ ├── perf │ │ ├── .gitignore │ │ ├── .kitchen.yml │ │ ├── Berksfile │ │ ├── CHANGELOG.md │ │ ├── Gemfile │ │ ├── LICENSE │ │ ├── README.md │ │ ├── Thorfile │ │ ├── Vagrantfile │ │ ├── chefignore │ │ ├── metadata.json │ │ ├── metadata.rb │ │ └── recipes │ │ │ └── default.rb │ ├── sbt-manual │ │ ├── .gitignore │ │ ├── CHANGELOG.md │ │ ├── README.md │ │ ├── metadata.json │ │ ├── metadata.rb │ │ └── recipes │ │ │ └── default.rb │ ├── sbt_credentials │ │ ├── CHANGELOG.md │ │ ├── README.md │ │ ├── metadata.json │ │ ├── metadata.rb │ │ └── recipes │ │ │ └── default.rb │ ├── update-hosts │ │ ├── CHANGELOG.md │ │ ├── README.md │ │ ├── files │ │ │ └── default │ │ │ │ ├── .gitkeep │ │ │ │ └── multi-node-test.hosts │ │ ├── metadata.json │ │ ├── metadata.rb │ │ ├── recipes │ │ │ └── default.rb │ │ └── templates │ │ │ └── default │ │ │ ├── multi-node-test.hosts │ │ │ ├── root-application.conf.rb │ │ │ ├── run-latency-benchmark.sh.rb │ │ │ ├── run-max-throughput-benchmark.sh.rb │ │ │ ├── run-multi-dc-test.sh.rb │ │ │ ├── run-multinode-benchmark.sh.rb │ │ │ ├── run-sharding-scalability-benchmark.sh.rb │ │ │ └── run-sharding-start-latency-benchmark.sh.rb │ └── wrk2 │ │ ├── .gitignore │ │ ├── .kitchen.yml │ │ ├── Berksfile │ │ ├── CHANGELOG.md │ │ ├── Gemfile │ │ ├── LICENSE │ │ ├── README.md │ │ ├── Thorfile │ │ ├── Vagrantfile │ │ ├── chefignore │ │ ├── metadata.json │ │ ├── metadata.rb │ │ └── recipes │ │ └── default.rb ├── ssh-config ├── update-multi-node-test.hosts-file.sh └── update-multinode-hosts.sh ├── lightbend.sbt ├── multidc ├── .gitignore ├── README.md ├── build.sbt ├── generate-writes.sh ├── kill-multi-dc-test-remote.sh ├── logs-speculative-writes.log ├── members.json ├── network-fleaky-node.sh ├── network-split-cassandra.sh ├── network-split-remoting.sh ├── nodes-bash-exports.sh ├── nodes-cassandra-bash-exports.sh ├── notes │ ├── cpu7c.png │ ├── cpu7w.png │ ├── jmx10.png │ ├── jmx12.png │ ├── jmx13.png │ ├── jmx21.png │ ├── jmx28.png │ ├── jmx29.png │ ├── jmx7.png │ ├── jmx8.png │ └── multi-dc-notes.md ├── project │ └── build.properties ├── rsync-local-src-to-akka-nodes.sh ├── run-multi-dc-test-remote.sh ├── run-multi-dc-test.sh └── src │ └── main │ ├── resources │ ├── application.conf │ └── logback.xml │ └── scala │ └── com │ └── lightbend │ └── multidc │ ├── HttpApi.scala │ ├── Incrementor.scala │ ├── ReplicatedCounter.scala │ ├── ReplicatedEntityApp.scala │ └── ReplicatedIntrospector.scala ├── nodes ├── re-akka-euwest-1a.json ├── re-cassandra-euwest-1a.json ├── re-cassandra-euwest-1b.json └── re-cassandra-euwest-1c.json ├── project ├── build.properties └── plugins.sbt ├── pubsub ├── README.md └── src │ └── main │ ├── resources │ └── application.conf │ └── scala │ └── com │ └── lightbend │ └── akka │ └── bench │ └── pubsub │ ├── BenchmarkCoordinator.scala │ ├── BenchmarkSession.scala │ ├── HttpApi.scala │ ├── PubSubBenchmark.scala │ ├── PubSubHost.scala │ ├── PubSubHostSession.scala │ ├── Publisher.scala │ ├── Subscriber.scala │ └── package.scala ├── scp-benchmark.sh ├── scp-sharding.sh └── sharding └── src └── main ├── resources └── application.conf └── scala └── com └── lightbend └── akka └── bench └── sharding ├── BenchSettings.scala ├── BenchmarkConfig.scala ├── InitCassandraApp.scala ├── latency ├── LatencyBenchEntity.scala ├── PersistenceHistograms.scala ├── PingLatencyCoordinator.scala └── ShardingStartLatencyApp.scala └── scalability ├── ActorCountingBenchmarkMaster.scala ├── ActorCountingEntity.scala └── ShardingActorCountScalabilityApp.scala /.gitignore: -------------------------------------------------------------------------------- 1 | *.pem 2 | 3 | log 4 | 5 | *# 6 | *.iml 7 | *.ipr 8 | *.iws 9 | *.pyc 10 | *.tm.epoch 11 | *.vim 12 | */project/boot 13 | */project/build/target 14 | */project/project.target.config-classes 15 | *~ 16 | .#* 17 | .*.swp 18 | .DS_Store 19 | .cache 20 | .cache 21 | .classpath 22 | .codefellow 23 | .ensime* 24 | .eprj 25 | .history 26 | .idea 27 | .manager 28 | .multi-jvm 29 | .project 30 | .scala_dependencies 31 | .scalastyle 32 | .settings 33 | .tags 34 | .tags_sorted_by_file 35 | .target 36 | .worksheet 37 | Makefile 38 | TAGS 39 | _akka_cluster/ 40 | _dump 41 | _mb 42 | activemq-data 43 | akka-contrib/rst_preprocessed/ 44 | akka-docs/_build/ 45 | akka-docs/exts/ 46 | akka-docs/rst_preprocessed/ 47 | akka-osgi/src/main/resources/*.conf 48 | akka.sublime-project 49 | akka.sublime-workspace 50 | akka.tmproj 51 | beanstalk/ 52 | var/ 53 | data 54 | deploy/*.jar 55 | etags 56 | lib_managed 57 | logs 58 | manifest.mf 59 | mongoDB/ 60 | multiverse.log 61 | out 62 | project/akka-build.properties 63 | project/boot/* 64 | project/plugins/project 65 | redis/ 66 | reports 67 | run-codefellow 68 | schoir.props 69 | semantic.cache 70 | src_managed 71 | storage 72 | tags 73 | target 74 | tm*.lck 75 | tm*.log 76 | tm.out 77 | worker*.log 78 | 79 | .internal-ips.tmp 80 | akka-diagnostics 81 | 82 | infra/.bundle 83 | infra/vendor 84 | infra/venv 85 | infra/cookbooks 86 | -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | style = defaultWithAlign 2 | maxColumn = 120 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # apps 2 | 3 | ## How to run locally 4 | 5 | ### Sharding 6 | 7 | * Start Cassandra with default configuration 8 | * Start 3 instances of the ShardingStartLatencyApp with different ports: 9 | * sbt -Dakka.remote.artery.canonical.hostname=127.0.0.1 -Dakka.remote.artery.canonical.port=2551 -Dshard-bench.minimum-nodes=3 -Dakka.cluster.seed-nodes.1=akka://ShardingStartLatencySystem@127.0.0.1:2551 "sharding/runMain com.lightbend.akka.bench.sharding.latency.ShardingStartLatencyApp" 10 | * sbt -Dakka.remote.artery.canonical.hostname=127.0.0.1 -Dakka.remote.artery.canonical.port=2552 -Dshard-bench.minimum-nodes=3 -Dakka.cluster.seed-nodes.1=akka://ShardingStartLatencySystem@127.0.0.1:2551 "sharding/runMain com.lightbend.akka.bench.sharding.latency.ShardingStartLatencyApp" 11 | * sbt -Dakka.remote.artery.canonical.hostname=127.0.0.1 -Dakka.remote.artery.canonical.port=2553 -Dshard-bench.minimum-nodes=3 -Dakka.cluster.seed-nodes.1=akka://ShardingStartLatencySystem@127.0.0.1:2551 "sharding/runMain com.lightbend.akka.bench.sharding.latency.ShardingStartLatencyApp" 12 | 13 | -------------------------------------------------------------------------------- /ddata/scripts/create-nodes-gcp.sh: -------------------------------------------------------------------------------- 1 | for node_number in 002 003 004 005 006 007 008 009 010 2 | do 3 | echo "Creating new node akka-sharding-${node_number}..." 4 | gcloud compute --project "akka-gcp" disks create "akka-ddata-${node_number}" --size "10" --zone "us-central1-a" --source-snapshot "ddata-node-snapshot" --type "pd-standard" 5 | 6 | gcloud compute --project "akka-gcp" instances create "akka-ddata-${node_number}" --zone "us-central1-a" \ 7 | --machine-type "n1-standard-4" --subnet "default" \ 8 | --no-address\ 9 | --maintenance-policy "MIGRATE" --service-account "7250250762-compute@developer.gserviceaccount.com" \ 10 | --scopes "https://www.googleapis.com/auth/devstorage.read_only","https://www.googleapis.com/auth/logging.write","https://www.googleapis.com/auth/monitoring.write","https://www.googleapis.com/auth/servicecontrol","https://www.googleapis.com/auth/service.management.readonly","https://www.googleapis.com/auth/trace.append" \ 11 | --tags “ddata”,”akka" --disk "name=akka-ddata-${node_number},device-name=akka-ddata-${node_number},mode=rw,boot=yes,auto-delete=yes" 12 | 13 | done 14 | 15 | 16 | 17 | gcloud compute --project "akka-gcp" instances create "akka-ddata-002" --zone "us-central1-a" \ 18 | --machine-type "n1-standard-4" --subnet "default" \ 19 | --no-address\ 20 | --maintenance-policy "MIGRATE" --service-account "7250250762-compute@developer.gserviceaccount.com" \ 21 | --scopes "https://www.googleapis.com/auth/devstorage.read_only","https://www.googleapis.com/auth/logging.write","https://www.googleapis.com/auth/monitoring.write","https://www.googleapis.com/auth/servicecontrol","https://www.googleapis.com/auth/service.management.readonly","https://www.googleapis.com/auth/trace.append" \ 22 | --tags "ddata","akka" --disk "name=akka-ddata-002,device-name=akka-ddata-002,mode=rw,boot=yes,auto-delete=yes" -------------------------------------------------------------------------------- /ddata/src/main/resources/application.conf: -------------------------------------------------------------------------------- 1 | akka { 2 | extensions = ["com.romix.akka.serialization.kryo.KryoSerializationExtension$"] 3 | 4 | diagnostics.checker.confirmed-typos = [ 5 | "akka.actor.kryo" 6 | ] 7 | 8 | actor { 9 | provider = "cluster" 10 | 11 | allow-java-serialization = no 12 | 13 | serializers { 14 | kryo = "com.romix.akka.serialization.kryo.KryoSerializer" 15 | } 16 | 17 | serialization-bindings { 18 | "com.lightbend.akka.bench.ddata.DDataHost$Added$" = kryo 19 | } 20 | 21 | # kryo for serialization for lazy reasons 22 | kryo { 23 | type = "nograph" 24 | idstrategy = "explicit" 25 | buffer-size = 4096 26 | max-buffer-size = -1 27 | use-manifests = false 28 | use-unsafe = false 29 | post-serialization-transformations = "off" 30 | implicit-registration-logging = false 31 | kryo-trace = false 32 | 33 | mappings { 34 | "com.lightbend.akka.bench.ddata.DDataHost$Added$" = 20 35 | } 36 | } 37 | } 38 | 39 | remote { 40 | artery { 41 | enabled = on 42 | 43 | # !! IMPORTANT !! 44 | # the below are provided by root-application.conf on our cloud nodes, 45 | # these values here serve only as local fallback 46 | canonical.hostname = "127.0.0.1" 47 | canonical.port = 2551 48 | # !! IMPORTANT !! 49 | } 50 | } 51 | 52 | cluster { 53 | # !! IMPORTANT !! 54 | # the below are provided by root-application.conf on our cloud nodes, 55 | # these values here serve only as local fallback 56 | seed-nodes = [ 57 | "akka://DistributedDataSystem@127.0.0.1:2551" 58 | ] 59 | # !! IMPORTANT !! 60 | 61 | distributed-data { 62 | # We want to be notified as quick as possible in the benchmark, so that we actually see 63 | # how long the dissemination takes and not this interval 64 | notify-subscribers-interval = 5 ms 65 | } 66 | 67 | downing-provider-class = "com.lightbend.akka.sbr.SplitBrainResolverProvider" 68 | 69 | split-brain-resolver.active-strategy = "keep-majority" 70 | } 71 | } 72 | 73 | bench.ddata { 74 | http-api.host = "localhost" 75 | http-api.port = 8080 76 | } 77 | -------------------------------------------------------------------------------- /ddata/src/main/scala/com/lightbend/akka/bench/ddata/DDataHost.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Lightbend Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.lightbend.akka.bench.ddata 18 | 19 | import akka.actor.{Actor, ActorLogging} 20 | import akka.cluster.Cluster 21 | import akka.cluster.ddata.Replicator._ 22 | import akka.cluster.ddata.{DistributedData, ORSet, ORSetKey, Replicator} 23 | 24 | object DDataHost { 25 | 26 | final val Name = "ddataHost" 27 | 28 | case class Add(el: String, writeConsistency: WriteConsistency) 29 | case object Added 30 | 31 | } 32 | 33 | class DDataHost extends Actor with ActorLogging { 34 | import DDataHost._ 35 | 36 | implicit val cluster = Cluster(context.system) 37 | 38 | val coordinator = context.actorOf( 39 | DDataBenchmarkCoordinator.singletonProxyProps(context.system), 40 | name = "coordinatorProxy") 41 | 42 | val replicator = DistributedData(context.system).replicator 43 | val DataKey = ORSetKey[String]("benchmark") 44 | 45 | replicator ! Replicator.Subscribe(DataKey, self) 46 | 47 | def receive = { 48 | case Add(el, writeConsistency) => 49 | replicator ! Replicator.Update(DataKey, ORSet.empty[String], writeConsistency)(_ + el) 50 | 51 | case c @ Replicator.Changed(DataKey) => 52 | // this is the subscribed change, not the ack on the update 53 | coordinator ! Added 54 | 55 | case c @ Replicator.UpdateTimeout(DataKey, _) => 56 | log.warning("Got write timeout") 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /ddata/src/main/scala/com/lightbend/akka/bench/ddata/DistributedDataBenchmark.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Lightbend Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.lightbend.akka.bench.ddata 18 | 19 | import akka.actor.{ ActorSystem, PoisonPill, Props } 20 | import akka.cluster.Cluster 21 | import java.io.File 22 | 23 | import akka.cluster.singleton.{ ClusterSingletonManager, ClusterSingletonManagerSettings } 24 | import com.typesafe.config.ConfigFactory 25 | 26 | import scala.util.Try 27 | 28 | object DistributedDataBenchmark extends App { 29 | 30 | final val CoordinatorManager = "coordinatorManager" 31 | 32 | // setup for cloud env ------------------------------------------------------------- 33 | val isMaster = Option(System.getenv("MASTER")).isDefined 34 | val rootConfFile = new File("/home/akka/root-application.conf") 35 | val rootConf = 36 | if (rootConfFile.exists) ConfigFactory.parseFile(rootConfFile) 37 | else ConfigFactory.empty("no-root-application-conf-found") 38 | 39 | val masterRole = 40 | if (isMaster) ConfigFactory.parseString("akka.cluster.roles=[master]") 41 | else ConfigFactory.empty() 42 | 43 | val conf = masterRole.withFallback(rootConf.withFallback(ConfigFactory.load())) 44 | // end of setup for cloud env ------------------------------------------------------ 45 | 46 | 47 | implicit val system = ActorSystem("DistributedDataSystem", conf) 48 | 49 | 50 | val cluster = Cluster(system) 51 | 52 | cluster.registerOnMemberUp { 53 | system.actorOf(Props[DDataHost], DDataHost.Name) 54 | 55 | // only the "master" node runs the singleton 56 | if (cluster.selfRoles("master")) { 57 | system.actorOf( 58 | ClusterSingletonManager.props( 59 | singletonProps = Props[DDataBenchmarkCoordinator], 60 | terminationMessage = PoisonPill, 61 | settings = ClusterSingletonManagerSettings(system).withRole("master")), 62 | name = CoordinatorManager) 63 | 64 | 65 | val httpHost = conf.getString("bench.ddata.http-api.host") 66 | val httpPort = conf.getInt("bench.ddata.http-api.port") 67 | 68 | HttpApi.startServer(httpHost, httpPort) 69 | 70 | } 71 | } 72 | 73 | scala.io.StdIn.readLine() // TODO not sure if readline will work well with starting it via scripts... 74 | system.terminate() 75 | } 76 | -------------------------------------------------------------------------------- /infra/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | littlechef.cfg 3 | berks-cookbooks 4 | tmp_* 5 | 6 | data_bags/* 7 | 8 | 9 | site-cookbooks/update-hosts/templates/default/multi-node-test.hosts 10 | -------------------------------------------------------------------------------- /infra/.ruby-version: -------------------------------------------------------------------------------- 1 | 2.1.10 2 | -------------------------------------------------------------------------------- /infra/Berksfile: -------------------------------------------------------------------------------- 1 | source 'https://supermarket.chef.io' 2 | source "http://api.berkshelf.com" 3 | 4 | cookbook 'apt' 5 | cookbook 'docker' 6 | cookbook 'cassandra' 7 | cookbook 'cassandra-dse', '~> 4.3.0' 8 | cookbook 'sudo' 9 | -------------------------------------------------------------------------------- /infra/Berksfile.lock: -------------------------------------------------------------------------------- 1 | DEPENDENCIES 2 | apt 3 | cassandra 4 | cassandra-dse (~> 4.3.0) 5 | docker 6 | sudo 7 | 8 | GRAPH 9 | apt (2.9.2) 10 | ark (3.1.0) 11 | build-essential (>= 0.0.0) 12 | seven_zip (>= 0.0.0) 13 | windows (>= 0.0.0) 14 | boost (2.1.0) 15 | build-essential (>= 0.0.0) 16 | build-essential (8.0.3) 17 | mingw (>= 1.1) 18 | seven_zip (>= 0.0.0) 19 | cassandra (0.2.4) 20 | apt (>= 0.0.0) 21 | install_from (>= 0.0.0) 22 | iptables (>= 0.0.0) 23 | java (>= 0.0.0) 24 | metachef (>= 0.0.0) 25 | runit (>= 0.0.0) 26 | thrift (>= 0.0.0) 27 | volumes (>= 0.0.0) 28 | cassandra-dse (4.3.0) 29 | apt (>= 0.0.0) 30 | ark (>= 0.0.0) 31 | java (>= 0.0.0) 32 | systemd (>= 0.0.0) 33 | ulimit (>= 0.0.0) 34 | yum (>= 0.0.0) 35 | compat_resource (12.5.14) 36 | docker (2.2.10) 37 | compat_resource (>= 12.5.14) 38 | homebrew (4.2.0) 39 | install_from (3.0.4) 40 | iptables (4.2.0) 41 | java (1.50.0) 42 | apt (>= 0.0.0) 43 | homebrew (>= 0.0.0) 44 | windows (>= 0.0.0) 45 | metachef (3.0.4) 46 | mingw (2.0.1) 47 | seven_zip (>= 0.0.0) 48 | ohai (5.1.0) 49 | packagecloud (0.3.0) 50 | python (1.4.6) 51 | build-essential (>= 0.0.0) 52 | yum-epel (>= 0.0.0) 53 | runit (3.0.5) 54 | packagecloud (>= 0.0.0) 55 | yum-epel (>= 0.0.0) 56 | seven_zip (2.0.2) 57 | windows (>= 1.2.2) 58 | sudo (3.5.0) 59 | systemd (2.1.3) 60 | thrift (1.2.0) 61 | boost (>= 0.0.0) 62 | build-essential (>= 0.0.0) 63 | python (>= 0.0.0) 64 | ulimit (0.4.0) 65 | volumes (3.0.4) 66 | metachef (>= 0.0.0) 67 | xfs (>= 0.0.0) 68 | windows (3.1.0) 69 | ohai (>= 4.0.0) 70 | xfs (2.0.1) 71 | yum (5.0.1) 72 | yum-epel (1.0.2) 73 | yum (>= 3.6) 74 | -------------------------------------------------------------------------------- /infra/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'chef' 4 | gem 'knife-solo' 5 | gem 'berkshelf' 6 | 7 | -------------------------------------------------------------------------------- /infra/README.md: -------------------------------------------------------------------------------- 1 | Akka Jenkins Infra 2 | ================== 3 | 4 | Dependencies 5 | ------------ 6 | 7 | ``` 8 | gem install bundler 9 | 10 | # install chef-dk: https://downloads.chef.io/chefdk 11 | bundle install --path vendor/bundle 12 | 13 | # Create and activate python virtual env 14 | virtualenv --python=/usr/bin/python2.7 venv 15 | source venv/bin/activate 16 | 17 | # install littlechef: https://github.com/tobami/littlechef 18 | pip install littlechef 19 | 20 | bundle exec berks vendor 21 | ``` 22 | 23 | Then prepare the littlechef file: 24 | 25 | ``` 26 | cp littlechef.cfg-example littlechef.cfg 27 | ``` 28 | 29 | Fix cookbooks directory: 30 | 31 | ``` 32 | mv berks-cookbooks cookbooks 33 | ``` 34 | 35 | Tryout the setup by running a command on a group of nodes: 36 | 37 | ``` 38 | bundle exec fix --concurrency 2 nodes_with_role:benchmark-ddata ssh:"hostname" 39 | ``` 40 | 41 | Preparing a node for cheffing 42 | ------------- 43 | 44 | ``` 45 | fix node:cassandra-01 deploy_chef 46 | ``` 47 | 48 | Adding a new akka node automatically 49 | ------------------------------------ 50 | 51 | ``` 52 | ./gcloud-new-akka-node.sh akka-node-002 "[\"10.132.0.7\", \"10.132.0.8\"]" 53 | ``` 54 | 55 | Cooking nodes 56 | ------------- 57 | 58 | Executing scripts on a given node: 59 | 60 | ``` 61 | fix node:moxie-a0 [role:specific-role] 62 | ``` 63 | 64 | **Cooking all nodes for the Akka team:** 65 | 66 | ``` 67 | # check which ones belong to akka_team: 68 | fix list_nodes_with_role:akka_team 69 | 70 | # cook their run_lists: 71 | fix nodes_with_role:akka_team 72 | ``` 73 | 74 | Read more here: https://github.com/tobami/littlechef 75 | 76 | Rsync problems 77 | -------------- 78 | Sometimes adds too many weird options to ssh used in rsync and fixing in 79 | `$ vim ~/.chefdk/gem/ruby/2.1.0/gems/knife-solo-0.5.1/lib/chef/knife/solo_cook.rb` is required, 80 | as shown here: https://github.com/ktoso/knife-solo/commit/78d47efe6d0b1399ae5da3a8cc5c6dd05ef10dfc 81 | -------------------------------------------------------------------------------- /infra/benchmark-run-ddata.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | declare -r CLAZZ=com.lightbend.akka.bench.ddata.DistributedDataBenchmark 4 | 5 | fix --verbose --concurrency 2 \ 6 | nodes_with_role:benchmark-ddata \ 7 | ssh:"cd /home/akka/apps; sudo -u akka /home/akka/sbt ';project apps; runMain $CLAZZ'" 8 | -------------------------------------------------------------------------------- /infra/benchmark-run-sharding-init-cassandra.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | declare -r CLAZZ="com.lightbend.akka.bench.sharding.InitCassandraApp" 4 | 5 | fix --verbose node:akka-cassandra-001 ssh:"cd /home/akka/apps; sudo -u akka /home/akka/sbt ';project sharding; runMain $CLAZZ'" 6 | -------------------------------------------------------------------------------- /infra/data_bags/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/infra/data_bags/.gitkeep -------------------------------------------------------------------------------- /infra/environments/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/infra/environments/.gitkeep -------------------------------------------------------------------------------- /infra/etc-hosts-akka-nodes-print.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | declare -r GREP=$1 4 | 5 | gcloud compute instances list | grep "$GREP" | grep RUNNING | awk ' { print $5," ",$1 } ' 6 | -------------------------------------------------------------------------------- /infra/etc-hosts-aws-re.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | aws ec2 describe-instances --query 'Reservations[].Instances[].[PublicIpAddress,Tags[?Key==`Name`].Value[]]' --output text | sed '$!N;s/\n/ /' | grep re -------------------------------------------------------------------------------- /infra/gcloud-clone-akka-sharding-nodes.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # NOTE: watch out to keep \ the last character (no trailing space) 4 | 5 | for node_number in 002 003 004 005 006 007 008 009 010 6 | do 7 | echo "Creating new node akka-sharding-${node_number}..." 8 | gcloud compute --project "akka-gcp" disks create "akka-sharding-${node_number}" --size "10" --zone "us-central1-a" \ 9 | --source-snapshot "akka-sharding-snapshot" --type "pd-standard" 10 | 11 | gcloud compute --project "akka-gcp" instances create "akka-sharding-${node_number}" --zone "us-central1-a" \ 12 | --machine-type "n1-standard-4" --subnet "default" \ 13 | --no-address\ 14 | --maintenance-policy "MIGRATE" --service-account "7250250762-compute@developer.gserviceaccount.com" \ 15 | --scopes "https://www.googleapis.com/auth/devstorage.read_only","https://www.googleapis.com/auth/logging.write","https://www.googleapis.com/auth/monitoring.write","https://www.googleapis.com/auth/servicecontrol","https://www.googleapis.com/auth/service.management.readonly","https://www.googleapis.com/auth/trace.append" \ 16 | --tags "sharding","akka" --disk "name=akka-sharding-${node_number},device-name=akka-sharding-${node_number},mode=rw,boot=yes,auto-delete=yes" 17 | 18 | done 19 | -------------------------------------------------------------------------------- /infra/gcloud-new-akka-node.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # NOTE: watch out to keep \ the last character (no trailing space) 4 | 5 | NAME="$1" 6 | SEED="$2" 7 | 8 | echo "Creating new node $NAME..." 9 | 10 | # we request the min cpu to be Skylake: 11 | # https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform 12 | 13 | gcloud compute --project "akka-gcp" \ 14 | instances create "$NAME" \ 15 | --zone "europe-west1-b" \ 16 | --machine-type "n1-standard-4" \ 17 | --subnet "default" --maintenance-policy "MIGRATE" --service-account "7250250762-compute@developer.gserviceaccount.com" \ 18 | --scopes "https://www.googleapis.com/auth/cloud-platform" \ 19 | --image "ubuntu-1704-zesty-v20170413" \ 20 | --image-project "ubuntu-os-cloud" \ 21 | --tags "akka","http-server","https-server" \ 22 | --boot-disk-size "20" --boot-disk-type "pd-ssd" \ 23 | --boot-disk-device-name "$NAME" \ 24 | --metadata startup-script='#! /bin/bash 25 | cd /home/akka 26 | echo "Running all startup-scripts in $(pwd)..." 27 | for script in $(ls *startup-script*); do 28 | echo "Running $script" 29 | ./$script 30 | done 31 | ' 32 | 33 | declare -r internal_ip=$(gcloud --project="akka-gcp" compute instances list | grep "$NAME" | head -n1 | awk '{ print $4 }') 34 | declare -r external_ip=$(gcloud --project="akka-gcp" compute instances list | grep "$NAME" | head -n1 | awk '{ print $5 }') 35 | 36 | cat nodes/akka-node.json.template | 37 | sed "s/NAME/$NAME/g" | 38 | sed "s/AKKA_SEED_NODES/$SEED/g" | 39 | sed "s/EXTERNAL_IP/$external_ip/g" | 40 | sed "s/INTERNAL_IP/$internal_ip/g" > nodes/$NAME.json 41 | 42 | 43 | echo "# -----------------------------------" 44 | echo "Check the file: ./nodes/$NAME.json" 45 | echo "# -----------------------------------" 46 | read 47 | 48 | 49 | echo "# -----------------------------------" 50 | echo "Seed nodes set to: $SEED" 51 | echo "# -----------------------------------" 52 | 53 | echo "------ UPDATE YOUR /etc/hosts -------" 54 | echo "# -----------------------------------" 55 | echo "$external_ip $NAME" 56 | echo "# -----------------------------------" 57 | echo " Press ENTER when ready... " 58 | read 59 | echo "# -----------------------------------" 60 | 61 | 62 | echo "Deploying chef..." 63 | sleep 5 64 | fix node:$NAME deploy_chef -y 65 | 66 | echo "Preparing node..." 67 | fix node:$NAME 68 | 69 | #echo "Deploying Chef to [$node]..." 70 | #knife solo prepare $NODE 71 | echo "------------------- DONE -------------------" 72 | -------------------------------------------------------------------------------- /infra/internal-ips.tmp: -------------------------------------------------------------------------------- 1 | 10.132.0.7 2 | 10.132.0.8 3 | 10.132.0.9 4 | -------------------------------------------------------------------------------- /infra/littlechef.cfg-example: -------------------------------------------------------------------------------- 1 | [userinfo] 2 | user = root 3 | password = 4 | #keypair-file = 5 | ssh-config = ~/.ssh/config 6 | #encrypted_data_bag_secret = 7 | [kitchen] 8 | node_work_path = /tmp/chef-solo/ 9 | 10 | [kitchen] 11 | berksfile = Berksfile 12 | berksfile_cookbooks_directory = berks-cookbooks 13 | -------------------------------------------------------------------------------- /infra/make-node-files-from-gcloud.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # examples: 4 | # 5 | # ./make-node-files-from-gcloud.sh akka-node-01 '["10.154.0.2"]' 6 | # 7 | # EXTRA_RUNLIST='"role[benchmark-sharding]",' ./make-node-files-from-gcloud.sh akka-node-00 '["10.132.0.7", "10.132.0.8"]' 8 | 9 | 10 | # -------------------- functions -------------------- 11 | json_append() { 12 | local old_json=$1; shift 13 | local count=0 14 | local -a args=( ) 15 | local elem_names='' 16 | while (( $# )); do 17 | args+=( --arg "element$((++count))" "$1" ) 18 | elem_names+="\$element$count, " 19 | shift 20 | done 21 | 22 | jq "${args[@]}" ". + [ ${elem_names%, } ]" <<<"${old_json:-[]}" 23 | } 24 | # ----------------- end of functions ----------------- 25 | 26 | if [[ "$#" != "2" ]]; then 27 | echo "USAGE: ./make-node-files-from-gcloud.sh GREP SEED_NODES" 28 | echo "EXTRA PARAMS:" 29 | echo " you can set: EXTRA_RUNLIST ( example: EXTRA_RUNLIST='"role[benchmark-ddata]",' )" 30 | fi 31 | 32 | declare -r GREP=$1 33 | declare -r SEED_NODES=$2 34 | 35 | declare -r minumum_nodes_NUM=$(gcloud compute instances list | grep $GREP | wc -l | awk '{print $1}') 36 | 37 | # cassandra contact points as json (just the first ... nodes) 38 | declare -r CASSANDRA_CONTACT_POINTS_NUM=2 39 | declare -r CASSANDRA_CONTACT_POINTS=$( 40 | gcloud compute instances list | 41 | grep "cassandra" | grep RUNNING | 42 | awk ' {print $4 } ' | 43 | head -n${CASSANDRA_CONTACT_POINTS_NUM} | 44 | awk ' BEGIN { ORS = ""; print "["; } { print "\/\@"$0"\/\@"; } END { print "]"; }' | sed "s^\"^\\\\\"^g;s^\/\@\/\@^\", \"^g;s^\/\@^\"^g" 45 | ) 46 | 47 | IFS=$'\n' # make newlines the only separator 48 | for node in $(gcloud compute instances list | grep "$GREP" | grep "RUNNING" | awk ' { print $1,$4,$5 } ') 49 | do 50 | name=$(echo $node | awk ' { print $1} ') 51 | internal_ip=$(echo $node | awk ' { print $2} ') 52 | external_ip=$(echo $node | awk ' { print $3} ') 53 | 54 | echo 55 | 56 | cat ./nodes/akka-node.json.template | 57 | sed "s/NAME/$name/g" | 58 | sed "s/INTERNAL_IP/$internal_ip/g" | 59 | sed "s/EXTERNAL_IP/$external_ip/g" | 60 | sed "s/AKKA_SEED_NODES/$SEED_NODES/g" | 61 | sed "s/EXTRA_RUNLIST/$EXTRA_RUNLIST/g" | 62 | sed "s/CASSANDRA_CONTACT_POINTS/$CASSANDRA_CONTACT_POINTS/g" | 63 | sed "s/minumum_nodes/$minumum_nodes_NUM/g" > ./nodes/$name.json 64 | 65 | echo "Generated: ./nodes/$name.json" 66 | done 67 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-01.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.4", 7 | "ipaddress": "", 8 | "name": "10.128.0.4", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-02.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.3", 7 | "ipaddress": "", 8 | "name": "10.128.0.3", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-03.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.6", 7 | "ipaddress": "", 8 | "name": "10.128.0.6", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-04.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.10", 7 | "ipaddress": "", 8 | "name": "10.128.0.10", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-05.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.7", 7 | "ipaddress": "", 8 | "name": "10.128.0.7", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-06.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.8", 7 | "ipaddress": "", 8 | "name": "10.128.0.8", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-07.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.5", 7 | "ipaddress": "", 8 | "name": "10.128.0.5", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-08.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.9", 7 | "ipaddress": "", 8 | "name": "10.128.0.9", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-09.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.11", 7 | "ipaddress": "", 8 | "name": "10.128.0.11", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-10.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.12", 7 | "ipaddress": "", 8 | "name": "10.128.0.12", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-11.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.13", 7 | "ipaddress": "", 8 | "name": "10.128.0.13", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-12.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.14", 7 | "ipaddress": "", 8 | "name": "10.128.0.14", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-13.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.15", 7 | "ipaddress": "", 8 | "name": "10.128.0.15", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-14.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.16", 7 | "ipaddress": "", 8 | "name": "10.128.0.16", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-15.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.17", 7 | "ipaddress": "", 8 | "name": "10.128.0.17", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-16.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.18", 7 | "ipaddress": "", 8 | "name": "10.128.0.18", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-17.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.19", 7 | "ipaddress": "", 8 | "name": "10.128.0.19", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-18.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.20", 7 | "ipaddress": "", 8 | "name": "10.128.0.20", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-19.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.21", 7 | "ipaddress": "", 8 | "name": "10.128.0.21", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra-20.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "10.128.0.22", 7 | "ipaddress": "", 8 | "name": "10.128.0.22", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "10.128.0.4", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-cassandra.json.template: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress-internal": "INTERNAL_IP", 7 | "ipaddress": "EXTERNAL_IP", 8 | "name": "NAME", 9 | "java": { 10 | "jdk_version": 8, 11 | "oracle": { 12 | "accept_oracle_download_terms": true 13 | }, 14 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 15 | }, 16 | "cassandra": { 17 | "seed_node": "CASSANDRA_SEED_IP", 18 | "install_java": false, 19 | "config": { 20 | "cluster_name": "akka-cassandra" 21 | } 22 | }, 23 | "run_list": [ 24 | "role[cassandra]" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /infra/nodes/akka-node-002.json: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "10.132.0.8", 4 | "ipaddress": "35.187.124.182", 5 | "name": "akka-node-002", 6 | 7 | "akka": { 8 | "system_name": "BenchmarkSystem", 9 | "port": 2551, 10 | "roles": ["shard"], 11 | 12 | "seed_nodes": ["10.132.0.7", "10.132.0.8"], 13 | "minumum_nodes": 3 14 | }, 15 | 16 | "cassandra": { 17 | "contact_points": ["10.132.0.4", "10.132.0.6"] 18 | }, 19 | 20 | "authorization": { 21 | "sudo": { 22 | "users": ["akka", "ubuntu", "ktoso", "patriknw"], 23 | "passwordless": "true" 24 | } 25 | }, 26 | 27 | "java": { 28 | "jdk_version": 8, 29 | "oracle": { 30 | "accept_oracle_download_terms": true 31 | }, 32 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 33 | }, 34 | "run_list": [ 35 | "role[benchmark]", "role[benchmark-sharding]", 36 | "role[gce-tools]" 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /infra/nodes/akka-node-003.json: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "10.132.0.9", 4 | "ipaddress": "130.211.96.195", 5 | "name": "akka-node-003", 6 | 7 | "akka": { 8 | "system_name": "BenchmarkSystem", 9 | "port": 2551, 10 | "roles": ["shard"], 11 | 12 | "seed_nodes": ["10.132.0.7", "10.132.0.8"], 13 | "minumum_nodes": 3 14 | }, 15 | 16 | "cassandra": { 17 | "contact_points": ["10.132.0.4", "10.132.0.6"] 18 | }, 19 | 20 | "authorization": { 21 | "sudo": { 22 | "users": ["akka", "ubuntu", "ktoso", "patriknw"], 23 | "passwordless": "true" 24 | } 25 | }, 26 | 27 | "java": { 28 | "jdk_version": 8, 29 | "oracle": { 30 | "accept_oracle_download_terms": true 31 | }, 32 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 33 | }, 34 | "run_list": [ 35 | "role[benchmark]", "role[benchmark-sharding]", 36 | "role[gce-tools]" 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /infra/nodes/akka-node-010.json: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "10.154.0.2", 4 | "ipaddress": "35.189.67.222", 5 | "name": "akka-node-010", 6 | 7 | "akka": { 8 | "system_name": "BenchmarkSystem", 9 | "port": 2551, 10 | "roles": [], 11 | 12 | "seed_nodes": ["10.154.0.2"], 13 | "minumum_nodes": 6 14 | }, 15 | 16 | "authorization": { 17 | "sudo": { 18 | "users": ["akka", "ubuntu", "ktoso", "patriknw"], 19 | "passwordless": "true" 20 | } 21 | }, 22 | "java": { 23 | "jdk_version": 8, 24 | "oracle": { 25 | "accept_oracle_download_terms": true 26 | }, 27 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 28 | }, 29 | "run_list": [ 30 | "role[benchmark]", "role[benchmark-ddata]", 31 | "role[gce-tools]" 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /infra/nodes/akka-node-011.json: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "10.154.0.3", 4 | "ipaddress": "35.189.72.215", 5 | "name": "akka-node-011", 6 | 7 | "akka": { 8 | "system_name": "BenchmarkSystem", 9 | "port": 2551, 10 | "roles": [], 11 | 12 | "seed_nodes": ["10.154.0.2"], 13 | "minumum_nodes": 6 14 | }, 15 | 16 | "authorization": { 17 | "sudo": { 18 | "users": ["akka", "ubuntu", "ktoso", "patriknw"], 19 | "passwordless": "true" 20 | } 21 | }, 22 | "java": { 23 | "jdk_version": 8, 24 | "oracle": { 25 | "accept_oracle_download_terms": true 26 | }, 27 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 28 | }, 29 | "run_list": [ 30 | "role[benchmark]", "role[benchmark-ddata]", 31 | "role[gce-tools]" 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /infra/nodes/akka-node-012.json: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "10.154.0.4", 4 | "ipaddress": "35.189.69.146", 5 | "name": "akka-node-012", 6 | 7 | "akka": { 8 | "system_name": "BenchmarkSystem", 9 | "port": 2551, 10 | "roles": [], 11 | 12 | "seed_nodes": ["10.154.0.2"], 13 | "minumum_nodes": 6 14 | }, 15 | 16 | "authorization": { 17 | "sudo": { 18 | "users": ["akka", "ubuntu", "ktoso", "patriknw"], 19 | "passwordless": "true" 20 | } 21 | }, 22 | "java": { 23 | "jdk_version": 8, 24 | "oracle": { 25 | "accept_oracle_download_terms": true 26 | }, 27 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 28 | }, 29 | "run_list": [ 30 | "role[benchmark]", "role[benchmark-ddata]", 31 | "role[gce-tools]" 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /infra/nodes/akka-node-013.json: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "10.154.0.5", 4 | "ipaddress": "35.189.69.177", 5 | "name": "akka-node-013", 6 | 7 | "akka": { 8 | "system_name": "BenchmarkSystem", 9 | "port": 2551, 10 | "roles": [], 11 | 12 | "seed_nodes": ["10.154.0.2"], 13 | "minumum_nodes": 6 14 | }, 15 | 16 | "authorization": { 17 | "sudo": { 18 | "users": ["akka", "ubuntu", "ktoso", "patriknw"], 19 | "passwordless": "true" 20 | } 21 | }, 22 | "java": { 23 | "jdk_version": 8, 24 | "oracle": { 25 | "accept_oracle_download_terms": true 26 | }, 27 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 28 | }, 29 | "run_list": [ 30 | "role[benchmark]", "role[benchmark-ddata]", 31 | "role[gce-tools]" 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /infra/nodes/akka-node-014.json: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "10.154.0.6", 4 | "ipaddress": "35.189.79.113", 5 | "name": "akka-node-014", 6 | 7 | "akka": { 8 | "system_name": "BenchmarkSystem", 9 | "port": 2551, 10 | "roles": [], 11 | 12 | "seed_nodes": ["10.154.0.2"], 13 | "minumum_nodes": 6 14 | }, 15 | 16 | "authorization": { 17 | "sudo": { 18 | "users": ["akka", "ubuntu", "ktoso", "patriknw"], 19 | "passwordless": "true" 20 | } 21 | }, 22 | "java": { 23 | "jdk_version": 8, 24 | "oracle": { 25 | "accept_oracle_download_terms": true 26 | }, 27 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 28 | }, 29 | "run_list": [ 30 | "role[benchmark]", "role[benchmark-ddata]", 31 | "role[gce-tools]" 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /infra/nodes/akka-node-015.json: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "10.154.0.7", 4 | "ipaddress": "35.189.79.50", 5 | "name": "akka-node-015", 6 | 7 | "akka": { 8 | "system_name": "BenchmarkSystem", 9 | "port": 2551, 10 | "roles": [], 11 | 12 | "seed_nodes": ["10.154.0.2"], 13 | "minumum_nodes": 6 14 | }, 15 | 16 | "authorization": { 17 | "sudo": { 18 | "users": ["akka", "ubuntu", "ktoso", "patriknw"], 19 | "passwordless": "true" 20 | } 21 | }, 22 | "java": { 23 | "jdk_version": 8, 24 | "oracle": { 25 | "accept_oracle_download_terms": true 26 | }, 27 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 28 | }, 29 | "run_list": [ 30 | "role[benchmark]", "role[benchmark-ddata]", 31 | "role[gce-tools]" 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /infra/nodes/akka-node.json.template: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "INTERNAL_IP", 4 | "ipaddress": "EXTERNAL_IP", 5 | "name": "NAME", 6 | 7 | "akka": { 8 | "system_name": "BenchmarkSystem", 9 | "port": 2551, 10 | "roles": [], 11 | 12 | "seed_nodes": AKKA_SEED_NODES, 13 | "minumum_nodes": minumum_nodes 14 | }, 15 | 16 | "cassandra": { 17 | "contact_points": CASSANDRA_CONTACT_POINTS 18 | }, 19 | 20 | "authorization": { 21 | "sudo": { 22 | "users": ["akka", "ubuntu", "ktoso", "patriknw"], 23 | "passwordless": "true" 24 | } 25 | }, 26 | 27 | "java": { 28 | "jdk_version": 8, 29 | "oracle": { 30 | "accept_oracle_download_terms": true 31 | }, 32 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 33 | }, 34 | "run_list": [ 35 | "role[benchmark]", EXTRA_RUNLIST 36 | "role[gce-tools]" 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /infra/nodes/akka-sharding-001.json: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "10.128.0.23", 4 | "ipaddress": "104.155.151.26", 5 | "name": "akka-sharding-001", 6 | 7 | "akka": { 8 | "system_name": "BenchmarkSystem", 9 | "port": 2551, 10 | "roles": ["master"], 11 | 12 | "seed_nodes": ["10.128.0.23"], 13 | "minumum_nodes": 3 14 | }, 15 | 16 | "cassandra": { 17 | "contact_points": ["10.128.0.4"] 18 | }, 19 | 20 | "authorization": { 21 | "sudo": { 22 | "users": ["akka", "ubuntu", "ktoso", "patriknw"], 23 | "passwordless": "true" 24 | } 25 | }, 26 | 27 | "java": { 28 | "jdk_version": 8, 29 | "oracle": { 30 | "accept_oracle_download_terms": true 31 | }, 32 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 33 | }, 34 | "run_list": [ 35 | "role[benchmark]", 36 | "role[benchmark-sharding]", 37 | "role[gce-tools]" 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /infra/nodes/moxie-a0.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "moxie-a0" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress": "10.0.1.230", 7 | "name": "moxie-a0", 8 | "run_list": [ 9 | "role[akka_team]", 10 | "role[jenkins_worker]" 11 | ] 12 | } -------------------------------------------------------------------------------- /infra/nodes/moxie-a1.json: -------------------------------------------------------------------------------- 1 | { 2 | "run_list": [ 3 | ], 4 | "automatic": { 5 | "ipaddress": "moxie-a1" 6 | } 7 | } -------------------------------------------------------------------------------- /infra/nodes/moxie-a2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moxie-a2", 3 | "run_list": [ 4 | "role[akka_team]", 5 | "role[jenkins_worker]" 6 | ], 7 | 8 | "automatic": { 9 | "ipaddress": "moxie-a2" 10 | }, 11 | "chef_environment": "_default", 12 | "ipaddress": "10.0.1.232" 13 | } 14 | -------------------------------------------------------------------------------- /infra/nodes/moxie-a3.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "moxie-a3" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress": "10.0.1.233", 7 | "name": "moxie-a3", 8 | "run_list": [ 9 | "role[akka_team]", 10 | "role[jenkins_worker]" 11 | ] 12 | } -------------------------------------------------------------------------------- /infra/nodes/moxie-a4.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "moxie-a4" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress": "10.0.1.234", 7 | "name": "moxie-a4", 8 | "run_list": [ 9 | "role[akka_team]", 10 | "role[jenkins_worker]" 11 | ] 12 | } -------------------------------------------------------------------------------- /infra/nodes/moxie-a5.json: -------------------------------------------------------------------------------- 1 | { 2 | "run_list": [ 3 | 4 | ], 5 | "automatic": { 6 | "ipaddress": "moxie-a5" 7 | } 8 | } -------------------------------------------------------------------------------- /infra/nodes/moxie-a6.json: -------------------------------------------------------------------------------- 1 | { 2 | "run_list": [ 3 | ], 4 | "automatic": { 5 | "ipaddress": "moxie-a6" 6 | } 7 | } -------------------------------------------------------------------------------- /infra/nodes/moxie-a7.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "moxie-a7" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress": "10.0.1.237", 7 | "name": "moxie-a7", 8 | "run_list": [ 9 | "role[akka_team]", 10 | "role[jenkins_worker]" 11 | ] 12 | } -------------------------------------------------------------------------------- /infra/nodes/moxie-a8.json: -------------------------------------------------------------------------------- 1 | { 2 | "automatic": { 3 | "ipaddress": "moxie-a8" 4 | }, 5 | "chef_environment": "_default", 6 | "ipaddress": "10.0.1.238", 7 | "name": "moxie-a8", 8 | "run_list": [ 9 | "role[akka_team]", 10 | "role[jenkins_master]", 11 | "role[jenkins_worker]" 12 | ] 13 | } -------------------------------------------------------------------------------- /infra/nodes/re-akka-eucentral-1a.json: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "172.31.12.178", 4 | "ipaddress-external": "18.194.173.185", 5 | "name": "re-akka-eucentral-1a", 6 | 7 | "akka": { 8 | "system_name": "MultiDcSystem", 9 | "port": 2551, 10 | "roles": [], 11 | "seed_nodes": [""], 12 | "min_nodes": 2, 13 | "dc": "" 14 | }, 15 | "cassandra": { 16 | "contact_points": ["35.158.122.93"] 17 | }, 18 | "authorization": { 19 | "sudo": { 20 | "users": ["akka", "ubuntu", "ktoso", "patriknw", "chbatey"], 21 | "passwordless": "true" 22 | } 23 | }, 24 | 25 | "java": { 26 | "jdk_version": 8, 27 | "oracle": { 28 | "accept_oracle_download_terms": true 29 | }, 30 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 31 | }, 32 | "run_list": [ 33 | "role[benchmark]", 34 | "role[akka-re]" 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /infra/nodes/re-akka-eucentral-1b.json: -------------------------------------------------------------------------------- 1 | { 2 | "akka": { 3 | "dc": "eu-central", 4 | "min_nodes": 2, 5 | "port": 2551, 6 | "roles": [], 7 | "seed_nodes": [ 8 | "54.154.72.165" 9 | ], 10 | "system_name": "MultiDcSystem" 11 | }, 12 | "authorization": { 13 | "sudo": { 14 | "passwordless": "true", 15 | "users": [ 16 | "akka", 17 | "ubuntu", 18 | "ktoso", 19 | "patriknw", 20 | "chbatey" 21 | ] 22 | } 23 | }, 24 | "cassandra": { 25 | "contact_points": [ 26 | "52.59.221.33" 27 | ] 28 | }, 29 | "chef_environment": "_default", 30 | "ipaddress": "172.31.17.21", 31 | "ipaddress-external": "52.59.241.114", 32 | "ipaddress-internal": "172.31.17.21", 33 | "java": { 34 | "jdk_version": 8, 35 | "oracle": { 36 | "accept_oracle_download_terms": true 37 | }, 38 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 39 | }, 40 | "name": "re-akka-eucentral-1b", 41 | "run_list": [ 42 | "role[benchmark]", 43 | "role[akka-re]" 44 | ] 45 | } -------------------------------------------------------------------------------- /infra/nodes/re-akka-eucentral-1c.json: -------------------------------------------------------------------------------- 1 | { 2 | "akka": { 3 | "dc": "eu-central", 4 | "min_nodes": 2, 5 | "port": 2551, 6 | "roles": [], 7 | "seed_nodes": [ 8 | "54.154.72.165" 9 | ], 10 | "system_name": "MultiDcSystem" 11 | }, 12 | "authorization": { 13 | "sudo": { 14 | "passwordless": "true", 15 | "users": [ 16 | "akka", 17 | "ubuntu", 18 | "ktoso", 19 | "patriknw", 20 | "chbatey" 21 | ] 22 | } 23 | }, 24 | "cassandra": { 25 | "contact_points": [ 26 | "52.59.221.33" 27 | ] 28 | }, 29 | "chef_environment": "_default", 30 | "ipaddress": "172.31.43.89", 31 | "ipaddress-external": "18.194.243.77", 32 | "ipaddress-internal": "172.31.43.89", 33 | "java": { 34 | "jdk_version": 8, 35 | "oracle": { 36 | "accept_oracle_download_terms": true 37 | }, 38 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 39 | }, 40 | "name": "re-akka-eucentral-1c", 41 | "run_list": [ 42 | "role[benchmark]", 43 | "role[akka-re]" 44 | ] 45 | } -------------------------------------------------------------------------------- /infra/nodes/re-akka-euwest-1a.json: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "172.31.24.65", 4 | "ipaddress-external": "54.171.108.87", 5 | "name": "re-akka-euwest-1a", 6 | 7 | "akka": { 8 | "system_name": "MultiDcSystem", 9 | "port": 2551, 10 | "roles": [], 11 | "seed_nodes": ["None"], 12 | "min_nodes": 2, 13 | "dc": "" 14 | }, 15 | "cassandra": { 16 | "contact_points": ["34.242.227.97"] 17 | }, 18 | "authorization": { 19 | "sudo": { 20 | "users": ["akka", "ubuntu", "ktoso", "patriknw", "chbatey"], 21 | "passwordless": "true" 22 | } 23 | }, 24 | 25 | "java": { 26 | "jdk_version": 8, 27 | "oracle": { 28 | "accept_oracle_download_terms": true 29 | }, 30 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 31 | }, 32 | "run_list": [ 33 | "role[benchmark]", 34 | "role[akka-re]" 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /infra/nodes/re-akka-euwest-1b.json: -------------------------------------------------------------------------------- 1 | { 2 | "akka": { 3 | "dc": "eu-west", 4 | "min_nodes": 2, 5 | "port": 2551, 6 | "roles": [], 7 | "seed_nodes": [ 8 | "54.154.72.165" 9 | ], 10 | "system_name": "MultiDcSystem" 11 | }, 12 | "authorization": { 13 | "sudo": { 14 | "passwordless": "true", 15 | "users": [ 16 | "akka", 17 | "ubuntu", 18 | "ktoso", 19 | "patriknw", 20 | "chbatey" 21 | ] 22 | } 23 | }, 24 | "cassandra": { 25 | "contact_points": [ 26 | "176.34.145.253" 27 | ] 28 | }, 29 | "chef_environment": "_default", 30 | "ipaddress": "172.31.35.187", 31 | "ipaddress-external": "34.253.229.136", 32 | "ipaddress-internal": "172.31.35.187", 33 | "java": { 34 | "jdk_version": 8, 35 | "oracle": { 36 | "accept_oracle_download_terms": true 37 | }, 38 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 39 | }, 40 | "name": "re-akka-euwest-1b", 41 | "run_list": [ 42 | "role[benchmark]", 43 | "role[akka-re]" 44 | ] 45 | } -------------------------------------------------------------------------------- /infra/nodes/re-akka-euwest-1c.json: -------------------------------------------------------------------------------- 1 | { 2 | "akka": { 3 | "dc": "eu-west", 4 | "min_nodes": 2, 5 | "port": 2551, 6 | "roles": [], 7 | "seed_nodes": [ 8 | "54.154.72.165" 9 | ], 10 | "system_name": "MultiDcSystem" 11 | }, 12 | "authorization": { 13 | "sudo": { 14 | "passwordless": "true", 15 | "users": [ 16 | "akka", 17 | "ubuntu", 18 | "ktoso", 19 | "patriknw", 20 | "chbatey" 21 | ] 22 | } 23 | }, 24 | "cassandra": { 25 | "contact_points": [ 26 | "176.34.145.253" 27 | ] 28 | }, 29 | "chef_environment": "_default", 30 | "ipaddress": "172.31.0.162", 31 | "ipaddress-external": "54.154.72.165", 32 | "ipaddress-internal": "172.31.0.162", 33 | "java": { 34 | "jdk_version": 8, 35 | "oracle": { 36 | "accept_oracle_download_terms": true 37 | }, 38 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 39 | }, 40 | "name": "re-akka-euwest-1c", 41 | "run_list": [ 42 | "role[benchmark]", 43 | "role[akka-re]" 44 | ] 45 | } -------------------------------------------------------------------------------- /infra/nodes/re-akka.json.template: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "INTERNAL_IP", 4 | "ipaddress-external": "EXTERNAL_IP", 5 | "name": "NAME", 6 | 7 | "akka": { 8 | "system_name": "MultiDcSystem", 9 | "port": 2551, 10 | "roles": [], 11 | "seed_nodes": ["AKKA_SEED_IP"], 12 | "min_nodes": 2, 13 | "dc": "DC" 14 | }, 15 | "cassandra": { 16 | "contact_points": ["CASSANDRA_CONTACT_IP"] 17 | }, 18 | "authorization": { 19 | "sudo": { 20 | "users": ["akka", "ubuntu", "ktoso", "patriknw", "chbatey"], 21 | "passwordless": "true" 22 | } 23 | }, 24 | 25 | "java": { 26 | "jdk_version": 8, 27 | "oracle": { 28 | "accept_oracle_download_terms": true 29 | }, 30 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 31 | }, 32 | "run_list": [ 33 | "role[benchmark]", 34 | "role[akka-re]" 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /infra/nodes/re-cassandra-eucentral-1a.json: -------------------------------------------------------------------------------- 1 | { 2 | "apt": { 3 | "confd": { 4 | "force_confask": false, 5 | "force_confold": true 6 | } 7 | }, 8 | "cassandra": { 9 | "config": { 10 | "cluster_name": "Test Cluster", 11 | "snitch": "Ec2MultiRegionSnitch" 12 | }, 13 | "install_java": false, 14 | "seed_node": "34.242.227.97" 15 | }, 16 | "chef_environment": "_default", 17 | "ipaddress": "172.31.14.182", 18 | "ipaddress-external": "35.158.122.93", 19 | "ipaddress-internal": "172.31.14.182", 20 | "java": { 21 | "jdk_version": 8, 22 | "oracle": { 23 | "accept_oracle_download_terms": true 24 | }, 25 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 26 | }, 27 | "name": "re-cassandra-eucentral-1a", 28 | "run_list": [ 29 | "role[cassandra-re]" 30 | ] 31 | } -------------------------------------------------------------------------------- /infra/nodes/re-cassandra-eucentral-1b.json: -------------------------------------------------------------------------------- 1 | { 2 | "apt": { 3 | "confd": { 4 | "force_confask": false, 5 | "force_confold": true 6 | } 7 | }, 8 | "cassandra": { 9 | "config": { 10 | "cluster_name": "Test Cluster", 11 | "snitch": "Ec2MultiRegionSnitch" 12 | }, 13 | "install_java": false, 14 | "seed_node": "34.242.227.97" 15 | }, 16 | "chef_environment": "_default", 17 | "ipaddress": "172.31.25.170", 18 | "ipaddress-external": "54.93.225.104", 19 | "ipaddress-internal": "172.31.25.170", 20 | "java": { 21 | "jdk_version": 8, 22 | "oracle": { 23 | "accept_oracle_download_terms": true 24 | }, 25 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 26 | }, 27 | "name": "re-cassandra-eucentral-1b", 28 | "run_list": [ 29 | "role[cassandra-re]" 30 | ] 31 | } -------------------------------------------------------------------------------- /infra/nodes/re-cassandra-eucentral-1c.json: -------------------------------------------------------------------------------- 1 | { 2 | "apt": { 3 | "confd": { 4 | "force_confask": false, 5 | "force_confold": true 6 | } 7 | }, 8 | "cassandra": { 9 | "config": { 10 | "cluster_name": "Test Cluster", 11 | "snitch": "Ec2MultiRegionSnitch" 12 | }, 13 | "install_java": false, 14 | "seed_node": "34.242.227.97" 15 | }, 16 | "chef_environment": "_default", 17 | "ipaddress": "172.31.45.92", 18 | "ipaddress-external": "18.195.21.84", 19 | "ipaddress-internal": "172.31.45.92", 20 | "java": { 21 | "jdk_version": 8, 22 | "oracle": { 23 | "accept_oracle_download_terms": true 24 | }, 25 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 26 | }, 27 | "name": "re-cassandra-eucentral-1c", 28 | "run_list": [ 29 | "role[cassandra-re]" 30 | ] 31 | } -------------------------------------------------------------------------------- /infra/nodes/re-cassandra-euwest-1a.json: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "172.31.28.174", 4 | "ipaddress-external": "54.194.79.5", 5 | "name": "re-cassandra-euwest-1a", 6 | "java": { 7 | "jdk_version": 8, 8 | "oracle": { 9 | "accept_oracle_download_terms": true 10 | }, 11 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 12 | }, 13 | "cassandra": { 14 | "seed_node": "34.242.227.97", 15 | "install_java": false, 16 | "config": { 17 | "cluster_name": "Test Cluster", 18 | "snitch": "Ec2MultiRegionSnitch" 19 | } 20 | }, 21 | "run_list": [ 22 | "role[cassandra-re]" 23 | ], 24 | "apt": { 25 | "confd": { 26 | "force_confold": true, 27 | "force_confask": false 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /infra/nodes/re-cassandra-euwest-1b.json: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "172.31.47.83", 4 | "ipaddress-external": "34.242.227.97", 5 | "name": "re-cassandra-euwest-1b", 6 | "java": { 7 | "jdk_version": 8, 8 | "oracle": { 9 | "accept_oracle_download_terms": true 10 | }, 11 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 12 | }, 13 | "cassandra": { 14 | "seed_node": "34.242.227.97", 15 | "install_java": false, 16 | "config": { 17 | "cluster_name": "Test Cluster", 18 | "snitch": "Ec2MultiRegionSnitch" 19 | } 20 | }, 21 | "run_list": [ 22 | "role[cassandra-re]" 23 | ], 24 | "apt": { 25 | "confd": { 26 | "force_confold": true, 27 | "force_confask": false 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /infra/nodes/re-cassandra-euwest-1c.json: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "172.31.13.199", 4 | "ipaddress-external": "34.242.248.117", 5 | "name": "re-cassandra-euwest-1c", 6 | "java": { 7 | "jdk_version": 8, 8 | "oracle": { 9 | "accept_oracle_download_terms": true 10 | }, 11 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 12 | }, 13 | "cassandra": { 14 | "seed_node": "34.242.227.97", 15 | "install_java": false, 16 | "config": { 17 | "cluster_name": "Test Cluster", 18 | "snitch": "Ec2MultiRegionSnitch" 19 | } 20 | }, 21 | "run_list": [ 22 | "role[cassandra-re]" 23 | ], 24 | "apt": { 25 | "confd": { 26 | "force_confold": true, 27 | "force_confask": false 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /infra/nodes/re-cassandra.json.template: -------------------------------------------------------------------------------- 1 | { 2 | "chef_environment": "_default", 3 | "ipaddress-internal": "INTERNAL_IP", 4 | "ipaddress-external": "EXTERNAL_IP", 5 | "name": "NAME", 6 | "java": { 7 | "jdk_version": 8, 8 | "oracle": { 9 | "accept_oracle_download_terms": true 10 | }, 11 | "url": "http://download.oracle.com/otn-pub/java/jdk/8u112-b15/server-jre-8u112-linux-x64.tar.gz" 12 | }, 13 | "cassandra": { 14 | "seed_node": "CASSANDRA_SEED_IP", 15 | "install_java": false, 16 | "config": { 17 | "cluster_name": "Test Cluster", 18 | "snitch": "Ec2MultiRegionSnitch" 19 | } 20 | }, 21 | "run_list": [ 22 | "role[cassandra-re]" 23 | ], 24 | "apt": { 25 | "confd": { 26 | "force_confold": true, 27 | "force_confask": false 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /infra/prepare-all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ $# != "1" ]; then 4 | echo "Usage: . prepare-all.sh SUDO_PASS" 5 | exit 1 6 | fi 7 | 8 | PASS="$1" 9 | 10 | for id in {0..8} 11 | do 12 | node="moxie-a$id" 13 | echo 14 | echo "Deploying Chef to [$node]..." 15 | # fix node:$node deploy_chef 16 | knife solo prepare --ssh-password "$PASS" local@$node 17 | echo 18 | done 19 | 20 | echo "Done!" -------------------------------------------------------------------------------- /infra/roles/akka-re.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "akka-re", 3 | "description": "", 4 | "json_class": "Chef::Role", 5 | 6 | "default_attributes": { 7 | 8 | }, 9 | 10 | "run_list": [ 11 | "recipe[akka-multi-dc]" 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /infra/roles/akka_team.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "akka_team", 3 | "description": "Marks a node used by the Akka Team exclusively.", 4 | "json_class": "Chef::Role", 5 | 6 | "default_attributes": { 7 | }, 8 | 9 | "run_list": [ 10 | ] 11 | } -------------------------------------------------------------------------------- /infra/roles/benchmark-ddata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "benchmark-ddata", 3 | "description": "", 4 | "json_class": "Chef::Role", 5 | 6 | "default_attributes": { 7 | }, 8 | 9 | "run_list": [ 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /infra/roles/benchmark-sharding.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "benchmark-ddata", 3 | "description": "", 4 | "json_class": "Chef::Role", 5 | 6 | "default_attributes": { 7 | }, 8 | 9 | "run_list": [ 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /infra/roles/benchmark.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "benchmark", 3 | "description": "", 4 | "json_class": "Chef::Role", 5 | 6 | "default_attributes": { 7 | }, 8 | 9 | "run_list": [ 10 | "recipe[akka-user]", 11 | "recipe[update-hosts]", 12 | "recipe[clone-akka-repos]", 13 | "recipe[sudo]", 14 | "recipe[sbt-manual]", 15 | "recipe[java-manual]", 16 | "recipe[perf]", 17 | "recipe[wrk2]", 18 | "recipe[ab]", 19 | "recipe[akka-linux-tweaks]" 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /infra/roles/cassandra-re.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cassandra", 3 | "description": "Make a cassandra node", 4 | "json_class": "Chef::Role", 5 | 6 | "default_attributes": { 7 | }, 8 | 9 | "run_list": [ 10 | "recipe[java-manual]", 11 | "recipe[cassandra-linux-tweaks]", 12 | "recipe[cassandra-manual]" 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /infra/roles/cassandra.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cassandra", 3 | "description": "Make a cassandra node", 4 | "json_class": "Chef::Role", 5 | 6 | "default_attributes": { 7 | }, 8 | 9 | "run_list": [ 10 | "recipe[java-manual]", 11 | "recipe[cassandra-linux-tweaks]", 12 | "recipe[cassandra-manual]" 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /infra/roles/gce-tools.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "benchmark", 3 | "description": "", 4 | "json_class": "Chef::Role", 5 | 6 | "default_attributes": { 7 | }, 8 | 9 | "run_list": [ 10 | "recipe[gce-logging-agent]", 11 | "recipe[sbt-manual]", 12 | "recipe[wrk2]", 13 | "recipe[ab]" 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /infra/roles/jenkins_master.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jenkins_master", 3 | "description": "", 4 | "json_class": "Chef::Role", 5 | 6 | "default_attributes": { 7 | }, 8 | 9 | "run_list": [ 10 | ] 11 | } -------------------------------------------------------------------------------- /infra/roles/jenkins_worker.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jenkins_worker", 3 | "description": "", 4 | "json_class": "Chef::Role", 5 | 6 | "default_attributes": { 7 | }, 8 | 9 | "run_list": [ 10 | "recipe[sbt-manual]", 11 | "recipe[sbt_credentials]", 12 | "recipe[java-manual]", 13 | "recipe[docker_simple]", 14 | "recipe[wrk2]", 15 | "recipe[ab]" 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /infra/site-cookbooks/ab/.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *# 3 | .#* 4 | \#*# 5 | .*.sw[a-z] 6 | *.un~ 7 | pkg/ 8 | 9 | # Berkshelf 10 | .vagrant 11 | /cookbooks 12 | Berksfile.lock 13 | 14 | # Bundler 15 | Gemfile.lock 16 | bin/* 17 | .bundle/* 18 | 19 | .kitchen/ 20 | .kitchen.local.yml 21 | -------------------------------------------------------------------------------- /infra/site-cookbooks/ab/.kitchen.yml: -------------------------------------------------------------------------------- 1 | --- 2 | driver: 3 | name: vagrant 4 | 5 | provisioner: 6 | name: chef_solo 7 | 8 | platforms: 9 | - name: ubuntu-14.04 10 | - name: centos-7.1 11 | 12 | suites: 13 | - name: default 14 | run_list: 15 | attributes: 16 | -------------------------------------------------------------------------------- /infra/site-cookbooks/ab/Berksfile: -------------------------------------------------------------------------------- 1 | source "https://supermarket.chef.io" 2 | 3 | metadata 4 | -------------------------------------------------------------------------------- /infra/site-cookbooks/ab/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 0.1.0 2 | 3 | Initial release of ab 4 | -------------------------------------------------------------------------------- /infra/site-cookbooks/ab/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'berkshelf' 4 | 5 | # Uncomment these lines if you want to live on the Edge: 6 | # 7 | # group :development do 8 | # gem "berkshelf", github: "berkshelf/berkshelf" 9 | # gem "vagrant", github: "mitchellh/vagrant", tag: "v1.6.3" 10 | # end 11 | # 12 | # group :plugins do 13 | # gem "vagrant-berkshelf", github: "berkshelf/vagrant-berkshelf" 14 | # gem "vagrant-omnibus", github: "schisamo/vagrant-omnibus" 15 | # end 16 | 17 | gem "test-kitchen" 18 | gem "kitchen-vagrant" 19 | -------------------------------------------------------------------------------- /infra/site-cookbooks/ab/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2015 YOUR_NAME 2 | 3 | All rights reserved - Do Not Redistribute 4 | -------------------------------------------------------------------------------- /infra/site-cookbooks/ab/README.md: -------------------------------------------------------------------------------- 1 | # ab-cookbook 2 | 3 | TODO: Enter the cookbook description here. 4 | 5 | ## Supported Platforms 6 | 7 | TODO: List your supported platforms. 8 | 9 | ## Attributes 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 |
KeyTypeDescriptionDefault
['ab']['bacon']Booleanwhether to include bacontrue
25 | 26 | ## Usage 27 | 28 | ### ab::default 29 | 30 | Include `ab` in your node's `run_list`: 31 | 32 | ```json 33 | { 34 | "run_list": [ 35 | "recipe[ab::default]" 36 | ] 37 | } 38 | ``` 39 | 40 | ## License and Authors 41 | 42 | Author:: YOUR_NAME () 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/ab/Thorfile: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'bundler' 4 | require 'bundler/setup' 5 | require 'berkshelf/thor' 6 | 7 | begin 8 | require "kitchen/thor_tasks" 9 | Kitchen::ThorTasks.new 10 | rescue LoadError 11 | puts ">>>>> Kitchen gem not loaded, omitting tasks" unless ENV["CI"] 12 | end 13 | -------------------------------------------------------------------------------- /infra/site-cookbooks/ab/chefignore: -------------------------------------------------------------------------------- 1 | .kitchen 2 | -------------------------------------------------------------------------------- /infra/site-cookbooks/ab/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ab", 3 | "description": "Installs/Configures ab", 4 | "long_description": "Installs/Configures ab", 5 | "maintainer": "YOUR_NAME", 6 | "maintainer_email": "YOUR_EMAIL", 7 | "license": "All rights reserved", 8 | "platforms": { 9 | 10 | }, 11 | "dependencies": { 12 | 13 | }, 14 | "recommendations": { 15 | 16 | }, 17 | "suggestions": { 18 | 19 | }, 20 | "conflicting": { 21 | 22 | }, 23 | "providing": { 24 | 25 | }, 26 | "replacing": { 27 | 28 | }, 29 | "attributes": { 30 | 31 | }, 32 | "groupings": { 33 | 34 | }, 35 | "recipes": { 36 | 37 | }, 38 | "version": "0.1.0", 39 | "source_url": "", 40 | "issues_url": "", 41 | "privacy": false 42 | } 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/ab/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'ab' 2 | maintainer 'YOUR_NAME' 3 | maintainer_email 'YOUR_EMAIL' 4 | license 'All rights reserved' 5 | description 'Installs/Configures ab' 6 | long_description 'Installs/Configures ab' 7 | version '0.1.0' 8 | -------------------------------------------------------------------------------- /infra/site-cookbooks/ab/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: ab 3 | # Recipe:: default 4 | # 5 | # Copyright (C) 2016 Konrad Malawski 6 | # 7 | # All rights reserved - Do Not Redistribute 8 | # 9 | 10 | package 'apache2-utils' do 11 | action :install 12 | end 13 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-linux-tweaks/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | gce-logging-agent CHANGELOG 2 | =========================== 3 | 4 | This file is used to list changes made in each version of the gce-logging-agent cookbook. 5 | 6 | 0.1.0 7 | ----- 8 | - [your_name] - Initial release of gce-logging-agent 9 | 10 | - - - 11 | Check the [Markdown Syntax Guide](http://daringfireball.net/projects/markdown/syntax) for help with Markdown. 12 | 13 | The [Github Flavored Markdown page](http://github.github.com/github-flavored-markdown/) describes the differences between markdown on github and standard markdown. 14 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-linux-tweaks/README.md: -------------------------------------------------------------------------------- 1 | akka linux tweaks 2 | ================= 3 | 4 | all kinds of OS settings that make akka run nicer. 5 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-linux-tweaks/files/default/startup-script-akka-tweaks.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script will be executed as startup-script (see gcloud-new-akka-node.sh), 4 | # by compute engine each time this node is started up. 5 | 6 | swapoff --all 7 | 8 | sysctl net.core.rmem_max=2097152 9 | sysctl net.core.wmem_max=2097152 10 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-linux-tweaks/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "akka-linux-tweaks", 3 | "description": "Installs/Configures gce-logging-agent", 4 | "long_description": "akka linux tweaks\n=================\n\nall kinds of OS settings that make akka run nicer.\n", 5 | "maintainer": "YOUR_COMPANY_NAME", 6 | "maintainer_email": "YOUR_EMAIL", 7 | "license": "All rights reserved", 8 | "platforms": { 9 | 10 | }, 11 | "dependencies": { 12 | 13 | }, 14 | "recommendations": { 15 | 16 | }, 17 | "suggestions": { 18 | 19 | }, 20 | "conflicting": { 21 | 22 | }, 23 | "providing": { 24 | 25 | }, 26 | "replacing": { 27 | 28 | }, 29 | "attributes": { 30 | 31 | }, 32 | "groupings": { 33 | 34 | }, 35 | "recipes": { 36 | 37 | }, 38 | "version": "0.1.0", 39 | "source_url": "", 40 | "issues_url": "", 41 | "privacy": false 42 | } 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-linux-tweaks/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'akka-linux-tweaks' 2 | maintainer 'YOUR_COMPANY_NAME' 3 | maintainer_email 'YOUR_EMAIL' 4 | license 'All rights reserved' 5 | description 'Installs/Configures gce-logging-agent' 6 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 7 | version '0.1.0' 8 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-linux-tweaks/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: gce-logging-agent 3 | # Recipe:: default 4 | # 5 | # Copyright 2017, Lightbend 6 | # 7 | # All rights reserved - Do Not Redistribute 8 | # 9 | 10 | cookbook_file "/home/akka/startup-script-akka-tweaks.sh" do 11 | source "startup-script-akka-tweaks.sh" 12 | end 13 | 14 | bash "chown akka.akka startup-script-akka-tweaks.sh" do 15 | code <<-EOH 16 | chown akka.akka /home/akka/startup-script-akka-tweaks.sh 17 | EOH 18 | end 19 | 20 | bash 'execute startup-script-akka-linux-tweaks.sh' do 21 | code <<-EOH 22 | cd /home/akka 23 | chmod +x ./startup-script-akka-tweaks.sh 24 | ./startup-script-akka-tweaks.sh 25 | EOH 26 | end 27 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-linux-tweaks/templates/default/akka.conf.rb: -------------------------------------------------------------------------------- 1 | 2 | type tail 3 | format none 4 | path <%= @log_path %> 5 | pos_file /var/lib/google-fluentd/pos/akka.pos 6 | read_from_head true 7 | tag akka 8 | 9 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-multi-dc/.gitignore: -------------------------------------------------------------------------------- 1 | files/* 2 | files/default/* 3 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-multi-dc/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | gce-logging-agent CHANGELOG 2 | =========================== 3 | 4 | This file is used to list changes made in each version of the akka-multi-dc cookbook. 5 | 6 | 0.1.0 7 | ----- 8 | - [your_name] - Initial release of gce-logging-agent 9 | 10 | - - - 11 | Check the [Markdown Syntax Guide](http://daringfireball.net/projects/markdown/syntax) for help with Markdown. 12 | 13 | The [Github Flavored Markdown page](http://github.github.com/github-flavored-markdown/) describes the differences between markdown on github and standard markdown. 14 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-multi-dc/README.md: -------------------------------------------------------------------------------- 1 | akka multi dc 2 | ================= 3 | 4 | all kinds of OS settings that make akka run nicer. 5 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-multi-dc/attributes/default.rb: -------------------------------------------------------------------------------- 1 | default['akka']['system_name'] = 'MultiDcCluster' 2 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-multi-dc/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "akka-multi-dc", 3 | "description": "...", 4 | "long_description": "", 5 | "maintainer": "YOUR_COMPANY_NAME", 6 | "maintainer_email": "YOUR_EMAIL", 7 | "license": "All rights reserved", 8 | "platforms": { 9 | 10 | }, 11 | "dependencies": { 12 | 13 | }, 14 | "recommendations": { 15 | 16 | }, 17 | "suggestions": { 18 | 19 | }, 20 | "conflicting": { 21 | 22 | }, 23 | "providing": { 24 | 25 | }, 26 | "replacing": { 27 | 28 | }, 29 | "attributes": { 30 | 31 | }, 32 | "groupings": { 33 | 34 | }, 35 | "recipes": { 36 | 37 | }, 38 | "version": "0.1.0", 39 | "source_url": "", 40 | "issues_url": "", 41 | "privacy": false 42 | } 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-multi-dc/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'akka-multi-dc' 2 | maintainer 'YOUR_COMPANY_NAME' 3 | maintainer_email 'YOUR_EMAIL' 4 | license 'All rights reserved' 5 | description '...' 6 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 7 | version '0.1.0' 8 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-multi-dc/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: akka-multi-dc 3 | # Recipe:: default 4 | # 5 | # Copyright 2017, Lightbend 6 | # 7 | # All rights reserved - Do Not Redistribute 8 | # 9 | 10 | directory '/home/akka/multidc' do 11 | owner 'akka' 12 | group 'akka' 13 | action :create 14 | end 15 | 16 | 17 | template "/home/akka/multidc/application.conf" do 18 | source "application.conf.erb" 19 | end 20 | 21 | bash "all akka users should trust replicated-entity.pem" do 22 | code <<-EOH 23 | sudo su - akka 24 | echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCXlAG/NOl6RMXCSvWvwZuqOMed3PFIkZvxfgJaSOhf2t50uSG1gJ6UEafDHuPf2wgkDr8Og8EtO5DV7X4s9oDbXII81U/4Mr0m7ar6n7KRR2e3YKrrJ1TpYMCivzrrABSqQYyWqpEh63YASNlyh8YQzGSpBbkALKkA4b4mR5F2/+6c/bXAthLMlElVs9Nvf3REXIZHmilLHpQzSsIi5DkqBXZmSl1GD4bjNuKUzVbjNNDXS0dRSztonOD4JWJRaOC4dagNL+OXVyycS3HKj0BnB3it4b03rFpr2U/oqp1mZEayTqb5wGlRBz9jUh6a5F94K3eJVR3Fxv1A54iKVAsd replicated-entity" >> $HOME/.ssh/authorized_keys 25 | EOH 26 | end 27 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-multi-dc/templates/default/application.conf.erb: -------------------------------------------------------------------------------- 1 | # ------------------------ WARNING -------------------------------- 2 | # This file is generated automatically via Chef scripts. 3 | # Do not edit manually - all your changes WILL be lost. 4 | # ------------------------ WARNING -------------------------------- 5 | akka { 6 | 7 | actor.provider = cluster 8 | 9 | remote { 10 | netty.tcp { 11 | # going across DCs so need the external IP 12 | hostname = "<%= node['ipaddress-external'] %>" 13 | port = <%= node['akka']['port'] %> 14 | bind-hostname = "0.0.0.0" 15 | } 16 | 17 | artery { 18 | # enabled = on this is up to the app to decide 19 | canonical.hostname = "<%= node['ipaddress-external'] %>" 20 | canonical.port = <%= node['akka']['port'] %> 21 | 22 | bind { 23 | port = <%= node['akka']['port'] %> 24 | hostname = "0.0.0.0" 25 | } 26 | } 27 | } 28 | 29 | cluster { 30 | 31 | min-nr-of-members = <%= node['akka']['min_nodes'] %> 32 | 33 | roles = [ 34 | <%= (node['akka']['roles'].map do |role| 35 | "\"#{role}\"" 36 | end).join(",\n ") %> 37 | ] 38 | 39 | seed-nodes = [ 40 | <%= (node['akka']['seed_nodes'].map do |seed| 41 | "\"akka://#{node['akka']['system_name']}@#{seed}:#{node['akka']['port']}\"" 42 | end).join(",\n ") %> 43 | ] 44 | 45 | http.management { 46 | hostname = "0.0.0.0" 47 | port = 19999 48 | } 49 | 50 | multi-data-center { 51 | self-data-center = "<%= node['akka']['dc'] %>" 52 | } 53 | } 54 | 55 | persistence { 56 | snapshot-store.plugin = "cassandra-snapshot-store" 57 | 58 | multi-data-center { 59 | all-data-centers = ["eu-central", "eu-west"] 60 | } 61 | } 62 | 63 | } 64 | 65 | cassandra-journal-multi-dc { 66 | contact-points = [ 67 | <%= (node['cassandra']['contact_points'].map do |node| 68 | "\"#{node}\"" 69 | end).join(",\n ") %> 70 | ] 71 | local-datacenter = "<%= node['akka']['dc'] %>" 72 | log-queries = on 73 | } 74 | 75 | cassandra-query-journal-multi-dc { 76 | contact-points = [ 77 | <%= (node['cassandra']['contact_points'].map do |node| 78 | "\"#{node}\"" 79 | end).join(",\n ") %> 80 | ] 81 | local-datacenter = "<%= node['akka']['dc'] %>" 82 | log-queries = on 83 | } 84 | 85 | cassandra-snapshot-store { 86 | # FQCN of the cassandra snapshot store plugin 87 | class = "akka.persistence.cassandra.snapshot.CassandraSnapshotStore" 88 | 89 | contact-points = [ 90 | <%= (node['cassandra']['contact_points'].map do |node| 91 | "\"#{node}\"" 92 | end).join(",\n ") %> 93 | ] 94 | 95 | local-datacenter = "<%= node['akka']['dc'] %>" 96 | log-queries = on 97 | } 98 | 99 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-user/.gitignore: -------------------------------------------------------------------------------- 1 | files/* 2 | files/default/* 3 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-user/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | gce-logging-agent CHANGELOG 2 | =========================== 3 | 4 | This file is used to list changes made in each version of the gce-logging-agent cookbook. 5 | 6 | 0.1.0 7 | ----- 8 | - [your_name] - Initial release of gce-logging-agent 9 | 10 | - - - 11 | Check the [Markdown Syntax Guide](http://daringfireball.net/projects/markdown/syntax) for help with Markdown. 12 | 13 | The [Github Flavored Markdown page](http://github.github.com/github-flavored-markdown/) describes the differences between markdown on github and standard markdown. 14 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-user/README.md: -------------------------------------------------------------------------------- 1 | akka linux tweaks 2 | ================= 3 | 4 | all kinds of OS settings that make akka run nicer. 5 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-user/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "akka-root-application-conf", 3 | "description": "...", 4 | "long_description": "", 5 | "maintainer": "YOUR_COMPANY_NAME", 6 | "maintainer_email": "YOUR_EMAIL", 7 | "license": "All rights reserved", 8 | "platforms": { 9 | 10 | }, 11 | "dependencies": { 12 | 13 | }, 14 | "recommendations": { 15 | 16 | }, 17 | "suggestions": { 18 | 19 | }, 20 | "conflicting": { 21 | 22 | }, 23 | "providing": { 24 | 25 | }, 26 | "replacing": { 27 | 28 | }, 29 | "attributes": { 30 | 31 | }, 32 | "groupings": { 33 | 34 | }, 35 | "recipes": { 36 | 37 | }, 38 | "version": "0.1.0", 39 | "source_url": "", 40 | "issues_url": "", 41 | "privacy": false 42 | } 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-user/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'akka-root-application-conf' 2 | maintainer 'YOUR_COMPANY_NAME' 3 | maintainer_email 'YOUR_EMAIL' 4 | license 'All rights reserved' 5 | description '...' 6 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 7 | version '0.1.0' 8 | -------------------------------------------------------------------------------- /infra/site-cookbooks/akka-user/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: gce-logging-agent 3 | # Recipe:: default 4 | # 5 | # Copyright 2017, Lightbend 6 | # 7 | # All rights reserved - Do Not Redistribute 8 | # 9 | 10 | user "akka" do 11 | home '/home/akka' 12 | shell '/bin/bash' 13 | password 'akka' 14 | action :create 15 | manage_home true 16 | end 17 | 18 | directory '/home/akka/.ssh' do 19 | owner 'akka' 20 | group 'akka' 21 | mode '0700' 22 | action :create 23 | end 24 | 25 | # set up mutual (naive, same key) trust between benchmark nodes 26 | bash "trust akka's public key" do 27 | code <<-EOH 28 | echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDg0jkMkullztlpthHQ8LB4fYvD9Fu2f0EdE6+slC9FjfqIdgkvC2OfOFNrjVniP9ghe7iVcIU+ZscbxiQO92VT6tj67XxgqMosYwmeonn43IR77JsgUKqlfHgGlrCwjg9b1IL2bhAY0Ib7Ot+1iqpxHIKk6uWyZZm6Nu3hMLIFBNA0C50DmuzC3y6sbUCVg/2d6seLeG6hFrhdtSWW/RkkxXEh5x798VgY0+WoU+SyYBEIcrc9ObNMzYKavg3jNJH+tEBUetVhzg2t9whmFWwZvFoX89pYF0vzK9lwh72AxqsjWa3tor4QHH39SBBPKsKT22vfX94Mzt0D2+panyzB akka@akka-nodes" >> /home/akka/.ssh/authorized_keys 29 | EOH 30 | end 31 | 32 | 33 | # these are secrets, any key will work though, just generate one 34 | cookbook_file '/home/akka/.ssh/id_rsa' do 35 | source 'akka-user_id_rsa' 36 | end 37 | cookbook_file '/home/akka/.ssh/id_rsa.pub' do 38 | source 'akka-user_id_rsa.pub' 39 | end 40 | 41 | bash "make sure right permissions on ssh keys" do 42 | code "chmod 400 /home/akka/.ssh/id_rsa" 43 | end 44 | 45 | # this is a special file to get commercial tools onto the nodes 46 | # use your key and put it in there 47 | bash "create $HOME/.lightbend" do 48 | code "mkdir -p /home/akka/.lightbend" 49 | end 50 | cookbook_file '/home/akka/.lightbend/commercial.credentials' do 51 | source 'lightbend-commercial.credentials' 52 | end 53 | 54 | -------------------------------------------------------------------------------- /infra/site-cookbooks/cassandra-linux-tweaks/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | gce-logging-agent CHANGELOG 2 | =========================== 3 | 4 | This file is used to list changes made in each version of the gce-logging-agent cookbook. 5 | 6 | 0.1.0 7 | ----- 8 | - [your_name] - Initial release of gce-logging-agent 9 | 10 | - - - 11 | Check the [Markdown Syntax Guide](http://daringfireball.net/projects/markdown/syntax) for help with Markdown. 12 | 13 | The [Github Flavored Markdown page](http://github.github.com/github-flavored-markdown/) describes the differences between markdown on github and standard markdown. 14 | -------------------------------------------------------------------------------- /infra/site-cookbooks/cassandra-linux-tweaks/README.md: -------------------------------------------------------------------------------- 1 | cassandra linux tweaks 2 | ====================== 3 | 4 | all kinds of OS settings for cassandra 5 | -------------------------------------------------------------------------------- /infra/site-cookbooks/cassandra-linux-tweaks/files/default/startup-script-cassandra-tweaks.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script will be executed as startup-script (see gcloud-new-akka-node.sh), 4 | # by compute engine each time this node is started up. 5 | 6 | swapoff --all 7 | 8 | sysctl net.core.rmem_max=16777216 9 | sysctl net.core.wmem_max=16777216 10 | 11 | sysctl net.core.rmem_default=16777216 12 | sysctl net.core.wmem_default=16777216 13 | 14 | sysctl net.core.optmem_max=40960 15 | 16 | sysctl net.ipv4.tcp_rmem="4096 87380 16777216" 17 | sysctl net.ipv4.tcp_wmem="4096 65536 16777216" 18 | 19 | echo 0 > /proc/sys/vm/zone_reclaim_mode 20 | -------------------------------------------------------------------------------- /infra/site-cookbooks/cassandra-linux-tweaks/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cassandra-linux-tweaks", 3 | "description": "Configures linux tweaks for running cassandra", 4 | "long_description": "Configures linux tweaks for running cassandra\n", 5 | "maintainer": "YOUR_COMPANY_NAME", 6 | "maintainer_email": "YOUR_EMAIL", 7 | "license": "All rights reserved", 8 | "platforms": { 9 | 10 | }, 11 | "dependencies": { 12 | 13 | }, 14 | "recommendations": { 15 | 16 | }, 17 | "suggestions": { 18 | 19 | }, 20 | "conflicting": { 21 | 22 | }, 23 | "providing": { 24 | 25 | }, 26 | "replacing": { 27 | 28 | }, 29 | "attributes": { 30 | 31 | }, 32 | "groupings": { 33 | 34 | }, 35 | "recipes": { 36 | 37 | }, 38 | "version": "0.1.0", 39 | "source_url": "", 40 | "issues_url": "", 41 | "privacy": false 42 | } 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/cassandra-linux-tweaks/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'cassandra-linux-tweaks' 2 | maintainer 'YOUR_COMPANY_NAME' 3 | maintainer_email 'YOUR_EMAIL' 4 | license 'All rights reserved' 5 | description 'Configures linux tweaks for running cassandra' 6 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 7 | version '0.1.0' 8 | -------------------------------------------------------------------------------- /infra/site-cookbooks/cassandra-linux-tweaks/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: cassandra-linux-tweaks 3 | # Recipe:: default 4 | # 5 | # Copyright 2017, Lightbend 6 | # 7 | # All rights reserved - Do Not Redistribute 8 | # 9 | 10 | cookbook_file "/home/ubuntu/startup-script-cassandra-tweaks.sh" do 11 | source "startup-script-cassandra-tweaks.sh" 12 | end 13 | 14 | template '/etc/security/limits.d/cassandra.conf' do 15 | source 'security-cassandra.conf' 16 | end 17 | 18 | bash "chown akka.akka startup-script-cassandra-tweaks.sh" do 19 | code <<-EOH 20 | chown ubuntu: /home/ubuntu/startup-script-cassandra-tweaks.sh 21 | EOH 22 | end 23 | 24 | bash 'execute startup-script-cassandra-linux-tweaks.sh' do 25 | code <<-EOH 26 | cd /home/ubuntu 27 | chmod +x ./startup-script-cassandra-tweaks.sh 28 | ./startup-script-cassandra-tweaks.sh 29 | EOH 30 | end 31 | -------------------------------------------------------------------------------- /infra/site-cookbooks/cassandra-linux-tweaks/templates/default/security-cassandra.conf: -------------------------------------------------------------------------------- 1 | cassandra - memlock unlimited 2 | cassandra - nofile 100000 3 | cassandra - as unlimited 4 | cassandra - nproc 32768 5 | -------------------------------------------------------------------------------- /infra/site-cookbooks/cassandra-manual/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | cassandra-manual CHANGELOG 2 | ========================== 3 | 4 | This file is used to list changes made in each version of the cassandra-manual cookbook. 5 | 6 | 0.1.0 7 | ----- 8 | - [your_name] - Initial release of cassandra-manual 9 | 10 | - - - 11 | Check the [Markdown Syntax Guide](http://daringfireball.net/projects/markdown/syntax) for help with Markdown. 12 | 13 | The [Github Flavored Markdown page](http://github.github.com/github-flavored-markdown/) describes the differences between markdown on github and standard markdown. 14 | -------------------------------------------------------------------------------- /infra/site-cookbooks/cassandra-manual/README.md: -------------------------------------------------------------------------------- 1 | cassandra-manual Cookbook 2 | ========================= 3 | TODO: Enter the cookbook description here. 4 | 5 | e.g. 6 | This cookbook makes your favorite breakfast sandwich. 7 | 8 | Requirements 9 | ------------ 10 | TODO: List your cookbook requirements. Be sure to include any requirements this cookbook has on platforms, libraries, other cookbooks, packages, operating systems, etc. 11 | 12 | e.g. 13 | #### packages 14 | - `toaster` - cassandra-manual needs toaster to brown your bagel. 15 | 16 | Attributes 17 | ---------- 18 | TODO: List your cookbook attributes here. 19 | 20 | e.g. 21 | #### cassandra-manual::default 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 |
KeyTypeDescriptionDefault
['cassandra-manual']['bacon']Booleanwhether to include bacontrue
36 | 37 | Usage 38 | ----- 39 | #### cassandra-manual::default 40 | TODO: Write usage instructions for each cookbook. 41 | 42 | e.g. 43 | Just include `cassandra-manual` in your node's `run_list`: 44 | 45 | ```json 46 | { 47 | "name":"my_node", 48 | "run_list": [ 49 | "recipe[cassandra-manual]" 50 | ] 51 | } 52 | ``` 53 | 54 | Contributing 55 | ------------ 56 | TODO: (optional) If this is a public cookbook, detail the process for contributing. If this is a private cookbook, remove this section. 57 | 58 | e.g. 59 | 1. Fork the repository on Github 60 | 2. Create a named feature branch (like `add_component_x`) 61 | 3. Write your change 62 | 4. Write tests for your change (if applicable) 63 | 5. Run the tests, ensuring they all pass 64 | 6. Submit a Pull Request using Github 65 | 66 | License and Authors 67 | ------------------- 68 | Authors: TODO: List authors 69 | -------------------------------------------------------------------------------- /infra/site-cookbooks/cassandra-manual/attributes/default.rb: -------------------------------------------------------------------------------- 1 | default['cassandra']['config']['snitch'] = "GossipingPropertyFileSnitch" -------------------------------------------------------------------------------- /infra/site-cookbooks/cassandra-manual/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cassandra-manual", 3 | "description": "Installs/Configures cassandra-manual", 4 | "long_description": "cassandra-manual Cookbook\n=========================\nTODO: Enter the cookbook description here.\n\ne.g.\nThis cookbook makes your favorite breakfast sandwich.\n\nRequirements\n------------\nTODO: List your cookbook requirements. Be sure to include any requirements this cookbook has on platforms, libraries, other cookbooks, packages, operating systems, etc.\n\ne.g.\n#### packages\n- `toaster` - cassandra-manual needs toaster to brown your bagel.\n\nAttributes\n----------\nTODO: List your cookbook attributes here.\n\ne.g.\n#### cassandra-manual::default\n\n \n \n \n \n \n \n \n \n \n \n \n \n
KeyTypeDescriptionDefault
['cassandra-manual']['bacon']Booleanwhether to include bacontrue
\n\nUsage\n-----\n#### cassandra-manual::default\nTODO: Write usage instructions for each cookbook.\n\ne.g.\nJust include `cassandra-manual` in your node's `run_list`:\n\n```json\n{\n \"name\":\"my_node\",\n \"run_list\": [\n \"recipe[cassandra-manual]\"\n ]\n}\n```\n\nContributing\n------------\nTODO: (optional) If this is a public cookbook, detail the process for contributing. If this is a private cookbook, remove this section.\n\ne.g.\n1. Fork the repository on Github\n2. Create a named feature branch (like `add_component_x`)\n3. Write your change\n4. Write tests for your change (if applicable)\n5. Run the tests, ensuring they all pass\n6. Submit a Pull Request using Github\n\nLicense and Authors\n-------------------\nAuthors: TODO: List authors\n", 5 | "maintainer": "YOUR_COMPANY_NAME", 6 | "maintainer_email": "YOUR_EMAIL", 7 | "license": "All rights reserved", 8 | "platforms": { 9 | 10 | }, 11 | "dependencies": { 12 | 13 | }, 14 | "recommendations": { 15 | 16 | }, 17 | "suggestions": { 18 | 19 | }, 20 | "conflicting": { 21 | 22 | }, 23 | "providing": { 24 | 25 | }, 26 | "replacing": { 27 | 28 | }, 29 | "attributes": { 30 | 31 | }, 32 | "groupings": { 33 | 34 | }, 35 | "recipes": { 36 | 37 | }, 38 | "version": "0.1.0", 39 | "source_url": "", 40 | "issues_url": "", 41 | "privacy": false 42 | } 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/cassandra-manual/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'cassandra-manual' 2 | maintainer 'YOUR_COMPANY_NAME' 3 | maintainer_email 'YOUR_EMAIL' 4 | license 'All rights reserved' 5 | description 'Installs/Configures cassandra-manual' 6 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 7 | version '0.1.0' 8 | -------------------------------------------------------------------------------- /infra/site-cookbooks/cassandra-manual/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: cassandra-manual 3 | # Recipe:: default 4 | # 5 | # Copyright 2017, YOUR_COMPANY_NAME 6 | # 7 | # All rights reserved - Do Not Redistribute 8 | 9 | 10 | user "cassandra" do 11 | home '/home/cassandra' 12 | shell '/bin/bash' 13 | password 'cassandra' 14 | action :create 15 | manage_home true 16 | end 17 | 18 | 19 | directory '/var/lib/cassandra/' do 20 | owner 'cassandra' 21 | group 'cassandra' 22 | action :create 23 | end 24 | 25 | directory '/var/log/cassandra/' do 26 | owner 'cassandra' 27 | group 'cassandra' 28 | action :create 29 | end 30 | 31 | 32 | bash 'install cassandra apt repo' do 33 | code <<-EOH 34 | echo "deb http://www.apache.org/dist/cassandra/debian 30x main" | sudo tee -a /etc/apt/sources.list.d/cassandra.sources.list 35 | 36 | curl https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add - 37 | 38 | sudo apt-get update 39 | 40 | sudo apt-key adv --keyserver pool.sks-keyservers.net --recv-key A278B781FE4B2BDA 41 | 42 | EOH 43 | end 44 | 45 | package 'ca-certificates-java' do 46 | action :install 47 | end 48 | 49 | package 'openjdk-8-jre-headless' do 50 | action :install 51 | end 52 | 53 | package 'cassandra' do 54 | action :install 55 | end 56 | 57 | template "/etc/cassandra/cassandra.yaml" do 58 | source "cassandra.yaml.rb" 59 | end 60 | 61 | bash "restart cassandra" do 62 | code "service cassandra restart" 63 | end 64 | -------------------------------------------------------------------------------- /infra/site-cookbooks/clone-akka-repos/.gitignore: -------------------------------------------------------------------------------- 1 | files/* 2 | files/default/* 3 | -------------------------------------------------------------------------------- /infra/site-cookbooks/clone-akka-repos/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | gce-logging-agent CHANGELOG 2 | =========================== 3 | 4 | This file is used to list changes made in each version of the gce-logging-agent cookbook. 5 | 6 | 0.1.0 7 | ----- 8 | - [your_name] - Initial release of gce-logging-agent 9 | 10 | - - - 11 | Check the [Markdown Syntax Guide](http://daringfireball.net/projects/markdown/syntax) for help with Markdown. 12 | 13 | The [Github Flavored Markdown page](http://github.github.com/github-flavored-markdown/) describes the differences between markdown on github and standard markdown. 14 | -------------------------------------------------------------------------------- /infra/site-cookbooks/clone-akka-repos/README.md: -------------------------------------------------------------------------------- 1 | clone-akka-repos 2 | ================ 3 | 4 | clone all repos ("refresh them") 5 | -------------------------------------------------------------------------------- /infra/site-cookbooks/clone-akka-repos/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "clone-akka-repos", 3 | "description": "...", 4 | "long_description": "", 5 | "maintainer": "YOUR_COMPANY_NAME", 6 | "maintainer_email": "YOUR_EMAIL", 7 | "license": "All rights reserved", 8 | "platforms": { 9 | 10 | }, 11 | "dependencies": { 12 | 13 | }, 14 | "recommendations": { 15 | 16 | }, 17 | "suggestions": { 18 | 19 | }, 20 | "conflicting": { 21 | 22 | }, 23 | "providing": { 24 | 25 | }, 26 | "replacing": { 27 | 28 | }, 29 | "attributes": { 30 | 31 | }, 32 | "groupings": { 33 | 34 | }, 35 | "recipes": { 36 | 37 | }, 38 | "version": "0.1.0", 39 | "source_url": "", 40 | "issues_url": "", 41 | "privacy": false 42 | } 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/clone-akka-repos/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'clone-akka-repos' 2 | maintainer 'YOUR_COMPANY_NAME' 3 | maintainer_email 'YOUR_EMAIL' 4 | license 'All rights reserved' 5 | description '...' 6 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 7 | version '0.1.0' 8 | -------------------------------------------------------------------------------- /infra/site-cookbooks/clone-akka-repos/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: gce-logging-agent 3 | # Recipe:: default 4 | # 5 | # Copyright 2017, Lightbend 6 | # 7 | # All rights reserved - Do Not Redistribute 8 | # 9 | 10 | package "git" 11 | 12 | # bash "clone akka" do 13 | # code <<-EOH 14 | # cd /home/akka 15 | # rm -rf akka 16 | # git clone https://github.com/akka/akka 17 | # 18 | # chown -R akka.akka akka 19 | # EOH 20 | # end 21 | # 22 | # bash "clone akka/apps" do 23 | # code <<-EOH 24 | # cd /home/akka 25 | # rm -rf apps 26 | # git clone https://github.com/akka/apps.git 27 | # 28 | # chown -R akka.akka apps 29 | # EOH 30 | # end 31 | 32 | bash "clone akka/apps/multidc" do 33 | code <<-EOH 34 | cd /home/akka/multidc 35 | rm -rf apps 36 | git clone -b replicated-entity-tests https://github.com/chbatey/apps.git 37 | chown -R akka.akka apps 38 | EOH 39 | end 40 | 41 | 42 | 43 | # bash "clone akka/apps/management" do 44 | # code <<-EOH 45 | # cd /home/akka/multidc 46 | # rm -rf apps-management 47 | # git clone -b rep-test https://github.com/chbatey/akka-management.git 48 | # 49 | # chown -R akka.akka akka-management 50 | # EOH 51 | # end 52 | 53 | -------------------------------------------------------------------------------- /infra/site-cookbooks/docker_simple/.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *# 3 | .#* 4 | \#*# 5 | .*.sw[a-z] 6 | *.un~ 7 | pkg/ 8 | 9 | # Berkshelf 10 | .vagrant 11 | /cookbooks 12 | Berksfile.lock 13 | 14 | # Bundler 15 | Gemfile.lock 16 | bin/* 17 | .bundle/* 18 | 19 | .kitchen/ 20 | .kitchen.local.yml 21 | -------------------------------------------------------------------------------- /infra/site-cookbooks/docker_simple/.kitchen.yml: -------------------------------------------------------------------------------- 1 | --- 2 | driver: 3 | name: vagrant 4 | 5 | provisioner: 6 | name: chef_solo 7 | 8 | platforms: 9 | - name: ubuntu-14.04 10 | - name: centos-7.1 11 | 12 | suites: 13 | - name: default 14 | run_list: 15 | attributes: 16 | -------------------------------------------------------------------------------- /infra/site-cookbooks/docker_simple/Berksfile: -------------------------------------------------------------------------------- 1 | source "https://supermarket.chef.io" 2 | 3 | metadata 4 | -------------------------------------------------------------------------------- /infra/site-cookbooks/docker_simple/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 0.1.0 2 | 3 | Initial release of docker_simple 4 | -------------------------------------------------------------------------------- /infra/site-cookbooks/docker_simple/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'berkshelf' 4 | 5 | # Uncomment these lines if you want to live on the Edge: 6 | # 7 | # group :development do 8 | # gem "berkshelf", github: "berkshelf/berkshelf" 9 | # gem "vagrant", github: "mitchellh/vagrant", tag: "v1.6.3" 10 | # end 11 | # 12 | # group :plugins do 13 | # gem "vagrant-berkshelf", github: "berkshelf/vagrant-berkshelf" 14 | # gem "vagrant-omnibus", github: "schisamo/vagrant-omnibus" 15 | # end 16 | 17 | gem "test-kitchen" 18 | gem "kitchen-vagrant" 19 | -------------------------------------------------------------------------------- /infra/site-cookbooks/docker_simple/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2015 YOUR_NAME 2 | 3 | All rights reserved - Do Not Redistribute 4 | -------------------------------------------------------------------------------- /infra/site-cookbooks/docker_simple/README.md: -------------------------------------------------------------------------------- 1 | # docker_simple-cookbook 2 | 3 | TODO: Enter the cookbook description here. 4 | 5 | ## Supported Platforms 6 | 7 | TODO: List your supported platforms. 8 | 9 | ## Attributes 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 |
KeyTypeDescriptionDefault
['docker_simple']['bacon']Booleanwhether to include bacontrue
25 | 26 | ## Usage 27 | 28 | ### docker_simple::default 29 | 30 | Include `docker_simple` in your node's `run_list`: 31 | 32 | ```json 33 | { 34 | "run_list": [ 35 | "recipe[docker_simple::default]" 36 | ] 37 | } 38 | ``` 39 | 40 | ## License and Authors 41 | 42 | Author:: YOUR_NAME () 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/docker_simple/Thorfile: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'bundler' 4 | require 'bundler/setup' 5 | require 'berkshelf/thor' 6 | 7 | begin 8 | require "kitchen/thor_tasks" 9 | Kitchen::ThorTasks.new 10 | rescue LoadError 11 | puts ">>>>> Kitchen gem not loaded, omitting tasks" unless ENV["CI"] 12 | end 13 | -------------------------------------------------------------------------------- /infra/site-cookbooks/docker_simple/chefignore: -------------------------------------------------------------------------------- 1 | .kitchen 2 | -------------------------------------------------------------------------------- /infra/site-cookbooks/docker_simple/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "docker_simple", 3 | "description": "Installs/Configures docker_simple", 4 | "long_description": "Installs/Configures docker_simple", 5 | "maintainer": "YOUR_NAME", 6 | "maintainer_email": "YOUR_EMAIL", 7 | "license": "All rights reserved", 8 | "platforms": { 9 | 10 | }, 11 | "dependencies": { 12 | 13 | }, 14 | "recommendations": { 15 | 16 | }, 17 | "suggestions": { 18 | 19 | }, 20 | "conflicting": { 21 | 22 | }, 23 | "providing": { 24 | 25 | }, 26 | "replacing": { 27 | 28 | }, 29 | "attributes": { 30 | 31 | }, 32 | "groupings": { 33 | 34 | }, 35 | "recipes": { 36 | 37 | }, 38 | "version": "0.1.0", 39 | "source_url": "", 40 | "issues_url": "", 41 | "privacy": false 42 | } 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/docker_simple/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'docker_simple' 2 | maintainer 'YOUR_NAME' 3 | maintainer_email 'YOUR_EMAIL' 4 | license 'All rights reserved' 5 | description 'Installs/Configures docker_simple' 6 | long_description 'Installs/Configures docker_simple' 7 | version '0.1.0' 8 | -------------------------------------------------------------------------------- /infra/site-cookbooks/docker_simple/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: docker_simple 3 | # Recipe:: default 4 | # 5 | # Copyright (C) 2015 Konrad Malawski 6 | # 7 | # All rights reserved - Do Not Redistribute 8 | # 9 | 10 | execute 'wget install docker' do 11 | command 'wget -qO- https://get.docker.com/ | sh' 12 | end 13 | 14 | group 'docker' do 15 | action :modify 16 | members ['jenkinsakka', 'local'] 17 | append true 18 | end -------------------------------------------------------------------------------- /infra/site-cookbooks/gce-logging-agent/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | gce-logging-agent CHANGELOG 2 | =========================== 3 | 4 | This file is used to list changes made in each version of the gce-logging-agent cookbook. 5 | 6 | 0.1.0 7 | ----- 8 | - [your_name] - Initial release of gce-logging-agent 9 | 10 | - - - 11 | Check the [Markdown Syntax Guide](http://daringfireball.net/projects/markdown/syntax) for help with Markdown. 12 | 13 | The [Github Flavored Markdown page](http://github.github.com/github-flavored-markdown/) describes the differences between markdown on github and standard markdown. 14 | -------------------------------------------------------------------------------- /infra/site-cookbooks/gce-logging-agent/README.md: -------------------------------------------------------------------------------- 1 | gce-logging-agent Cookbook 2 | ========================== 3 | TODO: Enter the cookbook description here. 4 | 5 | e.g. 6 | This cookbook makes your favorite breakfast sandwich. 7 | 8 | Requirements 9 | ------------ 10 | TODO: List your cookbook requirements. Be sure to include any requirements this cookbook has on platforms, libraries, other cookbooks, packages, operating systems, etc. 11 | 12 | e.g. 13 | #### packages 14 | - `toaster` - gce-logging-agent needs toaster to brown your bagel. 15 | 16 | Attributes 17 | ---------- 18 | TODO: List your cookbook attributes here. 19 | 20 | e.g. 21 | #### gce-logging-agent::default 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 |
KeyTypeDescriptionDefault
['gce-logging-agent']['bacon']Booleanwhether to include bacontrue
36 | 37 | Usage 38 | ----- 39 | #### gce-logging-agent::default 40 | TODO: Write usage instructions for each cookbook. 41 | 42 | e.g. 43 | Just include `gce-logging-agent` in your node's `run_list`: 44 | 45 | ```json 46 | { 47 | "name":"my_node", 48 | "run_list": [ 49 | "recipe[gce-logging-agent]" 50 | ] 51 | } 52 | ``` 53 | 54 | Contributing 55 | ------------ 56 | TODO: (optional) If this is a public cookbook, detail the process for contributing. If this is a private cookbook, remove this section. 57 | 58 | e.g. 59 | 1. Fork the repository on Github 60 | 2. Create a named feature branch (like `add_component_x`) 61 | 3. Write your change 62 | 4. Write tests for your change (if applicable) 63 | 5. Run the tests, ensuring they all pass 64 | 6. Submit a Pull Request using Github 65 | 66 | License and Authors 67 | ------------------- 68 | Authors: TODO: List authors 69 | -------------------------------------------------------------------------------- /infra/site-cookbooks/gce-logging-agent/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gce-logging-agent", 3 | "description": "Installs/Configures gce-logging-agent", 4 | "long_description": "gce-logging-agent Cookbook\n==========================\nTODO: Enter the cookbook description here.\n\ne.g.\nThis cookbook makes your favorite breakfast sandwich.\n\nRequirements\n------------\nTODO: List your cookbook requirements. Be sure to include any requirements this cookbook has on platforms, libraries, other cookbooks, packages, operating systems, etc.\n\ne.g.\n#### packages\n- `toaster` - gce-logging-agent needs toaster to brown your bagel.\n\nAttributes\n----------\nTODO: List your cookbook attributes here.\n\ne.g.\n#### gce-logging-agent::default\n\n \n \n \n \n \n \n \n \n \n \n \n \n
KeyTypeDescriptionDefault
['gce-logging-agent']['bacon']Booleanwhether to include bacontrue
\n\nUsage\n-----\n#### gce-logging-agent::default\nTODO: Write usage instructions for each cookbook.\n\ne.g.\nJust include `gce-logging-agent` in your node's `run_list`:\n\n```json\n{\n \"name\":\"my_node\",\n \"run_list\": [\n \"recipe[gce-logging-agent]\"\n ]\n}\n```\n\nContributing\n------------\nTODO: (optional) If this is a public cookbook, detail the process for contributing. If this is a private cookbook, remove this section.\n\ne.g.\n1. Fork the repository on Github\n2. Create a named feature branch (like `add_component_x`)\n3. Write your change\n4. Write tests for your change (if applicable)\n5. Run the tests, ensuring they all pass\n6. Submit a Pull Request using Github\n\nLicense and Authors\n-------------------\nAuthors: TODO: List authors\n", 5 | "maintainer": "YOUR_COMPANY_NAME", 6 | "maintainer_email": "YOUR_EMAIL", 7 | "license": "All rights reserved", 8 | "platforms": { 9 | 10 | }, 11 | "dependencies": { 12 | 13 | }, 14 | "recommendations": { 15 | 16 | }, 17 | "suggestions": { 18 | 19 | }, 20 | "conflicting": { 21 | 22 | }, 23 | "providing": { 24 | 25 | }, 26 | "replacing": { 27 | 28 | }, 29 | "attributes": { 30 | 31 | }, 32 | "groupings": { 33 | 34 | }, 35 | "recipes": { 36 | 37 | }, 38 | "version": "0.1.0", 39 | "source_url": "", 40 | "issues_url": "", 41 | "privacy": false 42 | } 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/gce-logging-agent/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'gce-logging-agent' 2 | maintainer 'YOUR_COMPANY_NAME' 3 | maintainer_email 'YOUR_EMAIL' 4 | license 'All rights reserved' 5 | description 'Installs/Configures gce-logging-agent' 6 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 7 | version '0.1.0' 8 | -------------------------------------------------------------------------------- /infra/site-cookbooks/gce-logging-agent/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: gce-logging-agent 3 | # Recipe:: default 4 | # 5 | # Copyright 2017, Lightbend 6 | # 7 | # All rights reserved - Do Not Redistribute 8 | # 9 | 10 | bash 'install logging agent (ubuntu)' do 11 | code <<-EOH 12 | curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh 13 | # sha256sum install-logging-agent.sh 14 | 15 | bash install-logging-agent.sh 16 | EOH 17 | end 18 | 19 | # create the config file so akka logs are collected () 20 | template "/etc/google-fluentd/config.d/akka.conf" do 21 | source "akka.conf.rb" 22 | variables :log_path => "/var/log/akka.log" # configure here where we log things 23 | end 24 | 25 | bash "force restart of google-logging agent" do 26 | code "service google-fluentd restart" 27 | end 28 | -------------------------------------------------------------------------------- /infra/site-cookbooks/gce-logging-agent/templates/default/akka.conf.rb: -------------------------------------------------------------------------------- 1 | 2 | type tail 3 | format none 4 | path /home/akka/logs/*/* 5 | pos_file /home/akka/gce-fluentd-akka.pos 6 | read_from_head true 7 | tag akka 8 | 9 | -------------------------------------------------------------------------------- /infra/site-cookbooks/java-manual/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | cassandra-manual CHANGELOG 2 | ========================== 3 | 4 | This file is used to list changes made in each version of the cassandra-manual cookbook. 5 | 6 | 0.1.0 7 | ----- 8 | - [your_name] - Initial release of cassandra-manual 9 | 10 | - - - 11 | Check the [Markdown Syntax Guide](http://daringfireball.net/projects/markdown/syntax) for help with Markdown. 12 | 13 | The [Github Flavored Markdown page](http://github.github.com/github-flavored-markdown/) describes the differences between markdown on github and standard markdown. 14 | -------------------------------------------------------------------------------- /infra/site-cookbooks/java-manual/README.md: -------------------------------------------------------------------------------- 1 | cassandra-manual Cookbook 2 | ========================= 3 | TODO: Enter the cookbook description here. 4 | 5 | e.g. 6 | This cookbook makes your favorite breakfast sandwich. 7 | 8 | Requirements 9 | ------------ 10 | TODO: List your cookbook requirements. Be sure to include any requirements this cookbook has on platforms, libraries, other cookbooks, packages, operating systems, etc. 11 | 12 | e.g. 13 | #### packages 14 | - `toaster` - cassandra-manual needs toaster to brown your bagel. 15 | 16 | Attributes 17 | ---------- 18 | TODO: List your cookbook attributes here. 19 | 20 | e.g. 21 | #### cassandra-manual::default 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 |
KeyTypeDescriptionDefault
['cassandra-manual']['bacon']Booleanwhether to include bacontrue
36 | 37 | Usage 38 | ----- 39 | #### cassandra-manual::default 40 | TODO: Write usage instructions for each cookbook. 41 | 42 | e.g. 43 | Just include `cassandra-manual` in your node's `run_list`: 44 | 45 | ```json 46 | { 47 | "name":"my_node", 48 | "run_list": [ 49 | "recipe[cassandra-manual]" 50 | ] 51 | } 52 | ``` 53 | 54 | Contributing 55 | ------------ 56 | TODO: (optional) If this is a public cookbook, detail the process for contributing. If this is a private cookbook, remove this section. 57 | 58 | e.g. 59 | 1. Fork the repository on Github 60 | 2. Create a named feature branch (like `add_component_x`) 61 | 3. Write your change 62 | 4. Write tests for your change (if applicable) 63 | 5. Run the tests, ensuring they all pass 64 | 6. Submit a Pull Request using Github 65 | 66 | License and Authors 67 | ------------------- 68 | Authors: TODO: List authors 69 | -------------------------------------------------------------------------------- /infra/site-cookbooks/java-manual/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cassandra-manual", 3 | "description": "Installs/Configures cassandra-manual", 4 | "long_description": "cassandra-manual Cookbook\n=========================\nTODO: Enter the cookbook description here.\n\ne.g.\nThis cookbook makes your favorite breakfast sandwich.\n\nRequirements\n------------\nTODO: List your cookbook requirements. Be sure to include any requirements this cookbook has on platforms, libraries, other cookbooks, packages, operating systems, etc.\n\ne.g.\n#### packages\n- `toaster` - cassandra-manual needs toaster to brown your bagel.\n\nAttributes\n----------\nTODO: List your cookbook attributes here.\n\ne.g.\n#### cassandra-manual::default\n\n \n \n \n \n \n \n \n \n \n \n \n \n
KeyTypeDescriptionDefault
['cassandra-manual']['bacon']Booleanwhether to include bacontrue
\n\nUsage\n-----\n#### cassandra-manual::default\nTODO: Write usage instructions for each cookbook.\n\ne.g.\nJust include `cassandra-manual` in your node's `run_list`:\n\n```json\n{\n \"name\":\"my_node\",\n \"run_list\": [\n \"recipe[cassandra-manual]\"\n ]\n}\n```\n\nContributing\n------------\nTODO: (optional) If this is a public cookbook, detail the process for contributing. If this is a private cookbook, remove this section.\n\ne.g.\n1. Fork the repository on Github\n2. Create a named feature branch (like `add_component_x`)\n3. Write your change\n4. Write tests for your change (if applicable)\n5. Run the tests, ensuring they all pass\n6. Submit a Pull Request using Github\n\nLicense and Authors\n-------------------\nAuthors: TODO: List authors\n", 5 | "maintainer": "YOUR_COMPANY_NAME", 6 | "maintainer_email": "YOUR_EMAIL", 7 | "license": "All rights reserved", 8 | "platforms": { 9 | 10 | }, 11 | "dependencies": { 12 | 13 | }, 14 | "recommendations": { 15 | 16 | }, 17 | "suggestions": { 18 | 19 | }, 20 | "conflicting": { 21 | 22 | }, 23 | "providing": { 24 | 25 | }, 26 | "replacing": { 27 | 28 | }, 29 | "attributes": { 30 | 31 | }, 32 | "groupings": { 33 | 34 | }, 35 | "recipes": { 36 | 37 | }, 38 | "version": "0.1.0", 39 | "source_url": "", 40 | "issues_url": "", 41 | "privacy": false 42 | } 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/java-manual/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'java-manual' 2 | maintainer 'YOUR_COMPANY_NAME' 3 | maintainer_email 'YOUR_EMAIL' 4 | license 'All rights reserved' 5 | description 'Java manual installation' 6 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 7 | version '0.1.0' 8 | -------------------------------------------------------------------------------- /infra/site-cookbooks/java-manual/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: cassandra-manual 3 | # Recipe:: default 4 | # 5 | # Copyright 2017, YOUR_COMPANY_NAME 6 | # 7 | # All rights reserved - Do Not Redistribute 8 | 9 | bash 'install oracle apt repo' do 10 | code <<-EOH 11 | add-apt-repository ppa:webupd8team/java 12 | 13 | apt-get update 14 | EOH 15 | end 16 | 17 | package 'debconf-utils' do 18 | action :install 19 | end 20 | 21 | bash 'accept oracle license' do 22 | code <<-EOH 23 | echo debconf shared/accepted-oracle-license-v1-1 select true | \ 24 | sudo debconf-set-selections 25 | 26 | echo debconf shared/accepted-oracle-license-v1-1 seen true | \ 27 | sudo debconf-set-selections 28 | EOH 29 | end 30 | 31 | package 'oracle-java8-installer' do 32 | action :install 33 | end 34 | 35 | package 'oracle-java8-set-default' do 36 | action :install 37 | end 38 | 39 | bash 'download open jdk9' do 40 | code <<-EOH 41 | cd /usr/lib/jvm 42 | rm -rf jdk-9 43 | wget 'http://download.oracle.com/otn-pub/java/jdk/9.0.1+11/jdk-9.0.1_linux-x64_bin.tar.gz' 44 | tar xzvf jdk-9.0.1_linux-x64_bin.tar.gz 45 | rm jdk-9.0.1_linux-x64_bin.tar.gz 46 | EOH 47 | end 48 | -------------------------------------------------------------------------------- /infra/site-cookbooks/perf/.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *# 3 | .#* 4 | \#*# 5 | .*.sw[a-z] 6 | *.un~ 7 | pkg/ 8 | 9 | # Berkshelf 10 | .vagrant 11 | /cookbooks 12 | Berksfile.lock 13 | 14 | # Bundler 15 | Gemfile.lock 16 | bin/* 17 | .bundle/* 18 | 19 | .kitchen/ 20 | .kitchen.local.yml 21 | -------------------------------------------------------------------------------- /infra/site-cookbooks/perf/.kitchen.yml: -------------------------------------------------------------------------------- 1 | --- 2 | driver: 3 | name: vagrant 4 | 5 | provisioner: 6 | name: chef_solo 7 | 8 | platforms: 9 | - name: ubuntu-14.04 10 | - name: centos-7.1 11 | 12 | suites: 13 | - name: default 14 | run_list: 15 | attributes: 16 | -------------------------------------------------------------------------------- /infra/site-cookbooks/perf/Berksfile: -------------------------------------------------------------------------------- 1 | source "https://supermarket.chef.io" 2 | 3 | metadata 4 | -------------------------------------------------------------------------------- /infra/site-cookbooks/perf/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 0.1.0 2 | 3 | Initial release of perf 4 | -------------------------------------------------------------------------------- /infra/site-cookbooks/perf/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'berkshelf' 4 | 5 | # Uncomment these lines if you want to live on the Edge: 6 | # 7 | # group :development do 8 | # gem "berkshelf", github: "berkshelf/berkshelf" 9 | # gem "vagrant", github: "mitchellh/vagrant", tag: "v1.6.3" 10 | # end 11 | # 12 | # group :plugins do 13 | # gem "vagrant-berkshelf", github: "berkshelf/vagrant-berkshelf" 14 | # gem "vagrant-omnibus", github: "schisamo/vagrant-omnibus" 15 | # end 16 | 17 | gem "test-kitchen" 18 | gem "kitchen-vagrant" 19 | -------------------------------------------------------------------------------- /infra/site-cookbooks/perf/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2015 YOUR_NAME 2 | 3 | All rights reserved - Do Not Redistribute 4 | -------------------------------------------------------------------------------- /infra/site-cookbooks/perf/README.md: -------------------------------------------------------------------------------- 1 | # perf-cookbook 2 | 3 | TODO: Enter the cookbook description here. 4 | 5 | ## Supported Platforms 6 | 7 | TODO: List your supported platforms. 8 | 9 | ## Attributes 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 |
KeyTypeDescriptionDefault
['perf']['bacon']Booleanwhether to include bacontrue
25 | 26 | ## Usage 27 | 28 | ### perf::default 29 | 30 | Include `perf` in your node's `run_list`: 31 | 32 | ```json 33 | { 34 | "run_list": [ 35 | "recipe[perf::default]" 36 | ] 37 | } 38 | ``` 39 | 40 | ## License and Authors 41 | 42 | Author:: YOUR_NAME () 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/perf/Thorfile: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'bundler' 4 | require 'bundler/setup' 5 | require 'berkshelf/thor' 6 | 7 | begin 8 | require "kitchen/thor_tasks" 9 | Kitchen::ThorTasks.new 10 | rescue LoadError 11 | puts ">>>>> Kitchen gem not loaded, omitting tasks" unless ENV["CI"] 12 | end 13 | -------------------------------------------------------------------------------- /infra/site-cookbooks/perf/chefignore: -------------------------------------------------------------------------------- 1 | .kitchen 2 | -------------------------------------------------------------------------------- /infra/site-cookbooks/perf/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "perf", 3 | "description": "Installs perf", 4 | "long_description": "Installs/Configures perf", 5 | "maintainer": "Konrad Malawski", 6 | "maintainer_email": "YOUR_EMAIL", 7 | "license": "All rights reserved", 8 | "platforms": { 9 | 10 | }, 11 | "dependencies": { 12 | 13 | }, 14 | "recommendations": { 15 | 16 | }, 17 | "suggestions": { 18 | 19 | }, 20 | "conflicting": { 21 | 22 | }, 23 | "providing": { 24 | 25 | }, 26 | "replacing": { 27 | 28 | }, 29 | "attributes": { 30 | 31 | }, 32 | "groupings": { 33 | 34 | }, 35 | "recipes": { 36 | 37 | }, 38 | "version": "0.1.0", 39 | "source_url": "", 40 | "issues_url": "", 41 | "privacy": false 42 | } 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/perf/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'perf' 2 | maintainer 'YOUR_NAME' 3 | maintainer_email 'YOUR_EMAIL' 4 | license 'All rights reserved' 5 | description 'Installs/Configures perf' 6 | long_description 'Installs/Configures perf' 7 | version '0.1.0' 8 | -------------------------------------------------------------------------------- /infra/site-cookbooks/perf/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: perf 3 | # Recipe:: default 4 | # 5 | # Copyright (C) 2016 Konrad Malawski 6 | # 7 | # All rights reserved - Do Not Redistribute 8 | # 9 | 10 | package 'linux-tools-common' do 11 | action :install 12 | end 13 | -------------------------------------------------------------------------------- /infra/site-cookbooks/sbt-manual/.gitignore: -------------------------------------------------------------------------------- 1 | files/* 2 | files/default/* 3 | -------------------------------------------------------------------------------- /infra/site-cookbooks/sbt-manual/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | gce-logging-agent CHANGELOG 2 | =========================== 3 | 4 | This file is used to list changes made in each version of the gce-logging-agent cookbook. 5 | 6 | 0.1.0 7 | ----- 8 | - [your_name] - Initial release of gce-logging-agent 9 | 10 | - - - 11 | Check the [Markdown Syntax Guide](http://daringfireball.net/projects/markdown/syntax) for help with Markdown. 12 | 13 | The [Github Flavored Markdown page](http://github.github.com/github-flavored-markdown/) describes the differences between markdown on github and standard markdown. 14 | -------------------------------------------------------------------------------- /infra/site-cookbooks/sbt-manual/README.md: -------------------------------------------------------------------------------- 1 | sbt-manual 2 | ========== 3 | -------------------------------------------------------------------------------- /infra/site-cookbooks/sbt-manual/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sbt-manual", 3 | "description": "...", 4 | "long_description": "", 5 | "maintainer": "YOUR_COMPANY_NAME", 6 | "maintainer_email": "YOUR_EMAIL", 7 | "license": "All rights reserved", 8 | "platforms": { 9 | 10 | }, 11 | "dependencies": { 12 | 13 | }, 14 | "recommendations": { 15 | 16 | }, 17 | "suggestions": { 18 | 19 | }, 20 | "conflicting": { 21 | 22 | }, 23 | "providing": { 24 | 25 | }, 26 | "replacing": { 27 | 28 | }, 29 | "attributes": { 30 | 31 | }, 32 | "groupings": { 33 | 34 | }, 35 | "recipes": { 36 | 37 | }, 38 | "version": "0.1.0", 39 | "source_url": "", 40 | "issues_url": "", 41 | "privacy": false 42 | } 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/sbt-manual/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'sbt-manual' 2 | maintainer 'YOUR_COMPANY_NAME' 3 | maintainer_email 'YOUR_EMAIL' 4 | license 'All rights reserved' 5 | description '...' 6 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 7 | version '0.1.0' 8 | -------------------------------------------------------------------------------- /infra/site-cookbooks/sbt-manual/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: gce-logging-agent 3 | # Recipe:: default 4 | # 5 | # Copyright 2017, Lightbend 6 | # 7 | # All rights reserved - Do Not Redistribute 8 | # 9 | 10 | user "akka" do 11 | home '/home/akka' 12 | shell '/bin/bash' 13 | password 'akka' 14 | action :create 15 | manage_home true 16 | end 17 | 18 | bash "clone akka" do 19 | code <<-EOH 20 | curl -Ls https://git.io/sbt > /home/akka/sbt 21 | chmod 0755 /home/akka/sbt 22 | chown akka.akka sbt 23 | 24 | echo '' >> /home/akka/.bashrc 25 | echo 'PATH=$PATH:/home/akka' >> /home/akka/.bashrc 26 | EOH 27 | end 28 | -------------------------------------------------------------------------------- /infra/site-cookbooks/sbt_credentials/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | sbt_credentials 2 | =============== 3 | -------------------------------------------------------------------------------- /infra/site-cookbooks/sbt_credentials/README.md: -------------------------------------------------------------------------------- 1 | sbt_credentials 2 | =============== 3 | -------------------------------------------------------------------------------- /infra/site-cookbooks/sbt_credentials/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sbt_credentials", 3 | "description": "Installs/Configures sbt_credentials", 4 | "long_description": "sbt_credentials\n===============\n", 5 | "maintainer": "YOUR_COMPANY_NAME", 6 | "maintainer_email": "YOUR_EMAIL", 7 | "license": "All rights reserved", 8 | "platforms": { 9 | 10 | }, 11 | "dependencies": { 12 | 13 | }, 14 | "recommendations": { 15 | 16 | }, 17 | "suggestions": { 18 | 19 | }, 20 | "conflicting": { 21 | 22 | }, 23 | "providing": { 24 | 25 | }, 26 | "replacing": { 27 | 28 | }, 29 | "attributes": { 30 | 31 | }, 32 | "groupings": { 33 | 34 | }, 35 | "recipes": { 36 | 37 | }, 38 | "version": "0.1.0", 39 | "source_url": "", 40 | "issues_url": "", 41 | "privacy": false 42 | } 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/sbt_credentials/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'sbt_credentials' 2 | maintainer 'YOUR_COMPANY_NAME' 3 | maintainer_email 'YOUR_EMAIL' 4 | license 'All rights reserved' 5 | description 'Installs/Configures sbt_credentials' 6 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 7 | version '0.1.0' 8 | -------------------------------------------------------------------------------- /infra/site-cookbooks/sbt_credentials/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: sbt_credentials 3 | # Recipe:: default 4 | # 5 | # Copyright 2017, YOUR_COMPANY_NAME 6 | # 7 | # All rights reserved - Do Not Redistribute 8 | 9 | bash 'link credentials for 1.0 to 0.13' do 10 | code <<-EOH 11 | cd /home/jenkinsakka/.sbt 12 | mkdir -p 1.0 13 | 14 | rm 1.0/credentials.sbt 15 | ln -s $(pwd)/0.13/credentials.sbt 1.0/credentials.sbt 16 | 17 | rm 1.0/global.sbt 18 | ln -s $(pwd)/0.13/global.sbt 1.0/ 19 | 20 | rm 1.0/repositories 21 | ln -s $(pwd)/0.13/repositories 1.0/ 22 | EOH 23 | end 24 | -------------------------------------------------------------------------------- /infra/site-cookbooks/update-hosts/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | gce-logging-agent CHANGELOG 2 | =========================== 3 | 4 | This file is used to list changes made in each version of the gce-logging-agent cookbook. 5 | 6 | 0.1.0 7 | ----- 8 | - [your_name] - Initial release of gce-logging-agent 9 | 10 | - - - 11 | Check the [Markdown Syntax Guide](http://daringfireball.net/projects/markdown/syntax) for help with Markdown. 12 | 13 | The [Github Flavored Markdown page](http://github.github.com/github-flavored-markdown/) describes the differences between markdown on github and standard markdown. 14 | -------------------------------------------------------------------------------- /infra/site-cookbooks/update-hosts/README.md: -------------------------------------------------------------------------------- 1 | akka linux tweaks 2 | ================= 3 | 4 | all kinds of OS settings that make akka run nicer. 5 | -------------------------------------------------------------------------------- /infra/site-cookbooks/update-hosts/files/default/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/infra/site-cookbooks/update-hosts/files/default/.gitkeep -------------------------------------------------------------------------------- /infra/site-cookbooks/update-hosts/files/default/multi-node-test.hosts: -------------------------------------------------------------------------------- 1 | 10.128.0.23 2 | -------------------------------------------------------------------------------- /infra/site-cookbooks/update-hosts/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "update-hosts", 3 | "description": "...", 4 | "long_description": "", 5 | "maintainer": "YOUR_COMPANY_NAME", 6 | "maintainer_email": "YOUR_EMAIL", 7 | "license": "All rights reserved", 8 | "platforms": { 9 | 10 | }, 11 | "dependencies": { 12 | 13 | }, 14 | "recommendations": { 15 | 16 | }, 17 | "suggestions": { 18 | 19 | }, 20 | "conflicting": { 21 | 22 | }, 23 | "providing": { 24 | 25 | }, 26 | "replacing": { 27 | 28 | }, 29 | "attributes": { 30 | 31 | }, 32 | "groupings": { 33 | 34 | }, 35 | "recipes": { 36 | 37 | }, 38 | "version": "0.1.0", 39 | "source_url": "", 40 | "issues_url": "", 41 | "privacy": false 42 | } 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/update-hosts/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'update-hosts' 2 | maintainer 'YOUR_COMPANY_NAME' 3 | maintainer_email 'YOUR_EMAIL' 4 | license 'All rights reserved' 5 | description '...' 6 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 7 | version '0.1.0' 8 | -------------------------------------------------------------------------------- /infra/site-cookbooks/update-hosts/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: update-hosts 3 | # Recipe:: default 4 | # 5 | # Copyright 2017, Lightbend 6 | # 7 | # All rights reserved - Do Not Redistribute 8 | # 9 | 10 | user "akka" do 11 | home '/home/akka' 12 | shell '/bin/bash' 13 | password 'akka' 14 | action :create 15 | manage_home true 16 | end 17 | 18 | template '/home/akka/root-application.conf' do 19 | source 'root-application.conf.rb' 20 | end 21 | 22 | template '/home/akka/run-multinode-benchmark.sh' do 23 | source 'run-multinode-benchmark.sh.rb' 24 | end 25 | 26 | template '/home/akka/run-max-throughput-benchmark.sh' do 27 | source 'run-max-throughput-benchmark.sh.rb' 28 | end 29 | 30 | template '/home/akka/run-latency-benchmark.sh' do 31 | source 'run-latency-benchmark.sh.rb' 32 | end 33 | 34 | template '/home/akka/run-multi-dc-test.sh' do 35 | source 'run-multi-dc-test.sh.rb' 36 | end 37 | 38 | template '/home/akka/run-latency-benchmark.sh' do 39 | source 'run-latency-benchmark.sh.rb' 40 | end 41 | 42 | cookbook_file '/home/akka/multi-node-test.hosts' do 43 | source 'multi-node-test.hosts' 44 | end 45 | 46 | 47 | bash "chown generated files for akka user" do 48 | code <<-EOH 49 | chown -R akka:akka /home/akka 50 | chmod +x /home/akka/*.sh 51 | EOH 52 | end 53 | -------------------------------------------------------------------------------- /infra/site-cookbooks/update-hosts/templates/default/multi-node-test.hosts: -------------------------------------------------------------------------------- 1 | 10.132.0.7 2 | 10.132.0.8 3 | 10.132.0.9 4 | -------------------------------------------------------------------------------- /infra/site-cookbooks/update-hosts/templates/default/root-application.conf.rb: -------------------------------------------------------------------------------- 1 | # ------------------------ WARNING -------------------------------- 2 | # This file is generated automatically via Chef scripts. 3 | # Do not edit manually - all your changes WILL be lost. 4 | # ------------------------ WARNING -------------------------------- 5 | akka { 6 | 7 | system-name = "<%= node['akka']['system_name'] %>" 8 | 9 | # total number of nodes this benchmark will run on (same as number of nodes in multi-node-test.hosts) 10 | minimum-nodes = <%= node['akka']['minumum_nodes'] %> 11 | 12 | remote { 13 | netty.tcp { 14 | hostname = "<%= node['ipaddress-internal'] %>" 15 | port = <%= node['akka']['port'] %> 16 | } 17 | 18 | artery { 19 | # enabled = on this is up to the app to decide 20 | canonical.hostname = "<%= node['ipaddress-internal'] %>" 21 | canonical.port = <%= node['akka']['port'] %> 22 | } 23 | } 24 | 25 | cluster { 26 | 27 | roles = [ 28 | <%= (node['akka']['roles'].map do |role| 29 | "\"#{role}\"" 30 | end).join(",\n ") %> 31 | ] 32 | 33 | seed-nodes = [ 34 | <%= (node['akka']['seed_nodes'].map do |seed| 35 | "\"akka://#{node['akka']['system_name']}@#{seed}:#{node['akka']['port']}\"" 36 | end).join(",\n ") %> 37 | ] 38 | 39 | http.management { 40 | hostname = "0.0.0.0" 41 | port = 19999 42 | } 43 | } 44 | 45 | } 46 | 47 | cassandra-journal { 48 | contact-points = [ 49 | <%= (node['cassandra']['contact_points'].map do |node| 50 | "\"#{node}\"" 51 | end).join(",\n ") %> 52 | ] 53 | } 54 | cassandra-snapshot-store { 55 | contact-points = [ 56 | <%= (node['cassandra']['contact_points'].map do |node| 57 | "\"#{node}\"" 58 | end).join(",\n ") %> 59 | ] 60 | } 61 | -------------------------------------------------------------------------------- /infra/site-cookbooks/update-hosts/templates/default/run-multi-dc-test.sh.rb: -------------------------------------------------------------------------------- 1 | echo 'Running $CLAZZ on $IP2' 2 | 3 | cd /home/akka/apps 4 | 5 | /home/akka/sbt -J-XX:+PrintGCDetails -J-XX:+PrintGCTimeStamps -J-Xms4G -J-Xmx4G -Dsbt.log.noformat=true '; project multidc; run' -------------------------------------------------------------------------------- /infra/site-cookbooks/update-hosts/templates/default/run-sharding-start-latency-benchmark.sh.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # -------------- functions ------------------------------------------------------------------------------------------------------- 4 | 5 | # $1 - IP to execute scripts on 6 | prepare() { 7 | declare -r IP2="$1" 8 | declare -r LOGS2="$LOGS_BASE/$IP2" 9 | 10 | echo "====== PREPARE $IP2 ======" 11 | 12 | cd /home/akka 13 | rsync -Pavuz /home/akka/apps akka@$IP2: > /dev/null 14 | rsync -Pavuz .ivy2 akka@$IP2: > /dev/null 15 | rsync -Pavuz .sbt akka@$IP2: > /dev/null 16 | 17 | ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $IP2 " 18 | cd $PWD 19 | 20 | rm -rf /dev/shm/aeron-* 21 | killall -9 java 22 | mkdir -p $LOGS2 23 | " & 24 | 25 | } 26 | 27 | # $1 - IP to execute scripts on 28 | # $2 - class name to run 29 | run() { 30 | declare -r IP2="$1" 31 | declare -r CLAZZ="$2" 32 | declare -r LOGS2="$LOGS_BASE/$IP2" 33 | 34 | echo "====== RUN $IP2 ======" 35 | 36 | ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $IP2 " 37 | cd $PWD 38 | 39 | mkdir -p $LOGS2 40 | 41 | cd /home/akka 42 | 43 | echo 'Running $CLAZZ on $IP2' 44 | cd /home/akka/apps 45 | /home/akka/sbt -Daeron.term.buffer.length=4194304 -J-Xms4G -J-Xmx4G -Dsbt.log.noformat=true '; project sharding; runMain $CLAZZ' > $LOGS2/log-sharding.txt 46 | " & 47 | 48 | sleep 2 49 | } 50 | 51 | # $1 - IP to execute scripts on 52 | finish() { 53 | declare -r IP2="$1" 54 | declare -r LOGS2="$LOGS_BASE/$IP2" 55 | 56 | echo "====== FINISH $IP2 ======" 57 | 58 | ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $IP2 " 59 | cd $PWD 60 | 61 | pkill -u akka java 62 | sleep 5 63 | pkill -9 -u akka java 64 | " & 65 | 66 | # avoid copy own files 67 | if [ ! -d "$LOGS_BASE/$IP2" ]; then 68 | rsync -Pavuz $IP2:$LOGS2 $LOGS_BASE/ 69 | fi 70 | } 71 | # -------------- end of functions ----------------------------------------------------------------------------------------------- 72 | 73 | 74 | declare -r LOGS_BASE="/home/akka/log/$(date '+20%y-%m-%d--%H%M')" 75 | declare -r IP1="$(head -n 1 /home/akka/multi-node-test.hosts)" 76 | declare -r LOGS1="$LOGS_BASE/$IP1" 77 | 78 | declare -r ARGS="$1" 79 | 80 | echo "logs in $LOGS_BASE" 81 | mkdir -p $LOGS1 82 | 83 | # description of test 84 | cd /home/akka/akka 85 | git log -n 1 >> $LOGS_BASE/readme.txt 86 | 87 | # prepare all nodes 88 | for ip in $(cat /home/akka/multi-node-test.hosts); do 89 | prepare $ip 90 | done 91 | 92 | sleep 10 93 | 94 | # run the tests ------------------------------------------ 95 | echo "running test...!" 96 | for ip in $(cat /home/akka/multi-node-test.hosts); do 97 | run $ip "com.lightbend.akka.bench.sharding.latency.ShardingStartLatencyApp" 98 | done 99 | 100 | echo "PRESS [ENTER] TO TERMINATE BENCHMARK" 101 | read 102 | 103 | # cleanup ------------------------------------------------ 104 | for ip in $(cat /home/akka/multi-node-test.hosts); do 105 | finish $ip 106 | done 107 | -------------------------------------------------------------------------------- /infra/site-cookbooks/wrk2/.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *# 3 | .#* 4 | \#*# 5 | .*.sw[a-z] 6 | *.un~ 7 | pkg/ 8 | 9 | # Berkshelf 10 | .vagrant 11 | /cookbooks 12 | Berksfile.lock 13 | 14 | # Bundler 15 | Gemfile.lock 16 | bin/* 17 | .bundle/* 18 | 19 | .kitchen/ 20 | .kitchen.local.yml 21 | -------------------------------------------------------------------------------- /infra/site-cookbooks/wrk2/.kitchen.yml: -------------------------------------------------------------------------------- 1 | --- 2 | driver: 3 | name: vagrant 4 | 5 | provisioner: 6 | name: chef_solo 7 | 8 | platforms: 9 | - name: ubuntu-14.04 10 | - name: centos-7.1 11 | 12 | suites: 13 | - name: default 14 | run_list: 15 | attributes: 16 | -------------------------------------------------------------------------------- /infra/site-cookbooks/wrk2/Berksfile: -------------------------------------------------------------------------------- 1 | source "https://supermarket.chef.io" 2 | 3 | metadata 4 | -------------------------------------------------------------------------------- /infra/site-cookbooks/wrk2/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 0.1.0 2 | 3 | Initial release of wrk2 4 | -------------------------------------------------------------------------------- /infra/site-cookbooks/wrk2/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'berkshelf' 4 | 5 | # Uncomment these lines if you want to live on the Edge: 6 | # 7 | # group :development do 8 | # gem "berkshelf", github: "berkshelf/berkshelf" 9 | # gem "vagrant", github: "mitchellh/vagrant", tag: "v1.6.3" 10 | # end 11 | # 12 | # group :plugins do 13 | # gem "vagrant-berkshelf", github: "berkshelf/vagrant-berkshelf" 14 | # gem "vagrant-omnibus", github: "schisamo/vagrant-omnibus" 15 | # end 16 | 17 | gem "test-kitchen" 18 | gem "kitchen-vagrant" 19 | -------------------------------------------------------------------------------- /infra/site-cookbooks/wrk2/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2015 YOUR_NAME 2 | 3 | All rights reserved - Do Not Redistribute 4 | -------------------------------------------------------------------------------- /infra/site-cookbooks/wrk2/README.md: -------------------------------------------------------------------------------- 1 | # wrk2-cookbook 2 | 3 | TODO: Enter the cookbook description here. 4 | 5 | ## Supported Platforms 6 | 7 | TODO: List your supported platforms. 8 | 9 | ## Attributes 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 |
KeyTypeDescriptionDefault
['wrk2']['bacon']Booleanwhether to include bacontrue
25 | 26 | ## Usage 27 | 28 | ### wrk2::default 29 | 30 | Include `wrk2` in your node's `run_list`: 31 | 32 | ```json 33 | { 34 | "run_list": [ 35 | "recipe[wrk2::default]" 36 | ] 37 | } 38 | ``` 39 | 40 | ## License and Authors 41 | 42 | Author:: YOUR_NAME () 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/wrk2/Thorfile: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'bundler' 4 | require 'bundler/setup' 5 | require 'berkshelf/thor' 6 | 7 | begin 8 | require "kitchen/thor_tasks" 9 | Kitchen::ThorTasks.new 10 | rescue LoadError 11 | puts ">>>>> Kitchen gem not loaded, omitting tasks" unless ENV["CI"] 12 | end 13 | -------------------------------------------------------------------------------- /infra/site-cookbooks/wrk2/chefignore: -------------------------------------------------------------------------------- 1 | .kitchen 2 | -------------------------------------------------------------------------------- /infra/site-cookbooks/wrk2/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "wrk2", 3 | "description": "Installs wrk2", 4 | "long_description": "Installs/Configures wrk2", 5 | "maintainer": "Konrad Malawski", 6 | "maintainer_email": "YOUR_EMAIL", 7 | "license": "All rights reserved", 8 | "platforms": { 9 | 10 | }, 11 | "dependencies": { 12 | 13 | }, 14 | "recommendations": { 15 | 16 | }, 17 | "suggestions": { 18 | 19 | }, 20 | "conflicting": { 21 | 22 | }, 23 | "providing": { 24 | 25 | }, 26 | "replacing": { 27 | 28 | }, 29 | "attributes": { 30 | 31 | }, 32 | "groupings": { 33 | 34 | }, 35 | "recipes": { 36 | 37 | }, 38 | "version": "0.1.0", 39 | "source_url": "", 40 | "issues_url": "", 41 | "privacy": false 42 | } 43 | -------------------------------------------------------------------------------- /infra/site-cookbooks/wrk2/metadata.rb: -------------------------------------------------------------------------------- 1 | name 'wrk2' 2 | maintainer 'YOUR_NAME' 3 | maintainer_email 'YOUR_EMAIL' 4 | license 'All rights reserved' 5 | description 'Installs/Configures wrk2' 6 | long_description 'Installs/Configures wrk2' 7 | version '0.1.0' 8 | -------------------------------------------------------------------------------- /infra/site-cookbooks/wrk2/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: wrk2 3 | # Recipe:: default 4 | # 5 | # Copyright (C) 2016 Konrad Malawski 6 | # 7 | # All rights reserved - Do Not Redistribute 8 | # 9 | 10 | package 'make' 11 | package 'gcc' 12 | 13 | execute 'cleanup wrk2' do 14 | command 'rm -rf wrk2' 15 | end 16 | 17 | execute 'clone wrk2' do 18 | command 'git clone https://github.com/giltene/wrk2' 19 | end 20 | 21 | package 'libssl-dev' do 22 | action :install 23 | end 24 | 25 | execute 'make wrk2' do 26 | command 'make' 27 | cwd 'wrk2' 28 | end 29 | 30 | execute 'copy to /usr/bin/wrk2' do 31 | command 'cp wrk /usr/bin/wrk' 32 | cwd 'wrk2' 33 | end 34 | -------------------------------------------------------------------------------- /infra/ssh-config: -------------------------------------------------------------------------------- 1 | host moxie-a0 2 | HostName moxie.typesafe.com 3 | port 5000 4 | user local 5 | 6 | host moxie-a1 7 | HostName moxie.typesafe.com 8 | port 5001 9 | user local 10 | 11 | host moxie-a2 12 | HostName moxie.typesafe.com 13 | port 5002 14 | user local 15 | 16 | host moxie-a3 17 | HostName moxie.typesafe.com 18 | port 5003 19 | user local 20 | 21 | host jenkins-moxie-a3 22 | HostName moxie.typesafe.com 23 | port 5003 24 | user jenkinsakka 25 | 26 | host moxie-a4 27 | HostName moxie.typesafe.com 28 | port 5004 29 | user local 30 | 31 | host moxie-a5 32 | HostName moxie.typesafe.com 33 | port 5005 34 | user local 35 | 36 | host moxie-a6 37 | HostName moxie.typesafe.com 38 | port 5006 39 | user local 40 | 41 | host moxie-a7 42 | HostName moxie.typesafe.com 43 | port 5007 44 | user local 45 | 46 | host moxie-a8 47 | HostName moxie.typesafe.com 48 | port 5008 49 | user local 50 | 51 | host moxie-a9 52 | HostName moxie.typesafe.com 53 | port 5009 54 | user local 55 | -------------------------------------------------------------------------------- /infra/update-multi-node-test.hosts-file.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # examples: 4 | # 5 | # ./update-multi-node-test.hosts-file.sh akka-shard 6 | 7 | if [[ "$#" != "1" ]]; then 8 | echo "USAGE: ./make-node-files-from-gcloud.sh GREP SEED_NODES" 9 | echo "EXTRA PARAMS:" 10 | echo " you can set: EXTRA_RUNLIST ( example: EXTRA_RUNLIST='"role[benchmark-ddata]",' )" 11 | fi 12 | 13 | declare -r GREP=$1 14 | 15 | declare -r nodes=$(gcloud compute instances list | grep $GREP | wc -l | awk '{print $4}') 16 | 17 | # cassandra contact points as json (just the first ... nodes) 18 | declare -r CASSANDRA_CONTACT_POINTS_NUM=2 19 | declare -r CASSANDRA_CONTACT_POINTS=$( 20 | gcloud compute instances list | 21 | grep "cassandra" | grep RUNNING | 22 | awk ' {print $4 } ' | 23 | head -n${CASSANDRA_CONTACT_POINTS_NUM} | 24 | awk ' BEGIN { ORS = ""; print "["; } { print "\/\@"$0"\/\@"; } END { print "]"; }' | sed "s^\"^\\\\\"^g;s^\/\@\/\@^\", \"^g;s^\/\@^\"^g" 25 | ) 26 | 27 | gcloud compute instances list | grep akka-shard | awk '{ print $4 }' 28 | -------------------------------------------------------------------------------- /infra/update-multinode-hosts.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [[ "$#" != "2" ]]; then 4 | echo "USAGE: ./update-hosts.sh node:akka-node-001 (which node to update) akka-node (which IPs to find and include)" 5 | exit 1 6 | fi 7 | 8 | # selector should be `node:akka-node-001` or `nodes_with_role:benchmark` 9 | declare -r SELECTOR=$1 10 | # node-grep should be as simple as "akka-node" for example 11 | declare -r NODE_GREP=$2 12 | 13 | echo "Looking for akka-nodes..." 14 | 15 | declare -r internal_ips=$(gcloud --project="akka-gcp" compute instances list | grep "$NODE_GREP" | awk '{ print $4 }') 16 | declare -r external_ips=$(gcloud --project="akka-gcp" compute instances list | grep "$NODE_GREP" | awk '{ print $5 }') 17 | 18 | rm ./internal-ips.tmp 19 | for ip in $internal_ips; do 20 | echo $ip >> ./internal-ips.tmp 21 | done 22 | 23 | echo "# ----------------------------------------------" 24 | echo "Found internal IPs, stored as ./internal-ips.tmp" 25 | echo "# ----------------------------------------------" 26 | cat ./internal-ips.tmp 27 | echo "# ----------------------------------------------" 28 | echo "IF YOU WANT, EDIT THAT FILE, WE WILL WRITE to all: $SELECTOR" 29 | echo "# ----------------------------------------------" 30 | read 31 | 32 | cat ./internal-ips.tmp > site-cookbooks/update-hosts/templates/default/multi-node-test.hosts 33 | 34 | declare -r PAR=5 35 | echo "EXECUTING WITH PARALLELISM: $PAR" 36 | 37 | # # usually not needed anymore: 38 | #echo "Preparing node..." 39 | #fix --concurrency $PAR $SELECTOR deploy_chef -y 40 | 41 | echo "Updating node..." 42 | fix --concurrency $PAR $SELECTOR recipe:update-hosts 43 | 44 | #echo "Deploying Chef to [$node]..." 45 | #knife solo prepare $NODE 46 | echo "------------------- DONE -------------------" 47 | -------------------------------------------------------------------------------- /lightbend.sbt: -------------------------------------------------------------------------------- 1 | // credentials can be retrieved from https://www.lightbend.com/product/lightbend-reactive-platform/credentials 2 | inThisBuild(Seq( 3 | credentials += Credentials(Path.userHome / ".lightbend" / "commercial.credentials"), 4 | resolvers += "com-mvn" at "https://repo.lightbend.com/commercial-releases/", 5 | resolvers += Resolver.url("com-ivy", 6 | url("https://repo.lightbend.com/commercial-releases/"))(Resolver.ivyStylePatterns) 7 | )) 8 | -------------------------------------------------------------------------------- /multidc/.gitignore: -------------------------------------------------------------------------------- 1 | *.pem 2 | -------------------------------------------------------------------------------- /multidc/build.sbt: -------------------------------------------------------------------------------- 1 | assemblyMergeStrategy in assembly := { 2 | case PathList("javax", "servlet", xs @ _*) => MergeStrategy.first 3 | case PathList(ps @ _*) if ps.last endsWith ".html" => MergeStrategy.first 4 | case "application.conf" => MergeStrategy.concat 5 | case props if props.endsWith(".properties") => MergeStrategy.concat 6 | case x => 7 | val oldStrategy = (assemblyMergeStrategy in assembly).value 8 | oldStrategy(x) 9 | } 10 | 11 | /* 12 | fork in run := true 13 | 14 | javaOptions in run ++= Seq( 15 | "-Daeron.mtu.length=1024" 16 | ) 17 | */ 18 | -------------------------------------------------------------------------------- /multidc/generate-writes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | # pull in the node ip addresses 4 | . nodes-bash-exports.sh 5 | 6 | for n in `seq 100 120`; do 7 | for d in `seq 1 10`; do 8 | curl 18.194.243.77:8080/introspector/pod-A-$n/write/data-$d 2> /dev/null > /dev/null 9 | # curl 34.253.229.136:8080/introspector/pod-B-$n/write/data-$d 2> /dev/null > /dev/null 10 | done 11 | done 12 | -------------------------------------------------------------------------------- /multidc/kill-multi-dc-test-remote.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # pull in the node ip addresses 4 | . nodes-bash-exports.sh 5 | 6 | for node in "${nodes_all[@]}"; do 7 | echo "============ Killing MultiDC App on: $node ... ============" 8 | ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i $HOME/.ssh/replicated-entity.pem akka@${node} " 9 | killall -9 java 10 | rm -rf /dev/shm/* 11 | " 12 | 13 | 14 | done 15 | 16 | ps aux | grep ssh | grep trictHostKeyChecking=no | awk '{ print $2 }' | xargs kill 17 | -------------------------------------------------------------------------------- /multidc/members.json: -------------------------------------------------------------------------------- 1 | { 2 | "selfNode": "akka://MultiDcSystem@54.154.90.153:2551", 3 | "leader": "akka://MultiDcSystem@54.154.90.153:2551", 4 | "oldest": "akka://MultiDcSystem@54.154.90.153:2551", 5 | "unreachable": [], 6 | "members": [ 7 | { 8 | "node": "akka://MultiDcSystem@54.154.90.153:2551", 9 | "nodeUid": "5406057317739130193", 10 | "status": "Up", 11 | "roles": [ 12 | "dc-eu-west" 13 | ] 14 | }, 15 | { 16 | "node": "akka://MultiDcSystem@54.229.246.13:2551", 17 | "nodeUid": "4214549075124656974", 18 | "status": "Up", 19 | "roles": [ 20 | "dc-eu-west" 21 | ] 22 | }, 23 | { 24 | "node": "akka://MultiDcSystem@54.229.45.233:2551", 25 | "nodeUid": "6938931469818103702", 26 | "status": "Up", 27 | "roles": [ 28 | "dc-eu-west" 29 | ] 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /multidc/network-fleaky-node.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | if [[ "$2" eq "fleaky" ]]; then 4 | echo "========= FLEAKY $2 node... =========" 5 | elif [[ "$2" eq "heal" ]]; then 6 | echo "========= HEAL $2 node... =========" 7 | fi 8 | 9 | 10 | nodeF_1=18.194.58.196 11 | nodeF_2=54.93.164.15 12 | nodeF_3=54.93.164.15 13 | nodeI_1=54.171.122.189 14 | nodeI_2=54.171.122.189 15 | nodeI_3=34.253.187.2 16 | 17 | all=( "$nodeF_1" "$nodeF_2" "$nodeF_3" "$nodeI_1" "$nodeI_2" "$nodeI_3" ) 18 | 19 | for node in "${all[@]}"; do 20 | echo "============ Staging MultiDC App on: $node ... ============" 21 | 22 | read -r -d '' COMMAND <<'EOF' 23 | tc qdisc change dev ens3 root netem reorder 0.02 duplicate 0.05 corrupt 0.01 delay 1 24 | EOF 25 | 26 | ssh -i $HOME/.ssh/replicated-entity.pem ubuntu@${node} $COMMAND 27 | ssh -i $HOME/.ssh/re-central.pem ubuntu@${node} $COMMAND 28 | 29 | done 30 | 31 | -------------------------------------------------------------------------------- /multidc/network-split-cassandra.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # pull in the node ip addresses 4 | . nodes-cassandra-bash-exports.sh 5 | 6 | echo "======= $1 =======" 7 | 8 | if [[ "$1" = "split" ]]; 9 | then 10 | echo "========= PARTITION FULL SPLIT... =========" 11 | 12 | for node in "${nodes_cassandra_central_ip[@]}"; do 13 | echo "============ Separating Central node: $node ... ============" 14 | for separate_from_node in "${nodes_cassandra_west_ip[@]}"; do 15 | echo "Splitting $node from $separate_from_node ... " 16 | ssh -i $HOME/.ssh/re-central.pem ubuntu@${node} "sudo iptables -A INPUT -p tcp -s $separate_from_node -j DROP" 17 | done 18 | done 19 | 20 | # uncomment if you want a "full split" 21 | for node in "${nodes_cassandra_west_ip[@]}"; do 22 | echo "============ Separating West node: $node ... ============" 23 | for separate_from_node in "${nodes_cassandra_central_ip[@]}"; do 24 | echo "Splitting $node from $separate_from_node ... " 25 | ssh -i $HOME/.ssh/replicated-entity.pem ubuntu@${node} "sudo iptables -A INPUT -p tcp -s $separate_from_node -j DROP" 26 | done 27 | done 28 | 29 | elif [[ "$1" = "heal" ]]; 30 | then 31 | echo "========= HEAL $2 to rejoin other nodes... =========" 32 | 33 | for node in "${nodes_cassandra_all_ip[@]}"; do 34 | echo "============ Healing node: $node ... ============" 35 | ssh -i $HOME/.ssh/re-central.pem ubuntu@${node} "sudo iptables -F" 36 | ssh -i $HOME/.ssh/replicated-entity.pem ubuntu@${node} "sudo iptables -F" 37 | done 38 | 39 | else 40 | echo "Usage: network-split-cassandra.sh [split | heal]" 41 | fi 42 | -------------------------------------------------------------------------------- /multidc/network-split-remoting.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # pull in the node ip addresses 4 | . nodes-bash-exports.sh 5 | 6 | echo "======= $1 =======" 7 | 8 | if [[ "$1" = "split" ]]; 9 | then 10 | echo "========= PARTITION FULL SPLIT... =========" 11 | 12 | for node in "${nodes_central[@]}"; do 13 | # echo "============ Separating Central node: $node ... ============" 14 | 15 | for separate_from_node in "${nodes_west_ip[@]}"; do 16 | ssh -i $HOME/.ssh/replicated-entity.pem ubuntu@${node} "sudo iptables -A INPUT -p udp -s $separate_from_node -j DROP" 17 | done 18 | done 19 | 20 | # uncomment if you want a "full split" 21 | for node in "${nodes_west[@]}"; do 22 | # echo "============ Separating West node: $node ... ============" 23 | for separate_from_node in "${nodes_central_ip[@]}"; do 24 | ssh -i $HOME/.ssh/replicated-entity.pem ubuntu@${node} "sudo iptables -A INPUT -p udp -s $separate_from_node -j DROP" 25 | done 26 | done 27 | 28 | elif [[ "$1" = "heal" ]]; 29 | then 30 | echo "========= HEAL $2 to rejoin other nodes... =========" 31 | 32 | for node in "${nodes_all[@]}"; do 33 | echo "============ Healing node: $node ... ============" 34 | ssh -i $HOME/.ssh/replicated-entity.pem ubuntu@${node} "sudo iptables -F" 35 | done 36 | 37 | else 38 | echo "Usage: network-split-remoting.sh [split | heal]" 39 | fi 40 | -------------------------------------------------------------------------------- /multidc/nodes-bash-exports.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | export node_central_1a="re-akka-eucentral-1a" 4 | export node_central_1a_ip=18.194.173.185 5 | export node_central_1b="re-akka-eucentral-1b" 6 | export node_central_1b_ip=52.59.241.114 7 | export node_central_1c="re-akka-eucentral-1c" 8 | export node_central_1c_ip=18.194.243.77 9 | 10 | export node_west_1a="re-akka-euwest-1a" 11 | export node_west_1a_ip=54.171.108.87 12 | export node_west_1b="re-akka-euwest-1b" 13 | export node_west_1b_ip=34.253.229.136 14 | export node_west_1c="re-akka-euwest-1c" 15 | export node_west_1c_ip=54.154.72.165 16 | 17 | # All 3 Akka nodes 18 | 19 | #export nodes_all=( "$node_central_1a" "$node_central_1b" "$node_central_1c" "$node_west_1a" "$node_west_1b" "$node_west_1c" ) 20 | #export nodes_all_ip=( "$node_central_1a_ip" "$node_central_1b_ip" "$node_central_1c_ip" "$node_west_1a_ip" "$node_west_1b_ip" "$node_west_1c_ip" ) 21 | 22 | #export nodes_central=( "$node_central_1a" "$node_central_1b" "$node_central_1c" ) 23 | #export nodes_central_ip=( "$node_central_1a_ip" "$node_central_1b_ip" "$node_central_1c_ip" ) 24 | 25 | #export nodes_west=( "$node_west_1a" "$node_west_1b" "$node_west_1c" ) 26 | #export nodes_west_ip=( "$node_west_1a_ip" "$node_west_1b_ip" "$node_west_1c_ip" ) 27 | 28 | # Only 1 Akka node in each DC 29 | 30 | export nodes_all=( "$node_central_1a" "$node_west_1a" ) 31 | export nodes_all_ip=( "$node_central_1a_ip" "$node_west_1a_ip" ) 32 | 33 | export nodes_central=( "$node_central_1a" ) 34 | export nodes_central_ip=( "$node_central_1a_ip" ) 35 | 36 | export nodes_west=( "$node_west_1a" ) 37 | export nodes_west_ip=( "$node_west_1a_ip" ) 38 | 39 | -------------------------------------------------------------------------------- /multidc/nodes-cassandra-bash-exports.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | export node_cassandra_central_1a_ip=35.158.122.93 4 | export node_cassandra_central_1b_ip=54.93.225.104 5 | export node_cassandra_central_1c_ip=18.195.21.84 6 | 7 | export node_cassandra_west_1a_ip=54.194.79.5 8 | export node_cassandra_west_1b_ip=34.242.227.97 9 | export node_cassandra_west_1c_ip=34.242.248.117 10 | 11 | export nodes_cassandra_all_ip=( "$node_cassandra_central_1a_ip" "$node_cassandra_central_1b_ip" "$node_cassandra_central_1c_ip" "$node_cassandra_west_1a_ip" "$node_cassandra_west_1b_ip" "$node_cassandra_west_1c_ip" ) 12 | 13 | export nodes_cassandra_central_ip=( "$node_cassandra_central_1a_ip" "$node_cassandra_central_1b_ip" "$node_cassandra_central_1c_ip" ) 14 | 15 | export nodes_cassandra_west_ip=( "$node_cassandra_west_1a_ip" "$node_cassandra_west_1b_ip" "$node_cassandra_west_1c_ip" ) 16 | 17 | -------------------------------------------------------------------------------- /multidc/notes/cpu7c.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/multidc/notes/cpu7c.png -------------------------------------------------------------------------------- /multidc/notes/cpu7w.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/multidc/notes/cpu7w.png -------------------------------------------------------------------------------- /multidc/notes/jmx10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/multidc/notes/jmx10.png -------------------------------------------------------------------------------- /multidc/notes/jmx12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/multidc/notes/jmx12.png -------------------------------------------------------------------------------- /multidc/notes/jmx13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/multidc/notes/jmx13.png -------------------------------------------------------------------------------- /multidc/notes/jmx21.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/multidc/notes/jmx21.png -------------------------------------------------------------------------------- /multidc/notes/jmx28.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/multidc/notes/jmx28.png -------------------------------------------------------------------------------- /multidc/notes/jmx29.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/multidc/notes/jmx29.png -------------------------------------------------------------------------------- /multidc/notes/jmx7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/multidc/notes/jmx7.png -------------------------------------------------------------------------------- /multidc/notes/jmx8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/multidc/notes/jmx8.png -------------------------------------------------------------------------------- /multidc/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.15 2 | -------------------------------------------------------------------------------- /multidc/rsync-local-src-to-akka-nodes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | # pull in the node ip addresses 4 | . nodes-bash-exports.sh 5 | 6 | for node in "${nodes_all[@]}"; do 7 | echo "============ syncing with: $node ... ============" 8 | 9 | ssh -i $HOME/.ssh/replicated-entity.pem akka@${node} "rm -rf multidc/apps/build.sbt" 10 | rsync -Pavuze "ssh -i $HOME/.ssh/replicated-entity.pem" ../build.sbt akka@${node}:multidc/apps/build.sbt & 11 | 12 | ssh -i $HOME/.ssh/replicated-entity.pem akka@${node} "rm -rf multidc/apps/multidc/build.sbt" 13 | rsync -Pavuze "ssh -i $HOME/.ssh/replicated-entity.pem" build.sbt akka@${node}:multidc/apps/multidc/build.sbt & 14 | 15 | ssh -i $HOME/.ssh/replicated-entity.pem akka@${node} "rm -rf multidc/apps/multidc/src" 16 | rsync -Pavuze "ssh -i $HOME/.ssh/replicated-entity.pem" src akka@${node}:multidc/apps/multidc/ & 17 | 18 | scp -i $HOME/.ssh/replicated-entity.pem '~/.ivy2/local/com.lightbend.akka/akka-persistence-multi-dc_2.11/1.1-M5+21-30d17edd+20171214-1234/jars/akka-persistence-multi-dc_2.11.jar' akka@${node}:/home/akka/akka-persistence-multi-dc_2.11.jar 19 | done 20 | -------------------------------------------------------------------------------- /multidc/run-multi-dc-test-remote.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # pull in the node ip addresses 4 | . nodes-bash-exports.sh 5 | 6 | for node in "${nodes_all[@]}"; do 7 | echo "============ Staging MultiDC App on: $node ... ============" 8 | ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i $HOME/.ssh/replicated-entity.pem akka@${node} " 9 | cd multidc/apps 10 | pwd 11 | 12 | killall -9 java 13 | sleep 1 14 | /home/akka/sbt '; project multidc; runMain com.lightbend.multidc.ReplicatedEntityApp ' 15 | " & 16 | 17 | done 18 | -------------------------------------------------------------------------------- /multidc/run-multi-dc-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | DC1=localhost:8080 4 | DC2=localhost:8081 5 | 6 | curl -v "$DC1/members" | jq 7 | 8 | curl -v "$DC1/single-counter-test?counter=first&updates=100" 9 | 10 | #curl -v "$DC1/single-counter-test?counter=first&updates=100" 11 | 12 | curl -v "$DC1/counter?id=first" 13 | 14 | curl -v "$DC1/test?counters=100&updates=100" 15 | curl -v "$DC1/counter?id=99" 16 | 17 | 18 | -------------------------------------------------------------------------------- /multidc/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} %-5level [%logger{36}] [%X{sourceThread}] - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | entity.log 11 | true 12 | 13 | true 14 | 16 | 17 | %-4relative [%thread] %-5level %logger{35} - %msg%n 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /multidc/src/main/scala/com/lightbend/multidc/Incrementor.scala: -------------------------------------------------------------------------------- 1 | package com.lightbend.multidc 2 | 3 | import java.util.concurrent.TimeUnit 4 | 5 | import akka.actor.{Actor, ActorLogging, ActorRef} 6 | import com.lightbend.multidc.ReplicatedCounter.{Increment, IncrementAck, ShardingEnvelope} 7 | 8 | import scala.concurrent.duration.Duration 9 | 10 | class Incrementor(id: String, nrIncrements: Int, clusterProxy: ActorRef) extends Actor with ActorLogging { 11 | import context._ 12 | 13 | val startTime = System.currentTimeMillis() 14 | 15 | log.info("Incrementor for id {} to do {} increments", id, nrIncrements) 16 | 17 | clusterProxy ! ShardingEnvelope(id, Increment("do it")) 18 | 19 | def receive = toGo(nrIncrements - 1) 20 | 21 | def toGo(nr: Int): Receive = { 22 | case IncrementAck => 23 | if (nr <= 0) { 24 | log.info("Incrementor for id {} finished. Took {} seconds", id, Duration(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS).toSeconds) 25 | system.stop(self) 26 | } else { 27 | clusterProxy ! ShardingEnvelope(id, Increment("do it")) 28 | become(toGo(nr - 1)) 29 | } 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /nodes/re-akka-euwest-1a.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/nodes/re-akka-euwest-1a.json -------------------------------------------------------------------------------- /nodes/re-cassandra-euwest-1a.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/nodes/re-cassandra-euwest-1a.json -------------------------------------------------------------------------------- /nodes/re-cassandra-euwest-1b.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/nodes/re-cassandra-euwest-1b.json -------------------------------------------------------------------------------- /nodes/re-cassandra-euwest-1c.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akka/apps/3a6a70abf5faa3190509f87892dafa250eb6c212/nodes/re-cassandra-euwest-1c.json -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.16 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | //addSbtPlugin("de.heikoseeberger" % "sbt-header" % "2.0.0") 2 | //addSbtPlugin("com.lucidchart" % "sbt-scalafmt" % "1.12") 3 | addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.2.0") 4 | addSbtPlugin("com.dwijnand" % "sbt-dynver" % "1.3.0") 5 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.5") 6 | -------------------------------------------------------------------------------- /pubsub/README.md: -------------------------------------------------------------------------------- 1 | # Distributed Pub Sub Benchmark 2 | 3 | Sample usage 4 | ```bash 5 | curl -X POST "http://localhost:8080/bench?numberOfNodes=1&messagesPerPublisher=10&numberOfTopics=200&numberOfPublishers=10&numberOfSubscribers=10" 6 | ``` 7 | 8 | To list the results of completed runs: 9 | ```bash 10 | curl http://localhost:8080/bench 11 | ``` 12 | 13 | Run additional nodes locally with: 14 | ``` 15 | sbt -J-Dakka.cluster.roles.0=banana -J-Dakka.remote.artery.canonical.port=0 16 | pubsub/run 17 | ``` 18 | 19 | The bench sends `messagesPerPublisher` messages to each topic from each publisher, but always at least one message per topic (in case `messagesPerPublisher` < `numberOfTopics`) 20 | 21 | Timing is static, so each run will pretty much take the exact same time: ~10 seconds waiting for all subscriptions to disseminate, ~20 seconds to wait for all messages to reach the subscribers, and then a timeout of 30s for result collection. If the entire bench session takes more than 2 minutes it fails. 22 | 23 | `numberOfTopics` determine the total number of topics published and subscribed to 24 | 25 | The `numberOfPublishers` determine the total number of publishers across the cluster, the publishers will be divided evenly across the cluster nodes so it will need to be evenly divisible over the number of nodes. Each publisher will then publish a message to a random topic, every 10 ms when the bench has started. 26 | 27 | The `numberOfSubscribers` determine how many subscribers each node starts, the topics are then evenly divided between the subscribers of a node. 28 | -------------------------------------------------------------------------------- /pubsub/src/main/resources/application.conf: -------------------------------------------------------------------------------- 1 | akka { 2 | extensions = [ 3 | "com.romix.akka.serialization.kryo.KryoSerializationExtension$" 4 | "akka.cluster.pubsub.DistributedPubSub" 5 | ] 6 | 7 | diagnostics.checker.confirmed-typos = [ 8 | "akka.actor.kryo" 9 | ] 10 | 11 | actor { 12 | provider = "cluster" 13 | 14 | allow-java-serialization = no 15 | 16 | serializers { 17 | kryo = "com.romix.akka.serialization.kryo.KryoSerializer" 18 | } 19 | 20 | serialization-bindings { 21 | "com.lightbend.akka.bench.pubsub.Payload" = kryo 22 | "com.lightbend.akka.bench.pubsub.Publisher$Started" = kryo 23 | "com.lightbend.akka.bench.pubsub.Publisher$Tick$" = kryo 24 | "com.lightbend.akka.bench.pubsub.Publisher$Start" = kryo 25 | "com.lightbend.akka.bench.pubsub.Subscriber$Subscribed" = kryo 26 | "com.lightbend.akka.bench.pubsub.Subscriber$CollectStats$" = kryo 27 | "com.lightbend.akka.bench.pubsub.PubSubHost$StartSession" = kryo 28 | "com.lightbend.akka.bench.pubsub.PubSubHost$StopRun" = kryo 29 | } 30 | 31 | # kryo for serialization for lazy reasons 32 | kryo { 33 | type = "nograph" 34 | idstrategy = "explicit" 35 | buffer-size = 4096 36 | max-buffer-size = -1 37 | use-manifests = false 38 | use-unsafe = false 39 | post-serialization-transformations = "off" 40 | implicit-registration-logging = false 41 | kryo-trace = false 42 | 43 | mappings { 44 | "com.lightbend.akka.bench.pubsub.Payload" = 21 45 | "com.lightbend.akka.bench.pubsub.Publisher$Started" = 22 46 | "com.lightbend.akka.bench.pubsub.Publisher$Start" = 23 47 | "com.lightbend.akka.bench.pubsub.Publisher$Tick$" = 24 48 | "com.lightbend.akka.bench.pubsub.Subscriber$Subscribed" = 25 49 | "com.lightbend.akka.bench.pubsub.Subscriber$CollectStats$" = 26 50 | "akka.actor.LocalActorRef" = 27 51 | "com.lightbend.akka.bench.pubsub.PubSubHost$StartSession" = 28 52 | "com.lightbend.akka.bench.pubsub.PubSubHost$StopRun" = 29 53 | } 54 | } 55 | } 56 | 57 | remote { 58 | artery { 59 | enabled = on 60 | 61 | # !! IMPORTANT !! 62 | # the below are provided by root-application.conf on our cloud nodes, 63 | # these values here serve only as local fallback 64 | canonical.hostname = "127.0.0.1" 65 | canonical.port = 2551 66 | canonical.port = ${?AKKA_CLUSTER_PORT} 67 | # !! IMPORTANT !! 68 | } 69 | } 70 | 71 | cluster { 72 | # !! IMPORTANT !! 73 | # the below are provided by root-application.conf on our cloud nodes, 74 | # these values here serve only as local fallback 75 | seed-nodes = [ 76 | "akka://Benchmark@127.0.0.1:2551" 77 | ] 78 | roles = ["master"] 79 | # !! IMPORTANT !! 80 | 81 | downing-provider-class = "com.lightbend.akka.sbr.SplitBrainResolverProvider" 82 | split-brain-resolver.active-strategy = "keep-majority" 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /pubsub/src/main/scala/com/lightbend/akka/bench/pubsub/BenchmarkCoordinator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Lightbend Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.lightbend.akka.bench.pubsub 18 | 19 | import akka.actor.{Actor, ActorLogging, Props, Terminated} 20 | 21 | object BenchmarkCoordinator { 22 | case class StartRun(numberOfNodes: Int, messagesPerPublisher: Int, numberOfTopics: Int, numberOfPublishers: Int, numberOfSubscribers: Int) { 23 | require(numberOfPublishers % numberOfNodes == 0, "Number of publishers must be evenly divisible over the number of nodes") 24 | require(numberOfSubscribers % numberOfNodes == 0, "Number of subscribers must be evenly divisible over the number of nodes") 25 | require(numberOfSubscribers <= numberOfTopics, "Number of subscribers cannot be more than the number of topics") 26 | require(numberOfTopics % numberOfSubscribers == 0, "Number of topics must be evenly divisible over the number of subscribers") 27 | } 28 | case class RunInitialized(id: Int) 29 | 30 | case class BenchResult( 31 | id: Int, 32 | numberOfNodes: Int, 33 | messagesPerPublisher: Int, 34 | numberOfTopics: Int, 35 | numberOfPublishers: Int, 36 | numberOfSubscribers: Int, 37 | messagesArrived: Int, 38 | totalNumberOfMessages: Int, 39 | failed: Int) 40 | 41 | case object GetResults 42 | case class AllResults(results: List[BenchResult]) 43 | 44 | def props(): Props = Props(new BenchmarkCoordinator()) 45 | } 46 | 47 | class BenchmarkCoordinator extends Actor with ActorLogging { 48 | 49 | import BenchmarkCoordinator._ 50 | 51 | var _runId = 0 52 | def nextRunId() = { 53 | _runId += 1 54 | _runId 55 | } 56 | 57 | var results = List.empty[BenchResult] 58 | 59 | def receive = idle 60 | 61 | def idle: Receive = { 62 | case StartRun(numberOfNodes, msgPerP, numTopic, numPubs, numSubs) => 63 | val id = nextRunId() 64 | context.watch(context.actorOf(BenchmarkSession.props(id, numberOfNodes, msgPerP, numTopic, numPubs, numSubs), s"run-$id")) 65 | sender() ! RunInitialized(id) 66 | context.become(running) 67 | 68 | case GetResults => sender() ! AllResults(results) 69 | 70 | } 71 | 72 | def running: Receive = { 73 | 74 | case GetResults => sender() ! AllResults(results) 75 | 76 | case result: BenchResult => 77 | results = result :: results 78 | 79 | case Terminated(_) => 80 | log.info("Run terminated") 81 | context.become(idle) 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /pubsub/src/main/scala/com/lightbend/akka/bench/pubsub/PubSubBenchmark.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Lightbend Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.lightbend.akka.bench.pubsub 18 | 19 | import scala.util.Try 20 | import akka.actor.{ActorSystem, PoisonPill, Props} 21 | import akka.cluster.Cluster 22 | import java.io.File 23 | 24 | import akka.cluster.singleton.{ClusterSingletonManager, ClusterSingletonManagerSettings, ClusterSingletonProxy, ClusterSingletonProxySettings} 25 | import com.typesafe.config.ConfigFactory 26 | 27 | object PubSubBenchmark extends App { 28 | final val CoordinatorManager = "pubSubCoordinatorManager" 29 | 30 | // setup for clound env ------------------------------------------------------------- 31 | val rootConfFile = new File("/home/akka/root-application.conf") 32 | val rootConf = 33 | if (rootConfFile.exists) ConfigFactory.parseFile(rootConfFile) 34 | else ConfigFactory.empty("no-root-application-conf-found") 35 | 36 | val conf = rootConf.withFallback(ConfigFactory.load()) 37 | // end of setup for clound env ------------------------------------------------------ 38 | 39 | val systemName = Try(conf.getString("akka.system-name")).getOrElse("Benchmark") 40 | implicit val system = ActorSystem(systemName, conf) 41 | 42 | system.actorOf(PubSubHost.props(), PubSubHost.Name) 43 | 44 | if (Cluster(system).selfRoles("master")) { 45 | system.actorOf( 46 | ClusterSingletonManager.props( 47 | singletonProps = BenchmarkCoordinator.props(), 48 | terminationMessage = PoisonPill, 49 | settings = ClusterSingletonManagerSettings(system).withRole("master")), 50 | name = CoordinatorManager) 51 | 52 | val proxy = system.actorOf(ClusterSingletonProxy.props( 53 | singletonManagerPath = "/user/" + PubSubBenchmark.CoordinatorManager, 54 | settings = ClusterSingletonProxySettings(system).withRole("master")), 55 | name = "coordinatorProxy") 56 | 57 | HttpApi.startServer("localhost", 8080, proxy) 58 | 59 | scala.io.StdIn.readLine() 60 | system.terminate() 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /pubsub/src/main/scala/com/lightbend/akka/bench/pubsub/PubSubHost.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Lightbend Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.lightbend.akka.bench.pubsub 18 | 19 | import akka.actor._ 20 | import akka.cluster.singleton._ 21 | 22 | object PubSubHost { 23 | 24 | case class StartSession(sessionId: Int, numberOfHosts: Int, numberOfTopics: Int, numberOfPublishers: Int, numberOfSubscribers: Int) 25 | case class StopRun(runId: Int) 26 | 27 | def props(): Props = Props(new PubSubHost()) 28 | val Name = "pub-sub-host" 29 | 30 | } 31 | 32 | /** 33 | * One of these runs on each node in the cluster 34 | */ 35 | class PubSubHost() extends Actor with ActorLogging { 36 | 37 | import PubSubHost._ 38 | 39 | log.info("PubSubHost started") 40 | 41 | val coordinator = context.actorOf( 42 | ClusterSingletonProxy.props( 43 | singletonManagerPath = "/user/" + PubSubBenchmark.CoordinatorManager, 44 | settings = ClusterSingletonProxySettings(context.system).withRole("master")), 45 | name = "coordinatorProxy") 46 | 47 | def receive = idle 48 | 49 | def idle: Receive = { 50 | case StartSession(id, numberOfHosts, numberOfTopics, numberOfPublishers, numberOfSubscribers) => 51 | val runner = context.watch(context.actorOf(PubSubHostSession.props(id, sender(), numberOfHosts, numberOfTopics, numberOfPublishers, numberOfSubscribers), s"session-$id")) 52 | context.become(running(runner, id)) 53 | 54 | } 55 | 56 | def running(runner: ActorRef, sessionId: Int): Receive = { 57 | case Terminated(_) => 58 | log.info("Host run {} stopped itself", sessionId) 59 | context.become(idle) 60 | 61 | case StopRun(`sessionId`) => 62 | context.unwatch(runner) 63 | runner ! PubSubHostSession.Stop(sessionId) 64 | log.info("Host run {} completed from coordinator", sessionId) 65 | context.become(idle) 66 | } 67 | } -------------------------------------------------------------------------------- /pubsub/src/main/scala/com/lightbend/akka/bench/pubsub/PubSubHostSession.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Lightbend Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.lightbend.akka.bench.pubsub 18 | 19 | import akka.actor.{Actor, ActorLogging, ActorRef, Props, Terminated} 20 | import akka.cluster.pubsub.DistributedPubSub 21 | 22 | object PubSubHostSession { 23 | 24 | case class Stop(runId: Int) 25 | 26 | def props(runId: Int, coordinator: ActorRef, numberOfNodes: Int, numberOfTopics: Int, numberOfPublishers: Int, numberOfSubscribers: Int): Props = 27 | Props(new PubSubHostSession(runId, coordinator, numberOfNodes, numberOfTopics, numberOfPublishers, numberOfSubscribers)) 28 | } 29 | 30 | class PubSubHostSession(sessionId: Int, coordinator: ActorRef, numberOfNodes: Int, numberOfTopics: Int, numberOfPublishers: Int, numberOfSubscribers: Int) extends Actor with ActorLogging { 31 | import PubSubHostSession._ 32 | log.info("Starting host session {}", sessionId) 33 | 34 | val mediator = DistributedPubSub(context.system).mediator 35 | 36 | val subscribersOnThisNode = numberOfSubscribers / numberOfNodes 37 | val publishersOnThisNode = numberOfPublishers / numberOfNodes 38 | 39 | val topicsPerSubscriber = numberOfTopics / subscribersOnThisNode 40 | 41 | (0 until numberOfTopics by topicsPerSubscriber).foreach(n => 42 | context.watch(context.actorOf(Subscriber.props(sessionId, n, Math.min(n + topicsPerSubscriber, numberOfTopics), mediator, coordinator), s"subscriber-$n")) 43 | ) 44 | log.info("Started {} subscribers, each subscribing to {} topics", subscribersOnThisNode, topicsPerSubscriber) 45 | 46 | (0 until publishersOnThisNode).foreach(n => 47 | context.watch(context.actorOf(Publisher.props(sessionId, numberOfTopics, coordinator), s"publisher-$n"))) 48 | log.info("Started {} publishers", publishersOnThisNode) 49 | 50 | def receive = { 51 | case Stop(`sessionId`) => 52 | log.debug("Shutting down session {}", sessionId) 53 | context.stop(self) 54 | 55 | case Terminated(ref) => 56 | if (context.children.isEmpty) { 57 | log.info("All publishers and subscribers shut down for session {}", sessionId) 58 | context.stop(self) 59 | } 60 | 61 | } 62 | } -------------------------------------------------------------------------------- /pubsub/src/main/scala/com/lightbend/akka/bench/pubsub/Publisher.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Lightbend Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.lightbend.akka.bench.pubsub 18 | 19 | import scala.util.Random 20 | import scala.concurrent.duration._ 21 | import akka.actor._ 22 | import akka.cluster.pubsub._ 23 | 24 | import scala.concurrent.ExecutionContext 25 | 26 | object Publisher { 27 | def props(runId: Int, maxTopic: Int, coordinator: ActorRef): Props = Props(new Publisher(runId, maxTopic, coordinator)) 28 | 29 | case class Started(runId: Int, publisher: ActorRef) 30 | case class Start(runId: Int, messagesToPublish: Int) 31 | case object Tick 32 | } 33 | class Publisher(runId: Int, maxTopic: Int, coordinator: ActorRef) extends Actor with ActorLogging { 34 | import Publisher._ 35 | import DistributedPubSubMediator.Publish 36 | 37 | implicit val ec: ExecutionContext = context.dispatcher 38 | 39 | override def preStart() = { 40 | coordinator ! Started(runId, self) 41 | } 42 | 43 | val mediator = DistributedPubSub(context.system).mediator 44 | val random = new Random() 45 | 46 | override def receive = { 47 | case Start(`runId`, messagesToPublish) => 48 | log.debug("publisher starting to publish") 49 | context.become(ticking(context.system.scheduler.schedule(0.seconds, 10.millis, self, Tick), messagesToPublish)) 50 | } 51 | 52 | def ticking(cancellable: Cancellable, ticksLeft: Int): Receive = { 53 | case Tick => 54 | mediator ! Publish(random.nextInt(maxTopic).toString, Payload(42)) 55 | if (ticksLeft == 1) { 56 | log.debug("Publisher done, shutting down") 57 | cancellable.cancel() 58 | context.stop(self) 59 | } else { 60 | context.become(ticking(cancellable, ticksLeft - 1)) 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /pubsub/src/main/scala/com/lightbend/akka/bench/pubsub/Subscriber.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Lightbend Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.lightbend.akka.bench.pubsub 18 | 19 | import scala.concurrent.{ExecutionContext, Future} 20 | import scala.concurrent.duration._ 21 | import scala.util.Random 22 | import akka.actor._ 23 | import akka.cluster.pubsub._ 24 | import akka.pattern.{ask, pipe} 25 | import akka.util.Timeout 26 | import DistributedPubSubMediator.{Subscribe, SubscribeAck} 27 | 28 | object Subscriber { 29 | def props(runId: Int, topic: Int, topicUntil: Int, mediator: ActorRef, coordinator: ActorRef): Props = 30 | Props(new Subscriber(runId, topic, topicUntil, mediator, coordinator)) 31 | 32 | case class Subscribed(runId: Int) 33 | case object CollectStats 34 | } 35 | class Subscriber(runId: Int, topic: Int, topicUntil: Int, mediator: ActorRef, coordinator: ActorRef) extends Actor with ActorLogging { 36 | import Subscriber._ 37 | 38 | var messagesReceived = 0 39 | val random = new Random() 40 | 41 | implicit val timeout: Timeout = 5.seconds 42 | implicit val ec: ExecutionContext = context.dispatcher 43 | 44 | Future.sequence((topic until topicUntil).map { n => 45 | mediator ? Subscribe(n.toString, self) 46 | }).pipeTo(self) 47 | 48 | override def receive = { 49 | case _: Seq[_] ⇒ 50 | log.info(s"Subscribed to topics $topic-$topicUntil") 51 | coordinator ! Subscribed(runId) 52 | case Payload(n) => 53 | messagesReceived += 1 54 | case CollectStats => 55 | // log.info("Collecting stats from this subscriber") 56 | sender() ! messagesReceived 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /pubsub/src/main/scala/com/lightbend/akka/bench/pubsub/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Lightbend Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.lightbend.akka.bench.pubsub 18 | 19 | case class Payload(i: Int) 20 | -------------------------------------------------------------------------------- /scp-benchmark.sh: -------------------------------------------------------------------------------- 1 | clear; gcloud compute scp infra/site-cookbooks/update-hosts/templates/default/run-sharding-start-latency-benchmark.sh.rb akka@akka-sharding-001:run-sharding-start-latency-benchmark.sh 2 | -------------------------------------------------------------------------------- /scp-sharding.sh: -------------------------------------------------------------------------------- 1 | clear; gcloud compute scp --recurse ../akka-apps/sharding/src akka@akka-sharding-001:/home/akka/apps/sharding/ 2 | -------------------------------------------------------------------------------- /sharding/src/main/scala/com/lightbend/akka/bench/sharding/BenchSettings.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Lightbend Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.lightbend.akka.bench.sharding 18 | 19 | import java.util.concurrent.TimeUnit 20 | 21 | import akka.actor.{ActorSystem, ExtendedActorSystem, Extension, ExtensionId} 22 | 23 | import scala.concurrent.duration._ 24 | 25 | object BenchSettings extends ExtensionId[BenchSettings] { 26 | override def createExtension(system: ExtendedActorSystem): BenchSettings = 27 | new BenchSettings(system) 28 | } 29 | 30 | sealed trait ShardingBenchmarkMode 31 | case object PersistShardingBenchmark extends ShardingBenchmarkMode 32 | case object RawPingPongShardingBenchmark extends ShardingBenchmarkMode 33 | 34 | class BenchSettings(system: ActorSystem) extends Extension { 35 | private val config = system.settings.config.getConfig("shard-bench") 36 | val Mode = config.getString("mode") match { 37 | case "persist" => PersistShardingBenchmark 38 | case _ => RawPingPongShardingBenchmark 39 | } 40 | val UniqueEntities = config.getInt("unique-entities") 41 | // val NumberOfPings = config.getInt("number-of-pings") 42 | val PingsPerSecond = config.getInt("pings-per-second") 43 | val NumberOfShards = config.getInt("number-of-shards") 44 | val MinimumNodes = config.getInt("minimum-nodes") 45 | 46 | val AddActorsPerBatch = config.getInt("add-actors-batch") 47 | val AddActorsInterval = config.getDuration("add-actors-interval", TimeUnit.SECONDS).seconds 48 | } 49 | -------------------------------------------------------------------------------- /sharding/src/main/scala/com/lightbend/akka/bench/sharding/BenchmarkConfig.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Lightbend Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.lightbend.akka.bench.sharding 18 | 19 | import java.io.File 20 | import java.net.InetAddress 21 | 22 | import com.typesafe.config.{ Config, ConfigFactory } 23 | 24 | object BenchmarkConfig { 25 | def load(): Config = { 26 | val bindAddressConf = ConfigFactory.parseString( 27 | s""" 28 | akka { 29 | cluster.http.management.hostname = "${InetAddress.getLocalHost.getHostAddress}" 30 | 31 | remote { 32 | artery.canonical.hostname = "${InetAddress.getLocalHost.getHostAddress}" 33 | } 34 | } 35 | """) 36 | 37 | val rootConfFile = new File("/home/akka/root-application.conf") 38 | val rootConf = 39 | if (rootConfFile.exists) ConfigFactory.parseFile(rootConfFile) 40 | else ConfigFactory.empty("no-root-application-conf-found") 41 | 42 | ConfigFactory.load(bindAddressConf.withFallback(rootConf).withFallback(ConfigFactory.defaultApplication())) 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /sharding/src/main/scala/com/lightbend/akka/bench/sharding/InitCassandraApp.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Lightbend Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.lightbend.akka.bench.sharding 18 | 19 | import java.io.File 20 | import java.util.concurrent.TimeUnit 21 | 22 | import akka.actor.{ ActorSystem, Props } 23 | import akka.pattern.ask 24 | import akka.cluster.Cluster 25 | import akka.cluster.http.management.ClusterHttpManagement 26 | import akka.persistence.PersistentActor 27 | import akka.util.Timeout 28 | import com.typesafe.config.ConfigFactory 29 | 30 | import scala.concurrent.Await 31 | import scala.concurrent.duration._ 32 | import scala.util.Try 33 | 34 | object InitCassandraApp extends App { 35 | 36 | // setup for clound env ------------------------------------------------------------- 37 | val rootConfFile = new File("/home/akka/root-application.conf") 38 | val rootConf = 39 | if (rootConfFile.exists) ConfigFactory.parseFile(rootConfFile) 40 | else ConfigFactory.empty("no-root-application-conf-found") 41 | 42 | val conf = rootConf.withFallback(ConfigFactory.load()) 43 | // end of setup for clound env ------------------------------------------------------ 44 | 45 | val systemName = Try(conf.getString("akka.system-name")).getOrElse("DistributedDataSystem") 46 | implicit val system = ActorSystem(systemName, conf) 47 | 48 | // management ----------- 49 | val cluster = Cluster(system) 50 | ClusterHttpManagement(cluster).start() 51 | // end of management ---- 52 | 53 | implicit val timeout = Timeout(45.seconds) 54 | 55 | val journalPluginId = "" 56 | val snapshotPluginId = "" 57 | 58 | val t0 = System.nanoTime() 59 | val initActor = system.actorOf(Props(classOf[AwaitPersistenceInit], journalPluginId, snapshotPluginId), "persistenceInit") 60 | val reply = initActor ? "hello" 61 | Await.ready(reply, 60.seconds) 62 | system.log.debug("awaitPersistenceInit took {} ms {}", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t0), system.name) 63 | 64 | } 65 | 66 | class AwaitPersistenceInit( 67 | override val journalPluginId: String, 68 | override val snapshotPluginId: String 69 | ) extends PersistentActor { 70 | def persistenceId: String = "persistenceInit" 71 | 72 | def receiveRecover: Receive = { 73 | case it => 74 | println(s"Recover: ${it}") 75 | } 76 | 77 | def receiveCommand: Receive = { 78 | case msg => 79 | persist(msg) { _ => 80 | sender() ! msg 81 | context.stop(self) 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /sharding/src/main/scala/com/lightbend/akka/bench/sharding/latency/PersistenceHistograms.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Lightbend Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.lightbend.akka.bench.sharding.latency 18 | 19 | import akka.actor.{ Actor, ActorLogging, Props } 20 | import org.HdrHistogram.Histogram 21 | 22 | import scala.concurrent.duration.Duration 23 | 24 | /** 25 | * Thread safe histograms updated from the persistent entity, an actor (should be a single one per actor system) 26 | * periodically printing the histograms to stdout 27 | */ 28 | object PersistenceHistograms { 29 | 30 | private val singlePersistTiming = new Histogram(20 * 1000 * 1000, 3) 31 | def recordSinglePersistTiming(t: Duration): Unit = singlePersistTiming.synchronized { 32 | try 33 | singlePersistTiming.recordValue(t.toMicros) 34 | catch { 35 | case ex: ArrayIndexOutOfBoundsException => 36 | throw new Exception(s"Tried to record ${PrettyDuration.format(t)}, which was out of bounds for the histogram!", ex) 37 | } 38 | } 39 | 40 | private val recoveryTiming = new Histogram(60 * 1000 * 1000, 3) 41 | def recordRecoveryPersistTiming(t: Duration): Unit = recoveryTiming.synchronized { 42 | try 43 | recoveryTiming.recordValue(t.toMicros) 44 | catch { 45 | case ex: ArrayIndexOutOfBoundsException => 46 | throw new Exception(s"Tried to record ${PrettyDuration.format(t)}, which was out of bounds for the histogram!", ex) 47 | } 48 | } 49 | 50 | object PrintHistograms 51 | 52 | def props() = Props[PersistenceHistograms] 53 | } 54 | 55 | class PersistenceHistograms extends Actor with ActorLogging { 56 | import scala.concurrent.duration._ 57 | import context.dispatcher 58 | 59 | context.system.scheduler.schedule(3.seconds, 3.seconds, self, "ping") 60 | 61 | override def receive: Receive = { 62 | case _ => 63 | println("========= Histogram of sharded actor wakeup times ======== ") 64 | PersistenceHistograms.recoveryTiming.synchronized { 65 | PersistenceHistograms.recoveryTiming.outputPercentileDistribution(System.out, 1.0) 66 | } 67 | // println("========= Histogram of persist times ======== ") 68 | // PersistenceHistograms.singlePersistTiming.synchronized { 69 | // PersistenceHistograms.singlePersistTiming.outputPercentileDistribution(System.out, 1.0) 70 | // } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /sharding/src/main/scala/com/lightbend/akka/bench/sharding/latency/ShardingStartLatencyApp.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Lightbend Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.lightbend.akka.bench.sharding.latency 18 | 19 | import java.net.InetAddress 20 | 21 | import akka.actor.{ ActorSystem, Props } 22 | import akka.cluster.Cluster 23 | import akka.cluster.http.management.ClusterHttpManagement 24 | import com.lightbend.akka.bench.sharding.BenchmarkConfig 25 | 26 | import scala.util.Try 27 | import akka.cluster.singleton.ClusterSingletonManager 28 | import akka.actor.PoisonPill 29 | import akka.cluster.singleton.ClusterSingletonManagerSettings 30 | 31 | object ShardingStartLatencyApp extends App { 32 | 33 | // setup for clound env ------------------------------------------------------------- 34 | val conf = BenchmarkConfig.load() 35 | // end of setup for clound env ------------------------------------------------------ 36 | 37 | val systemName = Try(conf.getString("akka.system-name")).getOrElse("ShardingStartLatencySystem") 38 | implicit val system = ActorSystem(systemName, conf) 39 | 40 | // management ----------- 41 | val cluster = Cluster(system) 42 | ClusterHttpManagement(cluster).start() 43 | // end of management ---- 44 | 45 | val region = LatencyBenchEntity.startRegion(system) 46 | 47 | val benchCoordinatorProps = 48 | ClusterSingletonManager.props(PingLatencyCoordinator.props(region), PoisonPill, 49 | ClusterSingletonManagerSettings(system)) 50 | system.actorOf(benchCoordinatorProps, "bench-coordinator") 51 | system.actorOf(PersistenceHistograms.props(), "persistence-histogram-printer") 52 | } 53 | -------------------------------------------------------------------------------- /sharding/src/main/scala/com/lightbend/akka/bench/sharding/scalability/ActorCountingEntity.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Lightbend Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.lightbend.akka.bench.sharding.scalability 18 | 19 | import akka.actor.{ Actor, ActorRef, ActorSystem, Props } 20 | import akka.cluster.sharding.{ ClusterSharding, ClusterShardingSettings, ShardRegion } 21 | import com.lightbend.akka.bench.sharding.BenchSettings 22 | 23 | object ActorCountingEntity { 24 | 25 | sealed trait HasId { def id: Long } 26 | // sent from master to entity 27 | final case class Start(batchCount: Int, override val id: Long, sentTimestamp: Long) extends HasId 28 | // entity replies with that once it has started, the time is the sender's time, 29 | // so the sender can calculate how long it took for the actor to get the message 30 | final case class Ready(batchCount: Int, sentTimestamp: Long) 31 | 32 | def props() = Props[ActorCountingEntity] 33 | 34 | // sharding config 35 | val typeName = "bench-entity" 36 | 37 | val extractEntityId: ShardRegion.ExtractEntityId = { 38 | case msg: ActorCountingEntity.HasId => (msg.id.toString, msg) 39 | } 40 | 41 | def extractShardId(numberOfShards: Int): ShardRegion.ExtractShardId = { 42 | case msg: ActorCountingEntity.HasId => (Math.abs(msg.id.hashCode) % numberOfShards).toString 43 | } 44 | 45 | def startRegion(system: ActorSystem) = 46 | ClusterSharding(system).start( 47 | typeName, 48 | ActorCountingEntity.props(), 49 | ClusterShardingSettings(system), 50 | extractEntityId, 51 | extractShardId(BenchSettings(system).NumberOfShards)) 52 | 53 | } 54 | 55 | /** Many many many instances of this Actor will be started in sharding. */ 56 | class ActorCountingEntity extends Actor { 57 | 58 | override def receive: Receive = { 59 | case ActorCountingEntity.Start(batchCount, _, senderTimestamp) => 60 | sender() ! ActorCountingEntity.Ready(batchCount, senderTimestamp) 61 | // stay around in memory, forevermore! 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /sharding/src/main/scala/com/lightbend/akka/bench/sharding/scalability/ShardingActorCountScalabilityApp.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Lightbend Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.lightbend.akka.bench.sharding.scalability 18 | 19 | import java.net.InetAddress 20 | 21 | import akka.actor.{ ActorSystem, Props } 22 | import akka.cluster.Cluster 23 | import akka.cluster.http.management.ClusterHttpManagement 24 | import com.lightbend.akka.bench.sharding.BenchmarkConfig 25 | 26 | import scala.util.Try 27 | import akka.actor.PoisonPill 28 | import akka.cluster.singleton.ClusterSingletonManagerSettings 29 | import akka.cluster.singleton.ClusterSingletonManager 30 | 31 | object ShardingActorCountScalabilityApp extends App { 32 | 33 | // setup for clound env ------------------------------------------------------------- 34 | val conf = BenchmarkConfig.load() 35 | // end of setup for clound env ------------------------------------------------------ 36 | 37 | val systemName = Try(conf.getString("akka.system-name")).getOrElse("ShardingActorCountScalabilitySystem") 38 | implicit val system = ActorSystem(systemName, conf) 39 | 40 | // management ----------- 41 | val cluster = Cluster(system) 42 | ClusterHttpManagement(cluster).start() 43 | // end of management ---- 44 | 45 | val region = ActorCountingEntity.startRegion(system) 46 | 47 | val benchCoordinatorProps = 48 | ClusterSingletonManager.props(ActorCountingBenchmarkMaster.props(region), PoisonPill, 49 | ClusterSingletonManagerSettings(system)) 50 | system.actorOf(benchCoordinatorProps, "bench-coordinator") 51 | 52 | } 53 | --------------------------------------------------------------------------------