├── cassandra ├── address.yaml ├── agent-run ├── cassandra-rackdc.properties ├── cassandra-run ├── Dockerfile └── cassandra.yaml ├── opscenter ├── cluster.conf └── Dockerfile ├── docker-compose.yml └── README.md /cassandra/address.yaml: -------------------------------------------------------------------------------- 1 | stomp_interface: cluster_nodeops_1.opscenter_compose.dev.docker -------------------------------------------------------------------------------- /cassandra/agent-run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd /opt/agent 4 | bin/datastax-agent -f 5 | -------------------------------------------------------------------------------- /cassandra/cassandra-rackdc.properties: -------------------------------------------------------------------------------- 1 | # indicate the rack and dc for this node 2 | dc={DC} 3 | rack=RAC1 -------------------------------------------------------------------------------- /opscenter/cluster.conf: -------------------------------------------------------------------------------- 1 | [cassandra] 2 | seed_hosts: cluster_seed1_1.cassandra_compose.dev.docker,cluster_seed2_1.cassandra_compose.dev.docker -------------------------------------------------------------------------------- /cassandra/cassandra-run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # set up datacenter name 4 | cat /modsrc/cassandra-rackdc.properties | sed -e s/{DC}/$DATACENTER/g > /opt/cassandra/conf/cassandra-rackdc.properties 5 | 6 | # Start server 7 | cd /opt/cassandra 8 | bin/cassandra -f -------------------------------------------------------------------------------- /opscenter/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Based of abh1nav/cassandra with modifications to work in docker-compose 3 | # 4 | FROM abh1nav/opscenter 5 | 6 | ADD . /modsrc 7 | 8 | RUN \ 9 | mkdir -p /opt/opscenter/conf/cluster; \ 10 | cp /modsrc/cluster.conf /opt/opscenter/conf/cluster/; 11 | 12 | -------------------------------------------------------------------------------- /cassandra/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Based of abh1nav/cassandra with modifications to work in docker-compose 3 | # 4 | FROM abh1nav/cassandra 5 | 6 | ADD . /modsrc 7 | 8 | RUN \ 9 | cp /modsrc/cassandra.yaml /opt/cassandra/conf/; \ 10 | cp /modsrc/address.yaml /opt/agent/conf/; \ 11 | cp /modsrc/agent-run /etc/service/agent/run; \ 12 | cp /modsrc/cassandra-run /etc/service/cassandra/run; 13 | 14 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | #SkyDNS and SkyDock to provide DNS among docker containers 2 | skydns: 3 | image: crosbymichael/skydns 4 | ports: 5 | - "172.17.42.1:53:53/udp" 6 | hostname: skydns 7 | command: "-nameserver 8.8.8.8:53 -domain docker" 8 | skydock: 9 | image: crosbymichael/skydock 10 | links: 11 | - skydns 12 | volumes: 13 | - /var/run/docker.sock:/docker.sock 14 | command: "-ttl=30 -environment=dev -s=/docker.sock -domain=docker -name=cluster_skydns_1" 15 | # 16 | # Datacenter 1 seed node 17 | # 18 | seed1: 19 | image: jdgoldie/cassandra_compose 20 | environment: 21 | - DATACENTER=DC1 22 | ports: 23 | - "9042:9042" 24 | - "9160:9160" 25 | # 26 | # Datacenter 1 non-seed node 27 | # 28 | nodedc1: 29 | image: jdgoldie/cassandra_compose 30 | environment: 31 | - DATACENTER=DC1 32 | # 33 | # Datacenter 2 seed node 34 | # 35 | seed2: 36 | image: jdgoldie/cassandra_compose 37 | environment: 38 | - DATACENTER=DC2 39 | ports: 40 | - "9043:9042" 41 | - "9161:9160" 42 | # 43 | # Datacenter 2 non-seed node 44 | # 45 | nodedc2: 46 | image: jdgoldie/cassandra_compose 47 | environment: 48 | - DATACENTER=DC2 49 | # 50 | # Ops Center Node 51 | # 52 | nodeops: 53 | image: jdgoldie/opscenter_compose 54 | ports: 55 | - "8888:8888" 56 | - "61620:61620" -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # A tiny multi-dc Cassandra cluster with Docker-compose 2 | 3 | :tada: Yes, another Docker-Cassandra cluster... 4 | 5 | ### The Docker Image 6 | 7 | The docker image is based off [abh1nav/cassandra](https://registry.hub.docker.com/u/abh1nav/cassandra/) 8 | with changes to support multiple data centers. 9 | 10 | Ops Center is currently disabled, but I hope to have time to get that working soon. 11 | 12 | ### The Cluster 13 | 14 | Most of the Docker-Casssandra clusters are built with shell scripts that use a combination of 15 | IP-discovery of running containers and environment variables to link containers together. 16 | 17 | I decided to try using [SkyDock](https://github.com/crosbymichael/skydock) and [SkyDNS](https://github.com/skynetservices/skydns1) 18 | to handle service discovery. 19 | 20 | SkyDock registers the IP for each docker container with SkyDNS using a simple, generated name. 21 | 22 | In this cluster, I have the domain configured to be `dev.docker`. 23 | 24 | ### Docker-compose 25 | 26 | [Docker-compose](https://docs.docker.com/compose/) is a nice way to orchestrate several containers 27 | from a single configuration file. Starting this cluster is as simple as 28 | 29 | ``` 30 | docker-compose -p cluster up -d 31 | ``` 32 | 33 | The `-p cluster` speficies the cluster name. This is important since compose will take the directory name by default. 34 | To make the DNS names of the containers perdictable, we specify the name here. This command should start the following: 35 | 36 | ``` 37 | cluster_seed2_1 38 | cluster_nodedc2_1 39 | cluster_nodedc1_1 40 | cluster_seed1_1 41 | cluster_skydns_1 42 | cluster_skydock_1 43 | ``` 44 | 45 | This means that the assigned name for the `seed1` service is `cluster_seed1_1.cassandra_compose.dev.docker`. If you 46 | check the `cassandra.yaml` file, you will see it listed as one of the seed nodes. 47 | 48 | Run 49 | ``` 50 | docker run -t --rm jdgoldie/cassandra_compose watch -n 3 bin/nodetool -f cluster_seed1_1.cassandra_compose.dev.docker status 51 | ``` 52 | to see the status of the cluster. Notice that, thanks to SkyDNS, the hostname resolves to the `seed1` container. 53 | 54 | Now add some nodes. 55 | 56 | ``` 57 | docker-compose -p cluster scale nodedc1=2 nodedc2=2 58 | ``` 59 | 60 | Check `nodetool` again and you will see another node join each datacenter. 61 | 62 | ### Future 63 | 64 | * Write a full blog entry to expand on `docker-compose.yml` 65 | * Get Ops Center working :+1: 66 | * Use labels for datacenters instead of using environment variables. 67 | * Sample application that writes to/from the cluster to demonstrate seamless failover. 68 | 69 | :octocat: 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /cassandra/cassandra.yaml: -------------------------------------------------------------------------------- 1 | cluster_name: 'cluster' 2 | num_tokens: 256 3 | hinted_handoff_enabled: true 4 | max_hint_window_in_ms: 10800000 # 3 hours 5 | hinted_handoff_throttle_in_kb: 1024 6 | max_hints_delivery_threads: 2 7 | batchlog_replay_throttle_in_kb: 1024 8 | authenticator: AllowAllAuthenticator 9 | authorizer: AllowAllAuthorizer 10 | permissions_validity_in_ms: 2000 11 | partitioner: org.apache.cassandra.dht.Murmur3Partitioner 12 | data_file_directories: 13 | - /opt/cassandra/data 14 | commitlog_directory: /opt/cassandra/commitlog 15 | disk_failure_policy: stop 16 | commit_failure_policy: stop 17 | key_cache_size_in_mb: 18 | key_cache_save_period: 14400 19 | row_cache_size_in_mb: 0 20 | row_cache_save_period: 0 21 | counter_cache_size_in_mb: 22 | counter_cache_save_period: 7200 23 | commitlog_sync: periodic 24 | commitlog_sync_period_in_ms: 10000 25 | commitlog_segment_size_in_mb: 32 26 | concurrent_reads: 32 27 | concurrent_writes: 32 28 | concurrent_counter_writes: 32 29 | memtable_allocation_type: heap_buffers 30 | index_summary_capacity_in_mb: 31 | index_summary_resize_interval_in_minutes: 60 32 | trickle_fsync: false 33 | trickle_fsync_interval_in_kb: 10240 34 | storage_port: 7000 35 | ssl_storage_port: 7001 36 | start_native_transport: true 37 | native_transport_port: 9042 38 | start_rpc: true 39 | rpc_port: 9160 40 | rpc_keepalive: true 41 | rpc_server_type: sync 42 | thrift_framed_transport_size_in_mb: 15 43 | incremental_backups: false 44 | snapshot_before_compaction: false 45 | auto_snapshot: true 46 | tombstone_warn_threshold: 1000 47 | tombstone_failure_threshold: 100000 48 | column_index_size_in_kb: 64 49 | batch_size_warn_threshold_in_kb: 5 50 | compaction_throughput_mb_per_sec: 16 51 | sstable_preemptive_open_interval_in_mb: 50 52 | read_request_timeout_in_ms: 5000 53 | range_request_timeout_in_ms: 10000 54 | write_request_timeout_in_ms: 2000 55 | counter_write_request_timeout_in_ms: 5000 56 | cas_contention_timeout_in_ms: 1000 57 | truncate_request_timeout_in_ms: 60000 58 | request_timeout_in_ms: 10000 59 | cross_node_timeout: false 60 | dynamic_snitch_update_interval_in_ms: 100 61 | dynamic_snitch_reset_interval_in_ms: 600000 62 | dynamic_snitch_badness_threshold: 0.1 63 | request_scheduler: org.apache.cassandra.scheduler.NoScheduler 64 | server_encryption_options: 65 | internode_encryption: none 66 | keystore: conf/.keystore 67 | keystore_password: cassandra 68 | truststore: conf/.truststore 69 | truststore_password: cassandra 70 | client_encryption_options: 71 | enabled: false 72 | keystore: conf/.keystore 73 | keystore_password: cassandra 74 | internode_compression: all 75 | inter_dc_tcp_nodelay: false 76 | seed_provider: 77 | - class_name: org.apache.cassandra.locator.SimpleSeedProvider 78 | parameters: 79 | - seeds: "cluster_seed1_1.cassandra_compose.dev.docker,cluster_seed2_1.cassandra_compose.dev.docker" 80 | endpoint_snitch: GossipingPropertyFileSnitch 81 | --------------------------------------------------------------------------------