├── OSSMETADATA ├── conf ├── recon_iv.pem ├── recon_key.pem ├── dynomite.yml ├── mc_single.yml ├── redis_single.yml ├── dynomite_dns_single.yml ├── dynomite_florida_single.yml ├── redis_node1.yml ├── redis_node2.yml ├── redis_dc1.yml ├── redis_dc2.yml ├── redis_rack1_node.yml ├── redis_rack2_node.yml ├── redis_rack3_node.yml ├── a_dc1.yml ├── a_dc2_rack1_node1.yml ├── a_dc2_rack2_node1.yml ├── a_dc2_rack2_node2.yml ├── a_dc2_rack1_node2.yml └── dynomite.pem ├── .travis.yml ├── init ├── systemd_environment__dynomite ├── systemd_service_rhel__dynomite.service ├── systemd_service_ubuntu__dynomite.service ├── README.md └── upstart_ubuntu__dynomite ├── m4 └── .gitignore ├── contrib ├── Makefile.am ├── yaml-0.1.4 │ └── .gitignore ├── yaml-0.1.4.tar.gz ├── murmur3 │ ├── makefile │ ├── murmur3.h │ └── README.md ├── fmemopen.h └── fmemopen.c ├── notes ├── kqueue.pdf └── memcache.txt ├── scripts ├── dynomite-manager │ ├── kill_redis.sh │ ├── launch_nfredis.sh │ ├── bash-alias │ ├── cassandra │ └── dynomite-manager ├── Florida │ ├── seeds.list │ ├── package.json │ └── florida.js ├── runall.sh ├── memcache │ └── populate_memcached.sh ├── dynomite │ ├── multi_get.sh │ ├── pipelined_read.sh │ ├── pipelined_write.sh │ ├── generate_yamls.py │ ├── dyn_redis_purge.py │ └── dyn_mc_test.py └── redis │ ├── redis-check.py │ └── dyno_redis_bgrewriteaof.sh ├── src ├── dyn_dict_msg_id.h ├── proto │ ├── stOHIQrl │ ├── Makefile.am │ └── dyn_proto.h ├── dyn_types.h ├── dyn_dnode_client.h ├── seedsprovider │ ├── Makefile.am │ ├── dyn_seeds_provider.h │ └── dyn_dns.c ├── dyn_node_snitch.h ├── dyn_dnode_proxy.h ├── event │ ├── Makefile.am │ └── dyn_event.h ├── entropy │ ├── Makefile.am │ └── dyn_entropy.h ├── hashkit │ ├── Makefile.am │ ├── dyn_murmur3.c │ ├── dyn_one_at_a_time.c │ ├── dyn_fnv.c │ ├── dyn_hsieh.c │ ├── dyn_murmur.c │ ├── dyn_crc16.c │ ├── dyn_hashkit.h │ ├── dyn_vnode.c │ └── dyn_random.c ├── dyn_setting.h ├── dyn_asciilogo.h ├── tools │ ├── Makefile.am │ └── dyn_hash_tool.c ├── dyn_client.h ├── dyn_histogram.h ├── dyn_dict_msg_id.c ├── dyn_setting.c ├── dyn_proxy.h ├── dyn_token.h ├── dyn_signal.h ├── dyn_dnode_peer.h ├── dyn_response_mgr.h ├── dyn_crypto.h ├── dyn_rbtree.h ├── dyn_ring_queue.h ├── dyn_array.h ├── dyn_gossip.h ├── dyn_dnode_msg.h ├── dyn_server.h ├── dyn_mbuf.h ├── dyn_node_snitch.c ├── dyn_signal.c ├── dyn_string.c ├── dyn_string.h ├── dyn_ring_queue.c ├── dyn_array.c ├── dyn_log.h └── dyn_cbuf.h ├── images ├── dynomite-logo.png └── dynomite-emblem.png ├── bin ├── kill_dynomite.sh ├── core_affinity.sh └── launch_dynomite.sh ├── test ├── load.py ├── dyno_cluster.py ├── utils.py ├── redis_node.py ├── node.py ├── dyno_node.py ├── dual_run.py └── func_test.py ├── Makefile.am ├── docker ├── HOWTO.md └── Dockerfile ├── .gitignore ├── docs ├── dyn_protocol.txt └── florida.md ├── CONTRIBUTING.md ├── travis.sh └── man └── dynomite.8 /OSSMETADATA: -------------------------------------------------------------------------------- 1 | osslifecycle=active 2 | -------------------------------------------------------------------------------- /conf/recon_iv.pem: -------------------------------------------------------------------------------- 1 | 01234567890123456 2 | -------------------------------------------------------------------------------- /conf/recon_key.pem: -------------------------------------------------------------------------------- 1 | 01234567890123456 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: c 2 | script: bash ./travis.sh 3 | -------------------------------------------------------------------------------- /init/systemd_environment__dynomite: -------------------------------------------------------------------------------- 1 | DYNOMITE_CONF=/etc/dynomite/dynomite.yml 2 | -------------------------------------------------------------------------------- /m4/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything 2 | * 3 | 4 | # Except me 5 | !.gitignore 6 | -------------------------------------------------------------------------------- /contrib/Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = yaml-0.1.4 2 | 3 | EXTRA_DIST = yaml-0.1.4.tar.gz 4 | -------------------------------------------------------------------------------- /notes/kqueue.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DynomiteDB/dynomite/HEAD/notes/kqueue.pdf -------------------------------------------------------------------------------- /scripts/dynomite-manager/kill_redis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sudo service redis_6379 stop 3 | 4 | -------------------------------------------------------------------------------- /src/dyn_dict_msg_id.h: -------------------------------------------------------------------------------- 1 | #include "dyn_dict.h" 2 | 3 | extern dictType msg_table_dict_type; 4 | -------------------------------------------------------------------------------- /src/proto/stOHIQrl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DynomiteDB/dynomite/HEAD/src/proto/stOHIQrl -------------------------------------------------------------------------------- /contrib/yaml-0.1.4/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything 2 | * 3 | 4 | # Except me 5 | !.gitignore 6 | -------------------------------------------------------------------------------- /scripts/dynomite-manager/launch_nfredis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sudo service redis_6379 start 3 | 4 | -------------------------------------------------------------------------------- /images/dynomite-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DynomiteDB/dynomite/HEAD/images/dynomite-logo.png -------------------------------------------------------------------------------- /contrib/yaml-0.1.4.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DynomiteDB/dynomite/HEAD/contrib/yaml-0.1.4.tar.gz -------------------------------------------------------------------------------- /images/dynomite-emblem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DynomiteDB/dynomite/HEAD/images/dynomite-emblem.png -------------------------------------------------------------------------------- /contrib/murmur3/makefile: -------------------------------------------------------------------------------- 1 | CFLAGS = -O3 -Wall 2 | 3 | all: example 4 | example: murmur3.o 5 | 6 | clean: 7 | rm -rf *.o 8 | -------------------------------------------------------------------------------- /scripts/Florida/seeds.list: -------------------------------------------------------------------------------- 1 | 192.168.6.41:8101:rc1:dc1:2147483647 2 | 192.168.6.42:8101:rc2:dc1:0 3 | 192.168.6.43:8101:rc2:dc1:2147483647 4 | -------------------------------------------------------------------------------- /conf/dynomite.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | listen: 127.0.0.1:8102 3 | dyn_listen: 127.0.0.1:8101 4 | tokens: '101134286' 5 | servers: 6 | - 127.0.0.1:22122:1 7 | data_store: 0 8 | 9 | -------------------------------------------------------------------------------- /bin/kill_dynomite.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cmd="pkill -f /apps/dynomite/bin/dynomite" 4 | if [ $USER != "root" ];then 5 | exec $cmd 6 | else 7 | su $userowner -c "$cmd" 8 | fi 9 | 10 | -------------------------------------------------------------------------------- /conf/mc_single.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | dyn_listen: 127.0.0.1:8101 3 | listen: 127.0.0.1:8102 4 | servers: 5 | - 127.0.0.1:22122:1 6 | tokens: 437425602 7 | data_store: 1 8 | stats_listen: 0.0.0.0:22222 9 | -------------------------------------------------------------------------------- /test/load.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | ## 4 | # A very simple basic program that will load data in a dynomite/redis server. 5 | ## 6 | import argparse 7 | import redis 8 | 9 | if __name__ == '__main__': 10 | 11 | -------------------------------------------------------------------------------- /conf/redis_single.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | dyn_listen: 127.0.0.1:8101 3 | data_store: 0 4 | listen: 127.0.0.1:8102 5 | dyn_seed_provider: simple_provider 6 | servers: 7 | - 127.0.0.1:22122:1 8 | tokens: 437425602 9 | stats_listen: 0.0.0.0:22222 10 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in aclocal.m4 configure config.h.in config.h.in~ stamp-h.in 2 | 3 | ACLOCAL_AMFLAGS = -I m4 4 | 5 | SUBDIRS = contrib src 6 | 7 | dist_man_MANS = man/dynomite.8 8 | 9 | EXTRA_DIST = README.md NOTICE LICENSE conf scripts notes 10 | -------------------------------------------------------------------------------- /scripts/dynomite-manager/bash-alias: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | alias cls=clear 4 | alias dlog='tail -f -n 2000 /logs/system/dynomite/dynomite.log' 5 | alias dmlog='tail -f -n 2000 /logs/system/dynomite-manager/dynomite-manager.log' 6 | alias rlog='tail -f -n 2000 /var/log/redis_6379.log' 7 | 8 | -------------------------------------------------------------------------------- /conf/dynomite_dns_single.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc 3 | rack: rack1 4 | dyn_listen: 0.0.0.0:8101 5 | dyn_seed_provider: dns_provider 6 | listen: 0.0.0.0:8102 7 | servers: 8 | - 127.0.0.1:6379:1 9 | tokens: '0' 10 | data_store: 0 11 | stats_listen: 0.0.0.0:22222 12 | -------------------------------------------------------------------------------- /src/dyn_types.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | typedef uint64_t msgid_t; 4 | typedef uint64_t msec_t; 5 | typedef uint64_t usec_t; 6 | typedef uint64_t sec_t; 7 | 8 | typedef enum { 9 | SECURE_OPTION_NONE, 10 | SECURE_OPTION_RACK, 11 | SECURE_OPTION_DC, 12 | SECURE_OPTION_ALL, 13 | }secure_server_option_t; 14 | -------------------------------------------------------------------------------- /test/dyno_cluster.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import redis 3 | import random 4 | #from dyno_node import DynoNode 5 | 6 | class DynoCluster(object): 7 | def __init__(self, nodes): 8 | self.nodes = nodes 9 | def get_connection(self): 10 | node = random.choice(self.nodes) 11 | return node.get_connection() 12 | -------------------------------------------------------------------------------- /test/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import random 3 | import string 4 | 5 | def string_generator(size=6, chars=string.letters + string.digits): 6 | return ''.join(random.choice(chars) for _ in range(size)) 7 | 8 | def number_generator(size=4, chars=string.digits): 9 | return ''.join(random.choice(chars) for _ in range(size)) 10 | -------------------------------------------------------------------------------- /src/dyn_dnode_client.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed storages. 3 | * Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | #include "dyn_core.h" 7 | 8 | 9 | #ifndef _DYN_DNODE_CLIENT_H_ 10 | #define _DYN_DNODE_CLIENT_H_ 11 | 12 | void init_dnode_client_conn(struct conn *conn); 13 | 14 | #endif 15 | -------------------------------------------------------------------------------- /src/proto/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | 3 | AM_CPPFLAGS = -I $(top_srcdir)/src 4 | 5 | AM_CFLAGS = -Wall -Wshadow 6 | AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value 7 | 8 | noinst_LIBRARIES = libproto.a 9 | 10 | noinst_HEADERS = dyn_proto.h 11 | 12 | libproto_a_SOURCES = \ 13 | dyn_memcache.c \ 14 | dyn_redis.c 15 | -------------------------------------------------------------------------------- /conf/dynomite_florida_single.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc 3 | rack: rack1 4 | dyn_listen: 0.0.0.0:8101 5 | dyn_seed_provider: florida_provider 6 | listen: 0.0.0.0:8102 7 | servers: 8 | - 127.0.0.1:6379:1 9 | tokens: '0' 10 | secure_server_option: datacenter 11 | pem_key_file: conf/dynomite.pem 12 | data_store: 0 13 | stats_listen: 0.0.0.0:22222 14 | -------------------------------------------------------------------------------- /scripts/runall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | N=$2 4 | O=$1 5 | 6 | if [ -z "$N" ]; then 7 | N=1 8 | fi 9 | 10 | if [ -z "$O" ]; then 11 | O=read 12 | fi 13 | 14 | echo "N=$N" 15 | echo "O=$O" 16 | 17 | for i in $( cat listnodes ); do 18 | echo 'processing: ' $i 19 | python dynomite/dyn_redis_test.py -P 22122 -H $i -o $O -n $N 20 | sleep 1 21 | done 22 | -------------------------------------------------------------------------------- /test/redis_node.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import redis 3 | from node import Node 4 | 5 | class RedisNode(Node): 6 | 7 | def __init__(self, host, ip, port): 8 | super(RedisNode, self).__init__(host, ip, port) 9 | self.name = "Redis" + self.name 10 | 11 | def get_connection(self): 12 | return redis.StrictRedis(self.host, self.port, db=0) 13 | -------------------------------------------------------------------------------- /src/seedsprovider/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | 3 | AM_CPPFLAGS = -I $(top_srcdir)/src 4 | 5 | AM_CFLAGS = -Wall -Wshadow 6 | AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value 7 | 8 | noinst_LIBRARIES = libseedsprovider.a 9 | 10 | noinst_HEADERS = dyn_seeds_provider.h 11 | 12 | libseedsprovider_a_SOURCES = \ 13 | dyn_florida.c \ 14 | dyn_dns.c 15 | -------------------------------------------------------------------------------- /test/node.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | class Node(object): 4 | def __init__(self, host="localhost", ip="127.0.0.1", port=1212): 5 | self.host=host 6 | self.port=port 7 | self.name="Node: %s:%d" % (host, port) 8 | def __name__(self): 9 | return self.name 10 | def start(self): 11 | return 12 | def stop(self): 13 | return 14 | -------------------------------------------------------------------------------- /conf/redis_node1.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc 3 | rack: rack 4 | dyn_listen: 127.0.0.1:8101 5 | dyn_seeds: 6 | - 127.0.0.2:8101:rack:dc:1383429731 7 | listen: 127.0.0.1:8102 8 | servers: 9 | - 127.0.0.1:22122:1 10 | tokens: '12345678' 11 | secure_server_option: datacenter 12 | pem_key_file: conf/dynomite.pem 13 | data_store: 0 14 | stats_listen: 0.0.0.0:22222 15 | -------------------------------------------------------------------------------- /conf/redis_node2.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc 3 | rack: rack 4 | dyn_listen: 127.0.0.2:8101 5 | dyn_seeds: 6 | - 127.0.0.1:8101:rack:dc:12345678 7 | listen: 127.0.0.2:8102 8 | servers: 9 | - 127.0.0.1:22123:1 10 | tokens: '1383429731' 11 | secure_server_option: datacenter 12 | pem_key_file: conf/dynomite.pem 13 | data_store: 0 14 | stats_listen: 0.0.0.0:22222 15 | -------------------------------------------------------------------------------- /conf/redis_dc1.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc1 3 | rack: rack1 4 | dyn_listen: 127.0.0.1:8101 5 | dyn_seeds: 6 | - 127.0.0.2:8101:rack2:dc2:1383429731 7 | listen: 127.0.0.1:8102 8 | servers: 9 | - 127.0.0.1:22122:1 10 | tokens: '12345678' 11 | secure_server_option: datacenter 12 | pem_key_file: conf/dynomite.pem 13 | data_store: 0 14 | stats_listen: 0.0.0.0:22222 15 | -------------------------------------------------------------------------------- /conf/redis_dc2.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc2 3 | rack: rack2 4 | dyn_listen: 127.0.0.2:8101 5 | dyn_seeds: 6 | - 127.0.0.1:8101:rack1:dc1:12345678 7 | listen: 127.0.0.2:8102 8 | servers: 9 | - 127.0.0.1:22123:1 10 | tokens: '1383429731' 11 | secure_server_option: datacenter 12 | pem_key_file: conf/dynomite.pem 13 | data_store: 0 14 | stats_listen: 0.0.0.0:22222 15 | -------------------------------------------------------------------------------- /src/dyn_node_snitch.h: -------------------------------------------------------------------------------- 1 | 2 | #include "dyn_core.h" 3 | 4 | 5 | #ifndef _DYN_SNITCH_H_ 6 | #define _DYN_SNITCH_H_ 7 | 8 | 9 | char *get_broadcast_address(struct server_pool *sp); 10 | char *get_public_hostname(struct server_pool *sp); 11 | char *get_public_ip4(struct server_pool *sp); 12 | char *get_private_ip4(struct server_pool *sp); 13 | char *hostname_to_private_ip4(char *hostname); 14 | 15 | #endif /* _DYN_SNITCH_H_s */ 16 | -------------------------------------------------------------------------------- /src/seedsprovider/dyn_seeds_provider.h: -------------------------------------------------------------------------------- 1 | 2 | #include "dyn_core.h" 3 | 4 | 5 | #ifndef _DYN_SEEDS_PROVIDER_H_ 6 | #define _DYN_SEEDS_PROVIDER_H_ 7 | 8 | 9 | #define SEEDS_CHECK_INTERVAL (30 * 1000) /* in msec */ 10 | 11 | 12 | uint8_t florida_get_seeds(struct context * ctx, struct mbuf *seeds_buf); 13 | uint8_t dns_get_seeds(struct context * ctx, struct mbuf *seeds_buf); 14 | 15 | 16 | #endif /* DYN_SEEDS_PROVIDER_H_ */ 17 | -------------------------------------------------------------------------------- /src/dyn_dnode_proxy.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed storages. 3 | * Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | 7 | #include "dyn_core.h" 8 | 9 | #ifndef _DYN_DNODE_SERVER_H_ 10 | #define _DYN_DNODE_SERVER_H_ 11 | 12 | rstatus_t dnode_init(struct context *ctx); 13 | void dnode_deinit(struct context *ctx); 14 | void init_dnode_proxy_conn(struct conn *conn); 15 | #endif 16 | 17 | -------------------------------------------------------------------------------- /conf/redis_rack1_node.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc 3 | rack: rack1 4 | dyn_listen: 127.0.0.1:8101 5 | dyn_seeds: 6 | - 127.0.0.2:8101:rack2:dc:1383429731 7 | - 127.0.0.3:8101:rack3:dc:1383429731 8 | listen: 127.0.0.1:8102 9 | servers: 10 | - 127.0.0.1:22121:1 11 | tokens: '1383429731' 12 | secure_server_option: datacenter 13 | pem_key_file: conf/dynomite.pem 14 | data_store: 0 15 | read_consistency : DC_ONE 16 | write_consistency : DC_ONE 17 | stats_listen: 0.0.0.0:22222 18 | -------------------------------------------------------------------------------- /conf/redis_rack2_node.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc 3 | rack: rack2 4 | dyn_listen: 127.0.0.2:8101 5 | dyn_seeds: 6 | - 127.0.0.1:8101:rack1:dc:1383429731 7 | - 127.0.0.3:8101:rack3:dc:1383429731 8 | listen: 127.0.0.2:8102 9 | servers: 10 | - 127.0.0.1:22122:1 11 | tokens: '1383429731' 12 | secure_server_option: datacenter 13 | pem_key_file: conf/dynomite.pem 14 | data_store: 0 15 | read_consistency : DC_ONE 16 | write_consistency : DC_ONE 17 | stats_listen: 0.0.0.0:22222 18 | -------------------------------------------------------------------------------- /conf/redis_rack3_node.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc 3 | rack: rack3 4 | dyn_listen: 127.0.0.3:8101 5 | dyn_seeds: 6 | - 127.0.0.1:8101:rack1:dc:1383429731 7 | - 127.0.0.2:8101:rack2:dc:1383429731 8 | listen: 127.0.0.3:8102 9 | servers: 10 | - 127.0.0.1:22123:1 11 | tokens: '1383429731' 12 | secure_server_option: datacenter 13 | pem_key_file: conf/dynomite.pem 14 | data_store: 0 15 | read_consistency : DC_ONE 16 | write_consistency : DC_ONE 17 | stats_listen: 0.0.0.0:22222 18 | -------------------------------------------------------------------------------- /conf/a_dc1.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc1 3 | rack: rack1 4 | dyn_listen: 127.0.0.1:8101 5 | dyn_seeds: 6 | - 127.0.0.2:8101:rack2:dc2:1383429731 7 | - 127.0.0.3:8101:rack2:dc2:12345678 8 | - 127.0.0.4:8101:rack1:dc2:1383429731 9 | - 127.0.0.5:8101:rack1:dc2:12345678 10 | listen: 127.0.0.1:8102 11 | servers: 12 | - 127.0.0.1:22121:1 13 | tokens: '1383429731' 14 | secure_server_option: datacenter 15 | pem_key_file: conf/dynomite.pem 16 | data_store: 0 17 | stats_listen: 0.0.0.0:22221 18 | -------------------------------------------------------------------------------- /conf/a_dc2_rack1_node1.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc2 3 | rack: rack1 4 | dyn_listen: 127.0.0.2:8101 5 | dyn_seeds: 6 | - 127.0.0.1:8101:rack1:dc1:1383429731 7 | - 127.0.0.4:8101:rack2:dc2:1383429731 8 | - 127.0.0.5:8101:rack2:dc2:12345678 9 | - 127.0.0.3:8101:rack1:dc2:12345678 10 | listen: 127.0.0.2:8102 11 | servers: 12 | - 127.0.0.1:22122:1 13 | tokens: '1383429731' 14 | secure_server_option: datacenter 15 | pem_key_file: conf/dynomite.pem 16 | data_store: 0 17 | stats_listen: 0.0.0.0:22222 18 | 19 | -------------------------------------------------------------------------------- /conf/a_dc2_rack2_node1.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc2 3 | rack: rack2 4 | dyn_listen: 127.0.0.4:8101 5 | dyn_seeds: 6 | - 127.0.0.1:8101:rack1:dc1:1383429731 7 | - 127.0.0.5:8101:rack2:dc2:12345678 8 | - 127.0.0.2:8101:rack1:dc2:1383429731 9 | - 127.0.0.3:8101:rack1:dc2:12345678 10 | listen: 127.0.0.4:8102 11 | servers: 12 | - 127.0.0.1:22124:1 13 | tokens: '1383429731' 14 | secure_server_option: datacenter 15 | pem_key_file: conf/dynomite.pem 16 | data_store: 0 17 | stats_listen: 0.0.0.0:22224 18 | 19 | -------------------------------------------------------------------------------- /conf/a_dc2_rack2_node2.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc2 3 | rack: rack2 4 | dyn_listen: 127.0.0.5:8101 5 | dyn_seeds: 6 | - 127.0.0.1:8101:rack1:dc1:1383429731 7 | - 127.0.0.4:8101:rack2:dc2:1383429731 8 | - 127.0.0.2:8101:rack1:dc2:1383429731 9 | - 127.0.0.3:8101:rack1:dc2:12345678 10 | listen: 127.0.0.5:8102 11 | servers: 12 | - 127.0.0.1:22125:1 13 | tokens: '12345678' 14 | secure_server_option: datacenter 15 | pem_key_file: conf/dynomite.pem 16 | data_store: 0 17 | stats_listen: 0.0.0.0:22225 18 | 19 | -------------------------------------------------------------------------------- /conf/a_dc2_rack1_node2.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc2 3 | rack: rack1 4 | dyn_listen: 127.0.0.3:8101 5 | dyn_seeds: 6 | - 127.0.0.1:8101:rack1:dc1:1383429731 7 | - 127.0.0.4:8101:rack2:dc2:1383429731 8 | - 127.0.0.5:8101:rack2:dc2:12345678 9 | - 127.0.0.2:8101:rack1:dc2:1383429731 10 | listen: 127.0.0.3:8102 11 | servers: 12 | - 127.0.0.1:22123:1 13 | tokens: '12345678' 14 | secure_server_option: datacenter 15 | pem_key_file: conf/dynomite.pem 16 | data_store: 0 17 | stats_listen: 0.0.0.0:22223 18 | 19 | 20 | -------------------------------------------------------------------------------- /docker/HOWTO.md: -------------------------------------------------------------------------------- 1 | # Build Image 2 | 3 | Create Dynomite plus Redis single server image with docker 4 | 5 | # Example: sudo docker build -t [name] . 6 | $ sudo docker build -t my_dynomite . 7 | 8 | # Running A Dynomite Instance 9 | 10 | Creating a container running Dynomite and Redis instance inside. To set the name, you can use the -name [name]. If a name is not set, an alphanumeric ID will be obtained. 11 | 12 | $ sudo docker run -name my_dynomite_instance -i -i my_dynomite 13 | 14 | To list all containers 15 | 16 | $ sudo docker ps -l 17 | 18 | Enjoy! 19 | 20 | -------------------------------------------------------------------------------- /scripts/memcache/populate_memcached.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | port=22123 4 | socatopt="-t 1 -T 1 -b 65537" 5 | 6 | val=`echo 6^6^6 | bc` 7 | val=`printf "%s\r\n" "${val}"` 8 | vallen=`printf "%s" "${val}" | wc -c` 9 | set_command="" 10 | 11 | # build 12 | for i in `seq 1 512`; do 13 | if [ `expr $i % 2` -eq "0" ]; then 14 | key="foo" 15 | else 16 | key="bar" 17 | fi 18 | key=`printf "%s%d" "${key}" "${i}"` 19 | 20 | set_command="set ${key} 0 0 ${vallen}\r\n${val}\r\n" 21 | 22 | printf "%b" "$set_command" | socat ${socatopt} - TCP:localhost:${port},nodelay,shut-down,nonblock=1 & 23 | done 24 | 25 | -------------------------------------------------------------------------------- /scripts/dynomite/multi_get.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | port=8102 4 | socatopt="-t 20 -T 20 -b 8193 -d -d " 5 | key="" 6 | keys="" 7 | get_command="" 8 | 9 | # build 10 | for i in `seq 1 512`; do 11 | if [ `expr $i % 2` -eq "0" ]; then 12 | key="foo" 13 | else 14 | key="bar" 15 | fi 16 | key=`printf "%s%d" "${key}" "${i}"` 17 | keys=`printf "%s %s" "${keys}" "${key}"` 18 | done 19 | 20 | get_command="get ${keys}\r\n" 21 | printf "%b" "$get_command" 22 | 23 | # read 24 | for i in `seq 1 16`; do 25 | printf "%b" "${get_command}" | socat ${socatopt} - TCP:localhost:${port},nodelay,shut-none,nonblock=1 1> /dev/null 2>&1 & 26 | done 27 | -------------------------------------------------------------------------------- /src/event/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | 3 | AM_CPPFLAGS = -I $(top_srcdir)/src 4 | 5 | AM_CFLAGS = -Wall -Wshadow 6 | AM_CFLAGS += -Wpointer-arith 7 | AM_CFLAGS += -Winline 8 | AM_CFLAGS += -Wunused-function -Wunused-variable -Wunused-value 9 | AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value 10 | AM_CFLAGS += -Wconversion -Wsign-compare 11 | AM_CFLAGS += -Wstrict-prototypes -Wmissing-prototypes -Wredundant-decls -Wmissing-declarations -Werror=implicit-function-declaration 12 | 13 | noinst_LIBRARIES = libevent.a 14 | 15 | noinst_HEADERS = dyn_event.h 16 | 17 | libevent_a_SOURCES = \ 18 | dyn_epoll.c \ 19 | dyn_kqueue.c \ 20 | dyn_evport.c 21 | 22 | -------------------------------------------------------------------------------- /src/entropy/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | 3 | AM_CPPFLAGS = -I $(top_srcdir)/src 4 | 5 | AM_CFLAGS = -Wall -Wshadow 6 | AM_CFLAGS += -Wpointer-arith 7 | AM_CFLAGS += -Winline 8 | AM_CFLAGS += -Wunused-function -Wunused-variable -Wunused-value 9 | AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value 10 | AM_CFLAGS += -Wconversion -Wsign-compare 11 | AM_CFLAGS += -Wstrict-prototypes -Wmissing-prototypes -Wredundant-decls -Wmissing-declarations -Wimplicit-function-declaration 12 | 13 | 14 | noinst_LIBRARIES = libentropy.a 15 | 16 | noinst_HEADERS = dyn_entropy.h 17 | 18 | libentropy_a_SOURCES = \ 19 | dyn_entropy_rcv.c \ 20 | dyn_entropy_snd.c \ 21 | dyn_entropy_util.c 22 | -------------------------------------------------------------------------------- /init/systemd_service_rhel__dynomite.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Dynomite server 3 | ; Services that should be started before Dynomite 4 | After=network.target nss-lookup.target time-sync.target 5 | Documentation=man:dynomite(8) 6 | 7 | [Service] 8 | User=dynomite 9 | ; Default environment variable values 10 | Environment=DYNOMITE_CONF=/etc/dynomite/dynomite.yaml 11 | Type=forking 12 | ; Environment vars to customize startup 13 | EnvironmentFile=-/etc/sysconfig/dynomite 14 | ExecStart=/usr/local/sbin/dynomite -d -c $DYNOMITE_CONF -p /var/run/dynomite/dynomite.pid 15 | ; Restart dynomite every 100ms 16 | Restart=always 17 | PIDFile=/var/run/dynomite/dynomite.pid 18 | 19 | [Install] 20 | WantedBy=multi-user.target 21 | -------------------------------------------------------------------------------- /init/systemd_service_ubuntu__dynomite.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Dynomite server 3 | ; Services that should be started before Dynomite 4 | After=network.target nss-lookup.target time-sync.target 5 | Documentation=man:dynomite(8) 6 | 7 | [Service] 8 | User=dynomite 9 | ; Default environment variable values 10 | Environment=DYNOMITE_CONF=/etc/dynomite/dynomite.yaml 11 | Type=forking 12 | ; Environment vars to customize startup 13 | EnvironmentFile=-/etc/default/dynomite 14 | ExecStart=/usr/local/sbin/dynomite -d -c $DYNOMITE_CONF -p /var/run/dynomite/dynomite.pid 15 | ; Restart dynomite every 100ms 16 | Restart=always 17 | PIDFile=/var/run/dynomite/dynomite.pid 18 | 19 | [Install] 20 | WantedBy=multi-user.target 21 | -------------------------------------------------------------------------------- /scripts/redis/redis-check.py: -------------------------------------------------------------------------------- 1 | import redis 2 | 3 | range=100 4 | factor=32 5 | port=22121 6 | 7 | r = redis.StrictRedis(host='localhost', port=port, db=0) 8 | 9 | # lrange 10 | print [r.lrange('lfoo', 0, x) for x in xrange(1, range)] 11 | print [r.lpush('lfoo', str(x)*factor) for x in xrange(1, range)] 12 | print [r.lrange('lfoo', 0, x) for x in xrange(1, range)] 13 | print r.delete('lfoo') 14 | 15 | # del 16 | print [r.set('foo' + str(x), str(x)*factor) for x in xrange(1, range)] 17 | keys = ['foo' + str(x) for x in xrange(1, range)] 18 | print [r.delete(keys) for x in xrange(1, range)] 19 | 20 | # mget 21 | print [r.set('foo' + str(x), str(x)*100) for x in xrange(1, range)] 22 | keys = ['foo' + str(x) for x in xrange(1, range)] 23 | print [r.mget(keys) for x in xrange(1, range)] 24 | -------------------------------------------------------------------------------- /scripts/dynomite/pipelined_read.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | #socatopt="-t 4 -T 4 -b 8193 -d -d " 4 | socatopt="-t 8 -T 8 -b 16384 -d -d" 5 | 6 | get_commands="" 7 | 8 | # build 9 | for i in `seq 1 64`; do 10 | if [ `expr $i % 2` -eq "0" ]; then 11 | key="foo" 12 | else 13 | key="bar" 14 | fi 15 | key=`printf "%s%d" "${key}" "${i}"` 16 | keylen=`printf "%s" "${key}" | wc -c` 17 | 18 | get_command="*2\r\n\$3\r\nget\r\n\$${keylen}\r\n${key}\r\n" 19 | get_commands=`printf "%s%s" "${get_commands}" "${get_command}"` 20 | done 21 | 22 | printf "%b" "$get_commands" > /tmp/socat.input 23 | # read 24 | for i in `seq 1 64`; do 25 | cat /tmp/socat.input | socat ${socatopt} - TCP:localhost:8102,nodelay,shut-down,nonblock=1 & #1 > /dev/null 2>&1 & 26 | done 27 | wait 28 | -------------------------------------------------------------------------------- /test/dyno_node.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import redis 3 | from node import Node 4 | from redis_node import RedisNode 5 | from dyno_cluster import DynoCluster 6 | 7 | class DynoNode(Node): 8 | def __init__(self, host="localhost", ip="127.0.0.1", port=8102, 9 | dnode_port=8101, data_store_port=22122): 10 | super(DynoNode, self).__init__(host, ip, port) 11 | self.name="Dyno" + self.name 12 | self.conf_file = None 13 | self.dnode_port = dnode_port 14 | self.data_store_node = RedisNode(host, ip, data_store_port) 15 | 16 | def get_connection(self): 17 | # should return the connection to the dyno port not the redis 18 | print "returning connection at %s:%d" % (self.host, self.port) 19 | return redis.StrictRedis(self.host, self.port, db=0) 20 | -------------------------------------------------------------------------------- /contrib/murmur3/murmur3.h: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // MurmurHash3 was written by Austin Appleby, and is placed in the 3 | // public domain. The author hereby disclaims copyright to this source 4 | // code. 5 | 6 | #ifndef _MURMURHASH3_H_ 7 | #define _MURMURHASH3_H_ 8 | 9 | #include 10 | 11 | //----------------------------------------------------------------------------- 12 | 13 | void MurmurHash3_x86_32 (const void *key, int len, uint32_t seed, void *out); 14 | 15 | void MurmurHash3_x86_128(const void *key, int len, uint32_t seed, void *out); 16 | 17 | void MurmurHash3_x64_128(const void *key, int len, uint32_t seed, void *out); 18 | 19 | //----------------------------------------------------------------------------- 20 | 21 | #endif // _MURMURHASH3_H_ 22 | -------------------------------------------------------------------------------- /scripts/dynomite/pipelined_write.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | socatopt="-t 1 -T 1 -b 16384" 4 | 5 | val=`echo 5^5^5 | bc` 6 | val=`printf "%s" "${val}"` 7 | vallen=`printf "%s" "${val}" | wc -c` 8 | set_command="" 9 | set_commands="" 10 | 11 | # build 12 | for i in `seq 1 64`; do 13 | if [ `expr $i % 2` -eq "0" ]; then 14 | key="foo" 15 | else 16 | key="bar" 17 | fi 18 | key=`printf "%s%d" "${key}" "${i}"` 19 | keylen=`printf "%s" "${key}" | wc -c` 20 | 21 | set_command="*3\r\n\$3\r\nset\r\n\$${keylen}\r\n${key}\r\n\$${vallen}\r\n${val}\r\n" 22 | set_commands=`printf "%s%s" "${set_commands}" "${set_command}"` 23 | done 24 | 25 | printf "%b" "$set_commands" > /tmp/socat.input 26 | 27 | # write 28 | for i in `seq 1 16`; do 29 | cat /tmp/socat.input | socat ${socatopt} - TCP:localhost:8102,nodelay,shut-down,nonblock=1 & 30 | done 31 | wait 32 | -------------------------------------------------------------------------------- /scripts/dynomite-manager/cassandra: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # chkconfig: 2345 95 20 3 | # description: This script does some stuff 4 | # processname: java 5 | 6 | start() { 7 | echo "Starting cassandra..." 8 | export JAVA_HOME=/home/ec2-user/jdk1.8.0_45 9 | export JRE_HOME=/home/ec2-user/jdk1.8.0_45/jre 10 | export PATH=$PATH:/home/ec2-user/jdk1.8.0_45/bin:/home/ec2-user/jdk1.8.0_45/jre/bin 11 | 12 | cd /home/ec2-user/apache-cassandra-2.1.14 13 | bin/cassandra start & 14 | } 15 | 16 | stop() { 17 | echo "stop" 18 | PID=`ps aux | grep cassandra | grep -v grep | awk '{print $2}'` 19 | if [[ "" != "$PID" ]]; then 20 | echo "killing $PID" 21 | sudo kill -9 $PID 22 | fi 23 | } 24 | 25 | case "$1" in start) 26 | start 27 | ;; 28 | stop) 29 | stop 30 | ;; 31 | *) 32 | 33 | echo $"Usage: $0 {start|stop}" 34 | RETVAL=1 35 | esac 36 | exit 0 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /src/hashkit/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | 3 | AM_CPPFLAGS = -I $(top_srcdir)/src -I $(top_srcdir)/contrib/murmur3 4 | 5 | AM_CFLAGS = -Wall -Wshadow 6 | AM_CFLAGS += -Wpointer-arith 7 | AM_CFLAGS += -Winline 8 | AM_CFLAGS += -Wunused-function -Wunused-variable -Wunused-value 9 | AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value 10 | AM_CFLAGS += -Wconversion -Wsign-compare 11 | AM_CFLAGS += -Wstrict-prototypes -Wmissing-prototypes -Wredundant-decls -Wmissing-declarations -Werror=implicit-function-declaration 12 | 13 | noinst_LIBRARIES = libhashkit.a 14 | 15 | noinst_HEADERS = dyn_hashkit.h 16 | 17 | libhashkit_a_SOURCES = \ 18 | dyn_crc16.c \ 19 | dyn_crc32.c \ 20 | dyn_fnv.c \ 21 | dyn_hsieh.c \ 22 | dyn_jenkins.c \ 23 | dyn_ketama.c \ 24 | dyn_md5.c \ 25 | dyn_modula.c \ 26 | dyn_murmur.c \ 27 | dyn_one_at_a_time.c \ 28 | dyn_random.c \ 29 | dyn_vnode.c \ 30 | dyn_murmur3.c 31 | -------------------------------------------------------------------------------- /src/dyn_setting.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed storages. 3 | * Copyright (C) 2014 Netflix, Inc. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | 19 | #ifndef _DYN_SETTING_H_ 20 | #define _DYN_SETTING_H_ 21 | 22 | 23 | uint32_t msgs_per_sec(void); 24 | void set_msgs_per_sec(uint32_t tokens_per_sec); 25 | 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /conf/dynomite.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIICWwIBAAKBgQCjijNAnd1/uFlrCtfO83Qo5idWbqLpyfQdKHd1CRE9AliA8pyM 3 | u9XqdyEbMJ229PPPWhlKui0moVcj1J8w/ZBzLFEQtqf+93Ytv9YR4tktyv0kWTnf 4 | yUzp9D74WHL2/lwfi0Nv+jSIWg5aQRqMzRsdErTwk+iaSuuZ/+3Jbp9JxwIDAQAB 5 | AoGAd3s70MTFlE+SfMMyQo/Z2Ru1t88jFV2oDTmIdShHwOQa1zLpr2R4eFHLDTtq 6 | rsWt71srSU5WOHf92z63g6ptwpxixb+HWIF49Ke0vtMUs+ah/E0jV9EyAwFmdACh 7 | Yxa/dwCTkl4O3q8/Lm1gXExpeif9PUtenIFU8Va2l1s87WECQQDV33ZLWatbAUQz 8 | qLXsRIOiSDAhTXyuAFX2I9HyKfAGxaOz7U8QYZYJVf/vmubWERHZhe8TurusYuEZ 9 | onh4zPuRAkEAw8CxzxKjGZB75Wz2+k9Qbkb6+HS4VBCVPHScwE2+kXnI28l+ATro 10 | MDE0F7VfYA2Cv8xDeBBFhR/rki/5MmhT1wJAMWa2il0iCZlXSZ90tih1R2zpQSgG 11 | 3qJRONO5UPiC6u2IDK2KD7yazXlB7vSW0WeL+fTH7oS5iO4mFKfEsBKZIQJAAZth 12 | WkvS3vDas4VzeZaRGHaotMVwhfAk6XwjlUiOGe+gAH3k9Omg4lRlExTAFa/GtQBe 13 | RWwMxgz/CBIwv4MvvQJAZxZuxzN7D5VhJlkL1IEXnPHgE+RgR0vTAQaqnSa9PNcv 14 | gAiXFRKkMHeAkHUIsZ382C5zzjUGQrCD8IrvpAwXBg== 15 | -----END RSA PRIVATE KEY----- 16 | -------------------------------------------------------------------------------- /bin/core_affinity.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Set core affinity for redis and dynomite processes 3 | # 4 | 5 | # Requires setting the EC2 Instance type as ENV variable. 6 | # If Dynomite is used outside of AWS environment, 7 | # the core affinity script can be configured accordingly. 8 | echo "$EC2_INSTANCE_TYPE" 9 | 10 | if [ "$EC2_INSTANCE_TYPE" == "r3.xlarge" ]; then 11 | dynomite_pid=`pgrep -f $DYN_DIR/bin/dynomite` 12 | echo "dynomite pid: $dynomite_pid" 13 | taskset -pac 2,3 $dynomite_pid 14 | 15 | redis_pid=`ps -ef | grep 22122 | grep redis | awk -F' ' '{print $2}'` 16 | echo "redis pid: $redis_pid" 17 | taskset -pac 1 $redis_pid 18 | 19 | else 20 | 21 | dynomite_pid=`pgrep -f $DYN_DIR/bin/dynomite` 22 | echo "dynomite pid: $dynomite_pid" 23 | taskset -pac 2,5,6 $dynomite_pid 24 | 25 | redis_pid=`ps -ef | grep 22122 | grep redis | awk -F' ' '{print $2}'` 26 | echo "redis pid: $redis_pid" 27 | taskset -pac 3,7 $redis_pid 28 | 29 | fi 30 | 31 | -------------------------------------------------------------------------------- /src/dyn_asciilogo.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed storages. 3 | * Copyright (C) 2015 Netflix, Inc. 4 | * 5 | * author: Ioannis Papapanagiotou 6 | */ 7 | 8 | 9 | char *ascii_logo = 10 | " \n" 11 | " # \ m \n" 12 | " mmm# m m mmmm mmm mmmmm mmm mm#mm mmm \n" 13 | " # # \\m m/ # # # # # # # # # # # \n" 14 | " # # #m# # # # # # # # # # #'''' \n" 15 | " \\#m## \\# # # #m# # # # mm#mm mm #mm \n" 16 | " m/ \n" 17 | " ## \n" 18 | " \n\n"; 19 | 20 | -------------------------------------------------------------------------------- /src/tools/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | AUTOMAKE_OPTIONS = subdir-objects 3 | 4 | AM_CPPFLAGS = 5 | if !OS_SOLARIS 6 | AM_CPPFLAGS += -D_GNU_SOURCE 7 | endif 8 | AM_CPPFLAGS += -I $(top_srcdir)/src 9 | AM_CPPFLAGS += -I $(top_srcdir)/src/hashkit 10 | 11 | AM_CFLAGS = 12 | AM_CFLAGS += -Wall -Wshadow 13 | AM_CFLAGS += -Wpointer-arith 14 | AM_CFLAGS += -Winline 15 | AM_CFLAGS += -Wunused-function -Wunused-variable -Wunused-value 16 | AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value 17 | AM_CFLAGS += -Wconversion -Wsign-compare 18 | AM_CFLAGS += -Wstrict-prototypes -Wmissing-prototypes -Wredundant-decls -Wmissing-declarations 19 | 20 | AM_LDFLAGS = 21 | 22 | if OS_SOLARIS 23 | AM_LDFLAGS += -lnsl -lsocket 24 | endif 25 | 26 | bin_PROGRAMS = dynomite-hash-tool 27 | 28 | dynomite_hash_tool_SOURCES = \ 29 | dyn_hash_tool.c \ 30 | ../dyn_token.c \ 31 | ../dyn_log.c \ 32 | ../dyn_util.c \ 33 | ../dyn_array.c 34 | 35 | dynomite_hash_tool_LDADD = $(top_builddir)/src/hashkit/libhashkit.a 36 | -------------------------------------------------------------------------------- /src/dyn_client.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed storages. 3 | * Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | /* 7 | * twemproxy - A fast and lightweight proxy for memcached protocol. 8 | * Copyright (C) 2011 Twitter, Inc. 9 | * 10 | * Licensed under the Apache License, Version 2.0 (the "License"); 11 | * you may not use this file except in compliance with the License. 12 | * You may obtain a copy of the License at 13 | * 14 | * http://www.apache.org/licenses/LICENSE-2.0 15 | * 16 | * Unless required by applicable law or agreed to in writing, software 17 | * distributed under the License is distributed on an "AS IS" BASIS, 18 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | * See the License for the specific language governing permissions and 20 | * limitations under the License. 21 | */ 22 | 23 | #include "dyn_core.h" 24 | 25 | #ifndef _DYN_CLIENT_H_ 26 | #define _DYN_CLIENT_H_ 27 | 28 | void init_client_conn(struct conn *conn); 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /src/dyn_histogram.h: -------------------------------------------------------------------------------- 1 | /* 2 | * dyn_histogram.h 3 | * 4 | * Created on: Feb 6, 2015 5 | * Author: mdo 6 | */ 7 | 8 | #ifndef DYN_HISTOGRAM_H_ 9 | #define DYN_HISTOGRAM_H_ 10 | 11 | 12 | #define BUCKET_SIZE 94 13 | 14 | 15 | struct histogram { 16 | uint64_t buckets[BUCKET_SIZE]; 17 | uint64_t mean; 18 | uint64_t val_95th; 19 | uint64_t val_99th; 20 | uint64_t val_999th; 21 | uint64_t val_max; 22 | }; 23 | 24 | 25 | rstatus_t histo_init(volatile struct histogram *histo); 26 | rstatus_t histo_reset(volatile struct histogram *histo); 27 | void histo_add(volatile struct histogram *histo, uint64_t val); 28 | uint64_t histo_get_bucket(volatile struct histogram *histo, int bucket); 29 | void histo_get_buckets(volatile struct histogram *histo, uint64_t* my_buckets); 30 | uint64_t histo_percentile(volatile struct histogram *histo, double percentile); 31 | uint64_t histo_mean(volatile struct histogram *histo); 32 | uint64_t histo_max(volatile struct histogram *histo); 33 | void histo_compute(volatile struct histogram *histo); 34 | 35 | 36 | #endif /* DYN_HISTOGRAM_H_ */ 37 | -------------------------------------------------------------------------------- /src/dyn_dict_msg_id.c: -------------------------------------------------------------------------------- 1 | #ifdef __APPLE__ 2 | // http://clc-wiki.net/wiki/C_standard_library:string.h:NULL 3 | // Portable C90 version: NULL ((void*)0) 4 | # ifndef NULL 5 | # define NULL 0 6 | # endif 7 | #else 8 | # include // For NULL 9 | #endif 10 | #include "dyn_types.h" 11 | #include "dyn_dict_msg_id.h" 12 | 13 | static unsigned int 14 | dict_msg_id_hash(const void *key) 15 | { 16 | msgid_t id = *(msgid_t*)key; 17 | return dictGenHashFunction(key, sizeof(id)); 18 | } 19 | 20 | static int 21 | dict_msg_id_cmp(void *privdata, const void *key1, const void *key2) 22 | { 23 | msgid_t id1 = *(msgid_t*)key1; 24 | msgid_t id2 = *(msgid_t*)key2; 25 | return id1 == id2; 26 | } 27 | 28 | dictType msg_table_dict_type = { 29 | dict_msg_id_hash, /* hash function */ 30 | NULL, /* key dup */ 31 | NULL, /* val dup */ 32 | dict_msg_id_cmp, /* key compare */ 33 | NULL, /* key destructor */ 34 | NULL /* val destructor */ 35 | }; 36 | 37 | 38 | -------------------------------------------------------------------------------- /scripts/Florida/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "florida.js", 3 | "version": "1.0.0", 4 | "description": "Service that returns list of seeds.", 5 | "keywords": [ 6 | "Dynomite", 7 | "Florida", 8 | "Netflix", 9 | "Redis", 10 | "Memcached", 11 | "Dynamo" 12 | ], 13 | "main": "florida.js", 14 | "scripts": { 15 | "test": "echo \"Error: no test specified\" && exit 1", 16 | "debug": "node florida.js ./seeds.list debug" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "git+https://github.com/Netflix/dynomite.git" 21 | }, 22 | "contributors": [ 23 | { 24 | "name": "Diego Pacheco", 25 | "email": "diego.pacheco.it@gmail.com", 26 | "url": "https://github.com/diegopacheco" 27 | }, 28 | { 29 | "name": "Akbar S. Ahmed", 30 | "email": "akbar501@gmail.com", 31 | "url": "https://grockdoc.com/@akbar501" 32 | } 33 | ], 34 | "license": "Apache-2.0", 35 | "bugs": { 36 | "url": "https://github.com/Netflix/dynomite/issues" 37 | }, 38 | "homepage": "https://github.com/Netflix/dynomite#readme" 39 | } 40 | -------------------------------------------------------------------------------- /src/dyn_setting.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed storages. 3 | * Copyright (C) 2014 Netflix, Inc. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | #include "dyn_core.h" 19 | #include "dyn_conf.h" 20 | 21 | static uint32_t conn_msg_rate = CONF_DEFAULT_CONN_MSG_RATE; //conn msgs per sec 22 | 23 | 24 | uint32_t msgs_per_sec(void) 25 | { 26 | return conn_msg_rate; 27 | } 28 | 29 | void set_msgs_per_sec(uint32_t tokens_per_sec) 30 | { 31 | conn_msg_rate = tokens_per_sec; 32 | } 33 | -------------------------------------------------------------------------------- /src/dyn_proxy.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed storages. 3 | * Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | /* 7 | * twemproxy - A fast and lightweight proxy for memcached protocol. 8 | * Copyright (C) 2011 Twitter, Inc. 9 | * 10 | * Licensed under the Apache License, Version 2.0 (the "License"); 11 | * you may not use this file except in compliance with the License. 12 | * You may obtain a copy of the License at 13 | * 14 | * http://www.apache.org/licenses/LICENSE-2.0 15 | * 16 | * Unless required by applicable law or agreed to in writing, software 17 | * distributed under the License is distributed on an "AS IS" BASIS, 18 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | * See the License for the specific language governing permissions and 20 | * limitations under the License. 21 | */ 22 | #include "dyn_core.h" 23 | 24 | 25 | #ifndef _DYN_PROXY_H_ 26 | #define _DYN_PROXY_H_ 27 | 28 | rstatus_t proxy_init(struct context *ctx); 29 | void proxy_deinit(struct context *ctx); 30 | void init_proxy_conn(struct conn *conn); 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /src/dyn_token.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed storages. 3 | * Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | 7 | #include "dyn_core.h" 8 | 9 | #ifndef _DYN_TOKEN_H_ 10 | #define _DYN_TOKEN_H_ 11 | 12 | 13 | struct dyn_token { 14 | uint32_t signum; 15 | uint32_t *mag; 16 | uint32_t len; 17 | }; 18 | 19 | 20 | void init_dyn_token(struct dyn_token *token); 21 | void deinit_dyn_token(struct dyn_token *token); 22 | rstatus_t size_dyn_token(struct dyn_token *token, uint32_t size); 23 | rstatus_t copy_dyn_token(const struct dyn_token * src, struct dyn_token * dst); 24 | 25 | /** 26 | * convenience function for setting a token whose value is just an int 27 | */ 28 | void set_int_dyn_token(struct dyn_token *token, uint32_t val); 29 | 30 | rstatus_t parse_dyn_token(uint8_t *start, uint32_t len, struct dyn_token *token); 31 | int32_t cmp_dyn_token(struct dyn_token *t1, struct dyn_token *t2); 32 | rstatus_t derive_tokens(struct array *tokens, uint8_t *start, uint8_t *end); 33 | rstatus_t derive_token(struct dyn_token *token, uint8_t *start, uint8_t *end); 34 | void print_dyn_token(struct dyn_token *token, int num_tabs); 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /src/dyn_signal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed storages. 3 | * Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | /* 7 | * twemproxy - A fast and lightweight proxy for memcached protocol. 8 | * Copyright (C) 2011 Twitter, Inc. 9 | * 10 | * Licensed under the Apache License, Version 2.0 (the "License"); 11 | * you may not use this file except in compliance with the License. 12 | * You may obtain a copy of the License at 13 | * 14 | * http://www.apache.org/licenses/LICENSE-2.0 15 | * 16 | * Unless required by applicable law or agreed to in writing, software 17 | * distributed under the License is distributed on an "AS IS" BASIS, 18 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | * See the License for the specific language governing permissions and 20 | * limitations under the License. 21 | */ 22 | 23 | 24 | #include "dyn_core.h" 25 | 26 | #ifndef _DYN_SIGNAL_H_ 27 | #define _DYN_SIGNAL_H_ 28 | 29 | 30 | /** 31 | * @brief POSIX signal 32 | */ 33 | struct signal { 34 | int signo; 35 | char *signame; 36 | int flags; 37 | void (*handler)(int signo); 38 | }; 39 | 40 | rstatus_t signal_init(void); 41 | void signal_deinit(void); 42 | void signal_handler(int signo); 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /scripts/Florida/florida.js: -------------------------------------------------------------------------------- 1 | var http = require('http'); 2 | var url = require('url'); 3 | var fs = require('fs'); 4 | 5 | // Settings 6 | var port = process.env.DYNOMITE_FLORIDA_PORT ? 7 | process.env.DYNOMITE_FLORIDA_PORT : 8080; 8 | 9 | var apiUrl = process.env.DYNOMITE_FLORIDA_REQUEST ? 10 | process.env.DYNOMITE_FLORIDA_REQUEST : '/REST/v1/admin/get_seeds'; 11 | 12 | // Parse command line options 13 | var seedsFilePath = process.argv[2] && process.argv[2].length > 0 ? 14 | process.argv[2] : '/etc/dynomite/seeds.list'; 15 | var enableDebug = process.argv[3] === 'debug' ? true : false; 16 | 17 | http.createServer(function(req, res) { 18 | var path = url.parse(req.url).pathname; 19 | enableDebug && console.log('Request: ' + path); 20 | 21 | res.writeHead(200, {'Content-Type': 'application/json'}); 22 | if (path === apiUrl) { 23 | fs.readFile(seedsFilePath, 'utf-8', function(err, data) { 24 | if (err) console.log(err); 25 | 26 | var now = (new Date()).toJSON(); 27 | var seeds = data.trim().replace(/\n/g, '|'); 28 | 29 | enableDebug && console.log(now + ' - get_seeds [' + seeds + ']'); 30 | res.write(seeds); 31 | res.end(); 32 | }); 33 | } else { 34 | res.end(); 35 | } 36 | }).listen(port); 37 | 38 | console.log('Server is listening on ' + port); 39 | -------------------------------------------------------------------------------- /src/hashkit/dyn_murmur3.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed storages. 3 | * Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | /* 7 | * twemproxy - A fast and lightweight proxy for memcached protocol. 8 | * Copyright (C) 2011 Twitter, Inc. 9 | * 10 | * Licensed under the Apache License, Version 2.0 (the "License"); 11 | * you may not use this file except in compliance with the License. 12 | * You may obtain a copy of the License at 13 | * 14 | * http://www.apache.org/licenses/LICENSE-2.0 15 | * 16 | * Unless required by applicable law or agreed to in writing, software 17 | * distributed under the License is distributed on an "AS IS" BASIS, 18 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | * See the License for the specific language governing permissions and 20 | * limitations under the License. 21 | */ 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | #define MURMUR3_SEED 0xc0a1e5ce 28 | 29 | rstatus_t 30 | hash_murmur3(const char *key, size_t length, struct dyn_token *token) 31 | { 32 | rstatus_t status = size_dyn_token(token, 4); 33 | if (status != DN_OK) { 34 | return status; 35 | } 36 | 37 | // MurmurHash3_x86_128(key, length, MURMUR3_SEED, token->mag); 38 | 39 | return DN_OK; 40 | } 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IDE 2 | .idea 3 | CMakeLists.txt 4 | 5 | # Compiled Object files 6 | *.lo 7 | *.o 8 | 9 | # Compiled Dynamic libraries 10 | *.so 11 | 12 | # Compiled Static libraries 13 | *.la 14 | *.a 15 | 16 | # Compiled misc 17 | *.dep 18 | *.gcda 19 | *.gcno 20 | *.gcov 21 | *.pyc 22 | 23 | # Packages 24 | *.tar.gz 25 | *.tar.bz2 26 | 27 | # Logs 28 | *.log 29 | 30 | # Temporary 31 | *.swp 32 | *.~ 33 | *~ 34 | *.project 35 | *.cproject 36 | nbproject/* 37 | TAGS 38 | TODO 39 | 40 | # Core and executable 41 | core* 42 | nutcracker 43 | dynomite 44 | dynomite-test 45 | dyno-hash-tool 46 | 47 | # extracted yaml 48 | !/contrib/yaml-0.1.4.tar.gz 49 | 50 | # Autotools 51 | .deps 52 | .libs 53 | 54 | /aclocal.m4 55 | /autom4te.cache 56 | /stamp-h1 57 | /autoscan.log 58 | /libtool 59 | 60 | /config/config.guess 61 | /config/config.sub 62 | /config/depcomp 63 | /config/install-sh 64 | /config/ltmain.sh 65 | /config/missing 66 | /config 67 | 68 | /config.h 69 | /config.h.in 70 | /config.h.in~ 71 | /config.log 72 | /config.status 73 | /configure.scan 74 | /configure 75 | 76 | Makefile 77 | Makefile.in 78 | /Debug 79 | /src/.dirstamp 80 | /src/test 81 | /src/tools 82 | /src/dynomite-test 83 | cscope.out 84 | /src/cscope.out 85 | /src/.dirstamp 86 | /src/tools/dyn_hash_tool 87 | /src/tools/dynomite-hash-tool 88 | /test/*.pyc 89 | /test/_binaries/* 90 | /test/_binaries 91 | /test/conf 92 | -------------------------------------------------------------------------------- /src/dyn_dnode_peer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed storages. 3 | * Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | #include "dyn_core.h" 6 | #include "dyn_server.h" 7 | 8 | 9 | #ifndef _DYN_DNODE_PEER_H_ 10 | #define _DYN_DNODE_PEER_H_ 11 | 12 | #define MAX_WAIT_BEFORE_RECONNECT_IN_SECS 10 13 | #define WAIT_BEFORE_UPDATE_PEERS_IN_MILLIS 30000 14 | 15 | msec_t dnode_peer_timeout(struct msg *msg, struct conn *conn); 16 | rstatus_t dnode_peer_init(struct context *ctx); 17 | void dnode_peer_deinit(struct array *nodes); 18 | void dnode_peer_connected(struct context *ctx, struct conn *conn); 19 | 20 | struct node *dnode_peer_pool_server(struct context *ctx, struct server_pool *pool, struct rack *rack, uint8_t *key, uint32_t keylen, uint8_t msg_type); 21 | struct conn *dnode_peer_pool_server_conn(struct context *ctx, struct node *server); 22 | rstatus_t dnode_peer_pool_preconnect(struct context *ctx); 23 | void dnode_peer_pool_disconnect(struct context *ctx); 24 | rstatus_t dnode_peer_forward_state(void *rmsg); 25 | rstatus_t dnode_peer_add(void *rmsg); 26 | rstatus_t dnode_peer_replace(void *rmsg); 27 | rstatus_t dnode_peer_handshake_announcing(void *rmsg); 28 | bool is_same_dc(struct server_pool *sp, struct node *peer_node); 29 | 30 | void init_dnode_peer_conn(struct conn *conn); 31 | void preselect_remote_rack_for_replication(struct context *ctx); 32 | #endif 33 | -------------------------------------------------------------------------------- /src/dyn_response_mgr.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef _DYN_RESPONSE_MGR_H_ 3 | #define _DYN_RESPONSE_MGR_H_ 4 | #define MAX_REPLICAS_PER_DC 3 5 | struct response_mgr { 6 | bool is_read; 7 | bool done; 8 | /* we could use the dynamic array 9 | here. But we have only 3 ASGs */ 10 | struct msg *responses[MAX_REPLICAS_PER_DC]; 11 | uint32_t checksums[MAX_REPLICAS_PER_DC]; 12 | uint8_t good_responses; // non-error responses received. (nil) is not an error 13 | uint8_t max_responses; // max responses expected. 14 | uint8_t quorum_responses; // responses expected to form a quorum 15 | uint8_t error_responses; // error responses received 16 | struct msg *err_rsp; // first error response 17 | struct conn *conn; 18 | struct msg *msg; 19 | }; 20 | 21 | void init_response_mgr(struct response_mgr *rspmgr, struct msg*, bool is_read, 22 | uint8_t max_responses, struct conn *conn); 23 | // DN_OK if response was accepted 24 | rstatus_t rspmgr_submit_response(struct response_mgr *rspmgr, struct msg *rsp); 25 | bool rspmgr_check_is_done(struct response_mgr *rspmgr); 26 | struct msg* rspmgr_get_response(struct response_mgr *rspmgr); 27 | void rspmgr_free_response(struct response_mgr *rspmgr, struct msg *dont_free); 28 | void rspmgr_free_other_responses(struct response_mgr *rspmgr, struct msg *dont_free); 29 | 30 | 31 | #endif 32 | -------------------------------------------------------------------------------- /scripts/dynomite-manager/dynomite-manager: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # chkconfig: 2345 95 20 3 | # description: This script does some stuff 4 | # processname: java 5 | 6 | export JAVA_HOME=/home/ec2-user/jdk1.8.0_45 7 | export JRE_HOME=/home/ec2-user/jdk1.8.0_45/jre 8 | export PATH=$PATH:/home/ec2-user/jdk1.8.0_45/bin:/home/ec2-user/jdk1.8.0_45/jre/bin 9 | 10 | export ASG_NAME="asg_dynomite" 11 | export EC2_REGION="us-west-2" 12 | export AUTO_SCALE_GROUP="asg_dynomite" 13 | 14 | start() { 15 | echo "Starting Dynomite Manager..." 16 | cd /home/ec2-user/dynomite-manager/dynomite-manager/ 17 | /home/ec2-user/dynomite-manager/dynomite-manager/gradlew jettyRun > /logs/system/dynomite-manager/dynomite-manager.log & 18 | } 19 | 20 | stop() { 21 | echo "stoping Dynomite Manager... " 22 | PID=`ps -ef | grep gradlew | awk '{print $2}' ORS=' ' | awk '{print $1}'` 23 | if [[ "" != "$PID" ]]; then 24 | echo "killing $PID" 25 | sudo kill -9 $PID 26 | fi 27 | } 28 | 29 | debug() { 30 | echo "Starting Dynomite Manager for DEBUG..." 31 | cd /home/ec2-user/dynomite-manager/dynomite-manager/ 32 | export GRADLE_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=n" 33 | ./gradlew jettyRun & 34 | } 35 | 36 | 37 | case "$1" in 38 | "start") 39 | start 40 | ;; 41 | "debug") 42 | debug 43 | ;; 44 | "stop") 45 | stop 46 | ;; 47 | *) 48 | 49 | echo $"Usage: $0 {start|stop|debug}" 50 | RETVAL=1 51 | esac 52 | exit 0 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /init/README.md: -------------------------------------------------------------------------------- 1 | # README 2 | 3 | The `init` directory is the location for all System V init, Upstart, systemd, etc. initialization scripts. 4 | 5 | ## systemd on RHEL 7 6 | 7 | Create the `dynomite` user. 8 | 9 | ```bash 10 | mkdir -p /usr/share/dynomite 11 | 12 | mkdir /var/run/dynomite 13 | 14 | useradd -r -M -c "Dynomite server" -s /sbin/nologin -d /usr/share/dynomite dynomite 15 | 16 | chown -R dynomite:dynomite /usr/share/dynomite 17 | 18 | chown -R dynomite:dynomite /var/run/dynomite 19 | ``` 20 | 21 | Install the Dynomite service file and the associated sysconfig file. 22 | 23 | ```bash 24 | cp init/systemd_environment__dynomite /etc/sysconfig/dynomite 25 | 26 | cp init/systemd_service_rhel__dynomite.service /usr/lib/systemd/system/dynomite.service 27 | 28 | systemctl daemon-reload 29 | 30 | systemctl enable dynomite 31 | 32 | systemctl status dynomite 33 | ``` 34 | 35 | ## systemd on Ubuntu 15.10 36 | 37 | Create the `dynomite` user. 38 | 39 | ```bash 40 | mkdir -p /usr/share/dynomite 41 | 42 | mkdir /var/run/dynomite 43 | 44 | useradd -r -M -c "Dynomite server" -s /sbin/nologin -d /usr/share/dynomite dynomite 45 | 46 | chown -R dynomite:dynomite /usr/share/dynomite 47 | 48 | chown -R dynomite:dynomite /var/run/dynomite 49 | ``` 50 | 51 | Install the Dynomite service file and the associated sysconfig file. 52 | 53 | ```bash 54 | cp init/systemd_environment__dynomite /etc/default/dynomite 55 | 56 | cp init/systemd_service_ubuntu__dynomite.service /lib/systemd/system/dynomite.service 57 | 58 | systemctl daemon-reload 59 | 60 | systemctl enable dynomite 61 | 62 | systemctl status dynomite 63 | ``` 64 | -------------------------------------------------------------------------------- /docs/dyn_protocol.txt: -------------------------------------------------------------------------------- 1 | Dyn message Protocol: 2 | 3 | I. Specification: 4 | 5 | 1. Structure format 6 | 7 | 8 | 9 | 10 | 11 | 12 | * 13 | 14 | * 15 | 16 | 17 | 2. Field descriptions 18 | a. MagicNumber: it is always 2014 19 | b. MessageId: sequence id of a message 20 | c. TypeId: please see the code for the list. 21 | d. BitField: indicate whether compression and/or encryption. Right most bit indicates encryption. 22 | The 2nd bit from the right indicates a compressed message. 23 | e. Version: version of the msg. This usually depends on the version of the dynomite. 24 | This is needed during upgrading a live cluster with mixed versions. 25 | f. Same-dc: bit to indicate whether the communication is inter-dcs. 26 | We should later to exchange node's metadata like rack, dc, and ip. 27 | 28 | 3. Example: 29 | This is a dynomite message that carries a Redis' "get foo" request: 30 | 31 | "2014 1344 5 1 1 0\r\n*4 minh\r\n*2\r\n$3\r\nget\r\n$3\r\nfoo\r\n" 32 | 33 | where "*2\r\n$3\r\nget\r\n$3\r\nfoo\r\n" is Redis' payload for "get foo" cmd. 34 | 35 | 36 | II. Bit field for encryption/decryption 37 | 38 | 39 | III. Compression 40 | We currently don't support compression but it should be similar to the way how 41 | encryption/decryption works. We will support/implement this feature in the near future 42 | if there is a demand for it. 43 | 44 | IV. Payload size 45 | Will fill in soon 46 | 47 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Dynomite 2 | 3 | Documentation for Dynomite is available in the following locations: 4 | - Dynomite docs 5 | - [wiki]( https://github.com/Netflix/dynomite/wiki ) 6 | 7 | The Dynomite team is following the Gitflow workflow. The active development branch is [dev](https://github.com/Netflix/dynomite/tree/dev), the stable branch is [master](https://github.com/Netflix/dynomite/tree/master). 8 | 9 | Contributions will be accepted to the [dev](https://github.com/Netflix/dynomite/tree/dev) only. 10 | 11 | 12 | ## How to provide a patch for a new feature 13 | 14 | 1. If it is a major feature, please create an [Issue]( https://github.com/Netflix/dynomite/issues ) and discuss with the project leaders. 15 | 16 | 2. If in step 1 you get an acknowledge from the project leaders, use the 17 | following procedure to submit a patch: 18 | 19 | a. Fork Dynomite on github ( http://help.github.com/fork-a-repo/ ) 20 | 21 | b. Create a topic branch (git checkout -b my_branch) 22 | 23 | c. Push to your branch (git push origin my_branch) 24 | 25 | d. Initiate a pull request on github ( http://help.github.com/send-pull-requests/ ) 26 | 27 | e. Done :) 28 | 29 | For minor fixes just open a pull request to the [dev]( https://github.com/Netflix/dynomite/tree/dev ) branch on Github. 30 | 31 | ## Questions 32 | 33 | If you have questions or want to report a bug please create an [Issue]( https://github.com/Netflix/dynomite/issues ) or chat with us on [![Dev chat at https://gitter.im/Netflix/dynomite](https://badges.gitter.im/Netflix/dynomite.svg)](https://gitter.im/Netflix/dynomite?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 34 | 35 | 36 | -------------------------------------------------------------------------------- /src/proto/dyn_proto.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed storages. 3 | * Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | /* 7 | * twemproxy - A fast and lightweight proxy for memcached protocol. 8 | * Copyright (C) 2011 Twitter, Inc. 9 | * 10 | * Licensed under the Apache License, Version 2.0 (the "License"); 11 | * you may not use this file except in compliance with the License. 12 | * You may obtain a copy of the License at 13 | * 14 | * http://www.apache.org/licenses/LICENSE-2.0 15 | * 16 | * Unless required by applicable law or agreed to in writing, software 17 | * distributed under the License is distributed on an "AS IS" BASIS, 18 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | * See the License for the specific language governing permissions and 20 | * limitations under the License. 21 | */ 22 | 23 | #include 24 | 25 | 26 | #ifndef _DN_PROTO_H_ 27 | #define _DN_PROTO_H_ 28 | 29 | 30 | 31 | 32 | void memcache_parse_req(struct msg *r); 33 | void memcache_parse_rsp(struct msg *r); 34 | void memcache_pre_splitcopy(struct mbuf *mbuf, void *arg); 35 | rstatus_t memcache_post_splitcopy(struct msg *r); 36 | void memcache_pre_coalesce(struct msg *r); 37 | void memcache_post_coalesce(struct msg *r); 38 | 39 | void redis_parse_req(struct msg *r); 40 | void redis_parse_rsp(struct msg *r); 41 | bool redis_failure(struct msg *r); 42 | void redis_pre_splitcopy(struct mbuf *mbuf, void *arg); 43 | rstatus_t redis_post_splitcopy(struct msg *r); 44 | rstatus_t redis_reply(struct msg *r); 45 | void redis_pre_coalesce(struct msg *r); 46 | void redis_post_coalesce(struct msg *r); 47 | void redis_swallow_msg(struct conn *conn, struct msg *pmsg, struct msg *msg); 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /src/dyn_crypto.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed storages. 3 | * Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | #ifndef DYN_CRYPTO_H_ 7 | #define DYN_CRYPTO_H_ 8 | 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | #include 15 | #include 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | 24 | #include "dyn_core.h" 25 | 26 | 27 | #define AES_KEYLEN 32 28 | 29 | 30 | rstatus_t crypto_init(struct server_pool *sp); 31 | rstatus_t crypto_init_for_test(void); 32 | rstatus_t crypto_deinit(void); 33 | 34 | char* base64_encode(const unsigned char *message, const size_t length); 35 | int base64_decode(const char *b64message, const size_t length, unsigned char **buffer); 36 | int calc_decode_length(const char *b64input, const size_t length); 37 | 38 | rstatus_t aes_encrypt(const unsigned char *msg, size_t msgLen, unsigned char **encMsg, unsigned char *aes_key); 39 | rstatus_t aes_decrypt(unsigned char *encMsg, size_t encMsgLen, unsigned char **decMsg, unsigned char *aes_key); 40 | 41 | rstatus_t dyn_aes_encrypt(const unsigned char *msg, size_t msgLen, 42 | struct mbuf *mbuf, unsigned char *aes_key); 43 | 44 | rstatus_t dyn_aes_decrypt(unsigned char *encMsg, size_t encMsgLen, 45 | struct mbuf *mbuf, unsigned char *aes_key); 46 | 47 | rstatus_t dyn_aes_encrypt_msg(struct msg *msg, unsigned char *aes_key); 48 | unsigned char* generate_aes_key(void); 49 | 50 | int dyn_rsa_size(void); 51 | 52 | rstatus_t dyn_rsa_encrypt(unsigned char *plain_msg, unsigned char *encrypted_buf); 53 | 54 | rstatus_t dyn_rsa_decrypt(unsigned char *encrypted_msg, unsigned char *decrypted_buf); 55 | 56 | 57 | #endif /* DYN_CRYPTO_H_ */ 58 | -------------------------------------------------------------------------------- /scripts/redis/dyno_redis_bgrewriteaof.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # The following script checks runs Redis background rewrite AOF. 3 | 4 | declare -i RESULT 5 | 6 | # handy logging and error handling functions 7 | function log() { printf '%s\n' "$*"; } 8 | 9 | # Start by checking if Redis is running 10 | REDIS_UP=`redis-cli -p 22122 ping | grep -c PONG` 11 | if [[ ${REDIS_UP} -ne 1 ]]; then 12 | ((RESULT++)) 13 | log "ERROR: Redis is not running" 14 | quit $RESULT 15 | fi 16 | 17 | log "OK: Redis BGREWRITEAOF starting" 18 | 19 | # check if BGREWRITEAOF can be competed successfully. 20 | REWRITEAOF=`redis-cli -p 22122 BGREWRITEAOF | grep ERR | wc -l` 21 | if (( $REWRITEAOF != 0)); then 22 | ((RESULT++)) 23 | log "ERROR: Redis BGREWRITEAOF failed" 24 | quit $RESULT 25 | fi 26 | 27 | SLEEPING=2 28 | log "OK: sleeping initial $SLEEPING seconds post bgrewriteaof" 29 | sleep $SLEEPING 30 | 31 | # If bgrewriteaof is still running, we iterate inside a loop that waits for bg_rewrite_aof to finish. 32 | # Exponential backoff adds 5 seconds to the sleeping time until the value aof_rewrite_in_progress is zero. 33 | # If the sleep takes too long (1800 seconds = 30 min), the process quits. 34 | REDIS_AOF_REWRITE_IN_PROGRESS=`redis-cli -p 22122 INFO | grep aof_rewrite_in_progress | awk -F ':' '{printf "%d\n",$2}'` 35 | while [[ ${REDIS_AOF_REWRITE_IN_PROGRESS} -gt 0 ]]; do 36 | sleep $SLEEPING 37 | log "OK: sleeping $SLEEPING because BGREWRITEAOF is pending" 38 | REDIS_AOF_REWRITE_IN_PROGRESS=`redis-cli -p 22122 INFO | grep aof_rewrite_in_progress | awk -F ':' '{printf "%d\n",$2}'` 39 | let SLEEPING=2*SLEEPING 40 | if [[ ${SLEEPING} -ge 1800 ]]; then 41 | log "ERROR: Redis BGREWRITEAOF takes more than 1800 seconds" 42 | ((RESULT++)) 43 | quit $RESULT 44 | fi 45 | done 46 | log "===================================================" 47 | -------------------------------------------------------------------------------- /src/hashkit/dyn_one_at_a_time.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed storages. 3 | * Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | /* 7 | * twemproxy - A fast and lightweight proxy for memcached protocol. 8 | * Copyright (C) 2011 Twitter, Inc. 9 | * 10 | * Licensed under the Apache License, Version 2.0 (the "License"); 11 | * you may not use this file except in compliance with the License. 12 | * You may obtain a copy of the License at 13 | * 14 | * http://www.apache.org/licenses/LICENSE-2.0 15 | * 16 | * Unless required by applicable law or agreed to in writing, software 17 | * distributed under the License is distributed on an "AS IS" BASIS, 18 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | * See the License for the specific language governing permissions and 20 | * limitations under the License. 21 | */ 22 | 23 | /* 24 | * HashKit 25 | * Copyright (C) 2009 Brian Aker 26 | * All rights reserved. 27 | * 28 | * Use and distribution licensed under the BSD license. See 29 | * the COPYING file in the parent directory for full text. 30 | */ 31 | 32 | /* 33 | * This has is Jenkin's "One at A time Hash". 34 | * http://en.wikipedia.org/wiki/Jenkins_hash_function 35 | */ 36 | 37 | #include 38 | #include 39 | 40 | rstatus_t 41 | hash_one_at_a_time(const char *key, size_t key_length, struct dyn_token *token) 42 | { 43 | const char *ptr = key; 44 | uint32_t value = 0; 45 | 46 | while (key_length--) { 47 | uint32_t val = (uint32_t) *ptr++; 48 | value += val; 49 | value += (value << 10); 50 | value ^= (value >> 6); 51 | } 52 | value += (value << 3); 53 | value ^= (value >> 11); 54 | value += (value << 15); 55 | 56 | size_dyn_token(token, 1); 57 | set_int_dyn_token(token, value); 58 | 59 | return DN_OK; 60 | } 61 | -------------------------------------------------------------------------------- /contrib/fmemopen.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2011-2014 NimbusKit 3 | // Originally ported from https://github.com/ingenuitas/python-tesseract/blob/master/fmemopen.c 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | // 17 | 18 | #ifndef FMEMOPEN_H_ 19 | #define FMEMOPEN_H_ 20 | 21 | #if defined __cplusplus 22 | extern "C" { 23 | #endif 24 | 25 | /** 26 | * A BSD port of the fmemopen Linux method using funopen. 27 | * 28 | * man docs for fmemopen: 29 | * http://linux.die.net/man/3/fmemopen 30 | * 31 | * man docs for funopen: 32 | * https://developer.apple.com/library/mac/#documentation/Darwin/Reference/ManPages/man3/funopen.3.html 33 | * 34 | * This method is ported from ingenuitas' python-tesseract project. 35 | * 36 | * You must call fclose on the returned file pointer or memory will be leaked. 37 | * 38 | * @param buf The data that will be used to back the FILE* methods. Must be at least 39 | * @c size bytes. 40 | * @param size The size of the @c buf data. 41 | * @param mode The permitted stream operation modes. 42 | * @return A pointer that can be used in the fread/fwrite/fseek/fclose family of methods. 43 | * If a failure occurred NULL will be returned. 44 | * @ingroup NimbusMemoryMappping 45 | */ 46 | FILE *fmemopen(void *buf, size_t size, const char *mode); 47 | 48 | #ifdef __cplusplus 49 | } 50 | #endif 51 | 52 | #endif // #ifndef FMEMOPEN_H_ 53 | -------------------------------------------------------------------------------- /test/dual_run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import redis 3 | 4 | class ResultMismatchError(Exception): 5 | def __init__(self, r_result, d_result, func, *args): 6 | self.r_result = r_result 7 | self.d_result = d_result 8 | self.func = func 9 | self.args = args 10 | def __str__(self): 11 | ret = "\n\t======Result Mismatch=======\n" 12 | ret += "\tQuery: %s %s" % (self.func, str(self.args)) 13 | ret += "\n\t===========================\n" 14 | ret += "\tRedis: %s" % str(self.r_result) 15 | ret += "\n\t===========================\n" 16 | ret += "\tDyno: %s" % str(self.d_result) 17 | return ret 18 | 19 | class dual_run(): 20 | def __init__(self, r, d, debug=None): 21 | self.r = r 22 | self.d = d 23 | self.debug = debug 24 | def run_verify(self, func, *args): 25 | r_result = None 26 | d_result = None 27 | r_func = getattr(self.r, func) 28 | d_func = getattr(self.d, func) 29 | r_result = r_func(*args) 30 | i = 0 31 | retry_limit = 3 32 | while i < retry_limit: 33 | try: 34 | d_result = d_func(*args) 35 | if i > 0: 36 | print "\tSucceeded in attempt {}".format(i+1) 37 | break 38 | except redis.exceptions.ResponseError, e: 39 | if "Peer Node is not connected" in str(e): 40 | i = i + 1 41 | print "\tGot error '{}' ... Retry effort {}/{}\n\tQuery '{} {}'".format(e, i, retry_limit, func, str(args)) 42 | continue 43 | print "\tGot error '{}'\n\tQuery '{} {}'".format(e, func, str(args)) 44 | break 45 | if self.debug: 46 | print "Query: %s %s" % (func, str(args)) 47 | print "Redis: %s" % str(r_result) 48 | print "Dyno : %s" % str(d_result) 49 | if r_result != d_result: 50 | raise ResultMismatchError(r_result, d_result, func, *args) 51 | return d_result 52 | -------------------------------------------------------------------------------- /src/dyn_rbtree.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed storages. 3 | * Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | /* 7 | * twemproxy - A fast and lightweight proxy for memcached protocol. 8 | * Copyright (C) 2011 Twitter, Inc. 9 | * 10 | * Licensed under the Apache License, Version 2.0 (the "License"); 11 | * you may not use this file except in compliance with the License. 12 | * You may obtain a copy of the License at 13 | * 14 | * http://www.apache.org/licenses/LICENSE-2.0 15 | * 16 | * Unless required by applicable law or agreed to in writing, software 17 | * distributed under the License is distributed on an "AS IS" BASIS, 18 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | * See the License for the specific language governing permissions and 20 | * limitations under the License. 21 | */ 22 | 23 | #ifndef _DYN_RBTREE_ 24 | #define _DYN_RBTREE_ 25 | #include "dyn_types.h" 26 | 27 | #define rbtree_red(_node) ((_node)->color = 1) 28 | #define rbtree_black(_node) ((_node)->color = 0) 29 | #define rbtree_is_red(_node) ((_node)->color) 30 | #define rbtree_is_black(_node) (!rbtree_is_red(_node)) 31 | #define rbtree_copy_color(_n1, _n2) ((_n1)->color = (_n2)->color) 32 | 33 | struct rbnode { 34 | struct rbnode *left; /* left link */ 35 | struct rbnode *right; /* right link */ 36 | struct rbnode *parent; /* parent link */ 37 | msec_t key; /* key for ordering */ 38 | msec_t timeout; /* timeout */ 39 | void *data; /* opaque data */ 40 | uint8_t color; /* red | black */ 41 | }; 42 | 43 | struct rbtree { 44 | struct rbnode *root; /* root node */ 45 | struct rbnode *sentinel; /* nil node */ 46 | }; 47 | 48 | void rbtree_node_init(struct rbnode *node); 49 | void rbtree_init(struct rbtree *tree, struct rbnode *node); 50 | struct rbnode *rbtree_min(struct rbtree *tree); 51 | void rbtree_insert(struct rbtree *tree, struct rbnode *node); 52 | void rbtree_delete(struct rbtree *tree, struct rbnode *node); 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /scripts/dynomite/generate_yamls.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | ''' 4 | script for generating dynomite yaml files for every node in a cluster. 5 | This script should be run per rack for all nodes in the rack and so the tokens are equally distributed. 6 | usage: