├── src ├── hashkit │ ├── .dirstamp │ ├── Makefile.am │ ├── dyn_hashkit.c │ ├── dyn_token.h │ ├── dyn_murmur3.c │ ├── dyn_one_at_a_time.c │ ├── dyn_hashkit.h │ ├── dyn_fnv.c │ ├── dyn_hsieh.c │ ├── dyn_murmur.c │ └── dyn_crc16.c ├── dyn_dict_msg_id.h ├── dyn_dnode_client.h ├── proto │ ├── Makefile.am │ └── dyn_proto.h ├── seedsprovider │ ├── Makefile.am │ ├── dyn_seeds_provider.h │ └── dyn_dns.c ├── dyn_node_snitch.h ├── dyn_dnode_proxy.h ├── dyn_connection_internal.h ├── event │ ├── Makefile.am │ └── dyn_event.h ├── dyn_vnode.h ├── entropy │ ├── Makefile.am │ └── dyn_entropy.h ├── dyn_dict_msg_id.c ├── dyn_asciilogo.h ├── dyn_setting.h ├── tools │ ├── Makefile.am │ └── dyn_hash_tool.c ├── dyn_types.c ├── dyn_histogram.h ├── dyn_setting.c ├── dyn_proxy.h ├── dyn_task.h ├── dyn_signal.h ├── dyn_client.h ├── dyn_ring_queue.h ├── dyn_dnode_peer.h ├── dyn_crypto.h ├── dyn_response_mgr.h ├── dyn_rbtree.h ├── dyn_gossip.h ├── dyn_array.h ├── dyn_connection_pool.h ├── dyn_dnode_msg.h ├── dyn_task.c ├── dyn_types.h ├── dyn_signal.c ├── dyn_server.h ├── dyn_node_snitch.c ├── dyn_mbuf.h ├── dyn_string.c ├── dyn_ring_queue.c ├── dyn_string.h ├── dyn_array.c ├── dyn_vnode.c └── dyn_log.h ├── OSSMETADATA ├── conf ├── recon_iv.pem ├── recon_key.pem ├── mc_single.yml ├── dynomite.yml ├── redis_single.yml ├── dynomite_dns_single.yml ├── dynomite_florida_single.yml ├── redis_node1.yml ├── redis_node2.yml ├── node1.yml ├── node2.yml ├── redis_dc1.yml ├── redis_dc2.yml ├── redis_rack1_node.yml ├── redis_rack2_node.yml ├── redis_rack3_node.yml └── dynomite.pem ├── init ├── systemd_environment__dynomite ├── systemd_service_rhel__dynomite.service ├── systemd_service_ubuntu__dynomite.service └── README.md ├── m4 └── .gitignore ├── contrib ├── Makefile.am ├── yaml-0.1.4 │ └── .gitignore ├── yaml-0.1.4.tar.gz ├── murmur3 │ ├── makefile │ ├── murmur3.h │ └── README.md ├── fmemopen.h └── fmemopen.c ├── notes ├── kqueue.pdf └── memcache.txt ├── scripts ├── dynomite-manager │ ├── kill_redis.sh │ ├── launch_nfredis.sh │ ├── bash-alias │ ├── cassandra │ └── dynomite-manager ├── Florida │ ├── seeds.list │ ├── package.json │ └── florida.js ├── runall.sh ├── memcache │ └── populate_memcached.sh ├── dynomite │ ├── multi_get.sh │ ├── pipelined_read.sh │ ├── pipelined_write.sh │ ├── generate_yamls.py │ ├── dyn_redis_purge.py │ └── dyn_mc_test.py └── redis │ ├── redis-check.py │ └── dyno_redis_bgrewriteaof.sh ├── images ├── dynomite-logo.png └── dynomite-emblem.png ├── test ├── supplemental.sh ├── ip_util.py ├── load.py ├── kill_cluster.py ├── no_quorum_request.yaml ├── safe_quorum_request.yaml ├── node.py ├── run_loop.py ├── redis_node.py ├── start_cluster.py ├── cluster_generator.py ├── dyno_node.py ├── utils.py └── dual_run.py ├── docker ├── scripts │ └── startup.sh ├── HOWTO.md └── Dockerfile ├── bin ├── kill_dynomite.sh ├── core_affinity.sh └── launch_dynomite.sh ├── Makefile.am ├── .travis.yml ├── travis.sh ├── .gitignore ├── docs ├── dyn_protocol.txt └── florida.md ├── CONTRIBUTING.md └── man └── dynomite.8 /src/hashkit/.dirstamp: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /OSSMETADATA: -------------------------------------------------------------------------------- 1 | osslifecycle=active 2 | -------------------------------------------------------------------------------- /conf/recon_iv.pem: -------------------------------------------------------------------------------- 1 | 01234567890123456 2 | -------------------------------------------------------------------------------- /conf/recon_key.pem: -------------------------------------------------------------------------------- 1 | 01234567890123456 2 | -------------------------------------------------------------------------------- /init/systemd_environment__dynomite: -------------------------------------------------------------------------------- 1 | DYNOMITE_CONF=/etc/dynomite/dynomite.yml 2 | -------------------------------------------------------------------------------- /m4/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything 2 | * 3 | 4 | # Except me 5 | !.gitignore 6 | -------------------------------------------------------------------------------- /contrib/Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = yaml-0.1.4 2 | 3 | EXTRA_DIST = yaml-0.1.4.tar.gz 4 | -------------------------------------------------------------------------------- /notes/kqueue.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/dynomite/HEAD/notes/kqueue.pdf -------------------------------------------------------------------------------- /scripts/dynomite-manager/kill_redis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sudo service redis_6379 stop 3 | 4 | -------------------------------------------------------------------------------- /src/dyn_dict_msg_id.h: -------------------------------------------------------------------------------- 1 | #include "dyn_dict.h" 2 | 3 | extern dictType msg_table_dict_type; 4 | -------------------------------------------------------------------------------- /contrib/yaml-0.1.4/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything 2 | * 3 | 4 | # Except me 5 | !.gitignore 6 | -------------------------------------------------------------------------------- /scripts/dynomite-manager/launch_nfredis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sudo service redis_6379 start 3 | 4 | -------------------------------------------------------------------------------- /contrib/yaml-0.1.4.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/dynomite/HEAD/contrib/yaml-0.1.4.tar.gz -------------------------------------------------------------------------------- /images/dynomite-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/dynomite/HEAD/images/dynomite-logo.png -------------------------------------------------------------------------------- /images/dynomite-emblem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/dynomite/HEAD/images/dynomite-emblem.png -------------------------------------------------------------------------------- /contrib/murmur3/makefile: -------------------------------------------------------------------------------- 1 | CFLAGS = -O3 -Wall 2 | 3 | all: example 4 | example: murmur3.o 5 | 6 | clean: 7 | rm -rf *.o 8 | -------------------------------------------------------------------------------- /test/supplemental.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # check a single stats port 3 | curl -s 127.0.1.2:22222/info | python -mjson.tool > /dev/null 4 | -------------------------------------------------------------------------------- /scripts/Florida/seeds.list: -------------------------------------------------------------------------------- 1 | 192.168.6.41:8101:rc1:dc1:2147483647 2 | 192.168.6.42:8101:rc2:dc1:0 3 | 192.168.6.43:8101:rc2:dc1:2147483647 4 | -------------------------------------------------------------------------------- /docker/scripts/startup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #Start redis server on 22122 4 | redis-server --port 22122 & 5 | 6 | src/dynomite --conf-file=conf/redis_single.yml -v5 7 | -------------------------------------------------------------------------------- /bin/kill_dynomite.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cmd="pkill -f /apps/dynomite/bin/dynomite" 4 | if [ $USER != "root" ];then 5 | exec $cmd 6 | else 7 | su $userowner -c "$cmd" 8 | fi 9 | 10 | -------------------------------------------------------------------------------- /conf/mc_single.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | dyn_listen: 127.0.0.1:8101 3 | listen: 127.0.0.1:8102 4 | servers: 5 | - 127.0.0.1:22122:1 6 | tokens: 437425602 7 | data_store: 1 8 | stats_listen: 0.0.0.0:22222 9 | -------------------------------------------------------------------------------- /test/ip_util.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import struct 3 | 4 | def quad2int(ip): 5 | return struct.unpack("!L", socket.inet_aton(ip))[0] 6 | 7 | def int2quad(ip): 8 | return socket.inet_ntoa(struct.pack('!L', ip)) 9 | -------------------------------------------------------------------------------- /test/load.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ## 4 | # A very simple basic program that will load data in a dynomite/redis server. 5 | ## 6 | import argparse 7 | import redis 8 | 9 | if __name__ == '__main__': 10 | 11 | -------------------------------------------------------------------------------- /conf/dynomite.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | listen: 127.0.0.1:8102 3 | dyn_listen: 127.0.0.1:8101 4 | tokens: '101134286' 5 | servers: 6 | - 127.0.0.1:22122:1 7 | data_store: 0 8 | mbuf_size: 16384 9 | max_msgs: 300000 10 | 11 | -------------------------------------------------------------------------------- /conf/redis_single.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | dyn_listen: 0.0.0.0:8101 3 | data_store: 0 4 | listen: 0.0.0.0:8102 5 | dyn_seed_provider: simple_provider 6 | servers: 7 | - 127.0.0.1:22122:1 8 | tokens: 437425602 9 | stats_listen: 0.0.0.0:22222 10 | -------------------------------------------------------------------------------- /test/kill_cluster.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | 4 | from utils import teardown_running_cluster 5 | 6 | def main(): 7 | teardown_running_cluster('running_cluster.yaml') 8 | 9 | if __name__ == '__main__': 10 | sys.exit(main()) 11 | -------------------------------------------------------------------------------- /test/no_quorum_request.yaml: -------------------------------------------------------------------------------- 1 | cluster_desc: 2 | - name: dc1 3 | racks: 4 | - [rack1, 1] 5 | - name: dc2 6 | racks: 7 | - [rack1, 1] 8 | - [rack2, 2] 9 | - [rack3, 4] 10 | conf: 11 | read_consistency: "DC_ONE" 12 | write_consistency: "DC_ONE" 13 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in aclocal.m4 configure config.h.in config.h.in~ stamp-h.in 2 | 3 | ACLOCAL_AMFLAGS = -I m4 4 | 5 | SUBDIRS = contrib src 6 | 7 | dist_man_MANS = man/dynomite.8 8 | 9 | EXTRA_DIST = README.md NOTICE LICENSE conf scripts notes 10 | -------------------------------------------------------------------------------- /scripts/dynomite-manager/bash-alias: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | alias cls=clear 4 | alias dlog='tail -f -n 2000 /logs/system/dynomite/dynomite.log' 5 | alias dmlog='tail -f -n 2000 /logs/system/dynomite-manager/dynomite-manager.log' 6 | alias rlog='tail -f -n 2000 /var/log/redis_6379.log' 7 | 8 | -------------------------------------------------------------------------------- /conf/dynomite_dns_single.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc 3 | rack: rack1 4 | dyn_listen: 0.0.0.0:8101 5 | dyn_seed_provider: dns_provider 6 | listen: 0.0.0.0:8102 7 | servers: 8 | - 127.0.0.1:6379:1 9 | tokens: '0' 10 | data_store: 0 11 | stats_listen: 0.0.0.0:22222 12 | -------------------------------------------------------------------------------- /test/safe_quorum_request.yaml: -------------------------------------------------------------------------------- 1 | cluster_desc: 2 | - name: dc1 3 | racks: 4 | - [rack1, 1] 5 | - [rack2, 1] 6 | - [rack3, 1] 7 | - name: dc2 8 | racks: 9 | - [rack1, 1] 10 | - [rack2, 1] 11 | - [rack3, 1] 12 | conf: 13 | read_consistency: "DC_SAFE_QUORUM" 14 | write_consistency: "DC_SAFE_QUORUM" 15 | -------------------------------------------------------------------------------- /src/dyn_dnode_client.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed 3 | * storages. Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | #ifndef _DYN_DNODE_CLIENT_H_ 7 | #define _DYN_DNODE_CLIENT_H_ 8 | 9 | // Forward declarations 10 | struct conn; 11 | 12 | void init_dnode_client_conn(struct conn *conn); 13 | 14 | #endif 15 | -------------------------------------------------------------------------------- /conf/dynomite_florida_single.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc 3 | rack: rack1 4 | dyn_listen: 0.0.0.0:8101 5 | dyn_seed_provider: florida_provider 6 | listen: 0.0.0.0:8102 7 | servers: 8 | - 127.0.0.1:6379:1 9 | tokens: '0' 10 | secure_server_option: datacenter 11 | pem_key_file: conf/dynomite.pem 12 | data_store: 0 13 | stats_listen: 0.0.0.0:22222 14 | -------------------------------------------------------------------------------- /test/node.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | class Node(object): 4 | def __init__(self, ip, port): 5 | self.ip = ip 6 | self.port=port 7 | self.name="Node: %s:%d" % (ip, port) 8 | 9 | def __name__(self): 10 | return self.name 11 | 12 | def start(self): 13 | return 14 | 15 | def stop(self): 16 | return 17 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: c 2 | script: bash ./travis.sh 3 | 4 | before_install: 5 | - sudo apt remove --purge python3-pip 6 | - curl -O https://bootstrap.pypa.io/pip/3.5/get-pip.py 7 | - sudo -E python3 get-pip.py 8 | - sudo -E python3 -m pip install --upgrade "pip < 22.3" 9 | 10 | addons: 11 | apt: 12 | update: true 13 | packages: 14 | - python3.9 15 | -------------------------------------------------------------------------------- /scripts/runall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | N=$2 4 | O=$1 5 | 6 | if [ -z "$N" ]; then 7 | N=1 8 | fi 9 | 10 | if [ -z "$O" ]; then 11 | O=read 12 | fi 13 | 14 | echo "N=$N" 15 | echo "O=$O" 16 | 17 | for i in $( cat listnodes ); do 18 | echo 'processing: ' $i 19 | python dynomite/dyn_redis_test.py -P 22122 -H $i -o $O -n $N 20 | sleep 1 21 | done 22 | -------------------------------------------------------------------------------- /src/proto/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | 3 | AM_CPPFLAGS = -I $(top_srcdir)/src 4 | 5 | AM_CFLAGS = -Wall -Wshadow 6 | AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value 7 | 8 | noinst_LIBRARIES = libproto.a 9 | 10 | noinst_HEADERS = dyn_proto.h 11 | 12 | libproto_a_SOURCES = \ 13 | dyn_memcache.c \ 14 | dyn_redis.c \ 15 | dyn_redis_repair.c 16 | -------------------------------------------------------------------------------- /src/seedsprovider/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | 3 | AM_CPPFLAGS = -I $(top_srcdir)/src 4 | 5 | AM_CFLAGS = -Wall -Wshadow 6 | AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value 7 | 8 | noinst_LIBRARIES = libseedsprovider.a 9 | 10 | noinst_HEADERS = dyn_seeds_provider.h 11 | 12 | libseedsprovider_a_SOURCES = \ 13 | dyn_florida.c \ 14 | dyn_dns.c 15 | -------------------------------------------------------------------------------- /conf/redis_node1.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc 3 | rack: rack 4 | dyn_listen: 127.0.0.1:8101 5 | dyn_seeds: 6 | - 127.0.0.2:8101:rack:dc:1383429731 7 | listen: 127.0.0.1:8102 8 | servers: 9 | - 127.0.0.1:22122:1 10 | tokens: '12345678' 11 | secure_server_option: datacenter 12 | pem_key_file: conf/dynomite.pem 13 | data_store: 0 14 | stats_listen: 0.0.0.0:22222 15 | -------------------------------------------------------------------------------- /conf/redis_node2.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc 3 | rack: rack 4 | dyn_listen: 127.0.0.2:8101 5 | dyn_seeds: 6 | - 127.0.0.1:8101:rack:dc:12345678 7 | listen: 127.0.0.2:8102 8 | servers: 9 | - 127.0.0.1:22123:1 10 | tokens: '1383429731' 11 | secure_server_option: datacenter 12 | pem_key_file: conf/dynomite.pem 13 | data_store: 0 14 | stats_listen: 0.0.0.0:22223 15 | -------------------------------------------------------------------------------- /conf/node1.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc1 3 | rack: rack1 4 | listen: 0.0.0.0:8102 5 | dyn_listen: 0.0.0.0:8101 6 | dyn_seeds: 7 | - 127.0.0.1:8113:rack1:dc2:101134286 8 | dyn_seed_provider: simple_provider 9 | tokens: '101134286' 10 | servers: 11 | - 127.0.0.1:11371:1 12 | data_store: 0 13 | stats_listen: 0.0.0.0:33331 14 | preconnect: true 15 | -------------------------------------------------------------------------------- /conf/node2.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc2 3 | rack: rack1 4 | listen: 0.0.0.0:8114 5 | dyn_listen: 0.0.0.0:8113 6 | dyn_seeds: 7 | - 127.0.0.1:8101:rack1:dc1:101134286 8 | dyn_seed_provider: simple_provider 9 | tokens: '101134286' 10 | servers: 11 | - 127.0.0.1:11370:1 12 | data_store: 0 13 | stats_listen: 0.0.0.0:33333 14 | preconnect: true 15 | -------------------------------------------------------------------------------- /src/seedsprovider/dyn_seeds_provider.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef _DYN_SEEDS_PROVIDER_H_ 3 | #define _DYN_SEEDS_PROVIDER_H_ 4 | 5 | #define SEEDS_CHECK_INTERVAL (30 * 1000) /* in msec */ 6 | 7 | // Forward declarations 8 | struct context; 9 | struct mbuf; 10 | 11 | uint8_t florida_get_seeds(struct context *ctx, struct mbuf *seeds_buf); 12 | uint8_t dns_get_seeds(struct context *ctx, struct mbuf *seeds_buf); 13 | 14 | #endif /* DYN_SEEDS_PROVIDER_H_ */ 15 | -------------------------------------------------------------------------------- /src/dyn_node_snitch.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef _DYN_SNITCH_H_ 3 | #define _DYN_SNITCH_H_ 4 | 5 | // Forward declarations 6 | struct server_pool; 7 | 8 | unsigned char *get_broadcast_address(struct server_pool *sp); 9 | char *get_public_hostname(struct server_pool *sp); 10 | char *get_public_ip4(struct server_pool *sp); 11 | char *get_private_ip4(struct server_pool *sp); 12 | unsigned char *hostname_to_private_ip4(char *hostname); 13 | 14 | #endif /* _DYN_SNITCH_H_s */ 15 | -------------------------------------------------------------------------------- /conf/redis_dc1.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc1 3 | rack: rack1 4 | dyn_listen: 127.0.0.1:8101 5 | dyn_seeds: 6 | - 127.0.0.2:8101:rack2:dc2:1383429731 7 | listen: 127.0.0.1:8102 8 | servers: 9 | - 127.0.0.1:22121:1 10 | tokens: '12345678' 11 | secure_server_option: datacenter 12 | pem_key_file: conf/dynomite.pem 13 | data_store: 0 14 | stats_listen: 0.0.0.0:22221 15 | datastore_connections: 3 16 | local_peer_connections: 3 17 | remote_peer_connections: 3 18 | -------------------------------------------------------------------------------- /conf/redis_dc2.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc2 3 | rack: rack2 4 | dyn_listen: 127.0.0.2:8101 5 | dyn_seeds: 6 | - 127.0.0.1:8101:rack1:dc1:12345678 7 | listen: 127.0.0.2:8102 8 | servers: 9 | - 127.0.0.1:22122:1 10 | tokens: '1383429731' 11 | secure_server_option: datacenter 12 | pem_key_file: conf/dynomite.pem 13 | data_store: 0 14 | stats_listen: 0.0.0.0:22222 15 | datastore_connections: 3 16 | local_peer_connections: 3 17 | remote_peer_connections: 3 18 | -------------------------------------------------------------------------------- /src/dyn_dnode_proxy.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed 3 | * storages. Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | #ifndef _DYN_DNODE_SERVER_H_ 7 | #define _DYN_DNODE_SERVER_H_ 8 | 9 | #include "dyn_types.h" 10 | 11 | // Forward declarations 12 | struct conn; 13 | struct context; 14 | 15 | rstatus_t dnode_proxy_init(struct context *ctx); 16 | void dnode_proxy_deinit(struct context *ctx); 17 | void init_dnode_proxy_conn(struct conn *conn); 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /conf/redis_rack1_node.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc 3 | rack: rack1 4 | dyn_listen: 127.0.0.1:8101 5 | dyn_seeds: 6 | - 127.0.0.2:8101:rack2:dc:1383429731 7 | - 127.0.0.3:8101:rack3:dc:1383429731 8 | listen: 127.0.0.1:8102 9 | servers: 10 | - 127.0.0.1:22121:1 11 | tokens: '1383429731' 12 | secure_server_option: datacenter 13 | pem_key_file: conf/dynomite.pem 14 | data_store: 0 15 | read_consistency : DC_SAFE_QUORUM 16 | write_consistency : DC_SAFE_QUORUM 17 | stats_listen: 0.0.0.0:22221 18 | -------------------------------------------------------------------------------- /conf/redis_rack2_node.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc 3 | rack: rack2 4 | dyn_listen: 127.0.0.2:8101 5 | dyn_seeds: 6 | - 127.0.0.1:8101:rack1:dc:1383429731 7 | - 127.0.0.3:8101:rack3:dc:1383429731 8 | listen: 127.0.0.2:8102 9 | servers: 10 | - 127.0.0.1:22122:1 11 | tokens: '1383429731' 12 | secure_server_option: datacenter 13 | pem_key_file: conf/dynomite.pem 14 | data_store: 0 15 | read_consistency : DC_SAFE_QUORUM 16 | write_consistency : DC_SAFE_QUORUM 17 | stats_listen: 0.0.0.0:22222 18 | -------------------------------------------------------------------------------- /conf/redis_rack3_node.yml: -------------------------------------------------------------------------------- 1 | dyn_o_mite: 2 | datacenter: dc 3 | rack: rack3 4 | dyn_listen: 127.0.0.3:8101 5 | dyn_seeds: 6 | - 127.0.0.1:8101:rack1:dc:1383429731 7 | - 127.0.0.2:8101:rack2:dc:1383429731 8 | listen: 127.0.0.3:8102 9 | servers: 10 | - 127.0.0.1:22123:1 11 | tokens: '1383429731' 12 | secure_server_option: datacenter 13 | pem_key_file: conf/dynomite.pem 14 | data_store: 0 15 | read_consistency : DC_SAFE_QUORUM 16 | write_consistency : DC_SAFE_QUORUM 17 | stats_listen: 0.0.0.0:22223 18 | -------------------------------------------------------------------------------- /src/dyn_connection_internal.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "dyn_types.h" 4 | 5 | // Forward declarations. 6 | struct conn; 7 | struct context; 8 | 9 | extern void _conn_deinit(void); 10 | extern void _conn_init(void); 11 | extern struct conn *_conn_get(void); 12 | extern void _conn_put(struct conn *conn); 13 | extern char *_conn_get_type_string(struct conn *conn); 14 | extern void _add_to_ready_q(struct context *ctx, struct conn *conn); 15 | extern void _remove_from_ready_q(struct context *ctx, struct conn *conn); 16 | extern rstatus_t _conn_reuse(struct conn *p); 17 | -------------------------------------------------------------------------------- /docker/HOWTO.md: -------------------------------------------------------------------------------- 1 | # Build Image 2 | 3 | Create Dynomite plus Redis single server image with docker 4 | 5 | # Example: sudo docker build -t [name] . 6 | $ sudo docker build -t my_dynomite . 7 | 8 | # Running A Dynomite Instance 9 | 10 | Creating a container running Dynomite and Redis instance inside. To set the name, you can use the -name [name]. If a name is not set, an alphanumeric ID will be obtained. 11 | 12 | $ sudo docker run -name my_dynomite_instance -i -i my_dynomite 13 | 14 | To list all containers 15 | 16 | $ sudo docker ps 17 | 18 | Enjoy! 19 | -------------------------------------------------------------------------------- /scripts/memcache/populate_memcached.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | port=22123 4 | socatopt="-t 1 -T 1 -b 65537" 5 | 6 | val=`echo 6^6^6 | bc` 7 | val=`printf "%s\r\n" "${val}"` 8 | vallen=`printf "%s" "${val}" | wc -c` 9 | set_command="" 10 | 11 | # build 12 | for i in `seq 1 512`; do 13 | if [ `expr $i % 2` -eq "0" ]; then 14 | key="foo" 15 | else 16 | key="bar" 17 | fi 18 | key=`printf "%s%d" "${key}" "${i}"` 19 | 20 | set_command="set ${key} 0 0 ${vallen}\r\n${val}\r\n" 21 | 22 | printf "%b" "$set_command" | socat ${socatopt} - TCP:localhost:${port},nodelay,shut-down,nonblock=1 & 23 | done 24 | 25 | -------------------------------------------------------------------------------- /scripts/dynomite/multi_get.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | port=8102 4 | socatopt="-t 20 -T 20 -b 8193 -d -d " 5 | key="" 6 | keys="" 7 | get_command="" 8 | 9 | # build 10 | for i in `seq 1 512`; do 11 | if [ `expr $i % 2` -eq "0" ]; then 12 | key="foo" 13 | else 14 | key="bar" 15 | fi 16 | key=`printf "%s%d" "${key}" "${i}"` 17 | keys=`printf "%s %s" "${keys}" "${key}"` 18 | done 19 | 20 | get_command="get ${keys}\r\n" 21 | printf "%b" "$get_command" 22 | 23 | # read 24 | for i in `seq 1 16`; do 25 | printf "%b" "${get_command}" | socat ${socatopt} - TCP:localhost:${port},nodelay,shut-none,nonblock=1 1> /dev/null 2>&1 & 26 | done 27 | -------------------------------------------------------------------------------- /src/event/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | 3 | AM_CPPFLAGS = -I $(top_srcdir)/src 4 | 5 | AM_CFLAGS = -Wall -Wshadow 6 | AM_CFLAGS += -Wpointer-arith 7 | AM_CFLAGS += -Winline 8 | AM_CFLAGS += -Wunused-function -Wunused-variable -Wunused-value 9 | AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value 10 | AM_CFLAGS += -Wconversion -Wsign-compare 11 | AM_CFLAGS += -Wstrict-prototypes -Wmissing-prototypes -Wredundant-decls -Wmissing-declarations -Werror=implicit-function-declaration 12 | 13 | noinst_LIBRARIES = libevent.a 14 | 15 | noinst_HEADERS = dyn_event.h 16 | 17 | libevent_a_SOURCES = \ 18 | dyn_epoll.c \ 19 | dyn_kqueue.c \ 20 | dyn_evport.c 21 | 22 | -------------------------------------------------------------------------------- /src/dyn_vnode.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed 3 | * storages. Copyright (C) 2019 Netflix, Inc. 4 | */ 5 | 6 | #pragma once 7 | #include 8 | 9 | // Initializes (on first call) and updates (on subsequent calls) the per rack continuums 10 | // and makes sure the tokens managed by the continuums are ascending. 11 | rstatus_t vnode_update(struct server_pool *pool); 12 | 13 | // Returns the index of the continuum from 'continuums' where 'token' falls. 14 | // If 'token' falls into interval (a,b], we return b. 15 | uint32_t vnode_dispatch(struct array *continuums, uint32_t ncontinuum, 16 | struct dyn_token *token); 17 | -------------------------------------------------------------------------------- /src/entropy/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | 3 | AM_CPPFLAGS = -I $(top_srcdir)/src 4 | 5 | AM_CFLAGS = -Wall -Wshadow 6 | AM_CFLAGS += -Wpointer-arith 7 | AM_CFLAGS += -Winline 8 | AM_CFLAGS += -Wunused-function -Wunused-variable -Wunused-value 9 | AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value 10 | AM_CFLAGS += -Wconversion -Wsign-compare 11 | AM_CFLAGS += -Wstrict-prototypes -Wmissing-prototypes -Wredundant-decls -Wmissing-declarations -Wimplicit-function-declaration 12 | 13 | 14 | noinst_LIBRARIES = libentropy.a 15 | 16 | noinst_HEADERS = dyn_entropy.h 17 | 18 | libentropy_a_SOURCES = \ 19 | dyn_entropy_rcv.c \ 20 | dyn_entropy_snd.c \ 21 | dyn_entropy_util.c 22 | -------------------------------------------------------------------------------- /init/systemd_service_rhel__dynomite.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Dynomite server 3 | ; Services that should be started before Dynomite 4 | After=network.target nss-lookup.target time-sync.target 5 | Documentation=man:dynomite(8) 6 | 7 | [Service] 8 | User=dynomite 9 | ; Default environment variable values 10 | Environment=DYNOMITE_CONF=/etc/dynomite/dynomite.yaml 11 | Type=forking 12 | ; Environment vars to customize startup 13 | EnvironmentFile=-/etc/sysconfig/dynomite 14 | ExecStart=/usr/local/sbin/dynomite -d -c $DYNOMITE_CONF -p /var/run/dynomite/dynomite.pid 15 | ; Restart dynomite every 100ms 16 | Restart=always 17 | PIDFile=/var/run/dynomite/dynomite.pid 18 | 19 | [Install] 20 | WantedBy=multi-user.target 21 | -------------------------------------------------------------------------------- /init/systemd_service_ubuntu__dynomite.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Dynomite server 3 | ; Services that should be started before Dynomite 4 | After=network.target nss-lookup.target time-sync.target 5 | Documentation=man:dynomite(8) 6 | 7 | [Service] 8 | User=dynomite 9 | ; Default environment variable values 10 | Environment=DYNOMITE_CONF=/etc/dynomite/dynomite.yaml 11 | Type=forking 12 | ; Environment vars to customize startup 13 | EnvironmentFile=-/etc/default/dynomite 14 | ExecStart=/usr/local/sbin/dynomite -d -c $DYNOMITE_CONF -p /var/run/dynomite/dynomite.pid 15 | ; Restart dynomite every 100ms 16 | Restart=always 17 | PIDFile=/var/run/dynomite/dynomite.pid 18 | 19 | [Install] 20 | WantedBy=multi-user.target 21 | -------------------------------------------------------------------------------- /scripts/redis/redis-check.py: -------------------------------------------------------------------------------- 1 | import redis 2 | 3 | range=100 4 | factor=32 5 | port=22121 6 | 7 | r = redis.StrictRedis(host='localhost', port=port, db=0) 8 | 9 | # lrange 10 | print([r.lrange('lfoo', 0, x) for x in range(1, range)]) 11 | print([r.lpush('lfoo', str(x)*factor) for x in range(1, range)]) 12 | print([r.lrange('lfoo', 0, x) for x in range(1, range)]) 13 | print(r.delete('lfoo')) 14 | 15 | # del 16 | print([r.set('foo' + str(x), str(x)*factor) for x in range(1, range)]) 17 | keys = ['foo' + str(x) for x in range(1, range)] 18 | print([r.delete(keys) for x in range(1, range)]) 19 | 20 | # mget 21 | print([r.set('foo' + str(x), str(x)*100) for x in range(1, range)]) 22 | keys = ['foo' + str(x) for x in range(1, range)] 23 | print([r.mget(keys) for x in range(1, range)]) 24 | -------------------------------------------------------------------------------- /src/dyn_dict_msg_id.c: -------------------------------------------------------------------------------- 1 | #include "dyn_dict_msg_id.h" 2 | #include // For NULL 3 | #include "dyn_types.h" 4 | 5 | static unsigned int dict_msg_id_hash(const void *key) { 6 | msgid_t id = *(msgid_t *)key; 7 | return dictGenHashFunction(key, sizeof(id)); 8 | } 9 | 10 | static int dict_msg_id_cmp(void *privdata, const void *key1, const void *key2) { 11 | msgid_t id1 = *(msgid_t *)key1; 12 | msgid_t id2 = *(msgid_t *)key2; 13 | return id1 == id2; 14 | } 15 | 16 | dictType msg_table_dict_type = { 17 | dict_msg_id_hash, /* hash function */ 18 | NULL, /* key dup */ 19 | NULL, /* val dup */ 20 | dict_msg_id_cmp, /* key compare */ 21 | NULL, /* key destructor */ 22 | NULL /* val destructor */ 23 | }; 24 | -------------------------------------------------------------------------------- /travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -n "$TRAVIS" ]; then 4 | sudo pip3 install redis plumbum pyyaml 5 | fi 6 | 7 | 8 | # parse options 9 | rebuild=true 10 | debug=false 11 | while [ "$#" -gt 0 ]; do 12 | arg=$1 13 | case $1 in 14 | -n|--no-rebuild) shift; rebuild=false;; 15 | -d|--debug) shift; debug=true;; 16 | -*) usage_fatal "unknown option: '$1'";; 17 | *) break;; # reached the list of file names 18 | esac 19 | done 20 | 21 | set -o errexit 22 | set -o nounset 23 | set -o pipefail 24 | 25 | 26 | #build Dynomite 27 | if [[ "${rebuild}" == "true" ]]; then 28 | CFLAGS="-ggdb3 -O0" autoreconf -fvi && ./configure --enable-debug=log && make 29 | else 30 | echo "not rebuilding Dynomite" 31 | fi 32 | 33 | exec test/cluster_generator.py 34 | -------------------------------------------------------------------------------- /scripts/dynomite/pipelined_read.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | #socatopt="-t 4 -T 4 -b 8193 -d -d " 4 | socatopt="-t 8 -T 8 -b 16384 -d -d" 5 | 6 | get_commands="" 7 | 8 | # build 9 | for i in `seq 1 64`; do 10 | if [ `expr $i % 2` -eq "0" ]; then 11 | key="foo" 12 | else 13 | key="bar" 14 | fi 15 | key=`printf "%s%d" "${key}" "${i}"` 16 | keylen=`printf "%s" "${key}" | wc -c` 17 | 18 | get_command="*2\r\n\$3\r\nget\r\n\$${keylen}\r\n${key}\r\n" 19 | get_commands=`printf "%s%s" "${get_commands}" "${get_command}"` 20 | done 21 | 22 | printf "%b" "$get_commands" > /tmp/socat.input 23 | # read 24 | for i in `seq 1 64`; do 25 | cat /tmp/socat.input | socat ${socatopt} - TCP:localhost:8102,nodelay,shut-down,nonblock=1 & #1 > /dev/null 2>&1 & 26 | done 27 | wait 28 | -------------------------------------------------------------------------------- /contrib/murmur3/murmur3.h: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // MurmurHash3 was written by Austin Appleby, and is placed in the 3 | // public domain. The author hereby disclaims copyright to this source 4 | // code. 5 | 6 | #ifndef _MURMURHASH3_H_ 7 | #define _MURMURHASH3_H_ 8 | 9 | #include 10 | 11 | //----------------------------------------------------------------------------- 12 | 13 | void MurmurHash3_x86_32 (const void *key, int len, uint32_t seed, void *out); 14 | 15 | void MurmurHash3_x86_128(const void *key, int len, uint32_t seed, void *out); 16 | 17 | void MurmurHash3_x64_128(const void *key, int len, uint32_t seed, void *out); 18 | 19 | //----------------------------------------------------------------------------- 20 | 21 | #endif // _MURMURHASH3_H_ 22 | -------------------------------------------------------------------------------- /src/dyn_asciilogo.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed 3 | * storages. Copyright (C) 2015 Netflix, Inc. 4 | * 5 | * author: Ioannis Papapanagiotou 6 | */ 7 | 8 | char *ascii_logo = 9 | " \n" 10 | " # \ m \n" 11 | " mmm# m m mmmm mmm mmmmm mmm mm#mm mmm \n" 12 | " # # \\m m/ # # # # # # # # # # # \n" 13 | " # # #m# # # # # # # # # # #'''' \n" 14 | " \\#m## \\# # # #m# # # # mm#mm mm #mm \n" 15 | " m/\n" 16 | " ##\n"; 17 | //" \n\n"; 18 | -------------------------------------------------------------------------------- /scripts/dynomite/pipelined_write.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | socatopt="-t 1 -T 1 -b 16384" 4 | 5 | val=`echo 5^5^5 | bc` 6 | val=`printf "%s" "${val}"` 7 | vallen=`printf "%s" "${val}" | wc -c` 8 | set_command="" 9 | set_commands="" 10 | 11 | # build 12 | for i in `seq 1 64`; do 13 | if [ `expr $i % 2` -eq "0" ]; then 14 | key="foo" 15 | else 16 | key="bar" 17 | fi 18 | key=`printf "%s%d" "${key}" "${i}"` 19 | keylen=`printf "%s" "${key}" | wc -c` 20 | 21 | set_command="*3\r\n\$3\r\nset\r\n\$${keylen}\r\n${key}\r\n\$${vallen}\r\n${val}\r\n" 22 | set_commands=`printf "%s%s" "${set_commands}" "${set_command}"` 23 | done 24 | 25 | printf "%b" "$set_commands" > /tmp/socat.input 26 | 27 | # write 28 | for i in `seq 1 16`; do 29 | cat /tmp/socat.input | socat ${socatopt} - TCP:localhost:8102,nodelay,shut-down,nonblock=1 & 30 | done 31 | wait 32 | -------------------------------------------------------------------------------- /scripts/dynomite-manager/cassandra: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # chkconfig: 2345 95 20 3 | # description: This script does some stuff 4 | # processname: java 5 | 6 | start() { 7 | echo "Starting cassandra..." 8 | export JAVA_HOME=/home/ec2-user/jdk1.8.0_45 9 | export JRE_HOME=/home/ec2-user/jdk1.8.0_45/jre 10 | export PATH=$PATH:/home/ec2-user/jdk1.8.0_45/bin:/home/ec2-user/jdk1.8.0_45/jre/bin 11 | 12 | cd /home/ec2-user/apache-cassandra-2.1.14 13 | bin/cassandra start & 14 | } 15 | 16 | stop() { 17 | echo "stop" 18 | PID=`ps aux | grep cassandra | grep -v grep | awk '{print $2}'` 19 | if [[ "" != "$PID" ]]; then 20 | echo "killing $PID" 21 | sudo kill -9 $PID 22 | fi 23 | } 24 | 25 | case "$1" in start) 26 | start 27 | ;; 28 | stop) 29 | stop 30 | ;; 31 | *) 32 | 33 | echo $"Usage: $0 {start|stop}" 34 | RETVAL=1 35 | esac 36 | exit 0 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /src/hashkit/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | 3 | AM_CPPFLAGS = -I $(top_srcdir)/src -I $(top_srcdir)/contrib/murmur3 4 | 5 | AM_CFLAGS = -Wall -Wshadow 6 | AM_CFLAGS += -Wpointer-arith 7 | AM_CFLAGS += -Winline 8 | AM_CFLAGS += -Wunused-function -Wunused-variable -Wunused-value 9 | AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value -Wno-missing-prototypes -Wno-missing-declarations 10 | AM_CFLAGS += -Wconversion -Wsign-compare 11 | AM_CFLAGS += -Wstrict-prototypes -Wredundant-decls -Werror=implicit-function-declaration 12 | 13 | noinst_LIBRARIES = libhashkit.a 14 | 15 | noinst_HEADERS = dyn_hashkit.h dyn_token.h 16 | 17 | libhashkit_a_SOURCES = \ 18 | dyn_hashkit.c \ 19 | dyn_crc16.c \ 20 | dyn_crc32.c \ 21 | dyn_fnv.c \ 22 | dyn_hsieh.c \ 23 | dyn_jenkins.c \ 24 | dyn_md5.c \ 25 | dyn_murmur.c \ 26 | dyn_one_at_a_time.c \ 27 | dyn_token.c \ 28 | dyn_murmur3.c 29 | -------------------------------------------------------------------------------- /src/dyn_setting.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed 3 | * storages. Copyright (C) 2014 Netflix, Inc. Licensed under the Apache License, 4 | * Version 2.0 (the "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #ifndef _DYN_SETTING_H_ 17 | #define _DYN_SETTING_H_ 18 | 19 | uint32_t msgs_per_sec(void); 20 | void set_msgs_per_sec(uint32_t tokens_per_sec); 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /conf/dynomite.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIICWwIBAAKBgQCjijNAnd1/uFlrCtfO83Qo5idWbqLpyfQdKHd1CRE9AliA8pyM 3 | u9XqdyEbMJ229PPPWhlKui0moVcj1J8w/ZBzLFEQtqf+93Ytv9YR4tktyv0kWTnf 4 | yUzp9D74WHL2/lwfi0Nv+jSIWg5aQRqMzRsdErTwk+iaSuuZ/+3Jbp9JxwIDAQAB 5 | AoGAd3s70MTFlE+SfMMyQo/Z2Ru1t88jFV2oDTmIdShHwOQa1zLpr2R4eFHLDTtq 6 | rsWt71srSU5WOHf92z63g6ptwpxixb+HWIF49Ke0vtMUs+ah/E0jV9EyAwFmdACh 7 | Yxa/dwCTkl4O3q8/Lm1gXExpeif9PUtenIFU8Va2l1s87WECQQDV33ZLWatbAUQz 8 | qLXsRIOiSDAhTXyuAFX2I9HyKfAGxaOz7U8QYZYJVf/vmubWERHZhe8TurusYuEZ 9 | onh4zPuRAkEAw8CxzxKjGZB75Wz2+k9Qbkb6+HS4VBCVPHScwE2+kXnI28l+ATro 10 | MDE0F7VfYA2Cv8xDeBBFhR/rki/5MmhT1wJAMWa2il0iCZlXSZ90tih1R2zpQSgG 11 | 3qJRONO5UPiC6u2IDK2KD7yazXlB7vSW0WeL+fTH7oS5iO4mFKfEsBKZIQJAAZth 12 | WkvS3vDas4VzeZaRGHaotMVwhfAk6XwjlUiOGe+gAH3k9Omg4lRlExTAFa/GtQBe 13 | RWwMxgz/CBIwv4MvvQJAZxZuxzN7D5VhJlkL1IEXnPHgE+RgR0vTAQaqnSa9PNcv 14 | gAiXFRKkMHeAkHUIsZ382C5zzjUGQrCD8IrvpAwXBg== 15 | -----END RSA PRIVATE KEY----- 16 | -------------------------------------------------------------------------------- /bin/core_affinity.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Set core affinity for redis and dynomite processes 3 | # 4 | 5 | # Requires setting the EC2 Instance type as ENV variable. 6 | # If Dynomite is used outside of AWS environment, 7 | # the core affinity script can be configured accordingly. 8 | echo "$EC2_INSTANCE_TYPE" 9 | 10 | if [ "$EC2_INSTANCE_TYPE" == "r3.xlarge" ]; then 11 | dynomite_pid=`pgrep -f $DYN_DIR/bin/dynomite` 12 | echo "dynomite pid: $dynomite_pid" 13 | taskset -pac 2,3 $dynomite_pid 14 | 15 | redis_pid=`ps -ef | grep 22122 | grep redis | awk -F' ' '{print $2}'` 16 | echo "redis pid: $redis_pid" 17 | taskset -pac 1 $redis_pid 18 | 19 | else 20 | 21 | dynomite_pid=`pgrep -f $DYN_DIR/bin/dynomite` 22 | echo "dynomite pid: $dynomite_pid" 23 | taskset -pac 2,5,6 $dynomite_pid 24 | 25 | redis_pid=`ps -ef | grep 22122 | grep redis | awk -F' ' '{print $2}'` 26 | echo "redis pid: $redis_pid" 27 | taskset -pac 3,7 $redis_pid 28 | 29 | fi 30 | 31 | -------------------------------------------------------------------------------- /src/tools/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | AUTOMAKE_OPTIONS = subdir-objects 3 | 4 | AM_CPPFLAGS = 5 | if !OS_SOLARIS 6 | AM_CPPFLAGS += -D_GNU_SOURCE 7 | endif 8 | AM_CPPFLAGS += -I $(top_srcdir)/src 9 | AM_CPPFLAGS += -I $(top_srcdir)/src/hashkit 10 | 11 | AM_CFLAGS = 12 | AM_CFLAGS += -Wall -Wshadow 13 | AM_CFLAGS += -Wpointer-arith 14 | AM_CFLAGS += -Winline 15 | AM_CFLAGS += -Wunused-function -Wunused-variable -Wunused-value 16 | AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value 17 | AM_CFLAGS += -Wconversion -Wsign-compare 18 | AM_CFLAGS += -Wstrict-prototypes -Wmissing-prototypes -Wredundant-decls -Wmissing-declarations 19 | 20 | AM_LDFLAGS = 21 | 22 | if OS_SOLARIS 23 | AM_LDFLAGS += -lnsl -lsocket 24 | endif 25 | 26 | bin_PROGRAMS = dynomite-hash-tool 27 | 28 | dynomite_hash_tool_SOURCES = \ 29 | dyn_hash_tool.c \ 30 | ../hashkit/dyn_token.c \ 31 | ../dyn_log.c \ 32 | ../dyn_util.c \ 33 | ../dyn_array.c 34 | 35 | dynomite_hash_tool_LDADD = $(top_builddir)/src/hashkit/libhashkit.a 36 | -------------------------------------------------------------------------------- /src/dyn_types.c: -------------------------------------------------------------------------------- 1 | #include "dyn_types.h" 2 | #include "stdio.h" 3 | 4 | #define OBJECT_MAGIC 0xdead 5 | 6 | void cleanup_charptr(char **ptr) { 7 | if (*ptr) free(*ptr); 8 | } 9 | 10 | void init_object(struct object *obj, object_type_t type, func_print_t print) { 11 | obj->magic = OBJECT_MAGIC; 12 | obj->type = type; 13 | obj->func_print = print; 14 | } 15 | 16 | char *print_obj(const void *ptr) { 17 | const object_t *obj = (const object_t *)ptr; 18 | static char buffer[PRINT_BUF_SIZE]; 19 | if (obj == NULL) { 20 | snprintf(buffer, PRINT_BUF_SIZE, ""); 21 | return buffer; 22 | } 23 | if (obj->magic != OBJECT_MAGIC) { 24 | snprintf(buffer, PRINT_BUF_SIZE, "addr:%p MAGIC NUMBER 0x%x", 25 | obj, obj->magic); 26 | return buffer; 27 | } 28 | if ((obj->type >= 0) && (obj->type < OBJ_LAST)) { 29 | return obj->func_print(obj); 30 | } else { 31 | snprintf(buffer, PRINT_BUF_SIZE, "addr:%p INVALID TYPE %d", 32 | obj, obj->type); 33 | return buffer; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/dyn_histogram.h: -------------------------------------------------------------------------------- 1 | /* 2 | * dyn_histogram.h 3 | * 4 | * Created on: Feb 6, 2015 5 | * Author: mdo 6 | */ 7 | 8 | #ifndef DYN_HISTOGRAM_H_ 9 | #define DYN_HISTOGRAM_H_ 10 | 11 | #define BUCKET_SIZE 94 12 | 13 | struct histogram { 14 | uint64_t buckets[BUCKET_SIZE]; 15 | uint64_t mean; 16 | uint64_t val_95th; 17 | uint64_t val_99th; 18 | uint64_t val_999th; 19 | uint64_t val_max; 20 | }; 21 | 22 | rstatus_t histo_init(volatile struct histogram *histo); 23 | rstatus_t histo_reset(volatile struct histogram *histo); 24 | void histo_add(volatile struct histogram *histo, uint64_t val); 25 | uint64_t histo_get_bucket(volatile struct histogram *histo, int bucket); 26 | void histo_get_buckets(volatile struct histogram *histo, uint64_t *my_buckets); 27 | uint64_t histo_percentile(volatile struct histogram *histo, double percentile); 28 | uint64_t histo_mean(volatile struct histogram *histo); 29 | uint64_t histo_max(volatile struct histogram *histo); 30 | void histo_compute(volatile struct histogram *histo); 31 | 32 | #endif /* DYN_HISTOGRAM_H_ */ 33 | -------------------------------------------------------------------------------- /test/run_loop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import subprocess, time 3 | 4 | def run_command(cmd): 5 | """Execute a shell command and return the output 6 | :param command: the command to be run and all of the arguments 7 | :returns: success_boolean, command_string, stdout, stderr 8 | """ 9 | 10 | proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 11 | for line in iter(proc.stdout.readline, b''): 12 | print(">>> " + line.rstrip()) 13 | (stdout, stderr) = proc.communicate() 14 | return proc.returncode == 0, proc 15 | 16 | def main(): 17 | cmd = "bash travis.sh -n" 18 | for i in range(0, 50): 19 | print("Running loop {}".format(i+1)) 20 | success, proc = run_command(cmd) 21 | if not success: 22 | for line in iter(proc.stderr.readline, b''): 23 | print(("--- " + line.rstrip())) 24 | break 25 | else: 26 | print("...........................Success\n") 27 | time.sleep(30) 28 | 29 | if __name__ == "__main__": 30 | main() 31 | -------------------------------------------------------------------------------- /scripts/Florida/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "florida.js", 3 | "version": "1.0.0", 4 | "description": "Service that returns list of seeds.", 5 | "keywords": [ 6 | "Dynomite", 7 | "Florida", 8 | "Netflix", 9 | "Redis", 10 | "Memcached", 11 | "Dynamo" 12 | ], 13 | "main": "florida.js", 14 | "scripts": { 15 | "test": "echo \"Error: no test specified\" && exit 1", 16 | "debug": "node florida.js ./seeds.list debug" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "git+https://github.com/Netflix/dynomite.git" 21 | }, 22 | "contributors": [ 23 | { 24 | "name": "Diego Pacheco", 25 | "email": "diego.pacheco.it@gmail.com", 26 | "url": "https://github.com/diegopacheco" 27 | }, 28 | { 29 | "name": "Akbar S. Ahmed", 30 | "email": "akbar501@gmail.com", 31 | "url": "https://grockdoc.com/@akbar501" 32 | } 33 | ], 34 | "license": "Apache-2.0", 35 | "bugs": { 36 | "url": "https://github.com/Netflix/dynomite/issues" 37 | }, 38 | "homepage": "https://github.com/Netflix/dynomite#readme" 39 | } 40 | -------------------------------------------------------------------------------- /test/redis_node.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import redis 3 | from node import Node 4 | 5 | from plumbum import BG 6 | from plumbum import local 7 | 8 | redis_bin = local.get('./test/_binaries/redis-server', 'redis-server') 9 | 10 | class RedisNode(Node): 11 | def __init__(self, ip, port): 12 | super(RedisNode, self).__init__(ip, port) 13 | self.name = "Redis" + self.name 14 | self.logfile = 'logs/redis_{}.log'.format(self.ip) 15 | self.proc_future = None 16 | 17 | def get_connection(self): 18 | return redis.Redis(self.ip, self.port, db=0) 19 | 20 | def get_pid(self): 21 | return self.proc_future.proc.pid 22 | 23 | def launch(self): 24 | self.proc_future = \ 25 | (redis_bin['--bind', self.ip, '--port', self.port, '--loglevel', 'verbose'] > self.logfile) & BG(-9) 26 | 27 | def teardown(self): 28 | self.proc_future.proc.kill() 29 | self.proc_future.wait() 30 | 31 | def __enter__(self): 32 | self.launch() 33 | 34 | def __exit__(self, type_, value, traceback): 35 | self.teardown() 36 | -------------------------------------------------------------------------------- /src/dyn_setting.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed 3 | * storages. Copyright (C) 2014 Netflix, Inc. Licensed under the Apache License, 4 | * Version 2.0 (the "License"); you may not use this file except in compliance 5 | * with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include "dyn_conf.h" 17 | #include "dyn_core.h" 18 | 19 | static uint32_t conn_msg_rate = CONF_DEFAULT_CONN_MSG_RATE; // conn msgs per 20 | // sec 21 | 22 | uint32_t msgs_per_sec(void) { return conn_msg_rate; } 23 | 24 | void set_msgs_per_sec(uint32_t tokens_per_sec) { 25 | conn_msg_rate = tokens_per_sec; 26 | } 27 | -------------------------------------------------------------------------------- /src/dyn_proxy.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed 3 | * storages. Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | /* 7 | * twemproxy - A fast and lightweight proxy for memcached protocol. 8 | * Copyright (C) 2011 Twitter, Inc. 9 | * 10 | * Licensed under the Apache License, Version 2.0 (the "License"); 11 | * you may not use this file except in compliance with the License. 12 | * You may obtain a copy of the License at 13 | * 14 | * http://www.apache.org/licenses/LICENSE-2.0 15 | * 16 | * Unless required by applicable law or agreed to in writing, software 17 | * distributed under the License is distributed on an "AS IS" BASIS, 18 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | * See the License for the specific language governing permissions and 20 | * limitations under the License. 21 | */ 22 | 23 | #ifndef _DYN_PROXY_H_ 24 | #define _DYN_PROXY_H_ 25 | 26 | #include "dyn_types.h" 27 | 28 | // Forward declarations 29 | struct conn; 30 | struct context; 31 | 32 | rstatus_t proxy_init(struct context *ctx); 33 | void proxy_deinit(struct context *ctx); 34 | void init_proxy_conn(struct conn *conn); 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /src/dyn_task.h: -------------------------------------------------------------------------------- 1 | #ifndef _DYN_TASK_H_ 2 | #define _DYN_TASK_H_ 3 | 4 | #include "dyn_rbtree.h" 5 | #include "dyn_types.h" 6 | 7 | struct task; 8 | 9 | /* handler that takes 1 argument */ 10 | typedef void (*task_handler_1)(void *arg1); 11 | 12 | /* Initialize the task manager */ 13 | rstatus_t task_mgr_init(void); 14 | 15 | /* creates a new task and adds to the internal datastructure 16 | * handler1 : a handler function that takes 1 arguemtn 17 | * arg1 : the argument that will be sent back to handler1 18 | * timeout : time in msec after which this task should get fired 19 | */ 20 | struct task *schedule_task_1(task_handler_1 handler1, void *arg1, 21 | msec_t timeout); 22 | 23 | /* Returns the time in msec to the next task */ 24 | msec_t time_to_next_task(void); 25 | 26 | /* Execute expired tasks one after other calling individual handlers */ 27 | /* limit = 0 (execute all expired tasks) 28 | * > 0 (up to limit tasks) 29 | */ 30 | void execute_expired_tasks(uint32_t limit); 31 | 32 | /* Cancel the provided task. The caller should keep track of the tasks scheduled 33 | * and use it to cancel */ 34 | void cancel_task(struct task *task); 35 | 36 | #endif /* _DYN_TASK_H_ */ 37 | -------------------------------------------------------------------------------- /src/dyn_signal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed 3 | * storages. Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | /* 7 | * twemproxy - A fast and lightweight proxy for memcached protocol. 8 | * Copyright (C) 2011 Twitter, Inc. 9 | * 10 | * Licensed under the Apache License, Version 2.0 (the "License"); 11 | * you may not use this file except in compliance with the License. 12 | * You may obtain a copy of the License at 13 | * 14 | * http://www.apache.org/licenses/LICENSE-2.0 15 | * 16 | * Unless required by applicable law or agreed to in writing, software 17 | * distributed under the License is distributed on an "AS IS" BASIS, 18 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | * See the License for the specific language governing permissions and 20 | * limitations under the License. 21 | */ 22 | 23 | #ifndef _DYN_SIGNAL_H_ 24 | #define _DYN_SIGNAL_H_ 25 | 26 | #include "dyn_types.h" 27 | 28 | /** 29 | * @brief POSIX signal 30 | */ 31 | struct signal { 32 | int signo; 33 | char *signame; 34 | int flags; 35 | void (*handler)(int signo); 36 | }; 37 | 38 | rstatus_t signal_init(void); 39 | void signal_deinit(void); 40 | void signal_handler(int signo); 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /src/hashkit/dyn_hashkit.c: -------------------------------------------------------------------------------- 1 | #include "dyn_hashkit.h" 2 | 3 | #include "../dyn_string.h" 4 | 5 | #define DEFINE_ACTION(_hash, _name) string(#_name), 6 | struct string hash_strings[] = {HASH_CODEC(DEFINE_ACTION) null_string}; 7 | #undef DEFINE_ACTION 8 | 9 | // Defines all the hashing functions 10 | #define DEFINE_ACTION(_hash, _name) \ 11 | rstatus_t hash_##_name(const unsigned char *key, size_t length, \ 12 | struct dyn_token *token); 13 | HASH_CODEC(DEFINE_ACTION) 14 | 15 | #undef DEFINE_ACTION 16 | 17 | // Creates an array of hash functions 18 | #define DEFINE_ACTION(_hash, _name) hash_##_name, 19 | static hash_func_t hash_algos[] = {HASH_CODEC(DEFINE_ACTION) NULL}; 20 | #undef DEFINE_ACTION 21 | 22 | hash_func_t get_hash_func(hash_type_t hash_type) { 23 | if ((hash_type >= 0) && (hash_type < HASH_INVALID)) 24 | return hash_algos[hash_type]; 25 | return NULL; 26 | } 27 | 28 | hash_type_t get_hash_type(struct string *hash_name) { 29 | struct string *hash_iter; 30 | for (hash_iter = hash_strings; hash_iter->len != 0; hash_iter++) { 31 | if (string_compare(hash_name, hash_iter) != 0) { 32 | continue; 33 | } 34 | 35 | return hash_iter - hash_strings; 36 | } 37 | return HASH_INVALID; 38 | } 39 | -------------------------------------------------------------------------------- /src/hashkit/dyn_token.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed 3 | * storages. Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | #ifndef _DYN_TOKEN_H_ 7 | #define _DYN_TOKEN_H_ 8 | 9 | #include "../dyn_types.h" 10 | 11 | // Forward declarations 12 | struct array; 13 | 14 | struct dyn_token { 15 | uint32_t signum; 16 | uint32_t mag[4]; 17 | uint32_t len; 18 | }; 19 | 20 | void init_dyn_token(struct dyn_token *token); 21 | void deinit_dyn_token(struct dyn_token *token); 22 | rstatus_t size_dyn_token(struct dyn_token *token, uint32_t size); 23 | rstatus_t copy_dyn_token(const struct dyn_token *src, struct dyn_token *dst); 24 | 25 | /** 26 | * convenience function for setting a token whose value is just an int 27 | */ 28 | void set_int_dyn_token(struct dyn_token *token, uint32_t val); 29 | 30 | rstatus_t parse_dyn_token(uint8_t *start, uint32_t len, 31 | struct dyn_token *token); 32 | int32_t cmp_dyn_token(const struct dyn_token *t1, const struct dyn_token *t2); 33 | rstatus_t derive_tokens(struct array *tokens, uint8_t *start, uint8_t *end); 34 | rstatus_t derive_token(struct dyn_token *token, uint8_t *start, uint8_t *end); 35 | void print_dyn_token(struct dyn_token *token, int num_tabs); 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /src/dyn_client.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed 3 | * storages. Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | /* 7 | * twemproxy - A fast and lightweight proxy for memcached protocol. 8 | * Copyright (C) 2011 Twitter, Inc. 9 | * 10 | * Licensed under the Apache License, Version 2.0 (the "License"); 11 | * you may not use this file except in compliance with the License. 12 | * You may obtain a copy of the License at 13 | * 14 | * http://www.apache.org/licenses/LICENSE-2.0 15 | * 16 | * Unless required by applicable law or agreed to in writing, software 17 | * distributed under the License is distributed on an "AS IS" BASIS, 18 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | * See the License for the specific language governing permissions and 20 | * limitations under the License. 21 | */ 22 | 23 | #include "dyn_core.h" 24 | 25 | #ifndef _DYN_CLIENT_H_ 26 | #define _DYN_CLIENT_H_ 27 | 28 | void init_client_conn(struct conn *conn); 29 | 30 | rstatus_t req_forward_to_peer(struct context *ctx, struct conn *c_conn, 31 | struct msg *req, struct node *peer, uint8_t* key, uint32_t keylen, 32 | struct mbuf *orig_mbuf, bool force_copy, bool force_swallow, 33 | dyn_error_t *dyn_error_code); 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /scripts/Florida/florida.js: -------------------------------------------------------------------------------- 1 | var http = require('http'); 2 | var url = require('url'); 3 | var fs = require('fs'); 4 | 5 | // Settings 6 | var port = process.env.DYNOMITE_FLORIDA_PORT ? 7 | process.env.DYNOMITE_FLORIDA_PORT : 8080; 8 | 9 | var apiUrl = process.env.DYNOMITE_FLORIDA_REQUEST ? 10 | process.env.DYNOMITE_FLORIDA_REQUEST : '/REST/v1/admin/get_seeds'; 11 | 12 | // Parse command line options 13 | var seedsFilePath = process.argv[2] && process.argv[2].length > 0 ? 14 | process.argv[2] : '/etc/dynomite/seeds.list'; 15 | var enableDebug = process.argv[3] === 'debug' ? true : false; 16 | 17 | http.createServer(function(req, res) { 18 | var path = url.parse(req.url).pathname; 19 | enableDebug && console.log('Request: ' + path); 20 | 21 | res.writeHead(200, {'Content-Type': 'application/json'}); 22 | if (path === apiUrl) { 23 | fs.readFile(seedsFilePath, 'utf-8', function(err, data) { 24 | if (err) console.log(err); 25 | 26 | var now = (new Date()).toJSON(); 27 | var seeds = data.trim().replace(/\n/g, '|'); 28 | 29 | enableDebug && console.log(now + ' - get_seeds [' + seeds + ']'); 30 | res.write(seeds); 31 | res.end(); 32 | }); 33 | } else { 34 | res.end(); 35 | } 36 | }).listen(port); 37 | 38 | console.log('Server is listening on ' + port); 39 | -------------------------------------------------------------------------------- /src/hashkit/dyn_murmur3.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed 3 | * storages. Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | /* 7 | * twemproxy - A fast and lightweight proxy for memcached protocol. 8 | * Copyright (C) 2011 Twitter, Inc. 9 | * 10 | * Licensed under the Apache License, Version 2.0 (the "License"); 11 | * you may not use this file except in compliance with the License. 12 | * You may obtain a copy of the License at 13 | * 14 | * http://www.apache.org/licenses/LICENSE-2.0 15 | * 16 | * Unless required by applicable law or agreed to in writing, software 17 | * distributed under the License is distributed on an "AS IS" BASIS, 18 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | * See the License for the specific language governing permissions and 20 | * limitations under the License. 21 | */ 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | #define MURMUR3_SEED 0xc0a1e5ce 28 | 29 | rstatus_t hash_murmur3(const unsigned char *key, size_t length, 30 | struct dyn_token *token) { 31 | rstatus_t status = size_dyn_token(token, 4); 32 | if (status != DN_OK) { 33 | return status; 34 | } 35 | 36 | // MurmurHash3_x86_128(key, length, MURMUR3_SEED, token->mag); 37 | 38 | return DN_OK; 39 | } 40 | -------------------------------------------------------------------------------- /test/start_cluster.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import argparse 3 | import sys 4 | 5 | from plumbum import local 6 | from time import sleep 7 | 8 | from dyno_cluster import DynoCluster 9 | from utils import generate_ips, setup_temp_dir, sleep_with_animation 10 | 11 | SETTLE_TIME = 3 12 | 13 | def main(): 14 | parser = argparse.ArgumentParser( 15 | description='Autogenerates a Dynomite cluster and runs functional ' + 16 | 'tests against it') 17 | parser.add_argument('request_file', default='test/safe_quorum_request.yaml', 18 | help='YAML file describing desired cluster', nargs='?') 19 | args = parser.parse_args() 20 | 21 | # Setup a temporary directory to store logs and configs for this cluster. 22 | temp = setup_temp_dir() 23 | 24 | # Generate IP addresses to be used by the nodes we will create. 25 | ips = generate_ips() 26 | 27 | # Create a Dynomite cluster. 28 | dynomite_cluster = DynoCluster(args.request_file, ips) 29 | 30 | # Make sure to change the working directory to the temp dir. 31 | with local.cwd(temp): 32 | # Launch the dynomite cluster. 33 | dynomite_cluster.launch() 34 | 35 | # Wait for a while for the above nodes to start. 36 | sleep_with_animation(SETTLE_TIME, "Waiting for cluster to start") 37 | 38 | if __name__ == '__main__': 39 | sys.exit(main()) 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IDE 2 | .idea 3 | CMakeLists.txt 4 | 5 | # Compiled Object files 6 | *.lo 7 | *.o 8 | 9 | # Compiled Dynamic libraries 10 | *.so 11 | 12 | # Compiled Static libraries 13 | *.la 14 | *.a 15 | 16 | # Compiled misc 17 | *.dep 18 | *.gcda 19 | *.gcno 20 | *.gcov 21 | *.pyc 22 | 23 | # Packages 24 | *.tar.gz 25 | *.tar.bz2 26 | 27 | # Logs 28 | *.log 29 | 30 | # Temporary 31 | *.swp 32 | *.~ 33 | *~ 34 | *.project 35 | *.cproject 36 | nbproject/* 37 | TAGS 38 | TODO 39 | 40 | # Core and executable 41 | core* 42 | nutcracker 43 | dynomite 44 | dynomite-test 45 | dyno-hash-tool 46 | 47 | # extracted yaml 48 | !/contrib/yaml-0.1.4.tar.gz 49 | 50 | # Autotools 51 | .deps 52 | .libs 53 | 54 | /aclocal.m4 55 | /autom4te.cache 56 | /stamp-h1 57 | /autoscan.log 58 | /libtool 59 | 60 | /config/config.guess 61 | /config/config.sub 62 | /config/depcomp 63 | /config/install-sh 64 | /config/ltmain.sh 65 | /config/missing 66 | /config 67 | 68 | /config.h 69 | /config.h.in 70 | /config.h.in~ 71 | /config.log 72 | /config.status 73 | /configure.scan 74 | /configure 75 | 76 | Makefile 77 | Makefile.in 78 | /Debug 79 | /src/.dirstamp 80 | /src/test 81 | /src/tools 82 | /src/dynomite-test 83 | cscope.out 84 | /src/cscope.out 85 | /src/.dirstamp 86 | /src/tools/dyn_hash_tool 87 | /src/tools/dynomite-hash-tool 88 | /test/*.pyc 89 | /test/_binaries/* 90 | /test/_binaries 91 | /test/conf 92 | -------------------------------------------------------------------------------- /scripts/dynomite-manager/dynomite-manager: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # chkconfig: 2345 95 20 3 | # description: This script does some stuff 4 | # processname: java 5 | 6 | export JAVA_HOME=/home/ec2-user/jdk1.8.0_45 7 | export JRE_HOME=/home/ec2-user/jdk1.8.0_45/jre 8 | export PATH=$PATH:/home/ec2-user/jdk1.8.0_45/bin:/home/ec2-user/jdk1.8.0_45/jre/bin 9 | 10 | export ASG_NAME="asg_dynomite" 11 | export EC2_REGION="us-west-2" 12 | export AUTO_SCALE_GROUP="asg_dynomite" 13 | 14 | start() { 15 | echo "Starting Dynomite Manager..." 16 | cd /home/ec2-user/dynomite-manager/dynomite-manager/ 17 | /home/ec2-user/dynomite-manager/dynomite-manager/gradlew jettyRun > /logs/system/dynomite-manager/dynomite-manager.log & 18 | } 19 | 20 | stop() { 21 | echo "stoping Dynomite Manager... " 22 | PID=`ps -ef | grep gradlew | awk '{print $2}' ORS=' ' | awk '{print $1}'` 23 | if [[ "" != "$PID" ]]; then 24 | echo "killing $PID" 25 | sudo kill -9 $PID 26 | fi 27 | } 28 | 29 | debug() { 30 | echo "Starting Dynomite Manager for DEBUG..." 31 | cd /home/ec2-user/dynomite-manager/dynomite-manager/ 32 | export GRADLE_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=n" 33 | ./gradlew jettyRun & 34 | } 35 | 36 | 37 | case "$1" in 38 | "start") 39 | start 40 | ;; 41 | "debug") 42 | debug 43 | ;; 44 | "stop") 45 | stop 46 | ;; 47 | *) 48 | 49 | echo $"Usage: $0 {start|stop|debug}" 50 | RETVAL=1 51 | esac 52 | exit 0 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /src/dyn_ring_queue.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed 3 | * storages. Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | #include "dyn_gossip.h" 7 | 8 | #ifndef _DYN_RING_QUEUE_ 9 | #define _DYN_RING_QUEUE_ 10 | 11 | #define C2G_InQ_SIZE 256 12 | #define C2G_OutQ_SIZE 256 13 | 14 | struct gossip_node; 15 | 16 | typedef rstatus_t (*callback_t)(void *msg); 17 | typedef void (*data_func_t)(void *); 18 | 19 | volatile struct { 20 | long m_getIdx; 21 | long m_putIdx; 22 | void *m_entry[C2G_InQ_SIZE]; 23 | } C2G_InQ; 24 | 25 | volatile struct { 26 | long m_getIdx; 27 | long m_putIdx; 28 | void *m_entry[C2G_OutQ_SIZE]; 29 | } C2G_OutQ; 30 | 31 | struct ring_msg { 32 | callback_t cb; 33 | uint8_t *data; /* place holder for a msg */ 34 | uint32_t capacity; /* max capacity */ 35 | uint32_t len; /* # of useful bytes in data (len =< capacity) */ 36 | struct array nodes; 37 | struct server_pool *sp; 38 | }; 39 | 40 | struct ring_msg *create_ring_msg(void); 41 | struct ring_msg *create_ring_msg_with_data(uint32_t capacity); 42 | struct ring_msg *create_ring_msg_with_size(uint32_t size, bool init_node); 43 | rstatus_t ring_msg_init(struct ring_msg *msg, uint32_t size, bool init_node); 44 | rstatus_t ring_msg_deinit(struct ring_msg *msg); 45 | 46 | struct gossip_node *create_node(void); 47 | rstatus_t node_init(struct gossip_node *node); 48 | rstatus_t node_deinit(struct gossip_node *node); 49 | rstatus_t node_copy(const struct gossip_node *src, struct gossip_node *dst); 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /init/README.md: -------------------------------------------------------------------------------- 1 | # README 2 | 3 | The `init` directory is the location for all System V init, Upstart, systemd, etc. initialization scripts. 4 | 5 | ## systemd on RHEL 7 6 | 7 | Create the `dynomite` user. 8 | 9 | ```bash 10 | mkdir -p /usr/share/dynomite 11 | 12 | mkdir /var/run/dynomite 13 | 14 | useradd -r -M -c "Dynomite server" -s /sbin/nologin -d /usr/share/dynomite dynomite 15 | 16 | chown -R dynomite:dynomite /usr/share/dynomite 17 | 18 | chown -R dynomite:dynomite /var/run/dynomite 19 | ``` 20 | 21 | Install the Dynomite service file and the associated sysconfig file. 22 | 23 | ```bash 24 | cp init/systemd_environment__dynomite /etc/sysconfig/dynomite 25 | 26 | cp init/systemd_service_rhel__dynomite.service /usr/lib/systemd/system/dynomite.service 27 | 28 | systemctl daemon-reload 29 | 30 | systemctl enable dynomite 31 | 32 | systemctl status dynomite 33 | ``` 34 | 35 | ## systemd on Ubuntu 15.10 36 | 37 | Create the `dynomite` user. 38 | 39 | ```bash 40 | mkdir -p /usr/share/dynomite 41 | 42 | mkdir /var/run/dynomite 43 | 44 | useradd -r -M -c "Dynomite server" -s /sbin/nologin -d /usr/share/dynomite dynomite 45 | 46 | chown -R dynomite:dynomite /usr/share/dynomite 47 | 48 | chown -R dynomite:dynomite /var/run/dynomite 49 | ``` 50 | 51 | Install the Dynomite service file and the associated sysconfig file. 52 | 53 | ```bash 54 | cp init/systemd_environment__dynomite /etc/default/dynomite 55 | 56 | cp init/systemd_service_ubuntu__dynomite.service /lib/systemd/system/dynomite.service 57 | 58 | systemctl daemon-reload 59 | 60 | systemctl enable dynomite 61 | 62 | systemctl status dynomite 63 | ``` 64 | -------------------------------------------------------------------------------- /docs/dyn_protocol.txt: -------------------------------------------------------------------------------- 1 | Dyn message Protocol: 2 | 3 | I. Specification: 4 | 5 | 1. Structure format 6 | 7 | 8 | 9 | 10 | 11 | 12 | * 13 | 14 | * 15 | 16 | 17 | 2. Field descriptions 18 | a. MagicNumber: it is always 2014 19 | b. MessageId: sequence id of a message 20 | c. TypeId: please see the code for the list. 21 | d. BitField: indicate whether compression and/or encryption. Right most bit indicates encryption. 22 | The 2nd bit from the right indicates a compressed message. 23 | e. Version: version of the msg. This usually depends on the version of the dynomite. 24 | This is needed during upgrading a live cluster with mixed versions. 25 | f. Same-dc: bit to indicate whether the communication is inter-dcs. 26 | We should later to exchange node's metadata like rack, dc, and ip. 27 | 28 | 3. Example: 29 | This is a dynomite message that carries a Redis' "get foo" request: 30 | 31 | "2014 1344 5 1 1 0\r\n*4 minh\r\n*2\r\n$3\r\nget\r\n$3\r\nfoo\r\n" 32 | 33 | where "*2\r\n$3\r\nget\r\n$3\r\nfoo\r\n" is Redis' payload for "get foo" cmd. 34 | 35 | 36 | II. Bit field for encryption/decryption 37 | 38 | 39 | III. Compression 40 | We currently don't support compression but it should be similar to the way how 41 | encryption/decryption works. We will support/implement this feature in the near future 42 | if there is a demand for it. 43 | 44 | IV. Payload size 45 | Will fill in soon 46 | 47 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Dynomite 2 | 3 | Documentation for Dynomite is available in the following locations: 4 | - Dynomite docs 5 | - [wiki]( https://github.com/Netflix/dynomite/wiki ) 6 | 7 | The Dynomite team is following the Gitflow workflow. The active development branch is [dev](https://github.com/Netflix/dynomite/tree/dev), the stable branch is [master](https://github.com/Netflix/dynomite/tree/master). 8 | 9 | Contributions will be accepted to the [dev](https://github.com/Netflix/dynomite/tree/dev) only. 10 | 11 | 12 | ## How to provide a patch for a new feature 13 | 14 | 1. If it is a major feature, please create an [Issue]( https://github.com/Netflix/dynomite/issues ) and discuss with the project leaders. 15 | 16 | 2. If in step 1 you get an acknowledge from the project leaders, use the 17 | following procedure to submit a patch: 18 | 19 | a. Fork Dynomite on github ( http://help.github.com/fork-a-repo/ ) 20 | 21 | b. Create a topic branch (git checkout -b my_branch) 22 | 23 | c. Push to your branch (git push origin my_branch) 24 | 25 | d. Initiate a pull request on github ( http://help.github.com/en/articles/creating-a-pull-request/ ) 26 | 27 | e. Done :) 28 | 29 | For minor fixes just open a pull request to the [dev]( https://github.com/Netflix/dynomite/tree/dev ) branch on Github. 30 | 31 | ## Questions 32 | 33 | If you have questions or want to report a bug please create an [Issue]( https://github.com/Netflix/dynomite/issues ) or chat with us on [![Dev chat at https://gitter.im/Netflix/dynomite](https://badges.gitter.im/Netflix/dynomite.svg)](https://gitter.im/Netflix/dynomite?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 34 | 35 | 36 | -------------------------------------------------------------------------------- /src/dyn_dnode_peer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed 3 | * storages. Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | #ifndef _DYN_DNODE_PEER_H_ 7 | #define _DYN_DNODE_PEER_H_ 8 | 9 | #include "dyn_message.h" 10 | #include "dyn_types.h" 11 | 12 | #define MAX_WAIT_BEFORE_RECONNECT_IN_SECS 10 13 | #define WAIT_BEFORE_UPDATE_PEERS_IN_MILLIS 30000 14 | 15 | // Forward declarations 16 | struct context; 17 | struct msg; 18 | struct rack; 19 | 20 | msec_t dnode_peer_timeout(struct msg *msg, struct conn *conn); 21 | rstatus_t dnode_initialize_peers(struct context *ctx); 22 | void dnode_peer_deinit(struct array *nodes); 23 | void dnode_peer_connected(struct context *ctx, struct conn *conn); 24 | 25 | struct node *dnode_peer_pool_server(struct context *ctx, 26 | struct server_pool *pool, struct rack *rack, 27 | uint8_t *key, uint32_t keylen, 28 | msg_routing_t msg_routing); 29 | struct conn *dnode_peer_get_conn(struct context *ctx, struct node *server, 30 | int tag); 31 | rstatus_t dnode_peer_pool_preconnect(struct context *ctx); 32 | void dnode_peer_pool_disconnect(struct context *ctx); 33 | uint32_t dnode_peer_idx_for_key_on_rack(struct server_pool *pool, 34 | struct rack *rack, uint8_t *key, 35 | uint32_t keylen); 36 | rstatus_t dnode_peer_forward_state(void *rmsg); 37 | rstatus_t dnode_peer_add(void *rmsg); 38 | rstatus_t dnode_peer_replace(void *rmsg); 39 | rstatus_t dnode_peer_handshake_announcing(void *rmsg); 40 | 41 | void init_dnode_peer_conn(struct conn *conn); 42 | void preselect_remote_rack_for_replication(struct context *ctx); 43 | #endif 44 | -------------------------------------------------------------------------------- /scripts/redis/dyno_redis_bgrewriteaof.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # The following script checks runs Redis background rewrite AOF. 3 | 4 | declare -i RESULT 5 | 6 | # handy logging and error handling functions 7 | function log() { printf '%s\n' "$*"; } 8 | 9 | # Start by checking if Redis is running 10 | REDIS_UP=`redis-cli -p 22122 ping | grep -c PONG` 11 | if [[ ${REDIS_UP} -ne 1 ]]; then 12 | ((RESULT++)) 13 | log "ERROR: Redis is not running" 14 | quit $RESULT 15 | fi 16 | 17 | log "OK: Redis BGREWRITEAOF starting" 18 | 19 | # check if BGREWRITEAOF can be competed successfully. 20 | REWRITEAOF=`redis-cli -p 22122 BGREWRITEAOF | grep ERR | wc -l` 21 | if (( $REWRITEAOF != 0)); then 22 | ((RESULT++)) 23 | log "ERROR: Redis BGREWRITEAOF failed" 24 | quit $RESULT 25 | fi 26 | 27 | SLEEPING=2 28 | log "OK: sleeping initial $SLEEPING seconds post bgrewriteaof" 29 | sleep $SLEEPING 30 | 31 | # If bgrewriteaof is still running, we iterate inside a loop that waits for bg_rewrite_aof to finish. 32 | # Exponential backoff adds 5 seconds to the sleeping time until the value aof_rewrite_in_progress is zero. 33 | # If the sleep takes too long (1800 seconds = 30 min), the process quits. 34 | REDIS_AOF_REWRITE_IN_PROGRESS=`redis-cli -p 22122 INFO | grep aof_rewrite_in_progress | awk -F ':' '{printf "%d\n",$2}'` 35 | while [[ ${REDIS_AOF_REWRITE_IN_PROGRESS} -gt 0 ]]; do 36 | sleep $SLEEPING 37 | log "OK: sleeping $SLEEPING because BGREWRITEAOF is pending" 38 | REDIS_AOF_REWRITE_IN_PROGRESS=`redis-cli -p 22122 INFO | grep aof_rewrite_in_progress | awk -F ':' '{printf "%d\n",$2}'` 39 | let SLEEPING=2*SLEEPING 40 | if [[ ${SLEEPING} -ge 1800 ]]; then 41 | log "ERROR: Redis BGREWRITEAOF takes more than 1800 seconds" 42 | ((RESULT++)) 43 | quit $RESULT 44 | fi 45 | done 46 | log "===================================================" 47 | -------------------------------------------------------------------------------- /src/hashkit/dyn_one_at_a_time.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed 3 | * storages. Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | /* 7 | * twemproxy - A fast and lightweight proxy for memcached protocol. 8 | * Copyright (C) 2011 Twitter, Inc. 9 | * 10 | * Licensed under the Apache License, Version 2.0 (the "License"); 11 | * you may not use this file except in compliance with the License. 12 | * You may obtain a copy of the License at 13 | * 14 | * http://www.apache.org/licenses/LICENSE-2.0 15 | * 16 | * Unless required by applicable law or agreed to in writing, software 17 | * distributed under the License is distributed on an "AS IS" BASIS, 18 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | * See the License for the specific language governing permissions and 20 | * limitations under the License. 21 | */ 22 | 23 | /* 24 | * HashKit 25 | * Copyright (C) 2009 Brian Aker 26 | * All rights reserved. 27 | * 28 | * Use and distribution licensed under the BSD license. See 29 | * the COPYING file in the parent directory for full text. 30 | */ 31 | 32 | /* 33 | * This has is Jenkin's "One at A time Hash". 34 | * http://en.wikipedia.org/wiki/Jenkins_hash_function 35 | */ 36 | 37 | #include 38 | #include 39 | 40 | rstatus_t hash_one_at_a_time(const unsigned char *key, size_t key_length, 41 | struct dyn_token *token) { 42 | const char *ptr = key; 43 | uint32_t value = 0; 44 | 45 | while (key_length--) { 46 | uint32_t val = (uint32_t)*ptr++; 47 | value += val; 48 | value += (value << 10); 49 | value ^= (value >> 6); 50 | } 51 | value += (value << 3); 52 | value ^= (value >> 11); 53 | value += (value << 15); 54 | 55 | size_dyn_token(token, 1); 56 | set_int_dyn_token(token, value); 57 | 58 | return DN_OK; 59 | } 60 | -------------------------------------------------------------------------------- /src/dyn_crypto.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed 3 | * storages. Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | #ifndef DYN_CRYPTO_H_ 7 | #define DYN_CRYPTO_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | #include 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "dyn_types.h" 23 | 24 | #define AES_KEYLEN 32 25 | 26 | // Forward declarations 27 | struct mbuf; 28 | struct msg; 29 | struct server_pool; 30 | 31 | rstatus_t crypto_init(struct server_pool *sp); 32 | rstatus_t crypto_deinit(void); 33 | 34 | char *base64_encode(const unsigned char *message, const size_t length); 35 | 36 | rstatus_t aes_encrypt(const unsigned char *msg, size_t msgLen, 37 | unsigned char **encMsg, unsigned char *aes_key); 38 | rstatus_t aes_decrypt(unsigned char *encMsg, size_t encMsgLen, 39 | unsigned char **decMsg, unsigned char *aes_key); 40 | 41 | rstatus_t dyn_aes_encrypt(const unsigned char *msg, size_t msgLen, 42 | struct mbuf *mbuf, unsigned char *aes_key); 43 | 44 | rstatus_t dyn_aes_decrypt(unsigned char *encMsg, size_t encMsgLen, 45 | struct mbuf *mbuf, unsigned char *aes_key); 46 | 47 | rstatus_t dyn_aes_encrypt_msg(struct msg *msg, unsigned char *aes_key, 48 | size_t *outlen); 49 | unsigned char *generate_aes_key(void); 50 | 51 | int dyn_rsa_size(void); 52 | 53 | rstatus_t dyn_rsa_encrypt(unsigned char *plain_msg, 54 | unsigned char *encrypted_buf); 55 | 56 | rstatus_t dyn_rsa_decrypt(unsigned char *encrypted_msg, 57 | unsigned char *decrypted_buf); 58 | 59 | #endif /* DYN_CRYPTO_H_ */ 60 | -------------------------------------------------------------------------------- /contrib/fmemopen.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2011-2014 NimbusKit 3 | // Originally ported from https://github.com/ingenuitas/python-tesseract/blob/master/fmemopen.c 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | // 17 | 18 | #ifndef FMEMOPEN_H_ 19 | #define FMEMOPEN_H_ 20 | 21 | #if defined __cplusplus 22 | extern "C" { 23 | #endif 24 | 25 | /** 26 | * A BSD port of the fmemopen Linux method using funopen. 27 | * 28 | * man docs for fmemopen: 29 | * http://linux.die.net/man/3/fmemopen 30 | * 31 | * man docs for funopen: 32 | * https://developer.apple.com/library/mac/#documentation/Darwin/Reference/ManPages/man3/funopen.3.html 33 | * 34 | * This method is ported from ingenuitas' python-tesseract project. 35 | * 36 | * You must call fclose on the returned file pointer or memory will be leaked. 37 | * 38 | * @param buf The data that will be used to back the FILE* methods. Must be at least 39 | * @c size bytes. 40 | * @param size The size of the @c buf data. 41 | * @param mode The permitted stream operation modes. 42 | * @return A pointer that can be used in the fread/fwrite/fseek/fclose family of methods. 43 | * If a failure occurred NULL will be returned. 44 | * @ingroup NimbusMemoryMappping 45 | */ 46 | FILE *fmemopen(void *buf, size_t size, const char *mode); 47 | 48 | #ifdef __cplusplus 49 | } 50 | #endif 51 | 52 | #endif // #ifndef FMEMOPEN_H_ 53 | -------------------------------------------------------------------------------- /src/dyn_response_mgr.h: -------------------------------------------------------------------------------- 1 | #ifndef _DYN_RESPONSE_MGR_H_ 2 | #define _DYN_RESPONSE_MGR_H_ 3 | 4 | #include "dyn_string.h" 5 | 6 | #define MAX_REPLICAS_PER_DC 3 7 | struct response_mgr { 8 | bool is_read; 9 | bool done; 10 | /* we could use the dynamic array 11 | here. But we have only 3 ASGs */ 12 | struct msg *responses[MAX_REPLICAS_PER_DC]; 13 | uint32_t checksums[MAX_REPLICAS_PER_DC]; 14 | // Number of non-error responses received. (nil) is not an error. 15 | uint8_t good_responses; 16 | // Maximum number of responses possible. 17 | uint8_t max_responses; 18 | // Number of responses required to form a quorum. 19 | uint8_t quorum_responses; 20 | // Number of error responses received. 21 | uint8_t error_responses; 22 | // First error response 23 | struct msg *err_rsp; 24 | struct conn *conn; 25 | // Corresponding request 26 | struct msg *msg; 27 | // The DC that this response manager is responsible for. 28 | struct string dc_name; 29 | }; 30 | 31 | rstatus_t init_response_mgr_all_dcs(struct context *ctx, struct msg *req, 32 | struct conn *c_conn, struct datacenter *local_dc); 33 | 34 | void init_response_mgr(struct msg *req, struct response_mgr *rspmgr, 35 | uint8_t max_responses_for_dc, struct conn *c_conn); 36 | 37 | // DN_OK if response was accepted 38 | rstatus_t rspmgr_submit_response(struct response_mgr *rspmgr, struct msg *rsp); 39 | bool rspmgr_check_is_done(struct response_mgr *rspmgr); 40 | struct msg *rspmgr_get_response(struct context *ctx, struct response_mgr *rspmgr); 41 | void rspmgr_free_response(struct response_mgr *rspmgr, struct msg *dont_free); 42 | void rspmgr_free_other_responses(struct response_mgr *rspmgr, 43 | struct msg *dont_free); 44 | rstatus_t msg_local_one_rsp_handler(struct context *ctx, struct msg *req, struct msg *rsp); 45 | rstatus_t rspmgr_clone_responses(struct response_mgr *src, 46 | struct array *responses); 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /src/dyn_rbtree.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dynomite - A thin, distributed replication layer for multi non-distributed 3 | * storages. Copyright (C) 2014 Netflix, Inc. 4 | */ 5 | 6 | /* 7 | * twemproxy - A fast and lightweight proxy for memcached protocol. 8 | * Copyright (C) 2011 Twitter, Inc. 9 | * 10 | * Licensed under the Apache License, Version 2.0 (the "License"); 11 | * you may not use this file except in compliance with the License. 12 | * You may obtain a copy of the License at 13 | * 14 | * http://www.apache.org/licenses/LICENSE-2.0 15 | * 16 | * Unless required by applicable law or agreed to in writing, software 17 | * distributed under the License is distributed on an "AS IS" BASIS, 18 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | * See the License for the specific language governing permissions and 20 | * limitations under the License. 21 | */ 22 | 23 | #ifndef _DYN_RBTREE_ 24 | #define _DYN_RBTREE_ 25 | #include "dyn_types.h" 26 | 27 | #define rbtree_red(_node) ((_node)->color = 1) 28 | #define rbtree_black(_node) ((_node)->color = 0) 29 | #define rbtree_is_red(_node) ((_node)->color) 30 | #define rbtree_is_black(_node) (!rbtree_is_red(_node)) 31 | #define rbtree_copy_color(_n1, _n2) ((_n1)->color = (_n2)->color) 32 | 33 | struct rbnode { 34 | struct rbnode *left; /* left link */ 35 | struct rbnode *right; /* right link */ 36 | struct rbnode *parent; /* parent link */ 37 | msec_t key; /* key for ordering */ 38 | msec_t timeout; /* timeout */ 39 | void *data; /* opaque data */ 40 | uint8_t color; /* red | black */ 41 | }; 42 | 43 | struct rbtree { 44 | struct rbnode *root; /* root node */ 45 | struct rbnode *sentinel; /* nil node */ 46 | }; 47 | 48 | void rbtree_node_init(struct rbnode *node); 49 | void rbtree_init(struct rbtree *tree, struct rbnode *node); 50 | struct rbnode *rbtree_min(struct rbtree *tree); 51 | void rbtree_insert(struct rbtree *tree, struct rbnode *node); 52 | void rbtree_delete(struct rbtree *tree, struct rbnode *node); 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /scripts/dynomite/generate_yamls.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ''' 4 | script for generating dynomite yaml files for every node in a cluster. 5 | This script should be run per rack for all nodes in the rack and so the tokens are equally distributed. 6 | usage: