├── .gitignore ├── CHANGELOG ├── LICENSE ├── Makefile ├── README.md ├── build_cluster.py ├── conf ├── node0.conf ├── node1.conf └── node2.conf ├── src ├── cluster.c ├── cluster.h ├── commands.c ├── commands.h ├── event.c ├── event.h ├── hashing.h ├── list.c ├── list.h ├── map.c ├── map.h ├── memento-benchmark.c ├── memento-cli.c ├── memento.c ├── networking.c ├── networking.h ├── persistence.c ├── persistence.h ├── serializer.c ├── serializer.h ├── util.c └── util.h └── tests ├── Makefile ├── dt_test.c └── unit.h /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | __pycache__ 3 | *.pyc 4 | *.d 5 | *.o 6 | *.ko 7 | *.obj 8 | *.elf 9 | *.ghc 10 | *.pch 11 | *.lib 12 | *.a 13 | *.la 14 | *.lo 15 | *.out 16 | *.hex 17 | *.dSYM/ 18 | *.su 19 | tags 20 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | * CHANGELOG 2 | ========================================= 3 | 4 | 2017-12-25 5 | 6 | - Fixed a bug that prevented cluster forming correctly and addressing of keys toward 7 | external nodes 8 | 9 | 2017-08-10 10 | 11 | - Fixed some problems regarding EPOLLET with a threadpool, using EPOLLONESHOT to 12 | manually rearm socket descriptor at every event, at the cost of an additional 13 | system call 14 | - Refactored command handling, trying to decouple a bit more modules 15 | 16 | 2017-02-11 17 | 18 | - Fixed some bugs 19 | - Added CLUSTERINFO commands, retrieve some informations regarding cluster state 20 | 21 | 2017-02-10 22 | 23 | - Almost complete refactoring of the code 24 | - Better organization, removed some useless commands 25 | - Rethink of the distribution part of the system 26 | - Rename of the project 27 | 28 | 2017-01-19 29 | 30 | - Rough support to distribution across a cluster of machines 31 | - Added murmur hashing algorithm to the hashing header 32 | 33 | 2016-09-06 34 | 35 | - Added new commands INCF and DECF, respectively increment/decrement by float quantity 36 | the float value represented by key specified 37 | - Added new command TTL, returning the time to live of a specified key 38 | - Minor fixes 39 | 40 | 2016-09-05 41 | 42 | - Added new command EXPIRE, set a timer in milliseconds to a key after that the key and his value 43 | is wiped 44 | - Added new comand GETP, return infos of a key-value pair 45 | - Improved shibui-cli, printing help and correctly handling SUB and TAIL command 46 | 47 | 2016-08-31 48 | 49 | - Started shibui-cli, supports basic commands without any human readability 50 | improvement 51 | 52 | 2016-08-30 53 | 54 | - Corrected some bug, added new commands `APPEND key value` and `PREPEND key value` 55 | 56 | 2016-08-29 57 | 58 | - Altered some commands, now DEL, SUB and UNSUB accept a list of keys, INC and 59 | DEC accept an optional value to be added to the current integer value of the 60 | key 61 | 62 | 2016-08-26 63 | 64 | - Added new command `FUZZYSCAN pattern`, perform a search through the keyspace 65 | returning all values associated to keys that match a given pattern by fuzzy 66 | search. 67 | 68 | - Added new command `UNSUB key` to stop receiving feeds from the key. 69 | 70 | - Added new command `FLUSH' to delete all keys in the keyspace 71 | 72 | - Fixed rehash_map code in order to update data history and the subscribers 73 | array as well when resizing the map. 74 | 75 | - Added new commands `KEYS` and `VALUES` respectively return all keys and all 76 | values stored in the hashmap 77 | 78 | - Added partitioning to keyspace distribution, this should simplify an aventual 79 | distribution across multiple nodes. 80 | 81 | 2016-08-25 82 | 83 | - Added new command `TAIL key number`, consume all values published to a key and 84 | subscribe to it. Number is the offset of the depletion, with 0 meaning "from 85 | the very first" and incresing number discard subsequents messages. 86 | 87 | - Fixed some bugs e refactored some code. 88 | 89 | - Added new command `PREFSCAN key_prefix`, find all values associated to keys that 90 | matches the prefix specified by key_prefix. 91 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016-2017 Andrea Giacomo Baldan 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so, 8 | subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CC=gcc 2 | CFLAGS=-std=gnu99 -Wall -lrt -lpthread -O3 -pedantic 3 | BIN=./bin 4 | SRC=src/map.c \ 5 | src/util.c \ 6 | src/commands.c \ 7 | src/persistence.c \ 8 | src/networking.c \ 9 | src/serializer.c \ 10 | src/hashing.h \ 11 | src/cluster.c \ 12 | src/event.c \ 13 | src/list.c 14 | 15 | memento: $(SRC) 16 | mkdir -p $(BIN) && $(CC) $(CFLAGS) $(SRC) src/memento.c -o $(BIN)/memento 17 | 18 | memento-cli: src/memento-cli.c 19 | mkdir -p $(BIN) && $(CC) $(CFLAGS) $(SRC) src/memento-cli.c -o $(BIN)/memento-cli 20 | 21 | memento-benchmark: src/memento-benchmark.c 22 | mkdir -p $(BIN) && $(CC) $(CFLAGS) $(SRC) src/memento-benchmark.c -o $(BIN)/memento-benchmark 23 | 24 | test: 25 | cd tests && $(MAKE) test 26 | 27 | clean: 28 | rm -f $(BIN)/memento $(BIN)/dt_test $(BIN)/memento-benchmark 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # memento 3 | 4 | Fairly simple hashmap implementation built on top of an epoll TCP server. A toy 5 | project just for learning purpose, fascinated by the topic, i decided to try to 6 | make a simpler version of redis/memcached to better understand how it works. 7 | The project is a minimal Redis-like implementation with a text based protocol, 8 | and like Redis, can be used as key-value in-memory store. 9 | 10 | 11 | Currently supports some basic operations, all commands are case insensitive, so 12 | not strictly uppercase. 13 | 14 | | Command | Args | Description | 15 | |---------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------- | 16 | | **SET** | `` `` | Sets `` to `` | 17 | | **GET** | `` | Get the value identified by `` | 18 | | **DEL** | `` ` .. ` | Delete values identified by ``..`` | 19 | | **INC** | `` `` | Increment by `` the value idenfied by ``, if no `` is specified increment by 1 | 20 | | **DEC** | `` `` | Decrement by `` the value idenfied by ``, if no `` is specified decrement by 1 | 21 | | **INCF** | `` `` | Increment by float `` the value identified by ``, if no `` is specified increment by 1.0 | 22 | | **DECF** | `` `` | Decrement by `` the value identified by ``, if no `` is specified decrement by 1.0 | 23 | | **GETP** | `` | Get all information of a key-value pair represented by ``, like key, value, creation time and expire time| 24 | | **APPEND** | `` `` | Append `` to `` | 25 | | **PREPEND** | `` `` | Prepend `` to `` | 26 | | **FLUSH** | | Delete all maps stored inside partitions | 27 | | **QUIT/EXIT** | | Close connection | 28 | 29 | 30 | ### Distribution 31 | 32 | Still in development and certainly bugged, it currently supports some commands 33 | in a distributed context across a cluster of machines. There isn't a replica 34 | system yet, planning to add it soon, currently the database can be started in 35 | cluster mode with the `-c` flag and by creating a configuration file on each 36 | node. As default behaviour, memento search for a `~/.memento` file in `$HOME`, 37 | by the way it is also possible to define a different path. 38 | 39 | The configuration file shall strictly follow some rules, a sample could be: 40 | 41 | ``` 42 | # configuration file 43 | 127.0.0.1 8181 node1 0 44 | 127.0.0.1 8182 node2 1 45 | 127.0.0.1 8183 node3 0 46 | ``` 47 | 48 | every line define a node, with an IP address, a port, his id-name and a flag to 49 | define if the `IP-PORT-ID` refers to self node (ie the machine where the file 50 | reside). `PORT` value refer to the actual listen port of the instance plus 100. 51 | So 8082 means a bus port 8182. 52 | 53 | It is possible to generate basic configurations using the helper script 54 | `build_cluster.py`, it accepts just the address and port of every node, so to 55 | create 3 configuration file as the previous example just run (all on the same 56 | machine, but can be easily used to generate configurations for different nodes 57 | in a LAN): 58 | 59 | $ python build_cluster.py 127.0.0.1:8081 127.0.0.1:8082 127.0.0.1:8083 60 | 61 | This instruction will generate files `node0.conf`, `node1.conf` and 62 | `node2.conf` containing each the right configuration. It can be renamed to 63 | `.memento` and dropped in the `$HOME` path of every node, or can be passed to 64 | the program as argument using `-f` option. 65 | Every node of the cluster, will try to connect to other, if all goes well, 66 | every node should print a log message like this: 67 | 68 | $ [INF][YYYY-mm-dd hh-mm-ss] - Cluster successfully formed 69 | 70 | From now on the system is ready to receive connections and commands. 71 | 72 | The distribution of keys and values follows a classic hashing addressing in a 73 | keyspace divided in equal buckets through the cluster. The total number of 74 | slots is 8192, so if the cluster is formed of 2 nodes, every node will get 75 | at most 4096 keys. 76 | 77 | ### Build 78 | 79 | To build the source just run `make`. A `memento` executable will be generated into 80 | a `bin` directory that can be started to listen on a defined `hostname`, 81 | ready to receive commands from any TCP client 82 | 83 | $ ./bin/memento -a -p 84 | 85 | `-c` for a cluster mode start 86 | 87 | $ ./bin/memento -a -p -c 88 | 89 | Parameter `` and `` fallback to 127.0.0.1 and 6737, memento accept also 90 | a -f parameter, it allows to specify a configuration file for cluster context. 91 | 92 | $ ./bin/memento -a -p -c -f ./conf/node0.conf 93 | 94 | The name of the node can be overridden with the `-i` option 95 | 96 | $ ./bin/memento -a -p -c -f -i 97 | 98 | It is also possible to stress-test the application by using `memento-benchmark`, previously 99 | generating it with `make memento-benchmark` command 100 | 101 | $ ./bin/memento-benchmark 102 | 103 | To build memento-cli just `make memento-cli` and run it like the following: 104 | 105 | $ ./bin/memento-cli 106 | 107 | ### Experimental 108 | 109 | The system is already blazing fast, testing it in a simple manner give already 110 | good results. Using a Linux box with an Intel(R) Core(TM) i5-4210U CPU @ 111 | 1.70Ghz and 4 GB ram I get 112 | 113 | ``` 114 | Nr. operations 100000 115 | 116 | 1) SET keyone valueone 117 | 2) GET keyone 118 | 3) SET with growing keys 119 | 4) All previous operations in 10 threads, each one performing 8000 requests per type 120 | 121 | [SET] - Elapsed time: 0.871047 s Op/s: 114804.37 122 | [GET] - Elapsed time: 0.889411 s Op/s: 112433.95 123 | [SET] - Elapsed time: 0.923016 s Op/s: 108340.48 124 | 125 | ``` 126 | 127 | #### Performance improvements 128 | 129 | Still experimenting different paths, the main bottleneck is the server 130 | interface where clients are served; initially I rewrote the network interface 131 | as a simple epoll main thread that handle EPOLLIN and EPOLLOUT events, using a 132 | thread-safe queue to dispatch incoming data on descriptor to a pool of reader 133 | threads. After being processed, their responsibility was to set the descriptor 134 | to EPOLLOUT, in this case the main thread enqueued the descriptor to a write 135 | queue, and a pool of writer threads constantly polling it, had the 136 | responsibility to send out data. 137 | 138 | Later I decided to adopt a model more nginx oriented, using a pool of threads 139 | to spin their own event loop, and using the main thread only for accepting 140 | incoming connection, demanding them to worker reader threads, this time, 141 | allowing the main thread to also handle internal communication between other 142 | nodes, semplifying a bit the process. Still a pool of writer threads is used to 143 | send out data to clients, this approach seems better, but it is still in 144 | development and testing. 145 | 146 | ### Changelog 147 | 148 | See the [CHANGELOG](CHANGELOG) file. 149 | 150 | ### TODO 151 | 152 | - [ ] Implementation of a real consistent-hashing ring 153 | - [ ] Runtime nodes joining and rebalance of the cluster 154 | - [ ] Implementation of a basic gossip protocol to spread cluster state 155 | - [ ] Implementation of redundancy (e.g. slave nodes) 156 | 157 | 158 | ### License 159 | 160 | See the [LICENSE](LICENSE) file for license rights and limitations (MIT). 161 | -------------------------------------------------------------------------------- /build_cluster.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Author: Andrea Giacomo Baldan 5 | # License: MIT 6 | # Description: Create cluster configurations for a defined number of nodes 7 | 8 | from glob import glob 9 | import os 10 | import sys 11 | import uuid 12 | 13 | 14 | def main(): 15 | """ 16 | Main access point, parse arguments and create configuration files needed 17 | to form a cluster 18 | """ 19 | 20 | # remove any conf file to avoid inconsistencies 21 | for filename in glob("*.conf"): 22 | os.remove(filename) 23 | 24 | # generate random uuids for each node 25 | uuids = {} 26 | for x in sys.argv[1:]: 27 | uid = uuid.uuid4() 28 | uuids[uid.hex] = x 29 | 30 | # write ip address, port and names to each conf file 31 | for idx, key in enumerate(uuids): 32 | count = 0 33 | with open("node{}.conf".format(idx), "a+") as fh: 34 | fh.write("# node{} configuration file\n".format(idx)) 35 | for k, v in uuids.items(): 36 | self = "0" 37 | peer = v.split(":") 38 | if count == idx: 39 | self = "1" 40 | fh.write(peer[0] + "\t" + str(int(peer[1]) + 100) + "\t" + k + "\t" + self + "\n") 41 | count += 1 42 | 43 | fh.close() 44 | 45 | 46 | 47 | if __name__ == '__main__': 48 | exit(main()) 49 | -------------------------------------------------------------------------------- /conf/node0.conf: -------------------------------------------------------------------------------- 1 | # node0 configuration file 2 | 127.0.0.1 8181 Node-A 1 3 | 127.0.0.1 8182 Node-B 0 4 | 127.0.0.1 8183 Node-C 0 5 | -------------------------------------------------------------------------------- /conf/node1.conf: -------------------------------------------------------------------------------- 1 | # node1 configuration file 2 | 127.0.0.1 8181 Node-A 0 3 | 127.0.0.1 8182 Node-B 1 4 | 127.0.0.1 8183 Node-C 0 5 | -------------------------------------------------------------------------------- /conf/node2.conf: -------------------------------------------------------------------------------- 1 | # node2 configuration file 2 | 127.0.0.1 8181 Node-A 0 3 | 127.0.0.1 8182 Node-B 0 4 | 127.0.0.1 8183 Node-C 1 5 | -------------------------------------------------------------------------------- /src/cluster.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to 6 | * deal in the Software without restriction, including without limitation the 7 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | * sell copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | * IN THE SOFTWARE. 21 | */ 22 | 23 | #include 24 | #include 25 | #include "util.h" 26 | #include "list.h" 27 | #include "event.h" 28 | #include "cluster.h" 29 | 30 | /* Global state store instance */ 31 | memento instance; 32 | 33 | /* Self reference in a cluster context */ 34 | cluster_node self; 35 | 36 | 37 | /* 38 | * Initialize the global shared structure, representing the cluster and the 39 | * store itself 40 | */ 41 | void init_system(int distributed, const char *id, 42 | const char *host, char *server_port, char *cluster_port) { 43 | 44 | /* Initialize instance containers */ 45 | instance.cluster_mode = distributed; 46 | instance.store = map_create(); 47 | instance.cluster = list_create(); 48 | instance.log_level = DEBUG; 49 | instance.verbose = 0; 50 | 51 | init_event_loop(host, server_port, cluster_port); 52 | 53 | /* check for distribution */ 54 | if (distributed == 1) { 55 | /* Initialize self reference */ 56 | self.state = REACHABLE; 57 | self.self = 1; 58 | self.addr = host; 59 | self.port = GETINT(cluster_port); 60 | self.name = id; 61 | /* lock for incoming connection, till the cluster is formed and ready */ 62 | instance.lock = 1; 63 | } 64 | else instance.lock = 0; 65 | } 66 | 67 | 68 | /* 69 | * Deallocate all structures in a cluster context 70 | */ 71 | void cluster_destroy(void) { 72 | /* deallocate instance containers */ 73 | map_release(instance.store); 74 | list_release(instance.cluster); 75 | free((char *) self.name); 76 | } 77 | 78 | 79 | /* 80 | * Add a cluster node to the global state memento 81 | */ 82 | void cluster_add_node(cluster_node *node) { 83 | instance.cluster = list_head_insert(instance.cluster, node); 84 | } 85 | 86 | 87 | /* 88 | * Check if the cluster node is already present in the list, just a linear 89 | * search on the cluster nodes list 90 | */ 91 | int cluster_contained(cluster_node *node) { 92 | /* Start from head node */ 93 | list_node *cursor = instance.cluster->head; 94 | /* cycle till cursor != NULL */ 95 | while (cursor) { 96 | cluster_node *n = (cluster_node *) cursor->data; 97 | if ((strncmp(n->addr, node->addr, strlen(node->addr))) == 0 98 | && n->port == node->port 99 | && (strncmp(n->name, node->name, strlen(node->name)) == 0)) { 100 | /* found a match */ 101 | return 1; 102 | } 103 | cursor = cursor->next; // move the pointer forward 104 | } 105 | /* node is not present */ 106 | return 0; 107 | } 108 | 109 | 110 | /* 111 | * Checks if the cluster node is in a REACHABLE state 112 | */ 113 | int cluster_reachable(cluster_node *node) { 114 | return node->state == REACHABLE ? 1 : 0; 115 | } 116 | 117 | 118 | /* 119 | * Count the number of cluster nodes in a state of UNREACHABLE 120 | */ 121 | int cluster_unreachable_count(void) { 122 | list_node *cursor = instance.cluster->head; 123 | int count = 0; 124 | 125 | while (cursor) { 126 | if (((cluster_node *) cursor->data)->state == UNREACHABLE) count++; 127 | cursor = cursor->next; 128 | } 129 | return count; 130 | } 131 | 132 | 133 | static cluster_node *find_node(const char *host, int port) { 134 | /* Start from head node */ 135 | list_node *cursor = instance.cluster->head; 136 | /* cycle till cursor != NULL */ 137 | while (cursor) { 138 | cluster_node *n = (cluster_node *) cursor->data; 139 | if ((strncmp(n->addr, host, strlen(host))) == 0 && n->port == port) { 140 | /* found a match */ 141 | return n; 142 | } 143 | cursor = cursor->next; // move the pointer forward 144 | } 145 | /* node is not present */ 146 | return NULL; 147 | } 148 | 149 | 150 | /* 151 | * Sets the cluster node contained in the cluster list to state st 152 | * instance. 153 | */ 154 | int cluster_set_state(cluster_node *node, state st) { 155 | cluster_node *n = find_node(node->addr, node->port); 156 | if (n) { 157 | n->state = st; 158 | return 1; 159 | } 160 | return 0; 161 | } 162 | 163 | 164 | /* 165 | * Return the associated cluster node to the host and port specified 166 | * FIXME: repeated code 167 | */ 168 | cluster_node *cluster_get_node(const char *host, char *port) { 169 | return find_node(host, GETINT(port)); 170 | } 171 | 172 | 173 | /* 174 | * Add a node to the cluster by adding it to the cluster list of nodes 175 | */ 176 | int cluster_join(const char *host, char *port) { 177 | int fd; 178 | if ((fd = connectto(host, port)) == -1) { 179 | //ERROR("Impossible connection to %s:%s\n", host, port); 180 | ERROR("Awaiting for peer %s:%s to accept connection\n", host, port); 181 | return -1; 182 | } 183 | /* If cluster node is present set the file descriptor */ 184 | cluster_node *n = cluster_get_node(host, port); 185 | if (n) { 186 | n->fd = fd; 187 | if (cluster_reachable(n) == 0) cluster_set_state(n, REACHABLE); 188 | } else { 189 | /* create a new node and add it to the list */ 190 | cluster_node *new_node = shb_malloc(sizeof(cluster_node)); 191 | new_node->addr = host; 192 | new_node->port = GETINT(port); 193 | new_node->fd = fd; 194 | new_node->state = REACHABLE; 195 | new_node->self = 0; 196 | instance.cluster = 197 | list_head_insert(instance.cluster, new_node); 198 | } 199 | ADD_FD(instance.el.epollfd, fd); 200 | return 1; 201 | } 202 | 203 | 204 | /* 205 | * Balance the cluster by giving a correct key range to every node, resulting 206 | * in a sorted list of balanced nodes 207 | */ 208 | void cluster_balance(void) { 209 | 210 | /* Define a step to obtain the ranges */ 211 | int range = 0; 212 | unsigned long len = instance.cluster->len; 213 | int step = PARTITIONS / len; 214 | 215 | /* sort the list */ 216 | instance.cluster->head = merge_sort(instance.cluster->head); 217 | list_node *cursor = instance.cluster->head; 218 | 219 | /* Cycle through the cluster nodes */ 220 | while (cursor) { 221 | cluster_node *n = (cluster_node *) cursor->data; 222 | n->range_min = range; 223 | range += step; 224 | n->range_max = range - 1; 225 | if (cursor->next == NULL) n->range_max = PARTITIONS; 226 | cursor = cursor->next; 227 | } 228 | } 229 | 230 | 231 | /* 232 | * Sets the name of the self node of the cluster, just for utility usages 233 | */ 234 | void cluster_set_selfname(const char *name) { 235 | self.name = strdup(name); 236 | } 237 | -------------------------------------------------------------------------------- /src/cluster.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to 6 | * deal in the Software without restriction, including without limitation the 7 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | * sell copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | * IN THE SOFTWARE. 21 | */ 22 | 23 | #ifndef CLUSTER_H 24 | #define CLUSTER_H 25 | 26 | #include "map.h" 27 | #include "list.h" 28 | #include "util.h" 29 | #include "networking.h" 30 | 31 | 32 | #define PARTITIONS 8192 // whole keyspace 33 | 34 | 35 | typedef enum { REACHABLE, UNREACHABLE } state; 36 | 37 | 38 | /* Cluster node structure */ 39 | typedef struct { 40 | const char *name; // node name, a 64 byte len string 41 | const char *addr; // node ip address 42 | int port; // node port 43 | int fd; // node file descriptor 44 | state state; // current node state in the cluster 45 | unsigned int self : 1; // define if the node is a seed or not 46 | unsigned int range_min; // key range lower bound 47 | unsigned int range_max; // key range upper bound 48 | } cluster_node; 49 | 50 | /* Global shared state of the system, it represents the distributed map */ 51 | typedef struct { 52 | event_loop el; // event_loop structure, must be initialized 53 | unsigned int lock : 1; // global lock, used in a cluster context 54 | unsigned int cluster_mode : 1; // distributed flag 55 | map *store; // items of the DB 56 | list *cluster; // map of cluster nodes 57 | loglevel log_level; // log level of the entire system 58 | unsigned int verbose : 1; // verbosity for logs 59 | } memento; 60 | 61 | 62 | extern memento instance; // global shared instance for the current node in the cluster 63 | 64 | extern cluster_node self; // a reference to the current node 65 | 66 | 67 | void init_system(int, const char *, const char *, char *, char *); 68 | void cluster_destroy(void); 69 | void cluster_add_node(cluster_node *); 70 | cluster_node *cluster_get_node(const char *, char *); 71 | int cluster_contained(cluster_node *); 72 | int cluster_reachable(cluster_node *); 73 | int cluster_unreachable_count(void); 74 | int cluster_set_state(cluster_node *, state); 75 | int cluster_join(const char *, char *); 76 | void cluster_balance(void); 77 | void cluster_set_selfname(const char *); 78 | 79 | #endif 80 | -------------------------------------------------------------------------------- /src/commands.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to 6 | * deal in the Software without restriction, including without limitation the 7 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | * sell copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | * IN THE SOFTWARE. 21 | */ 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include "commands.h" 30 | #include "networking.h" 31 | #include "serializer.h" 32 | #include "hashing.h" 33 | #include "cluster.h" 34 | #include "util.h" 35 | 36 | 37 | command command_entries[] = { 38 | {"set", set_command, reply_default}, 39 | {"get", get_command, reply_data}, 40 | {"del", del_command, reply_default}, 41 | {"inc", inc_command, reply_default}, 42 | {"dec", dec_command, reply_default}, 43 | {"incf", incf_command, reply_default}, 44 | {"decf", decf_command, reply_default}, 45 | {"append", append_command, reply_default}, 46 | {"prepend", prepend_command, reply_default}, 47 | {"getp", getp_command, reply_data}, 48 | {"flush", flush_command, reply_default} 49 | }; 50 | 51 | 52 | int commands_len(void) { 53 | return sizeof(command_entries) / sizeof(command); 54 | } 55 | 56 | 57 | void *reply_data(reply *rep) { 58 | peer_t *p = (peer_t *) malloc(sizeof(peer_t)); 59 | p->fd = rep->sfd; 60 | p->alloc = 0; 61 | if (rep->data == NULL) 62 | rep->data = S_NIL; 63 | 64 | if (instance.cluster_mode == 1 && rep->fp == 1) { 65 | /* adding some informations about the node host */ 66 | char response[strlen((char *) rep->data) + strlen(self.addr) + strlen(self.name) + 9]; 67 | sprintf(response, "%s:%s:%d> %s", self.name, self.addr, self.port, (char *) rep->data); 68 | struct message m = { response, rep->rfd, 1 }; 69 | char *payload = serialize(m); 70 | p->data = payload; 71 | p->size = strlen(response) + S_OFFSET; 72 | p->tocli = 0; 73 | if (instance.verbose) DEBUG("Reply data to peer\n"); 74 | } 75 | else { 76 | if (instance.verbose) DEBUG("Reply data to client %s\n", rep->data); 77 | rep->data = append_string(rep->data, "\r\n"); 78 | p->data = rep->data; 79 | p->size = strlen(rep->data); 80 | p->tocli = 1; 81 | } 82 | schedule_write(p); 83 | return NULL; 84 | } 85 | 86 | 87 | void *reply_default(reply *rep) { 88 | peer_t *p = (peer_t *) malloc(sizeof(peer_t)); 89 | int resp = *((int *) rep->data); 90 | 91 | if (rep->fp == 1) { 92 | int len = 0; 93 | char *payload = NULL; 94 | struct message msg; 95 | switch (resp) { 96 | case MAP_OK: 97 | msg = (struct message) { S_OK, rep->rfd, 1 }; 98 | len = strlen(S_OK) + S_OFFSET; 99 | break; 100 | case MAP_ERR: 101 | msg = (struct message) { S_NIL, rep->rfd, 1 }; 102 | len = strlen(S_NIL) + S_OFFSET; 103 | break; 104 | case COMMAND_NOT_FOUND: 105 | msg = (struct message) { S_UNK, rep->rfd, 1 }; 106 | len = strlen(S_UNK) + S_OFFSET; 107 | break; 108 | case END: 109 | return NULL; 110 | break; 111 | default: 112 | break; 113 | } 114 | payload = serialize(msg); 115 | p->fd = rep->sfd; 116 | p->data = payload; 117 | p->size = len; 118 | p->alloc = 1; 119 | p->tocli = 0; 120 | if (instance.verbose) { 121 | DEBUG("Response to peer node at %d with %s\n", rep->sfd, msg.content); 122 | } 123 | } 124 | else { 125 | switch (resp) { 126 | case MAP_OK: 127 | p->data = S_OK; 128 | break; 129 | case MAP_ERR: 130 | p->data = S_NIL; 131 | break; 132 | case COMMAND_NOT_FOUND: 133 | p->data = S_UNK; 134 | break; 135 | case END: 136 | return NULL; 137 | break; 138 | default: 139 | return 0; 140 | break; 141 | } 142 | p->size = strlen(p->data); 143 | p->fd = rep->sfd; 144 | p->alloc = 0; 145 | p->tocli = 1; 146 | } 147 | schedule_write(p); 148 | free(rep->data); 149 | return NULL; 150 | } 151 | 152 | 153 | void execute(char *buffer, int sfd, int rfd, int fp) { 154 | reply *rep = malloc(sizeof(reply)); 155 | rep->sfd = sfd; 156 | rep->rfd = rfd; 157 | rep->fp = fp; 158 | char *command = strtok(buffer, " \r\n"); 159 | if (command) { 160 | /* check if the buffer contains a valid command */ 161 | for (int i = 0; i < commands_len(); i++) { 162 | if (strncasecmp(command, command_entries[i].name, 163 | strlen(command_entries[i].name)) == 0 164 | && strlen(command_entries[i].name) == strlen(command)) { 165 | rep->data = (*command_entries[i].func)(buffer + strlen(command) + 1); 166 | (*command_entries[i].callback)(rep); 167 | } 168 | } 169 | } 170 | free(rep); 171 | } 172 | 173 | 174 | /* 175 | * Auxiliary function used to check the command coming from clients 176 | * FIXME: repeated code 177 | */ 178 | int check_command(char *buffer) { 179 | 180 | char *command = strtok(buffer, " \r\n"); 181 | 182 | if (command) { 183 | // in case of 'QUIT' or 'EXIT' close the connection 184 | if (strlen(command) == 4 185 | && (strncasecmp(command, "quit", 4) == 0 186 | || strncasecmp(command, "exit", 4) == 0)) 187 | return END; 188 | 189 | for (int i = 0; i < commands_len(); i++) { 190 | if (strncasecmp(command, command_entries[i].name, 191 | strlen(command_entries[i].name)) == 0 192 | && strlen(command_entries[i].name) == strlen(command)) { 193 | return 1; 194 | } 195 | } 196 | } 197 | return COMMAND_NOT_FOUND; 198 | } 199 | 200 | 201 | /* 202 | * Utility function, get a valid index in the range of the cluster buckets, so 203 | * it is possible to route command to the correct node 204 | */ 205 | static int hash(char *key) { 206 | uint32_t seed = 65133; // initial seed for murmur hashing 207 | if (key) { 208 | char *holder = malloc(strlen(key)); 209 | strcpy(holder, key); 210 | trim(holder); 211 | /* retrieve an index in the bucket range */ 212 | int idx = murmur3_32((const uint8_t *) holder, 213 | strlen(holder), seed) % PARTITIONS; 214 | 215 | if (instance.verbose) { 216 | DEBUG("Destination node: %d for key %s\r\n", idx, holder); 217 | } 218 | free(holder); 219 | return idx; 220 | } else return -1; 221 | } 222 | 223 | /* 224 | * Helper function to route command to the node into which keyspace contains 225 | * idx, find the node by performing a simple linear search 226 | */ 227 | static void route_command(int idx, int fd, char *b, struct message *msg) { 228 | /* Send the message serialized to the right node according the routing 229 | table cluster */ 230 | list_node *cursor = instance.cluster->head; 231 | while(cursor) { 232 | cluster_node *n = (cluster_node *) cursor->data; 233 | if (idx >= n->range_min && idx <= n->range_max) { 234 | /* check if the range is in the current node */ 235 | if (n->self == 1) { 236 | /* commit command directly if the range is handled by the 237 | current node */ 238 | execute(b, fd, fd, 0); 239 | break; 240 | } else { 241 | msg->content = b; 242 | msg->fd = fd; 243 | msg->ready = 0; // comes from client, i'm peer 244 | char *payload = serialize(*msg); 245 | peer_t *p = (peer_t *) malloc(sizeof(peer_t)); 246 | p->fd = n->fd; // dest fd 247 | p->alloc = 1; 248 | p->data = payload; 249 | p->size = strlen(b) + S_OFFSET; 250 | p->tocli = 0; 251 | if (instance.verbose) { 252 | DEBUG("Redirect toward cluster member %s\n", n->name); 253 | } 254 | schedule_write(p); 255 | break; 256 | } 257 | } 258 | cursor = cursor->next; 259 | } 260 | } 261 | 262 | 263 | int peer_command_handler(int fd) { 264 | char buf[instance.el.bufsize]; 265 | memset(buf, 0x00, instance.el.bufsize); 266 | int ret = 0; 267 | 268 | /* read data from file descriptor socket */ 269 | // TODO: need a circular buffer or something similar to handle all incoming 270 | // data in a loop 271 | int bytes = recv(fd, buf, instance.el.bufsize, 0); 272 | 273 | if (bytes == -1) { 274 | if (errno != EAGAIN) ret = 1; 275 | } 276 | else if (bytes == 0) { 277 | ret = 1; 278 | } 279 | else { 280 | peer_t *p = (peer_t *) malloc(sizeof(peer_t)); 281 | /* message came from a peer node, so it is serialized */ 282 | struct message m = deserialize(buf); // deserialize into message structure 283 | p->alloc = 1; 284 | p->tocli = 1; 285 | p->fd = m.fd; 286 | if (instance.verbose) { 287 | DEBUG("Received data from peer node, message: %s\n", m.content); 288 | } 289 | if (strcmp(m.content, S_OK) == 0 290 | || strcmp(m.content, S_NIL) == 0 291 | || strcmp(m.content, S_UNK) == 0) { 292 | if (instance.verbose) DEBUG("Answer to client\n"); 293 | p->data = m.content; 294 | p->size = strlen(p->data); 295 | schedule_write(p); 296 | } else if (m.ready == 1) { 297 | if (instance.verbose) { 298 | DEBUG("Answer to client from a query %s\n", m.content); 299 | } 300 | m.content = append_string(m.content, "\r\n"); 301 | /* answer to a query operations to the original client */ 302 | p->data = m.content; 303 | p->size = strlen(m.content); 304 | schedule_write(p); 305 | } else { 306 | /* message from another node */ 307 | if (instance.verbose) { 308 | DEBUG("Answer to another node: %s\n", m.content); 309 | } 310 | execute(m.content, fd, m.fd, 1); 311 | } 312 | } 313 | return ret; 314 | } 315 | 316 | 317 | int client_command_handler(int fd) { 318 | struct message msg; 319 | char buf[instance.el.bufsize]; 320 | memset(buf, 0x00, instance.el.bufsize); 321 | int ret = 0; 322 | 323 | /* read data from file descriptor socket */ 324 | // TODO: need a circular buffer or something similar to handle all incoming 325 | // data in a loop 326 | int bytes = recv(fd, buf, instance.el.bufsize, 0); 327 | 328 | if (bytes == -1) { 329 | if (errno != EAGAIN) ret = 1; 330 | } 331 | else if (bytes == 0) { 332 | ret = 1; 333 | } 334 | else { 335 | /* check the if the command is genuine */ 336 | char *dupstr = strdup(buf); 337 | int check = check_command(dupstr); 338 | free(dupstr); 339 | 340 | if (check == 1) { 341 | if (instance.cluster_mode == 1) { 342 | /* message came directly from a client */ 343 | char *b = strdup(buf); // payload to send 344 | strtok(buf, " \r\n"); 345 | /* command is handled and it isn't an informative one */ 346 | char *arg_1 = strtok(NULL, " "); 347 | int idx = hash(arg_1); 348 | /* route the command to the correct node */ 349 | route_command(idx, fd, b, &msg); 350 | free(b); 351 | } 352 | else { 353 | /* Single node instance, cluster is not enabled */ 354 | execute(buf, fd, fd, 0); 355 | } 356 | } else { 357 | /* command received is not recognized or is a quit command */ 358 | DEBUG("Unrecognized command\n"); 359 | peer_t *p = (peer_t *) malloc(sizeof(peer_t)); 360 | switch (check) { 361 | case COMMAND_NOT_FOUND: 362 | p->fd = fd; 363 | p->alloc = 0; 364 | p->tocli = 1; 365 | p->data = S_UNK; 366 | p->size = strlen(S_UNK); 367 | schedule_write(p); 368 | break; 369 | case END: 370 | ret = END; 371 | break; 372 | default: 373 | break; 374 | } 375 | } 376 | } 377 | return ret; 378 | } 379 | 380 | 381 | /*************************** -- COMMANDS -- ***************************/ 382 | 383 | 384 | void *set_command(char *cmd) { 385 | int *ret = malloc(sizeof(int)); 386 | void *key = strtok(cmd, " "); 387 | if (key) { 388 | void *val = (char *) key + strlen(key) + 1; 389 | remove_newline(val); 390 | if (val) *ret = map_put(instance.store, strdup(key), strdup(val)); 391 | } 392 | return ret; 393 | } 394 | 395 | 396 | void *get_command(char *cmd) { 397 | void *key = strtok(cmd, " "); 398 | if (key) { 399 | trim(key); 400 | void *val = map_get(instance.store, key); 401 | if (val) { 402 | return val; 403 | } 404 | } 405 | return NULL; 406 | } 407 | 408 | 409 | /* 410 | * DEL command handler delete a key-value pair if present, return 411 | * MAP_ERR reply code otherwise. 412 | * 413 | * Require one argument: 414 | * 415 | * DEL 416 | */ 417 | void *del_command(char *cmd) { 418 | int *ret = malloc(sizeof(int)); 419 | *ret = MAP_ERR; 420 | void *key = strtok(cmd, " "); 421 | while (key) { 422 | trim(key); 423 | *ret = map_del(instance.store, key); 424 | key = strtok(NULL, " "); 425 | } 426 | return ret; 427 | } 428 | 429 | 430 | /* 431 | * INC command handler, increment (add 1) to the integer value to it if 432 | * present, return MAP_ERR reply code otherwise. 433 | * 434 | * Requires at least one arguments, but also accept optionally the amount to be 435 | * added to the specified key: 436 | * 437 | * INC // +1 to 438 | * INC 5 // +5 to 439 | */ 440 | void *inc_command(char *cmd) { 441 | int *ret = malloc(sizeof(int)); 442 | *ret = MAP_ERR; 443 | void *key = strtok(cmd, " "); 444 | if (key) { 445 | trim(key); 446 | void *val = map_get(instance.store, key); 447 | if (val) { 448 | char *s = (char *) val; 449 | if (ISINT(s) == 1) { 450 | int v = GETINT(s); 451 | char *by = strtok(NULL, " "); 452 | if (by && ISINT(by)) { 453 | v += GETINT(by); 454 | } else v++; 455 | sprintf(val, "%d\n", v); 456 | *ret = MAP_OK; 457 | } 458 | } 459 | } 460 | return ret; 461 | } 462 | 463 | 464 | /* 465 | * INCF command handler, increment (add 1.00) to the float value to it if 466 | * present, return MAP_ERR reply code otherwise. 467 | * 468 | * Requires at least one arguments, but also accept optionally the amount to be 469 | * added to the specified key: 470 | * 471 | * INCF // +1.0 to 472 | * INCF 5.0 // +5.0 to 473 | */ 474 | void *incf_command(char *cmd) { 475 | int *ret = malloc(sizeof(int)); 476 | *ret = MAP_ERR; 477 | void *key = strtok(cmd, " "); 478 | if (key) { 479 | trim(key); 480 | void *val = map_get(instance.store, key); 481 | if (val) { 482 | char *s = (char *) val; 483 | if (is_float(s) == 1) { 484 | double v = GETDOUBLE(s); 485 | char *by = strtok(NULL, " "); 486 | if (by && is_float(by)) { 487 | v += GETDOUBLE(by); 488 | } else v += 1.0; 489 | sprintf(val, "%lf\n", v); 490 | *ret = MAP_OK; 491 | } 492 | } 493 | } 494 | return ret; 495 | } 496 | 497 | 498 | /* 499 | * DEC command handler, decrement (subtract 1) to the integer value to it if 500 | * present, return MAP_ERR reply code otherwise. 501 | * 502 | * Requires at least one arguments, but also accept optionally the amount to be 503 | * subtracted to the specified key: 504 | * 505 | * DEC // -1 to 506 | * DEC 5 // -5 to 507 | */ 508 | void *dec_command(char *cmd) { 509 | int *ret = malloc(sizeof(int)); 510 | *ret = MAP_ERR; 511 | void *key = strtok(cmd, " "); 512 | if (key) { 513 | trim(key); 514 | void *val = map_get(instance.store, key); 515 | if (val) { 516 | char *s = (char *) val; 517 | if(ISINT(s) == 1) { 518 | int v = GETINT(s); 519 | char *by = strtok(NULL, " "); 520 | if (by != NULL && ISINT(by)) { 521 | v -= GETINT(by); 522 | } else v--; 523 | sprintf(val, "%d\n", v); 524 | *ret = MAP_OK; 525 | } 526 | } 527 | } 528 | return ret; 529 | } 530 | 531 | 532 | /* 533 | * DECF command handler, decrement (subtract 1.00) to the foat value to it if 534 | * present, return MAP_ERR reply code otherwise. 535 | * 536 | * Requires at least one arguments, but also accept optionally the amount to be 537 | * subtracted to the specified key: 538 | * 539 | * DECF // -1.0 to 540 | * DECF 5.0 // -5.0 to 541 | */ 542 | void *decf_command(char *cmd) { 543 | int *ret = malloc(sizeof(int)); 544 | *ret = MAP_ERR; 545 | void *key = strtok(cmd, " "); 546 | if (key) { 547 | trim(key); 548 | void *val = map_get(instance.store, key); 549 | if (val) { 550 | char *s = (char *) val; 551 | if(is_float(s) == 1) { 552 | double v = GETDOUBLE(s); 553 | char *by = strtok(NULL, " "); 554 | if (by != NULL && ISINT(by)) { 555 | v -= GETDOUBLE(by); 556 | } else v -= 1.0; 557 | sprintf(val, "%lf\n", v); 558 | *ret = MAP_OK; 559 | } 560 | } 561 | } 562 | return ret; 563 | } 564 | 565 | 566 | /* 567 | * APPEND command handler, append a suffix to the value associated to the found 568 | * key, returning MISSING reply code if no key were found. 569 | * 570 | * Require two arguments: 571 | * 572 | * APPEND 573 | */ 574 | void *append_command(char *cmd) { 575 | int *ret = malloc(sizeof(int)); 576 | *ret = MAP_ERR; 577 | void *key = strtok(cmd, " "); 578 | void *val = (char *) key + strlen(key) + 1; 579 | if (key && val) { 580 | char *key_holder = strdup(key); 581 | char *val_holder = strdup(val); 582 | void *_val = map_get(instance.store, key_holder); 583 | if (_val) { 584 | remove_newline(_val); 585 | char *append = append_string(_val, val_holder); 586 | *ret = map_put(instance.store, key_holder, append); 587 | } 588 | } 589 | return ret; 590 | } 591 | 592 | 593 | 594 | /* 595 | * PREPEND command handler, prepend a prefix to the value associated to the 596 | * found key, returning MISSING reply code if no key were found. 597 | * 598 | * Require two arguments: 599 | * 600 | * PREPEND 601 | */ 602 | void *prepend_command(char *cmd) { 603 | int *ret = malloc(sizeof(int)); 604 | *ret = MAP_ERR; 605 | void *key = strtok(cmd, " "); 606 | void *val = (char *) key + strlen(key) + 1; 607 | if (key && val) { 608 | char *key_holder = strdup(key); 609 | char *val_holder = strdup(val); 610 | void *_val = map_get(instance.store, key_holder); 611 | if (_val) { 612 | remove_newline(val_holder); 613 | char *append = append_string(val_holder, _val); 614 | *ret = map_put(instance.store, key_holder, append); 615 | } 616 | } 617 | return ret; 618 | } 619 | 620 | 621 | /* 622 | * GETP command handler gather all the informations associated to the key-value 623 | * pair if present, including expire time and creation time, NULL reply 624 | * otherwise. 625 | * 626 | * Require one argument: 627 | * 628 | * GETP 629 | */ 630 | void *getp_command(char *cmd) { 631 | void *key = strtok(cmd, " "); 632 | if (key) { 633 | trim(key); 634 | map_entry *kv = map_get_entry(instance.store, key); 635 | if (kv) { 636 | size_t kvstrsize = strlen(kv->key) 637 | + strlen((char *) kv->val) + (sizeof(long) * 2) + 128; 638 | char *kvstring = malloc(kvstrsize); // long numbers 639 | /* check if expire time is set */ 640 | char expire_time[19]; 641 | if (kv->has_expire_time) 642 | sprintf(expire_time, "%ld\n", kv->expire_time / 1000); 643 | else 644 | sprintf(expire_time, "%d\n", -1); 645 | 646 | /* format answer */ 647 | sprintf(kvstring, "key: %s\nvalue: %screation_time: %ld\nexpire_time: %s", 648 | (char *) kv->key, (char *) kv->val, kv->creation_time, expire_time); 649 | return kvstring; 650 | } 651 | } 652 | return NULL; 653 | } 654 | 655 | 656 | /* 657 | * FLUSH command handler, delete the entire keyspace. 658 | * 659 | * Doesn't require any argument. 660 | */ 661 | void *flush_command(char *cmd) { 662 | int *ret = malloc(sizeof(int)); 663 | if (instance.store != NULL) { 664 | map_release(instance.store); 665 | instance.store = NULL; 666 | instance.store = map_create(); 667 | } 668 | *ret = OK; 669 | return ret; 670 | } 671 | -------------------------------------------------------------------------------- /src/commands.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to 6 | * deal in the Software without restriction, including without limitation the 7 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | * sell copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | * IN THE SOFTWARE. 21 | */ 22 | 23 | #ifndef COMMANDS_H 24 | #define COMMANDS_H 25 | 26 | //#include "map.h" 27 | #include "networking.h" 28 | 29 | /* Server base responses */ 30 | #define S_OK "OK\r\n" 31 | #define S_NIL "(null)\r\n" 32 | #define S_OOM "(Out of memory)\r\n" 33 | #define S_UNK "(Unknown command)\r\n" 34 | 35 | 36 | typedef enum { OK, PAYLOAD_OK, ITERATE_OK, 37 | MISSING, FULL, OOM, COMMAND_NOT_FOUND, END} reply_code; 38 | 39 | 40 | typedef struct { 41 | void *data; /* data payload */ 42 | int sfd; /* file descriptor sender, can also be a node */ 43 | int rfd; /* file descriptor original client sender */ 44 | int size; /* size of the payload */ 45 | unsigned int fp : 1; /* message from another node flag */ 46 | } reply; 47 | 48 | 49 | typedef struct { 50 | char *name; /* name of the command */ 51 | void *(*func)(char *); /* command implementation function */ 52 | void *(*callback)(reply *); /* callback handler function for result */ 53 | } command; 54 | 55 | 56 | void *reply_default(reply *); 57 | void *reply_data(reply *); 58 | int check_command(char *); 59 | int peer_command_handler(int); 60 | int client_command_handler(int); 61 | 62 | void *set_command(char *); 63 | void *get_command(char *); 64 | void *del_command(char *); 65 | void *inc_command(char *); 66 | void *dec_command(char *); 67 | void *incf_command(char *); 68 | void *decf_command(char *); 69 | void *getp_command(char *); 70 | void *append_command(char *); 71 | void *prepend_command(char *); 72 | void *flush_command(char *); 73 | 74 | #endif 75 | -------------------------------------------------------------------------------- /src/event.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "event.h" 4 | 5 | 6 | void add_epollin(int efd, int fd) { 7 | struct epoll_event ev; 8 | ev.data.fd = fd; 9 | ev.events = EPOLLIN | EPOLLET | EPOLLONESHOT; 10 | 11 | if (epoll_ctl(efd, EPOLL_CTL_ADD, fd, &ev) < 0) { 12 | perror("epoll_ctl(2): add epollin"); 13 | } 14 | } 15 | 16 | void set_epollout(int efd, int fd, void *data) { 17 | struct epoll_event ev; 18 | ev.data.fd = fd; 19 | if (data) 20 | ev.data.ptr = data; 21 | ev.events = EPOLLOUT | EPOLLET | EPOLLONESHOT; 22 | 23 | if (epoll_ctl(efd, EPOLL_CTL_MOD, fd, &ev) < 0) { 24 | perror("epoll_ctl(2): set epollout"); 25 | } 26 | } 27 | 28 | void set_epollin(int efd, int fd) { 29 | struct epoll_event ev; 30 | ev.data.fd = fd; 31 | ev.events = EPOLLIN | EPOLLET | EPOLLONESHOT; 32 | 33 | if (epoll_ctl(efd, EPOLL_CTL_MOD, fd, &ev) < 0) { 34 | perror("epoll_ctl(2): set epollin"); 35 | } 36 | } 37 | 38 | 39 | -------------------------------------------------------------------------------- /src/event.h: -------------------------------------------------------------------------------- 1 | #ifndef EVENT_H 2 | #define EVENT_H 3 | 4 | #define ADD_FD(efd, fd) add_epollin(efd, fd) 5 | #define SET_FD_OUT(efd, fd, data) set_epollout(efd, fd, data) 6 | #define SET_FD_IN(efd, fd) set_epollin(efd, fd) 7 | 8 | 9 | void add_epollin(int, int); 10 | void set_epollout(int, int, void*); 11 | void set_epollin(int, int); 12 | 13 | #endif 14 | -------------------------------------------------------------------------------- /src/hashing.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #ifndef HASHING_H 23 | #define HASHING_H 24 | 25 | #include 26 | #include 27 | #include 28 | #include "util.h" 29 | 30 | 31 | /* 32 | * Basic hasing function, uses CRC32 33 | */ 34 | int _hash(char *keystring) { 35 | 36 | unsigned long key = CRC32((unsigned char *) (keystring), strlen(keystring)); 37 | 38 | /* Robert Jenkins' 32 bit Mix Function */ 39 | key += (key << 12); 40 | key ^= (key >> 22); 41 | key += (key << 4); 42 | key ^= (key >> 9); 43 | key += (key << 10); 44 | key ^= (key >> 2); 45 | key += (key << 7); 46 | key ^= (key >> 12); 47 | 48 | /* Knuth's Multiplicative Method */ 49 | key = (key >> 3) * 2654435761; 50 | 51 | return key; 52 | } 53 | 54 | 55 | /* 56 | * Jenkins hash function 57 | */ 58 | uint32_t jenkins_one_at_a_time_hash(const uint8_t* key, size_t length) { 59 | size_t i = 0; 60 | uint32_t hash = 0; 61 | while (i != length) { 62 | hash += key[i++]; 63 | hash += hash << 10; 64 | hash ^= hash >> 6; 65 | } 66 | hash += hash << 3; 67 | hash ^= hash >> 11; 68 | hash += hash << 15; 69 | return hash; 70 | } 71 | 72 | 73 | /* 74 | * Murmur 3 function, should be initialized using a random seed 75 | */ 76 | uint32_t murmur3_32(const uint8_t *key, size_t len, uint32_t seed) { 77 | uint32_t h = seed; 78 | if (len > 3) { 79 | const uint32_t *key_x4 = (const uint32_t *) key; 80 | size_t i = len >> 2; 81 | do { 82 | uint32_t k = *key_x4++; 83 | k *= 0xcc9e2d51; 84 | k = (k << 15) | (k >> 17); 85 | k *= 0x1b873593; 86 | h ^= k; 87 | h = (h << 13) | (h >> 19); 88 | h += (h << 2) + 0xe6546b64; 89 | } while (--i); 90 | key = (const uint8_t *) key_x4; 91 | } 92 | if (len & 3) { 93 | size_t i = len & 3; 94 | uint32_t k = 0; 95 | key = &key[i - 1]; 96 | do { 97 | k <<= 8; 98 | k |= *key--; 99 | } while (--i); 100 | k *= 0xcc9e2d51; 101 | k = (k << 15) | (k >> 17); 102 | k *= 0x1b873593; 103 | h ^= k; 104 | } 105 | h ^= len; 106 | h ^= h >> 16; 107 | h *= 0x85ebca6b; 108 | h ^= h >> 13; 109 | h *= 0xc2b2ae35; 110 | h ^= h >> 16; 111 | return h; 112 | } 113 | 114 | 115 | #endif 116 | -------------------------------------------------------------------------------- /src/list.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 21 | */ 22 | 23 | #include 24 | #include 25 | #include 26 | #include "list.h" 27 | #include "cluster.h" 28 | 29 | 30 | /* 31 | * Create a list, initializing all fields 32 | */ 33 | list *list_create(void) { 34 | list *l = shb_malloc(sizeof(list)); 35 | // set default values to the list structure fields 36 | l->head = l->tail = NULL; 37 | l->len = 0L; 38 | return l; 39 | } 40 | 41 | 42 | /* 43 | * Attach a node to the head of a new list 44 | */ 45 | list *list_attach(list *l, list_node *head, unsigned long len) { 46 | // set default values to the list structure fields 47 | l->head = head; 48 | l->len = len; 49 | return l; 50 | } 51 | 52 | /* 53 | * Destroy a list, releasing all allocated memory 54 | */ 55 | void list_release(list *l) { 56 | if (!l) return; 57 | list_node *h = l->head; 58 | list_node *tmp; 59 | // free all nodes 60 | while (l->len--) { 61 | tmp = h->next; 62 | if (h) { 63 | if (h->data) free(h->data); 64 | free(h); 65 | } 66 | h = tmp; 67 | } 68 | // free list structure pointer 69 | free(l); 70 | } 71 | 72 | 73 | /* 74 | * Insert value at the front of the list 75 | * Complexity: O(1) 76 | */ 77 | list *list_head_insert(list *l, void *val) { 78 | list_node *new_node = shb_malloc(sizeof(list_node)); 79 | new_node->data = val; 80 | if (l->len == 0) { 81 | l->head = l->tail = new_node; 82 | new_node->next = NULL; 83 | } else { 84 | new_node->next = l->head; 85 | l->head = new_node; 86 | } 87 | l->len++; 88 | return l; 89 | } 90 | 91 | 92 | /* 93 | * Insert value at the back of the list 94 | * Complexity: O(1) 95 | */ 96 | list *list_tail_insert(list *l, void *val) { 97 | list_node *new_node = shb_malloc(sizeof(list_node)); 98 | new_node->data = val; 99 | new_node->next = NULL; 100 | if (l->len == 0) l->head = l->tail = new_node; 101 | else { 102 | l->tail->next = new_node; 103 | l->tail = new_node; 104 | } 105 | l->len++; 106 | return l; 107 | } 108 | 109 | 110 | /* 111 | * Returns a pointer to a node near the middle of the list, 112 | * after having truncated the original list before that point. 113 | */ 114 | static list_node *bisect_list(list_node *head) { 115 | /* The fast pointer moves twice as fast as the slow pointer. */ 116 | /* The prev pointer points to the node preceding the slow pointer. */ 117 | list_node *fast = head, *slow = head, *prev = NULL; 118 | 119 | while (fast != NULL && fast->next != NULL) { 120 | fast = fast->next->next; 121 | prev = slow; 122 | slow = slow->next; 123 | } 124 | 125 | if (prev != NULL) 126 | prev->next = NULL; 127 | 128 | return slow; 129 | } 130 | 131 | 132 | /* 133 | * Merges two list by using the head node of the two, sorting them according to 134 | * lexigraphical ordering of the node names. 135 | */ 136 | static list_node *merge_list(list_node *list1, list_node *list2) { 137 | list_node dummy_head = { NULL, NULL }, *tail = &dummy_head; 138 | 139 | while ((list1 != NULL) && (list2 != NULL)) { 140 | 141 | /* cast to cluster_node */ 142 | cluster_node *n1 = 143 | (cluster_node *) list1->data; 144 | cluster_node *n2 = 145 | (cluster_node *) list2->data; 146 | 147 | list_node **min = ((strcmp(n1->name, n2->name)) < 0) ? &list1 : &list2; 148 | list_node *next = (*min)->next; 149 | tail = tail->next = *min; 150 | *min = next; 151 | } 152 | 153 | tail->next = list1 ? list1 : list2; 154 | return dummy_head.next; 155 | } 156 | 157 | 158 | /* 159 | * Merge sort for cluster nodes list, based on the name field of every node 160 | */ 161 | list_node *merge_sort(list_node *head) { 162 | list_node *list1 = head; 163 | if ( (list1 == NULL) || (list1->next == NULL) ) 164 | return list1; 165 | /* find the middle */ 166 | list_node *list2 = bisect_list(list1); 167 | 168 | return merge_list(merge_sort(list1), merge_sort(list2)); 169 | } 170 | -------------------------------------------------------------------------------- /src/list.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to 6 | * deal in the Software without restriction, including without limitation the 7 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | * sell copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | * IN THE SOFTWARE. Copyright (c) 2016-2017 Andrea Giacomo Baldan 21 | */ 22 | 23 | #ifndef LIST_H 24 | #define LIST_H 25 | 26 | 27 | struct _list_node { 28 | void *data; 29 | struct _list_node *next; 30 | }; 31 | 32 | 33 | typedef struct _list_node list_node; 34 | 35 | 36 | typedef struct { 37 | list_node *head; 38 | list_node *tail; 39 | unsigned long len; 40 | } list; 41 | 42 | 43 | list *list_create(void); 44 | list *list_attach(list *, list_node *, unsigned long); 45 | void list_release(list *); 46 | list *list_head_insert(list *, void *); 47 | list *list_tail_insert(list *, void *); 48 | list_node *merge_sort(list_node *); 49 | 50 | #endif 51 | -------------------------------------------------------------------------------- /src/map.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include "map.h" 27 | #include "util.h" 28 | 29 | 30 | const unsigned int INITIAL_SIZE = 256; 31 | const unsigned int MAX_CHAIN_LENGTH = 8; 32 | 33 | 34 | /* 35 | * Hashing function for a string 36 | */ 37 | static unsigned int hashmap_hash_int(map *m, char *keystr) { 38 | 39 | unsigned long key = CRC32((unsigned char *) (keystr), strlen(keystr)); 40 | 41 | /* Robert Jenkins' 32 bit Mix Function */ 42 | key += (key << 12); 43 | key ^= (key >> 22); 44 | key += (key << 4); 45 | key ^= (key >> 9); 46 | key += (key << 10); 47 | key ^= (key >> 2); 48 | key += (key << 7); 49 | key ^= (key >> 12); 50 | 51 | /* Knuth's Multiplicative Method */ 52 | key = (key >> 3) * 2654435761; 53 | 54 | return key % m->table_size; 55 | } 56 | 57 | 58 | /* 59 | * Return the integer of the location in entries to store the point to the item, 60 | * or MAP_FULL. 61 | */ 62 | static int hashmap_hash(map *in, void *key) { 63 | /* If full, return immediately */ 64 | if (in->size >= (in->table_size / 2)) return MAP_FULL; 65 | /* Find the best index */ 66 | int curr = hashmap_hash_int(in, key); 67 | /* Linear probing */ 68 | for(int i = 0; i < MAX_CHAIN_LENGTH; i++){ 69 | if (in->entries[curr].in_use == 0) 70 | return curr; 71 | 72 | if (in->entries[curr].in_use == 1 && (strcmp(in->entries[curr].key, key) == 0)) 73 | return curr; 74 | 75 | curr = (curr + 1) % in->table_size; 76 | } 77 | 78 | return MAP_FULL; 79 | } 80 | 81 | 82 | /* 83 | * Doubles the size of the hashmap, and rehashes all the elements 84 | */ 85 | static int hashmap_rehash(map *m) { 86 | unsigned long old_size; 87 | map_entry *curr; 88 | 89 | /* Setup the new elements */ 90 | map_entry *temp = calloc(2 * m->table_size, sizeof(map_entry)); 91 | if (!temp) return MAP_ERR; 92 | 93 | /* Update the array */ 94 | curr = m->entries; 95 | m->entries = temp; 96 | 97 | /* Update the size */ 98 | old_size = m->table_size; 99 | m->table_size = 2 * m->table_size; 100 | m->size = 0; 101 | 102 | /* Rehash the elements */ 103 | for(unsigned long i = 0; i < old_size; i++) { 104 | int status; 105 | 106 | if (curr[i].in_use == 0) 107 | continue; 108 | 109 | status = map_put(m, curr[i].key, curr[i].val); 110 | if (status != MAP_OK) 111 | return status; 112 | } 113 | free(curr); 114 | return MAP_OK; 115 | } 116 | 117 | 118 | /* 119 | * Return an empty hashmap, or NULL on failure. The newly create hashmap is 120 | * dynamically allocated on the heap memory, so it must be released manually. 121 | */ 122 | map *map_create(void) { 123 | map *m = shb_malloc(sizeof(map)); 124 | if(!m) return NULL; 125 | 126 | m->entries = (map_entry *) calloc(INITIAL_SIZE, sizeof(map_entry)); 127 | if(!m->entries) { 128 | if (m) map_release(m); 129 | return NULL; 130 | } 131 | 132 | m->table_size = INITIAL_SIZE; 133 | m->size = 0; 134 | 135 | return m; 136 | } 137 | 138 | 139 | /* 140 | * Add a pointer to the hashmap with some key 141 | */ 142 | int map_put(map *m, void *key, void *val) { 143 | /* Find a place to put our value */ 144 | int index = hashmap_hash(m, key); 145 | while(index == MAP_FULL){ 146 | if (hashmap_rehash(m) == MAP_ERR) return MAP_ERR; 147 | index = hashmap_hash(m, key); 148 | } 149 | /* Set the entries */ 150 | m->entries[index].val = val; 151 | m->entries[index].key = key; 152 | if (m->entries[index].in_use == 0) { 153 | m->entries[index].in_use = 1; 154 | m->entries[index].has_expire_time = 0; 155 | m->entries[index].expire_time = -1; 156 | m->entries[index].creation_time = current_timestamp(); 157 | m->size++; 158 | } 159 | 160 | return MAP_OK; 161 | } 162 | 163 | 164 | /* 165 | * Get your pointer out of the hashmap with a key 166 | */ 167 | void *map_get(map *m, void *key) { 168 | /* Find data location */ 169 | int curr = hashmap_hash_int(m, key); 170 | /* Linear probing, if necessary */ 171 | for(int i = 0; i < MAX_CHAIN_LENGTH; i++){ 172 | if (m->entries[curr].in_use == 1) { 173 | if (strcmp(m->entries[curr].key, key) == 0) 174 | return (m->entries[curr].val); 175 | } 176 | curr = (curr + 1) % m->table_size; 177 | } 178 | /* Not found */ 179 | return NULL; 180 | } 181 | 182 | 183 | /* 184 | * Return the key-value pair represented by a key in the map 185 | */ 186 | map_entry *map_get_entry(map *m, void *key) { 187 | /* Find data location */ 188 | int curr = hashmap_hash_int(m, key); 189 | 190 | /* Linear probing, if necessary */ 191 | for(int i = 0; i < MAX_CHAIN_LENGTH; i++) { 192 | if (m->entries[curr].in_use == 1) { 193 | if (strcmp(m->entries[curr].key, key) == 0) 194 | return &m->entries[curr]; 195 | } 196 | 197 | curr = (curr + 1) % m->table_size; 198 | } 199 | /* Not found */ 200 | return NULL; 201 | } 202 | 203 | 204 | /* 205 | * Remove an element with that key from the map 206 | */ 207 | int map_del(map *m, void *key) { 208 | /* Find key */ 209 | int curr = hashmap_hash_int(m, key); 210 | /* Linear probing, if necessary */ 211 | for(int i = 0; i < MAX_CHAIN_LENGTH; i++) { 212 | // check wether the position in array is in use 213 | int in_use = m->entries[curr].in_use; 214 | if (in_use == 1) { 215 | if (strcmp(m->entries[curr].key, key) == 0) { 216 | /* Blank out the fields */ 217 | m->entries[curr].in_use = 0; 218 | m->entries[curr].has_expire_time = 0; 219 | m->entries[curr].expire_time = -1; 220 | m->entries[curr].creation_time = -1; 221 | /* Reduce the size */ 222 | m->size--; 223 | return MAP_OK; 224 | } 225 | } 226 | curr = (curr + 1) % m->table_size; 227 | } 228 | /* Data not found */ 229 | return MAP_ERR; 230 | } 231 | 232 | 233 | /* 234 | * Iterate the function parameter over each element in the hashmap. The 235 | * additional any_t argument is passed to the function as its first 236 | * argument and the pair is the second. 237 | */ 238 | int map_iterate2(map *m, func f, void *arg1) { 239 | /* On empty hashmap, return immediately */ 240 | if (m->size <= 0) return MAP_ERR; 241 | /* Linear probing */ 242 | for(int i = 0; i < m->table_size; i++) { 243 | if (m->entries[i].in_use != 0) { 244 | map_entry data = m->entries[i]; 245 | int status = f(arg1, &data); 246 | if (status != MAP_OK) return status; 247 | } 248 | } 249 | return MAP_OK; 250 | } 251 | 252 | 253 | /* 254 | * Iterate the function parameter over each element in the hashmap. The 255 | * additional any_t argument is passed to the function as its first 256 | * argument and the pair is the second. 257 | */ 258 | int map_iterate3(map *m, func3 f, void *arg1, void *arg2) { 259 | /* On empty hashmap, return immediately */ 260 | if (m->size <= 0) return MAP_ERR; 261 | /* Linear probing */ 262 | for(int i = 0; i < m->table_size; i++) { 263 | if (m->entries[i].in_use != 0) { 264 | map_entry data = m->entries[i]; 265 | int status = f(arg1, arg2, &data); 266 | if (status != MAP_OK) return status; 267 | } 268 | } 269 | return MAP_OK; 270 | } 271 | 272 | 273 | /* callback function used with iterate to clean up the hashmap */ 274 | static int destroy(void *t1, void *t2) { 275 | 276 | map_entry *kv = (map_entry *) t2; 277 | 278 | if (kv) { 279 | // free key field 280 | if (kv->key) 281 | free(kv->key); 282 | // free value field 283 | if (kv->val) 284 | free(kv->val); 285 | } else return MAP_ERR; 286 | 287 | return MAP_OK; 288 | } 289 | 290 | /* Deallocate the hashmap */ 291 | void map_release(map *m){ 292 | map_iterate2(m, destroy, NULL); 293 | if (m) { 294 | if (m->entries) 295 | free(m->entries); 296 | free(m); 297 | } 298 | } 299 | -------------------------------------------------------------------------------- /src/map.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #ifndef MAP_H 23 | #define MAP_H 24 | 25 | 26 | #define MAP_OK 0 27 | #define MAP_ERR -1 28 | #define MAP_FULL -2 29 | 30 | 31 | typedef int (*func)(void *, void *); 32 | typedef int (*func3)(void *, void *, void *); 33 | 34 | 35 | /* We need to keep keys and values */ 36 | typedef struct { 37 | void *key; 38 | void *val; 39 | unsigned int in_use : 1; 40 | unsigned int has_expire_time : 1; 41 | long creation_time; 42 | long expire_time; 43 | } map_entry; 44 | 45 | 46 | /* 47 | * An hashmap has some maximum size and current size, as well as the data to 48 | * hold. 49 | */ 50 | typedef struct { 51 | unsigned long table_size; 52 | unsigned long size; 53 | map_entry *entries; 54 | } map; 55 | 56 | 57 | /* Map API */ 58 | map *map_create(void); 59 | void map_release(map *); 60 | int map_put(map *, void *, void *); 61 | void *map_get(map *, void *); 62 | map_entry *map_get_entry(map *, void *); 63 | int map_del(map *, void *); 64 | int map_iterate2(map *, func, void *); 65 | int map_iterate3(map *, func3, void *, void *); 66 | 67 | 68 | #endif 69 | -------------------------------------------------------------------------------- /src/memento-benchmark.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include "networking.h" 30 | #include "util.h" 31 | 32 | 33 | 34 | struct connection { 35 | char *host; 36 | char *port; 37 | }; 38 | 39 | 40 | static void *make_requests(void *t) { 41 | 42 | struct connection *c = (struct connection *) t; 43 | 44 | int fd = connectto(c->host, c->port); 45 | int requests = 8000; 46 | char real_keys[requests][16]; 47 | char buf[5]; 48 | float time_elapsed, start_time, end_time, result; 49 | int size = 0; 50 | 51 | for (int j = 0; j < requests; ++j) 52 | snprintf(real_keys[j], 16, "set %d value\r\n", j); 53 | 54 | memset(buf, 0x00, 5); 55 | 56 | /* `SET keyone valueone` benchmark */ 57 | 58 | char *cmd = "set keyone valueone\r\n"; 59 | start_time = (float) clock()/CLOCKS_PER_SEC; 60 | for (int i = 0; i < requests; ++i) { 61 | size = strlen(cmd); 62 | if (send_all(fd, cmd, &size) < 0) 63 | perror("send"); 64 | if (read(fd, buf, 5) < 0) 65 | perror("read"); 66 | } 67 | 68 | end_time = (float)clock()/CLOCKS_PER_SEC; 69 | time_elapsed = end_time - start_time; 70 | result = requests / time_elapsed; 71 | 72 | printf(" [SET]"); 73 | printf(" - Thread: %lu", pthread_self()); 74 | printf(" - Elapsed time: %f s Op/s: %.2f\n", time_elapsed, result); 75 | 76 | /* `GET keyone` benchmark */ 77 | 78 | char getr[10]; 79 | memset(getr, 0x00, 10); 80 | char *get = "get keyone\r\n"; 81 | start_time = (float) clock()/CLOCKS_PER_SEC; 82 | for (int i = 0; i < requests; ++i) { 83 | size = strlen(get); 84 | if (send_all(fd, get, &size) < 0) 85 | perror("send"); 86 | if (read(fd, getr, 10) < 0) 87 | perror("read"); 88 | } 89 | 90 | end_time = (float)clock()/CLOCKS_PER_SEC; 91 | time_elapsed = end_time - start_time; 92 | result = requests / time_elapsed; 93 | printf(" [GET]"); 94 | printf(" - Thread: %lu", pthread_self()); 95 | printf(" - Elapsed time: %f s Op/s: %.2f\n", time_elapsed, result); 96 | 97 | /* `SET xxxxx value` benchmark */ 98 | 99 | start_time = (float) clock()/CLOCKS_PER_SEC; 100 | for (int i = 0; i < requests; ++i) { 101 | size = strlen(real_keys[i]); 102 | if (send_all(fd, real_keys[i], &size) < 0) 103 | perror("send"); 104 | if (read(fd, buf, 5) < 0) 105 | perror("read"); 106 | } 107 | 108 | end_time = (float) clock()/CLOCKS_PER_SEC; 109 | time_elapsed = end_time - start_time; 110 | result = requests / time_elapsed; 111 | printf(" [SET]"); 112 | printf(" - Thread: %lu", pthread_self()); 113 | printf(" - Elapsed time: %f s Op/s: %.2f\n", time_elapsed, result); 114 | 115 | return NULL; 116 | } 117 | 118 | 119 | int main(int argc, char **argv) { 120 | 121 | char *host = "127.0.0.1"; 122 | char *port = "8082"; 123 | int thread_nr = 50; 124 | pthread_t th[thread_nr]; 125 | 126 | if (argc > 2) { 127 | host = argv[1]; 128 | port = argv[2]; 129 | } 130 | 131 | int fd = connectto(host, port); 132 | char real_keys[100000][18]; 133 | char buf[5]; 134 | float time_elapsed, start_time, end_time, result; 135 | int size = 0; 136 | 137 | printf("\n"); 138 | printf(" Nr. operations %d\n\n", 100000); 139 | printf(" 1) SET keyone valueone\n"); 140 | printf(" 2) GET keyone\n"); 141 | printf(" 3) SET with growing keys\n"); 142 | printf(" 4) All previous operations in %d threads, each one performing 8000 requests per type\n", thread_nr); 143 | printf("\n"); 144 | 145 | for (int j = 0; j < 100000; ++j) 146 | snprintf(real_keys[j], 18, "set %d value\r\n", j); 147 | 148 | memset(buf, 0x00, 5); 149 | 150 | /* `SET keyone valueone` benchmark */ 151 | 152 | char *cmd = "set keyone valueone\r\n"; 153 | start_time = (float) clock()/CLOCKS_PER_SEC; 154 | for (int i = 0; i < 100000; ++i) { 155 | size = strlen(cmd); 156 | if (send_all(fd, cmd, &size) < 0) 157 | perror("send"); 158 | if (read(fd, buf, 5) < 0) 159 | perror("read"); 160 | } 161 | 162 | end_time = (float)clock()/CLOCKS_PER_SEC; 163 | time_elapsed = end_time - start_time; 164 | result = 100000 / time_elapsed; 165 | 166 | printf(" [SET]"); 167 | printf(" - Elapsed time: %f s Op/s: %.2f\n", time_elapsed, result); 168 | 169 | sleep(1); 170 | 171 | /* `GET keyone` benchmark */ 172 | 173 | char getr[10]; 174 | memset(getr, 0x00, 10); 175 | char *get = "get keyone\r\n"; 176 | start_time = (float) clock()/CLOCKS_PER_SEC; 177 | for (int i = 0; i < 100000; ++i) { 178 | size = strlen(get); 179 | if (send_all(fd, get, &size) < 0) 180 | perror("send"); 181 | if (read(fd, getr, 10) < 0) 182 | perror("read"); 183 | } 184 | 185 | end_time = (float)clock()/CLOCKS_PER_SEC; 186 | time_elapsed = end_time - start_time; 187 | result = 100000 / time_elapsed; 188 | printf(" [GET]"); 189 | printf(" - Elapsed time: %f s Op/s: %.2f\n", time_elapsed, result); 190 | 191 | sleep(1); 192 | 193 | /* `SET xxxxx value` benchmark */ 194 | 195 | start_time = (float) clock()/CLOCKS_PER_SEC; 196 | for (int i = 0; i < 100000; ++i) { 197 | size = 18; 198 | if (send_all(fd, real_keys[i], &size) < 0) 199 | perror("send"); 200 | if (read(fd, buf, 5) < 0) 201 | perror("read"); 202 | } 203 | 204 | end_time = (float)clock()/CLOCKS_PER_SEC; 205 | time_elapsed = end_time - start_time; 206 | result = 100000 / time_elapsed; 207 | printf(" [SET]"); 208 | printf(" - Elapsed time: %f s Op/s: %.2f\n", time_elapsed, result); 209 | printf(" --------------------------------------------------------------------\n"); 210 | 211 | sleep(1); 212 | 213 | /* Parallel requests */ 214 | struct connection conn; 215 | conn.host = host; 216 | conn.port = port; 217 | 218 | for (int i = 0; i < thread_nr; ++i) { 219 | if (pthread_create(&th[i], NULL, make_requests, &conn) < 0) 220 | perror("pthread"); 221 | usleep(10000); 222 | } 223 | 224 | for (int i = 0; i < thread_nr; ++i) 225 | pthread_join(th[i], NULL); 226 | 227 | printf("\n"); 228 | 229 | return 0; 230 | } 231 | -------------------------------------------------------------------------------- /src/memento-cli.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "networking.h" 13 | 14 | #define CMD_BUFSIZE 1024 15 | #define VERSION "0.0.1" 16 | 17 | void banner(void); 18 | void help(void); 19 | void command_loop(int, char *, char *); 20 | char *read_command(void); 21 | 22 | int main(int argc, char **argv) { 23 | int sock_fd, port; 24 | struct sockaddr_in serveraddr; 25 | struct hostent *server; 26 | char *hostname; 27 | 28 | /* check command line arguments */ 29 | if (argc != 3) { 30 | fprintf(stderr,"usage: %s \n", argv[0]); 31 | exit(0); 32 | } 33 | hostname = argv[1]; 34 | port = atoi(argv[2]); 35 | 36 | /* socket: create the socket */ 37 | /* sock_fd = socket(AF_INET, SOCK_STREAM, 0); */ 38 | /* if (sock_fd < 0) */ 39 | /* perror("ERROR opening socket"); */ 40 | /* */ 41 | /* #<{(| gethostbyname: get the server's DNS entry |)}># */ 42 | /* server = gethostbyname(hostname); */ 43 | /* if (server == NULL) { */ 44 | /* fprintf(stderr,"ERROR, no such host as %s\n", hostname); */ 45 | /* exit(0); */ 46 | /* } */ 47 | /* */ 48 | /* #<{(| build the server's address |)}># */ 49 | /* bzero((char *) &serveraddr, sizeof(serveraddr)); */ 50 | /* serveraddr.sin_family = AF_INET; */ 51 | /* bcopy((char *)server->h_addr, */ 52 | /* (char *)&serveraddr.sin_addr.s_addr, server->h_length); */ 53 | /* serveraddr.sin_port = htons(port); */ 54 | /* */ 55 | /* #<{(| connect: create a connection with the server |)}># */ 56 | /* if (connect(sock_fd, (const struct sockaddr *) &serveraddr, sizeof(serveraddr)) < 0) { */ 57 | /* perror("ERROR connecting"); */ 58 | /* exit(0); */ 59 | /* } */ 60 | 61 | char c_port[4]; 62 | sprintf(c_port, "%d", port); 63 | banner(); 64 | sock_fd = connectto(hostname, c_port); 65 | command_loop(sock_fd, hostname, c_port); 66 | return EXIT_SUCCESS; 67 | } 68 | 69 | // loop for accepting, parsing and sending commands to the memento server 70 | // instance, reading response 71 | void command_loop(int fd, char *hostname, char *port) { 72 | char *line, buf[CMD_BUFSIZE]; 73 | int status = 1; 74 | // while condition: status > 0 75 | while (status > 0) { 76 | printf("%s:%s> ", hostname, port); 77 | line = read_command(); 78 | char *supp_command = strdup(line); 79 | char *comm = strtok(supp_command, " "); 80 | if (strlen(line) > 1) { 81 | char token[4]; 82 | strncpy(token, line, 4); 83 | if (strncasecmp(comm, "KEYS", 4) == 0 || 84 | strncasecmp(comm, "VALUES", 6) == 0 || 85 | strncasecmp(comm, "KEYSPACE", 8) == 0) { 86 | fcntl(fd, F_SETFL, O_NONBLOCK); 87 | status = send(fd, line, strlen(line), 0); 88 | /* usleep(100000); */ 89 | if (status < 0) 90 | perror("ERROR writing to socket"); 91 | while(recv(fd, buf, CMD_BUFSIZE, 0) > 0) { 92 | printf("%s", buf); 93 | bzero(buf, CMD_BUFSIZE); 94 | } 95 | /* fcntl(fd, F_SETFL, O_BLOCK); */ 96 | } else if (token[0] == '?' || 97 | token[0] == 'h' || 98 | token[0] == 'H' || 99 | strncasecmp(token, "HELP", 4) == 0) { 100 | help(); 101 | bzero(buf, CMD_BUFSIZE); 102 | } else { 103 | // write to server 104 | status = send(fd, line, strlen(line), 0); 105 | if (status < 0) 106 | perror("ERROR writing to socket"); 107 | /* print the server's reply */ 108 | bzero(buf, CMD_BUFSIZE); 109 | status = recv(fd, buf, CMD_BUFSIZE, 0); 110 | if (status < 0) 111 | perror("ERROR reading from socket"); 112 | printf("%s", buf); 113 | } 114 | free(supp_command); 115 | free(line); 116 | } 117 | } 118 | } 119 | // read a single string containing command 120 | char *read_command(void) { 121 | char *line = NULL; 122 | size_t bufsize = 0; 123 | getline(&line, &bufsize, stdin); 124 | return line; 125 | } 126 | // print initial banner 127 | void banner(void) { 128 | printf("\n"); 129 | printf("Memento CLI v %s\n", VERSION); 130 | printf("Type h or ? or help to print command list\n\n"); 131 | } 132 | // print help 133 | void help(void) { 134 | printf("\n"); 135 | printf("SET key value Sets to \n"); 136 | printf("GET key Get the value identified by \n"); 137 | printf("DEL key key2 .. keyN Delete values identified by ..\n"); 138 | printf("INC key qty Increment by the value idenfied by , if\n"); 139 | printf(" no is specified increment by 1\n"); 140 | printf("DEC key qty Decrement by the value idenfied by , if\n"); 141 | printf(" no is specified decrement by 1\n"); 142 | printf("INCF key qty Increment by float the value identified\n"); 143 | printf(" by , if no is specified increment by 1.0\n"); 144 | printf("DECF key qty Decrement by the value identified by ,\n"); 145 | printf(" if no is specified decrement by 1.0\n"); 146 | printf("GETP key Get all information of a key-value pair represented by\n"); 147 | printf(" , like key, value, creation time and expire time\n"); 148 | printf("APPEND key value Append to \n"); 149 | printf("PREPEND key value Prepend to \n"); 150 | printf("FLUSH Delete all maps stored inside partitions\n"); 151 | printf("QUIT/EXIT Close connection\n"); 152 | printf("\n"); 153 | } 154 | -------------------------------------------------------------------------------- /src/memento.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to 6 | * deal in the Software without restriction, including without limitation the 7 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | * sell copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | * IN THE SOFTWARE. 21 | */ 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include "util.h" 32 | #include "cluster.h" 33 | #include "networking.h" 34 | 35 | 36 | static void exit_handler(int sig) { 37 | signal(sig, SIG_IGN); 38 | cluster_destroy(); 39 | INFO("Shutting down, closing all listening sockets\n"); 40 | } 41 | 42 | 43 | /* 44 | * The only thread used in the system, just for connect all nodes defined 45 | * in the configuration file 46 | */ 47 | static void *form_cluster_thread(void *p) { 48 | int len = 0; 49 | while (instance.lock == 1) { 50 | list_node *cursor = instance.cluster->head; 51 | while (cursor) { 52 | /* count for UNREACHABLE nodes */ 53 | len = cluster_unreachable_count(); 54 | cluster_node *n = (cluster_node *) cursor->data; 55 | if (cluster_reachable(n) == 0) { 56 | DEBUG("Trying to connect to cluster node %s:%d\n", 57 | n->addr, n->port); 58 | char p[5]; 59 | sprintf(p, "%d", n->port); 60 | if (cluster_join(n->addr, p) == 1) len--; 61 | } 62 | cursor = cursor->next; 63 | } 64 | if (len <= 0) instance.lock = 0; // all nodes are connected, release lock 65 | sleep(3); 66 | } 67 | DEBUG("Cluster node succesfully joined to the cluster\n"); 68 | cluster_balance(); 69 | DEBUG("Keyspace correctly balanced\n"); 70 | // debug check 71 | // FIXME: remove 72 | list_node *cursor = instance.cluster->head; 73 | while (cursor) { 74 | cluster_node *n = (cluster_node *) cursor->data; 75 | DEBUG("\t%s:%d -> [%u,%u]\n", n->addr, n->port, n->range_min, n->range_max); 76 | cursor = cursor->next; 77 | } 78 | INFO("Cluster succesfully formed\n"); 79 | return NULL; 80 | } 81 | 82 | 83 | int main(int argc, char **argv) { 84 | signal(SIGINT, exit_handler); 85 | 86 | /* Initialize random seed */ 87 | srand((unsigned int) time(NULL)); 88 | const char *home = get_homedir(); 89 | char *address = "127.0.0.1"; 90 | char *port = "8082"; 91 | char *confpath = "/.memento"; // default path for configuration ~/.memento 92 | char *filename = malloc(strlen(home) + strlen(confpath)); 93 | sprintf(filename, "%s%s", home, confpath); 94 | char *id = NULL; 95 | int opt, cluster_mode = 0; 96 | static pthread_t thread; 97 | 98 | while((opt = getopt(argc, argv, "a:i:p:cf:")) != -1) { 99 | switch(opt) { 100 | case 'a': 101 | address = optarg; 102 | break; 103 | case 'p': 104 | port = optarg; 105 | break; 106 | case 'i': 107 | id = optarg; 108 | break; 109 | case 'c': 110 | cluster_mode = 1; 111 | break; 112 | case 'f': 113 | filename = optarg; 114 | cluster_mode = 1; 115 | break; 116 | default: 117 | cluster_mode = 0; 118 | break; 119 | } 120 | } 121 | 122 | if (GETINT(port) > 65435) { 123 | fprintf(stderr, "Port must be at lesser than or equal to 65435\n"); 124 | exit(EXIT_FAILURE); 125 | } 126 | 127 | char bus_port[20]; 128 | int bport = GETINT(port) + 100; 129 | sprintf(bus_port, "%d", bport); 130 | 131 | INFO("Memento version v1.0.0\n"); 132 | INFO("Listening on %s:%s\n", address, port); 133 | 134 | /* If cluster mode is enabled Initialize cluster map */ 135 | if (cluster_mode == 1) { 136 | 137 | INFO("Cluster mode enabled; configuration=%s\n", filename); 138 | 139 | init_system(1, id, address, port, bus_port); 140 | 141 | /* read cluster configuration */ 142 | FILE *file = fopen(filename, "r"); 143 | char line[256]; 144 | int linenr = 0; 145 | 146 | if (file == NULL) { 147 | perror("Couldn't open the configuration file\n"); 148 | free(filename); 149 | exit(EXIT_FAILURE); 150 | } 151 | 152 | INFO("Applying cluster configuration:\n"); 153 | 154 | while (fgets(line, 256, (FILE *) file) != NULL) { 155 | 156 | char *ip = malloc(15); 157 | char *pt = malloc(5); 158 | char *name = calloc(1, 256); 159 | int self_flag = 0; 160 | linenr++; 161 | 162 | /* skip comments line */ 163 | if (line[0] == '#') continue; 164 | 165 | if (sscanf(line, "%s %s %s %d", ip, pt, name, &self_flag) != 4) { 166 | fprintf(stderr, "Syntax error, line %d\n", linenr); 167 | continue; 168 | } 169 | 170 | DEBUG("\tNode #%d ip %s port %s name %s self %d\n", 171 | linenr, ip, pt, name, self_flag); 172 | 173 | if (GETINT(pt) > 65435) { 174 | fprintf(stderr, "[CFG] - Port must be at lesser than or equal to 65435\n"); 175 | exit(EXIT_FAILURE); 176 | } 177 | 178 | 179 | /* create a new node and add it to the list */ 180 | cluster_node *new_node = shb_malloc(sizeof(cluster_node)); 181 | new_node->addr = ip; 182 | new_node->port = GETINT(pt) + 100; 183 | new_node->state = UNREACHABLE; 184 | free(pt); 185 | 186 | if (self_flag == 1) { 187 | /* check if the node is already present */ 188 | if (cluster_contained(new_node) == 1) { 189 | free(new_node); 190 | if (!id) cluster_set_selfname(name); 191 | continue; 192 | } else { 193 | new_node->self = 1; 194 | new_node->state = REACHABLE; 195 | } 196 | if (!id) cluster_set_selfname(name); 197 | } 198 | else new_node->self = 0; 199 | 200 | new_node->name = name; 201 | 202 | /* add the node to the cluster list */ 203 | cluster_add_node(new_node); 204 | } 205 | 206 | fclose(file); 207 | 208 | /* start forming the thread */ 209 | if (pthread_create(&thread, NULL, &form_cluster_thread, NULL) != 0) 210 | perror("ERROR pthread"); 211 | 212 | } else { 213 | 214 | /* single node mode */ 215 | init_system(0, id, address, port, bus_port); 216 | } 217 | 218 | /* start the main listen loop */ 219 | start_loop(); 220 | return 0; 221 | } 222 | -------------------------------------------------------------------------------- /src/networking.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to 6 | * deal in the Software without restriction, including without limitation the 7 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | * sell copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | * IN THE SOFTWARE. 21 | */ 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include "networking.h" 37 | #include "commands.h" 38 | #include "cluster.h" 39 | #include "event.h" 40 | 41 | 42 | void init_event_loop(const char *host, 43 | const char *server_port, const char *cluster_port) { 44 | instance.el.host = host; 45 | instance.el.server_port = server_port; 46 | instance.el.cluster_port = cluster_port; 47 | instance.el.epoll_workers = EPOLL_WORKERS; 48 | instance.el.max_events = MAX_EVENTS; 49 | instance.el.bufsize = BUFSIZE; 50 | 51 | /* initialize global epollfd */ 52 | if ((instance.el.epollfd = epoll_create1(0)) == -1) { 53 | perror("epoll_create1"); 54 | exit(EXIT_FAILURE); 55 | } 56 | 57 | /* initialize global bus epollfd */ 58 | if ((instance.el.bepollfd = epoll_create1(0)) == -1) { 59 | perror("bepollfd not created"); 60 | exit(EXIT_FAILURE); 61 | } 62 | 63 | instance.el.server_handler = client_command_handler; 64 | instance.el.cluster_handler = peer_command_handler; 65 | } 66 | 67 | 68 | int send_all(int sfd, char *buf, int *len) { 69 | 70 | int total = 0; 71 | int bytesleft = *len; 72 | int n; 73 | 74 | while (total < *len) { 75 | n = send(sfd, buf + total, bytesleft, 0); 76 | if (n == -1) break; 77 | total += n; 78 | bytesleft -= n; 79 | } 80 | 81 | *len = total; 82 | 83 | return n == -1 ? -1 : 0; 84 | } 85 | 86 | 87 | /* Set non-blocking socket */ 88 | int set_nonblocking(int fd) { 89 | int flags, result; 90 | flags = fcntl(fd, F_GETFL, 0); 91 | if (flags == -1) { 92 | perror("fcntl"); 93 | return -1; 94 | } 95 | result = fcntl(fd, F_SETFL, flags | O_NONBLOCK); 96 | if (result == -1) { 97 | perror("fcntl"); 98 | return -1; 99 | } 100 | return 0; 101 | } 102 | 103 | 104 | /* Auxiliary function for creating epoll server */ 105 | static int create_and_bind(const char *host, const char *port) { 106 | struct addrinfo hints; 107 | struct addrinfo *result, *rp; 108 | int sfd; 109 | 110 | memset(&hints, 0, sizeof (struct addrinfo)); 111 | hints.ai_family = AF_UNSPEC; 112 | hints.ai_socktype = SOCK_STREAM; 113 | hints.ai_flags = AI_PASSIVE; /* 0.0.0.0 all interfaces */ 114 | 115 | if (getaddrinfo(host, port, &hints, &result) != 0) { 116 | perror("getaddrinfo error"); 117 | return -1; 118 | } 119 | 120 | for (rp = result; rp != NULL; rp = rp->ai_next) { 121 | sfd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); 122 | 123 | if (sfd == -1) continue; 124 | 125 | /* set SO_REUSEADDR so the socket will be reusable after process kill */ 126 | if (setsockopt(sfd, SOL_SOCKET, (SO_REUSEPORT | SO_REUSEADDR), 127 | &(int) { 1 }, sizeof(int)) < 0) 128 | perror("SO_REUSEADDR"); 129 | 130 | if ((bind(sfd, rp->ai_addr, rp->ai_addrlen)) == 0) { 131 | /* Succesful bind */ 132 | break; 133 | } 134 | close(sfd); 135 | } 136 | 137 | if (rp == NULL) { 138 | perror("Could not bind"); 139 | return -1; 140 | } 141 | 142 | freeaddrinfo(result); 143 | return sfd; 144 | } 145 | 146 | 147 | /* 148 | * Create a non-blocking socket and make it listen on the specfied address and 149 | * port 150 | */ 151 | int listento(const char *host, const char *port) { 152 | int sfd; 153 | 154 | if ((sfd = create_and_bind(host, port)) == -1) 155 | abort(); 156 | 157 | if ((set_nonblocking(sfd)) == -1) 158 | abort(); 159 | 160 | if ((listen(sfd, SOMAXCONN)) == -1) { 161 | perror("listen"); 162 | abort(); 163 | } 164 | 165 | return sfd; 166 | } 167 | 168 | 169 | /* 170 | * Create a socket and use it to connect to the specified host and port 171 | */ 172 | int connectto(const char *host, const char *port) { 173 | 174 | int p = atoi(port); 175 | struct sockaddr_in serveraddr; 176 | struct hostent *server; 177 | 178 | /* socket: create the socket */ 179 | int sfd = socket(AF_INET, SOCK_STREAM, 0); 180 | if (sfd < 0) perror("ERROR opening socket"); 181 | 182 | /* gethostbyname: get the server's DNS entry */ 183 | server = gethostbyname(host); 184 | if (server == NULL) { 185 | fprintf(stderr, "ERROR, no such host as %s\n", host); 186 | exit(EXIT_FAILURE); 187 | } 188 | 189 | /* build the server's address */ 190 | bzero((char *) &serveraddr, sizeof(serveraddr)); 191 | serveraddr.sin_family = AF_INET; 192 | bcopy((char *) server->h_addr, 193 | (char *) &serveraddr.sin_addr.s_addr, server->h_length); 194 | serveraddr.sin_port = htons(p); 195 | 196 | /* connect: create a connection with the server */ 197 | if (connect(sfd, (const struct sockaddr *) &serveraddr, 198 | sizeof(serveraddr)) < 0) { 199 | ERROR("Connection error: %s\n", strerror(errno)); 200 | return -1; 201 | } 202 | return sfd; 203 | } 204 | 205 | 206 | static void *worker(void *args) { 207 | 208 | int done = 0; 209 | struct epoll_event *events = calloc(instance.el.max_events, sizeof(*events)); 210 | if (events == NULL) { 211 | perror("calloc(3) failed when attempting to allocate events buffer"); 212 | pthread_exit(NULL); 213 | } 214 | 215 | int events_cnt; 216 | while ((events_cnt = 217 | epoll_wait(instance.el.bepollfd, events, instance.el.max_events, -1)) > 0) { 218 | for (int i = 0; i < events_cnt; i++) { 219 | 220 | if ((events[i].events & EPOLLERR) || 221 | (events[i].events & EPOLLHUP)) { 222 | /* An error has occured on this fd */ 223 | fprintf(stderr, "worker epoll error\n"); 224 | close(events[i].data.fd); 225 | continue; 226 | } 227 | 228 | if (events[i].events & EPOLLIN) { 229 | /* There's some data to be processed */ 230 | if (events[i].data.fd < 0) 231 | continue; 232 | done = (instance.el.server_handler)(events[i].data.fd); 233 | if (done == END || done == 1) { 234 | /* close the connection */ 235 | DEBUG("Closing connection\n"); 236 | close(events[i].data.fd); 237 | // TODO: remove fd from epoll 238 | } 239 | } else if (events[i].events & EPOLLOUT) { 240 | 241 | peer_t *p = (peer_t *) events[i].data.ptr; 242 | if (instance.verbose) { 243 | DEBUG("Answering to client from worker %d\n", p->fd); 244 | } 245 | SET_FD_IN(instance.el.bepollfd, p->fd); 246 | 247 | if (send_all(p->fd, p->data, (int *) &p->size) < 0) 248 | perror("Send data failed"); 249 | 250 | /* Check if struct udata contains allocated memory */ 251 | if (p->alloc == 1) free(p->data); 252 | free(p); 253 | } 254 | } 255 | } 256 | 257 | if (events_cnt == 0) { 258 | fprintf(stderr, "epoll_wait(2) returned 0, but timeout was not specified...?"); 259 | } else { 260 | perror("epoll_wait(2) error"); 261 | } 262 | 263 | free(events); 264 | return NULL; 265 | } 266 | 267 | 268 | static void handle_connection(int fd, int server, int bus) { 269 | int accept_socket; 270 | struct sockaddr addr; 271 | socklen_t addrlen = sizeof addr; 272 | char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV]; 273 | 274 | if ((accept_socket = accept(fd, (struct sockaddr *) &addr, &addrlen)) == -1) { 275 | perror("accept"); 276 | close(fd); 277 | } 278 | 279 | getnameinfo(&addr, addrlen, hbuf, sizeof(hbuf), 280 | sbuf, sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV); 281 | 282 | set_nonblocking(accept_socket); 283 | 284 | /* Client connection check, this case must add the descriptor 285 | to the next worker thread in the list */ 286 | if (fd == server) { 287 | ADD_FD(instance.el.bepollfd, accept_socket); 288 | SET_FD_IN(instance.el.epollfd, server); 289 | DEBUG("Connection %s:%s\n", hbuf, sbuf); 290 | } 291 | 292 | /* Bus connection check, in this case, the descriptor 293 | represents another node connecting, must be added to the 294 | global epoll instance */ 295 | if (fd == bus) { 296 | ADD_FD(instance.el.epollfd, accept_socket); 297 | SET_FD_IN(instance.el.epollfd, bus); 298 | DEBUG("New node connected %s:%s\n", hbuf, sbuf); 299 | } 300 | } 301 | 302 | 303 | static int handle_input(int fd) { 304 | /* There's some data from peer nodes to be processed, check if 305 | * the lock is released (i.e. the cluster has succesfully formed) and handle 306 | * incoming messages 307 | */ 308 | int done = 0; 309 | if (instance.lock == 0) { 310 | if (instance.verbose) DEBUG("Handling request from %d\n", fd); 311 | done = (instance.el.cluster_handler)(fd); 312 | if (done == END) { 313 | /* close the connection */ 314 | DEBUG("Closing connection\n"); 315 | close(fd); 316 | return -1; 317 | } 318 | } 319 | return 0; 320 | } 321 | 322 | 323 | static void handle_output(void *ptr) { 324 | peer_t *p = (peer_t *) ptr; 325 | if (instance.verbose) DEBUG("Send to fd %d\n", p->fd); 326 | SET_FD_IN(instance.el.epollfd, p->fd); 327 | if (send_all(p->fd, p->data, (int *) &p->size) < 0) 328 | perror("Send data failed"); 329 | /* Check if struct udata contains allocated memory */ 330 | if (p->alloc == 1) free(p->data); 331 | free(p); 332 | } 333 | 334 | 335 | /* 336 | * Main event loop thread, awaits for incoming connections using the global 337 | * epoll instance, his main responsibility is to pass incoming client 338 | * connections descriptor to a worker thread according to a simple round robin 339 | * scheduling, other than this, it is the sole responsible of the communication 340 | * between nodes if the system is started in cluster mode. 341 | */ 342 | int start_loop(void) { 343 | struct epoll_event *events = calloc(instance.el.max_events, sizeof(*events)); 344 | if (events == NULL) { 345 | perror("calloc(3) failed when attempting to allocate events buffer"); 346 | pthread_exit(NULL); 347 | } 348 | /* initialize two sockets: 349 | * - one for incoming client connections 350 | * - a second for intercommunication between nodes 351 | */ 352 | int fds[2] = { 353 | listento(instance.el.host, instance.el.server_port), 354 | listento(instance.el.host, instance.el.cluster_port) 355 | }; 356 | /* worker thread pool */ 357 | pthread_t workers[instance.el.epoll_workers]; 358 | /* I/0 thread pool initialization, allocating a worker_epool structure for 359 | each one. A worker_epool structure is formed of an epoll descriptor and 360 | his event queue. Every worker_epoll is added to a list, in order to 361 | reuse them in the event loop to add connecting descriptors in a round 362 | robin scheduling */ 363 | for (int i = 0; i < instance.el.epoll_workers; ++i) 364 | pthread_create(&workers[i], NULL, worker, NULL); 365 | 366 | int nfds; 367 | 368 | /* Add two already listening nonblocking descriptors to the loop, the first 369 | one represent the main point of access for clients, the second one is 370 | responsible for the communication between nodes (bus) */ 371 | for (int n = 0; n < 2; ++n) { 372 | ADD_FD(instance.el.epollfd, fds[n]); 373 | } 374 | /* Start the main event loop, epoll_wait blocks until an event occur */ 375 | while (1) { 376 | /* Reset the cycle of the round-robin selection of epoll fds */ 377 | if ((nfds = epoll_wait(instance.el.epollfd, 378 | events, instance.el.max_events, -1)) == -1) { 379 | perror("epoll_wait"); 380 | exit(EXIT_FAILURE); 381 | } 382 | for (int i = 0; i < nfds; ++i) { 383 | if ((events[i].events & EPOLLERR) || 384 | (events[i].events & EPOLLHUP)) { 385 | /* An error has occured on this fd, or the socket is not 386 | ready for reading */ 387 | perror ("epoll error"); 388 | close(events[i].data.fd); 389 | continue; 390 | } 391 | /* If fdescriptor is main server or bus server add it to worker 392 | threads or to the global epoll instance respectively */ 393 | if (events[i].data.fd == fds[0] 394 | || events[i].data.fd == fds[1]) { 395 | handle_connection(events[i].data.fd, fds[0], fds[1]); 396 | } else if (events[i].events & EPOLLIN) { 397 | if (handle_input(events[i].data.fd) == -1) 398 | break; 399 | } 400 | else if (events[i].events & EPOLLOUT) { 401 | handle_output(events[i].data.ptr); 402 | } 403 | } 404 | } 405 | free(events); 406 | return 0; 407 | } 408 | 409 | 410 | /* 411 | * Add peer_t structure to the global write queue, a writer worker will 412 | * handle it 413 | */ 414 | void schedule_write(peer_t *p) { 415 | if (p->tocli == 0) { 416 | if (instance.verbose) { 417 | DEBUG("Scheduled write to peer fd: %d\n", p->fd); 418 | } 419 | SET_FD_OUT(instance.el.epollfd, p->fd, p); 420 | } else { 421 | if (instance.verbose) { 422 | DEBUG("Scheduled write to client\n"); 423 | } 424 | SET_FD_OUT(instance.el.bepollfd, p->fd, p); 425 | } 426 | } 427 | -------------------------------------------------------------------------------- /src/networking.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to 6 | * deal in the Software without restriction, including without limitation the 7 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | * sell copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | * IN THE SOFTWARE. 21 | */ 22 | 23 | #ifndef NETWORKING_H 24 | #define NETWORKING_H 25 | 26 | 27 | #include 28 | 29 | 30 | #define EPOLL_WORKERS 4 31 | #define MAX_EVENTS 64 32 | #define BUFSIZE 2048 33 | 34 | 35 | /* 36 | * File descriptor handler, a functor used when incoming data from clients must 37 | * be handled 38 | */ 39 | typedef int (*fd_handler)(int, int); 40 | 41 | 42 | typedef struct peer { 43 | int fd; 44 | unsigned int size; 45 | unsigned int alloc : 1; 46 | unsigned int tocli : 1; 47 | char *data; 48 | } peer_t; 49 | 50 | 51 | typedef struct { 52 | const char *host; /* hostname to listen */ 53 | const char *server_port; /* main entry point for connecting clients */ 54 | const char *cluster_port; /* cluster service connection port for nodes */ 55 | int epoll_workers; /* epoll workers threadpool number */ 56 | int max_events; /* max number of events handled by epoll loop */ 57 | int bufsize; /* buffer size for reading data from sockets */ 58 | int epollfd; /* file descriptor for epoll */ 59 | int bepollfd; /* file descriptor for epoll on the bus */ 60 | int (*cluster_handler)(int); /* function pointer to cluster communication implementation function */ 61 | int (*server_handler)(int); /* function pointer to client-server communication implementation function */ 62 | } event_loop; 63 | 64 | 65 | void init_event_loop(const char *, const char *, const char *); 66 | int send_all(int, char *, int *); 67 | int set_nonblocking(int); 68 | int listento(const char *, const char *); 69 | int connectto(const char *, const char *); 70 | int start_loop(); 71 | void schedule_write(peer_t *); 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /src/persistence.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to 6 | * deal in the Software without restriction, including without limitation the 7 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | * sell copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | * IN THE SOFTWARE. 21 | */ 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include "persistence.h" 33 | 34 | 35 | //TODO: still untouched 36 | int async_write(char *str) { 37 | int file = open(PERSISTENCE_LOG, O_CREAT|O_RDWR|O_APPEND, 0644); 38 | if (file == -1) { 39 | perror("log"); 40 | exit(1); 41 | } 42 | 43 | struct aiocb cb; 44 | 45 | char *buf = malloc(strlen(str)); 46 | strcpy(buf, str); 47 | 48 | memset(&cb, 0, sizeof(struct aiocb)); 49 | cb.aio_nbytes = strlen(str); 50 | cb.aio_fildes = file; 51 | cb.aio_buf = str; 52 | cb.aio_offset = 0; 53 | 54 | if (aio_write(&cb) != 0) { 55 | perror("aiocb"); 56 | return -1; 57 | } 58 | 59 | while (aio_error(&cb) == EINPROGRESS) {} 60 | 61 | 62 | close(file); 63 | return 0; 64 | } 65 | -------------------------------------------------------------------------------- /src/persistence.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to 6 | * deal in the Software without restriction, including without limitation the 7 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | * sell copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | * IN THE SOFTWARE. 21 | */ 22 | 23 | #ifndef PERSISTENCE_H 24 | #define PERSISTENCE_H 25 | 26 | #define PERSISTENCE_LOG "/tmp/memento/" 27 | 28 | int async_write(char *); 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /src/serializer.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to 6 | * deal in the Software without restriction, including without limitation the 7 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | * sell copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | * IN THE SOFTWARE. 21 | */ 22 | 23 | 24 | #include 25 | #include 26 | #include "serializer.h" 27 | 28 | 29 | /* 30 | * Pack a message structure in an array of bytes, following a simple convention: 31 | * 32 | * [ 4 bytes ] | [ 4 bytes ] | [ 2 bytes ] | [ content len bytes ] 33 | * ----------- | ----------- | ----------- | -------------------- 34 | * cont. len | desc. len | flag len | remanining content 35 | * ----------- | ----------- | ----------- | -------------------- 36 | * 37 | * This way it's easier to pass it around through sockets 38 | */ 39 | char *serialize(struct message msg) { 40 | int mlen = strlen(msg.content); 41 | int flen = sizeof(int) + sizeof(unsigned int) + mlen ; // structure whole len 42 | char *serialized = malloc(sizeof(char) * (flen + sizeof(int))); // serialization whole len 43 | char *metadata = serialized; 44 | char *fd = serialized + sizeof(int); // space for descriptor 45 | char *fp = fd + sizeof(int); // space for flag 46 | char *content = fp + sizeof(unsigned int); // space for content message 47 | *((int*) metadata) = mlen; 48 | *((int*) fd) = msg.fd; 49 | *((unsigned int*) fp) = msg.ready; 50 | strcpy(content, msg.content); 51 | return serialized; 52 | } 53 | 54 | 55 | /* 56 | * Unpack an array of bytes reconstructing the original message structure 57 | */ 58 | struct message deserialize(char *serialized_msg) { 59 | char *metadata = serialized_msg; 60 | char *fd = metadata + sizeof(int); 61 | char *fp = fd + sizeof(int); 62 | char *content = fp + sizeof(unsigned int); 63 | struct message msg; 64 | int mlen = *((int *) metadata); 65 | msg.fd = *((int *) fd); 66 | msg.ready = *((unsigned int *) fp); 67 | msg.content = malloc((mlen + 1) * sizeof(char)); 68 | strcpy(msg.content, content); 69 | return msg; 70 | } 71 | -------------------------------------------------------------------------------- /src/serializer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to 6 | * deal in the Software without restriction, including without limitation the 7 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | * sell copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in 12 | * all copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | * IN THE SOFTWARE. 21 | */ 22 | 23 | 24 | #ifndef SERIALIZER_H 25 | #define SERIALIZER_H 26 | 27 | 28 | #define S_OFFSET (sizeof(int) * 2) + sizeof(unsigned int) 29 | 30 | 31 | /* 32 | * Structure representing a message, used to exchange data between nodes 33 | * using internal bus 34 | */ 35 | struct message { 36 | char *content; 37 | int fd; 38 | unsigned int ready : 1; 39 | }; 40 | 41 | /* serialization API */ 42 | char *serialize(struct message); 43 | struct message deserialize(char *); 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /src/util.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include "util.h" 31 | #include "cluster.h" 32 | 33 | 34 | #define CHECK_PTR(ptr) \ 35 | if (!ptr) { \ 36 | perror("No memory available"); \ 37 | return NULL; \ 38 | } \ 39 | return ptr; 40 | 41 | 42 | /* The implementation here was originally done by Gary S. Brown. Slighltly 43 | * modified by Pete Warden, without any imposition on the reuse of the code. 44 | */ 45 | 46 | /* ============================================================= */ 47 | /* COPYRIGHT (C) 1986 Gary S. Brown. You may use this program, or */ 48 | /* code or tables extracted from it, as desired without restriction. */ 49 | /* */ 50 | /* First, the polynomial itself and its table of feedback terms. The */ 51 | /* polynomial is */ 52 | /* X^32+X^26+X^23+X^22+X^16+X^12+X^11+X^10+X^8+X^7+X^5+X^4+X^2+X^1+X^0 */ 53 | /* */ 54 | /* Note that we take it "backwards" and put the highest-order term in */ 55 | /* the lowest-order bit. The X^32 term is "implied"; the LSB is the */ 56 | /* X^31 term, etc. The X^0 term (usually shown as "+1") results in */ 57 | /* the MSB being 1. */ 58 | /* */ 59 | /* Note that the usual hardware shift register implementation, which */ 60 | /* is what we're using (we're merely optimizing it by doing eight-bit */ 61 | /* chunks at a time) shifts bits into the lowest-order term. In our */ 62 | /* implementation, that means shifting towards the right. Why do we */ 63 | /* do it this way? Because the calculated CRC must be transmitted in */ 64 | /* order from highest-order term to lowest-order term. UARTs transmit */ 65 | /* characters in order from LSB to MSB. By storing the CRC this way, */ 66 | /* we hand it to the UART in the order low-byte to high-byte; the UART */ 67 | /* sends each low-bit to hight-bit; and the result is transmission bit */ 68 | /* by bit from highest- to lowest-order term without requiring any bit */ 69 | /* shuffling on our part. Reception works similarly. */ 70 | /* */ 71 | /* The feedback terms table consists of 256, 32-bit entries. Notes: */ 72 | /* */ 73 | /* The table can be generated at runtime if desired; code to do so */ 74 | /* is shown later. It might not be obvious, but the feedback */ 75 | /* terms simply represent the results of eight shift/xor opera- */ 76 | /* tions for all combinations of data and CRC register values. */ 77 | /* */ 78 | /* The values must be right-shifted by eight bits by the "updcrc" */ 79 | /* logic; the shift must be unsigned (bring in zeroes). On some */ 80 | /* hardware you could probably optimize the shift in assembler by */ 81 | /* using byte-swap instructions. */ 82 | /* polynomial $edb88320 */ 83 | /* */ 84 | /* -------------------------------------------------------------------- */ 85 | 86 | static unsigned long crc32_tab[] = { 87 | 0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L, 88 | 0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L, 89 | 0xe0d5e91eL, 0x97d2d988L, 0x09b64c2bL, 0x7eb17cbdL, 0xe7b82d07L, 90 | 0x90bf1d91L, 0x1db71064L, 0x6ab020f2L, 0xf3b97148L, 0x84be41deL, 91 | 0x1adad47dL, 0x6ddde4ebL, 0xf4d4b551L, 0x83d385c7L, 0x136c9856L, 92 | 0x646ba8c0L, 0xfd62f97aL, 0x8a65c9ecL, 0x14015c4fL, 0x63066cd9L, 93 | 0xfa0f3d63L, 0x8d080df5L, 0x3b6e20c8L, 0x4c69105eL, 0xd56041e4L, 94 | 0xa2677172L, 0x3c03e4d1L, 0x4b04d447L, 0xd20d85fdL, 0xa50ab56bL, 95 | 0x35b5a8faL, 0x42b2986cL, 0xdbbbc9d6L, 0xacbcf940L, 0x32d86ce3L, 96 | 0x45df5c75L, 0xdcd60dcfL, 0xabd13d59L, 0x26d930acL, 0x51de003aL, 97 | 0xc8d75180L, 0xbfd06116L, 0x21b4f4b5L, 0x56b3c423L, 0xcfba9599L, 98 | 0xb8bda50fL, 0x2802b89eL, 0x5f058808L, 0xc60cd9b2L, 0xb10be924L, 99 | 0x2f6f7c87L, 0x58684c11L, 0xc1611dabL, 0xb6662d3dL, 0x76dc4190L, 100 | 0x01db7106L, 0x98d220bcL, 0xefd5102aL, 0x71b18589L, 0x06b6b51fL, 101 | 0x9fbfe4a5L, 0xe8b8d433L, 0x7807c9a2L, 0x0f00f934L, 0x9609a88eL, 102 | 0xe10e9818L, 0x7f6a0dbbL, 0x086d3d2dL, 0x91646c97L, 0xe6635c01L, 103 | 0x6b6b51f4L, 0x1c6c6162L, 0x856530d8L, 0xf262004eL, 0x6c0695edL, 104 | 0x1b01a57bL, 0x8208f4c1L, 0xf50fc457L, 0x65b0d9c6L, 0x12b7e950L, 105 | 0x8bbeb8eaL, 0xfcb9887cL, 0x62dd1ddfL, 0x15da2d49L, 0x8cd37cf3L, 106 | 0xfbd44c65L, 0x4db26158L, 0x3ab551ceL, 0xa3bc0074L, 0xd4bb30e2L, 107 | 0x4adfa541L, 0x3dd895d7L, 0xa4d1c46dL, 0xd3d6f4fbL, 0x4369e96aL, 108 | 0x346ed9fcL, 0xad678846L, 0xda60b8d0L, 0x44042d73L, 0x33031de5L, 109 | 0xaa0a4c5fL, 0xdd0d7cc9L, 0x5005713cL, 0x270241aaL, 0xbe0b1010L, 110 | 0xc90c2086L, 0x5768b525L, 0x206f85b3L, 0xb966d409L, 0xce61e49fL, 111 | 0x5edef90eL, 0x29d9c998L, 0xb0d09822L, 0xc7d7a8b4L, 0x59b33d17L, 112 | 0x2eb40d81L, 0xb7bd5c3bL, 0xc0ba6cadL, 0xedb88320L, 0x9abfb3b6L, 113 | 0x03b6e20cL, 0x74b1d29aL, 0xead54739L, 0x9dd277afL, 0x04db2615L, 114 | 0x73dc1683L, 0xe3630b12L, 0x94643b84L, 0x0d6d6a3eL, 0x7a6a5aa8L, 115 | 0xe40ecf0bL, 0x9309ff9dL, 0x0a00ae27L, 0x7d079eb1L, 0xf00f9344L, 116 | 0x8708a3d2L, 0x1e01f268L, 0x6906c2feL, 0xf762575dL, 0x806567cbL, 117 | 0x196c3671L, 0x6e6b06e7L, 0xfed41b76L, 0x89d32be0L, 0x10da7a5aL, 118 | 0x67dd4accL, 0xf9b9df6fL, 0x8ebeeff9L, 0x17b7be43L, 0x60b08ed5L, 119 | 0xd6d6a3e8L, 0xa1d1937eL, 0x38d8c2c4L, 0x4fdff252L, 0xd1bb67f1L, 120 | 0xa6bc5767L, 0x3fb506ddL, 0x48b2364bL, 0xd80d2bdaL, 0xaf0a1b4cL, 121 | 0x36034af6L, 0x41047a60L, 0xdf60efc3L, 0xa867df55L, 0x316e8eefL, 122 | 0x4669be79L, 0xcb61b38cL, 0xbc66831aL, 0x256fd2a0L, 0x5268e236L, 123 | 0xcc0c7795L, 0xbb0b4703L, 0x220216b9L, 0x5505262fL, 0xc5ba3bbeL, 124 | 0xb2bd0b28L, 0x2bb45a92L, 0x5cb36a04L, 0xc2d7ffa7L, 0xb5d0cf31L, 125 | 0x2cd99e8bL, 0x5bdeae1dL, 0x9b64c2b0L, 0xec63f226L, 0x756aa39cL, 126 | 0x026d930aL, 0x9c0906a9L, 0xeb0e363fL, 0x72076785L, 0x05005713L, 127 | 0x95bf4a82L, 0xe2b87a14L, 0x7bb12baeL, 0x0cb61b38L, 0x92d28e9bL, 128 | 0xe5d5be0dL, 0x7cdcefb7L, 0x0bdbdf21L, 0x86d3d2d4L, 0xf1d4e242L, 129 | 0x68ddb3f8L, 0x1fda836eL, 0x81be16cdL, 0xf6b9265bL, 0x6fb077e1L, 130 | 0x18b74777L, 0x88085ae6L, 0xff0f6a70L, 0x66063bcaL, 0x11010b5cL, 131 | 0x8f659effL, 0xf862ae69L, 0x616bffd3L, 0x166ccf45L, 0xa00ae278L, 132 | 0xd70dd2eeL, 0x4e048354L, 0x3903b3c2L, 0xa7672661L, 0xd06016f7L, 133 | 0x4969474dL, 0x3e6e77dbL, 0xaed16a4aL, 0xd9d65adcL, 0x40df0b66L, 134 | 0x37d83bf0L, 0xa9bcae53L, 0xdebb9ec5L, 0x47b2cf7fL, 0x30b5ffe9L, 135 | 0xbdbdf21cL, 0xcabac28aL, 0x53b39330L, 0x24b4a3a6L, 0xbad03605L, 136 | 0xcdd70693L, 0x54de5729L, 0x23d967bfL, 0xb3667a2eL, 0xc4614ab8L, 137 | 0x5d681b02L, 0x2a6f2b94L, 0xb40bbe37L, 0xc30c8ea1L, 0x5a05df1bL, 138 | 0x2d02ef8dL 139 | }; 140 | 141 | /* Return a 32-bit CRC of the contents of the buffer. */ 142 | unsigned long crc32(const unsigned char *s, unsigned int len) { 143 | unsigned int i; 144 | unsigned long crc32val; 145 | 146 | crc32val = 0; 147 | for (i = 0; i < len; i ++) { 148 | crc32val = crc32_tab[(crc32val ^ s[i]) & 0xff] ^ (crc32val >> 8); 149 | } 150 | return crc32val; 151 | } 152 | 153 | 154 | /* 155 | * Safely memory allocation function 156 | */ 157 | void *shb_malloc(size_t size) { 158 | void *ptr = malloc(size); 159 | CHECK_PTR(ptr); 160 | } 161 | 162 | 163 | /* 164 | * Return the current timestamp in milliseconds 165 | */ 166 | long long current_timestamp(void) { 167 | struct timeval te; 168 | gettimeofday(&te, NULL); 169 | long long ms = te.tv_sec * 1000LL + te.tv_usec / 1000; 170 | return ms; 171 | } 172 | 173 | /* 174 | * trim string, removing leading and trailing spaces 175 | */ 176 | void trim(char *str) { 177 | int i; 178 | int begin = 0; 179 | int end = strlen(str) - 1; 180 | 181 | while (isspace(str[begin])) 182 | begin++; 183 | 184 | while ((end >= begin) && isspace(str[end])) 185 | end--; 186 | 187 | // Shift all characters back to the start of the string array. 188 | for (i = begin; i <= end; i++) 189 | str[i - begin] = str[i]; 190 | 191 | str[i - begin] = '\0'; // Null terminate string. 192 | } 193 | 194 | /* auxiliary function to check wether a string is an integer */ 195 | int is_integer(const char *s) { 196 | size_t ln = strlen(s) - 1; 197 | for(unsigned long i = 0; i < ln; i++) 198 | if(!isdigit(s[i])) return 0; 199 | return 1; 200 | } 201 | 202 | /* auxiliary function to check wether a string is a floating number */ 203 | int is_float(const char *s) { 204 | if (is_integer(s)) 205 | return 0; 206 | double dnum; 207 | if (sscanf(s, "%lf", &dnum) == 0) return 0; 208 | return 1; 209 | } 210 | 211 | /* auxiliary function to convert integer contained inside string into int */ 212 | int to_int(const char *s) { 213 | int len = 0; 214 | while(isdigit(*s)) { 215 | len = (len * 10) + (*s - '0'); 216 | s++; 217 | } 218 | return len; 219 | } 220 | 221 | 222 | /* auxiliary function to convert double contained inside a string int a double */ 223 | double to_double(const char *s) { 224 | double dnum; 225 | if (sscanf(s, "%lf", &dnum) == 1) return dnum; 226 | return 0.0; 227 | } 228 | 229 | 230 | /* 231 | * Define the current node name inside the cluster 232 | */ 233 | const char *node_name(unsigned int len) { 234 | 235 | char *pool = "abcdefghijklmnopqrstwxyz0123456789"; 236 | /* Length of the string */ 237 | int i = 0; 238 | 239 | char *node_name = malloc(len); 240 | 241 | /* build name using random positions in the poll */ 242 | while(len--) { 243 | node_name[i++] = pool[(rand() % strlen(pool))]; 244 | } 245 | 246 | return node_name; 247 | 248 | } 249 | 250 | 251 | /* 252 | * Find the home directory, checking if $HOME is set 253 | */ 254 | const char *get_homedir(void) { 255 | const char *homedir; 256 | 257 | if ((homedir = getenv("HOME")) == NULL) { 258 | homedir = getpwuid(getuid())->pw_dir; 259 | } 260 | 261 | return homedir; 262 | } 263 | 264 | 265 | void s_log(loglevel level, const char *info, ...) { 266 | /* Print log only if level is the same of the instance loglevel */ 267 | if (level <= instance.log_level) { 268 | va_list argptr; 269 | va_start(argptr, info); 270 | char time_buff[50]; 271 | char prefix[50]; 272 | time_t now = time(0); 273 | if (1 == instance.verbose) { 274 | switch(level) { 275 | case DEBUG: 276 | sprintf(prefix, "[DBG]"); 277 | break; 278 | case ERR: 279 | sprintf(prefix, "\033[1m[ERR]"); 280 | break; 281 | case INFO: 282 | sprintf(prefix, "[INF]"); 283 | break; 284 | default: 285 | sprintf(prefix, "[INF]"); 286 | break; 287 | } 288 | strftime(time_buff, 50, "[%Y-%m-%d %H:%M:%S] - ", localtime(&now)); 289 | } else { 290 | strftime(prefix, 50, "%Y-%m-%d %H:%M:%S", localtime(&now)); 291 | sprintf(time_buff, " "); 292 | } 293 | char content[strlen(prefix) + strlen(info) + strlen(time_buff)]; 294 | memset(content, 0x00, strlen(content)); 295 | strcat(content, prefix); 296 | strcat(content, time_buff); 297 | strcat(content, info); 298 | strcat(content, "\033[0m"); 299 | vfprintf(stdout, content, argptr); 300 | va_end(argptr); 301 | } 302 | } 303 | 304 | 305 | /* Utility function, concat two strings togheter */ 306 | char *append_string(const char *str, const char *token) { 307 | size_t len = strlen(str) + strlen(token); 308 | char *ret = malloc(len * sizeof(char) + 1); 309 | *ret = '\0'; 310 | return strcat(strcat(ret, str), token); 311 | } 312 | 313 | 314 | /* Utility function, remove trailing newline */ 315 | void remove_newline(char *str) { 316 | str[strcspn(str, "\r\n")] = 0; 317 | } 318 | 319 | 320 | config *create_config(void) { 321 | config *conf = malloc(sizeof(config)); 322 | conf->log_level = DEBUG; 323 | conf->name = NULL; 324 | conf->host = "127.0.0.1"; 325 | conf->port = 8081; 326 | return conf; 327 | } 328 | 329 | 330 | void read_config(config *conf, char *filename) { 331 | FILE *file = fopen(filename, "r"); 332 | 333 | if (file == NULL) { 334 | perror("Couldn't open the configuration file\n"); 335 | free(filename); 336 | exit(EXIT_FAILURE); 337 | } 338 | 339 | char line[256]; 340 | int i = 0; 341 | 342 | while (fgets(line, 256, (FILE *) file) != NULL) { 343 | 344 | if (line[0] == '#') { 345 | i++; 346 | continue; 347 | } 348 | 349 | char *cfline; 350 | cfline = strstr((char *) line, DELIM); 351 | cfline = strstr((char *) line, DELIM); 352 | cfline = cfline + strlen(DELIM); 353 | 354 | if (strcmp(line, "loglevel") == 0) { 355 | if (strcmp(cfline, "debug") == 0) 356 | conf->log_level = DEBUG; 357 | else if (strcmp(cfline, "info") == 0) 358 | conf->log_level = INFO; 359 | else conf->log_level = ERR; 360 | } else if (strcmp(line, "nodename") == 0) { 361 | cfline = cfline + strlen(DELIM); 362 | conf->name = strdup(cfline); 363 | } else if (strcmp(line, "host") == 0) { 364 | conf->host = strdup(cfline); 365 | } else if (strcmp(line, "port") == 0) { 366 | conf->port = GETINT(cfline); 367 | } 368 | } 369 | fclose(file); 370 | DEBUG("Configuration file %s succesfully read", filename); 371 | } 372 | -------------------------------------------------------------------------------- /src/util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2017 Andrea Giacomo Baldan 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | * this software and associated documentation files (the "Software"), to deal in 6 | * the Software without restriction, including without limitation the rights to 7 | * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | * the Software, and to permit persons to whom the Software is furnished to do so, 9 | * subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | */ 21 | 22 | #ifndef UTIL_H 23 | #define UTIL_H 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | 30 | #define GETINT(x) to_int(x) 31 | #define GETFLOAT(x) to_float(x) 32 | #define GETDOUBLE(x) to_double(x) 33 | #define ISINT(x) is_integer(x) 34 | #define CRC32(c, x) crc32(c, x) 35 | #define RANDBETWEEN(A,B) A + rand()/(RAND_MAX/(B - A)) 36 | 37 | 38 | /* Log level */ 39 | typedef enum { INFO, ERR, DEBUG } loglevel; 40 | 41 | void *shb_malloc(size_t); 42 | unsigned long crc32(const unsigned char *, unsigned int); 43 | long long current_timestamp(void); 44 | void trim(char *); 45 | int is_integer(const char *); 46 | int is_float(const char *); 47 | int to_int(const char *); 48 | double to_double(const char *); 49 | const char *node_name(unsigned int); 50 | const char *get_homedir(void); 51 | char *append_string(const char *, const char *); 52 | void remove_newline(char *); 53 | 54 | /* conf */ 55 | 56 | #define DELIM "=" 57 | 58 | typedef struct { 59 | loglevel log_level; 60 | const char *name; 61 | const char *host; 62 | int port; 63 | } config; 64 | 65 | config *create_config(); 66 | void read_config(config *, char *); 67 | 68 | /* logging */ 69 | 70 | void s_log(loglevel, const char *, ...); 71 | 72 | #define LOG(...) s_log( __VA_ARGS__ ) 73 | #define DEBUG(...) LOG(DEBUG, __VA_ARGS__) 74 | #define ERROR(...) LOG(ERR, __VA_ARGS__) 75 | #define INFO(...) LOG(INFO, __VA_ARGS__) 76 | 77 | #endif 78 | -------------------------------------------------------------------------------- /tests/Makefile: -------------------------------------------------------------------------------- 1 | CC=gcc 2 | CFLAGS=-std=gnu99 -Wall -lrt -lpthread 3 | RELEASE=../release 4 | SRC=../src/map.c \ 5 | ../src/util.c \ 6 | ../src/hashing.h \ 7 | ../src/cluster.c \ 8 | ../src/commands.c \ 9 | ../src/persistence.c\ 10 | ../src/networking.c \ 11 | ../src/serializer.c \ 12 | ../src/list.c 13 | 14 | 15 | memento: $(SRC) 16 | $(CC) $(CFLAGS) $(SRC) -o memento 17 | 18 | test: dt_test.c 19 | mkdir -p $(RELEASE) && $(CC) $(CFLAGS) $(SRC) -o $(RELEASE)/dt_test dt_test.c && $(RELEASE)/dt_test 20 | -------------------------------------------------------------------------------- /tests/dt_test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "unit.h" 5 | #include "../src/map.h" 6 | #include "../src/list.h" 7 | 8 | 9 | int tests_run = 0; 10 | 11 | 12 | /* 13 | * Tests the creation of a map 14 | */ 15 | static char *test_map_create(void) { 16 | map *m = map_create(); 17 | ASSERT("[! create]: map not created", m != NULL); 18 | map_release(m); 19 | return 0; 20 | } 21 | 22 | 23 | /* 24 | * Tests the release of a map 25 | */ 26 | static char *test_map_release(void) { 27 | map *m = map_create(); 28 | map_release(m); 29 | ASSERT("[! release]: map not released", m->size == 0); 30 | return 0; 31 | } 32 | 33 | 34 | /* 35 | * Tests the insertion function of the map 36 | */ 37 | static char *test_map_put(void) { 38 | map *m = map_create(); 39 | char *key = "hello"; 40 | char *val = "world"; 41 | int status = map_put(m, strdup(key), strdup(val)); 42 | ASSERT("[! put]: map size = 0", m->size == 1); 43 | ASSERT("[! put]: put didn't work as expected", status == MAP_OK); 44 | char *val1 = "WORLD"; 45 | map_put(m, strdup(key), strdup(val1)); 46 | void *ret = map_get(m, key); 47 | ASSERT("[! put]: put didn't update the value", strcmp(val1, ret) == 0); 48 | map_release(m); 49 | return 0; 50 | } 51 | 52 | 53 | /* 54 | * Tests lookup function of the map 55 | */ 56 | static char *test_map_get(void) { 57 | map *m = map_create(); 58 | char *key = "hello"; 59 | char *val = "world"; 60 | map_put(m, strdup(key), strdup(val)); 61 | char *ret = (char *) map_get(m, key); 62 | ASSERT("[! get]: get didn't work as expected", strcmp(ret, val) == 0); 63 | map_release(m); 64 | return 0; 65 | } 66 | 67 | 68 | /* 69 | * Tests the deletion function of the map 70 | */ 71 | static char *test_map_del(void) { 72 | map *m = map_create(); 73 | char *key = "hello"; 74 | char *val = "world"; 75 | map_put(m, strdup(key), strdup(val)); 76 | int status = map_del(m, key); 77 | ASSERT("[! del]: map size = 1", m->size == 0); 78 | ASSERT("[! del]: del didn't work as expected", status == MAP_OK); 79 | map_release(m); 80 | return 0; 81 | } 82 | 83 | 84 | /* 85 | * Tests the iteration of map_iterate2 86 | */ 87 | static int destroy_map(void *arg1, void *arg2) { 88 | map_entry *entry = (map_entry *) arg2; 89 | char *alt = "altered"; 90 | strcpy(entry->val, alt); 91 | return 0; 92 | } 93 | 94 | static char *test_map_iterate2(void) { 95 | map *m = map_create(); 96 | char *key0 = "hello"; 97 | char *val0 = "world"; 98 | char *key1 = "this"; 99 | char *val1 = "time"; 100 | map_put(m, strdup(key0), strdup(val0)); 101 | map_put(m, strdup(key1), strdup(val1)); 102 | map_iterate2(m, destroy_map, NULL); 103 | char *v0 = (char *) map_get(m, key0); 104 | char *v1 = (char *) map_get(m, key1); 105 | char *alt = "altered"; 106 | ASSERT("[! iterate2]: iteration function is not correctly applied", 107 | (strncmp(v0, alt, 7) == 0) && (strncmp(v1, alt, 7) == 0)); 108 | map_release(m); 109 | return 0; 110 | } 111 | 112 | 113 | /* 114 | * Tests the creation of a list 115 | */ 116 | static char *test_list_create(void) { 117 | list *l = list_create(); 118 | ASSERT("[! create]: list not created", l != NULL); 119 | list_release(l); 120 | return 0; 121 | } 122 | 123 | 124 | /* 125 | * Tests the release of a list 126 | */ 127 | static char *test_list_release(void) { 128 | list *l = list_create(); 129 | list_release(l); 130 | ASSERT("[! release]: list not released", l != NULL); 131 | return 0; 132 | } 133 | 134 | 135 | /* 136 | * Tests the insertion function of the list to the head node 137 | */ 138 | static char *test_list_head_insert(void) { 139 | list *l = list_create(); 140 | char *val = "hello world"; 141 | list *ll = list_head_insert(l, strdup(val)); 142 | ASSERT("[! head insert]: list size = 0", l->len == 1); 143 | ASSERT("[! head insert]: head insert didn't work as expected", ll != NULL); 144 | list_release(ll); 145 | return 0; 146 | } 147 | 148 | 149 | /* 150 | * Tests the insertion function of the list to the tail node 151 | */ 152 | static char *test_list_tail_insert(void) { 153 | list *l = list_create(); 154 | char *val = "hello world"; 155 | list *ll = list_tail_insert(l, strdup(val)); 156 | ASSERT("[! tail insert]: list size = 0", l->len == 1); 157 | ASSERT("[! tail insert]: tail insert didn't work as expected", ll != NULL); 158 | list_release(ll); 159 | return 0; 160 | } 161 | 162 | 163 | /* 164 | * All datastructure tests 165 | */ 166 | static char *all_tests() { 167 | RUN_TEST(test_map_create); 168 | RUN_TEST(test_map_release); 169 | RUN_TEST(test_map_put); 170 | RUN_TEST(test_map_get); 171 | RUN_TEST(test_map_del); 172 | RUN_TEST(test_map_iterate2); 173 | RUN_TEST(test_list_create); 174 | RUN_TEST(test_list_release); 175 | RUN_TEST(test_list_head_insert); 176 | RUN_TEST(test_list_tail_insert); 177 | 178 | return 0; 179 | } 180 | 181 | int main(int argc, char **argv) { 182 | char *result = all_tests(); 183 | if (result != 0) { 184 | printf(" %s\n", result); 185 | } 186 | else { 187 | printf("\n [*] ALL TESTS PASSED\n"); 188 | } 189 | printf(" [*] Tests run: %d\n\n", tests_run); 190 | 191 | return result != 0; 192 | } 193 | -------------------------------------------------------------------------------- /tests/unit.h: -------------------------------------------------------------------------------- 1 | #ifndef UNIT_H 2 | #define UNIT_H 3 | 4 | #define ASSERT(message, test) do { if (!(test)) return message; } while (0) 5 | 6 | #define RUN_TEST(test) do { char *message = test(); tests_run++; \ 7 | if (message) return message; } while (0) 8 | 9 | extern int tests_run; 10 | 11 | #endif 12 | --------------------------------------------------------------------------------