├── docs ├── doxygen │ ├── .gitignore │ ├── Makefile │ └── template │ │ ├── footer.html │ │ └── header.html ├── arch.png ├── arch.graffle ├── shardcached.png └── shardcached.graffle ├── deps ├── .gitignore └── Makefile ├── storage_plugins ├── deps │ ├── .gitignore │ └── Makefile ├── sqlite │ ├── Makefile │ └── sqlite.c ├── mysql │ ├── Makefile │ └── mysql.c ├── Makefile ├── redis │ ├── Makefile │ └── redis.c ├── riak │ ├── Makefile │ └── riak.c └── README.md ├── src ├── storage_fs.h ├── storage_mem.h ├── shcd_http.h ├── storage.h ├── acl.h ├── ini.h ├── acl.c ├── storage.c ├── storage_mem.c ├── ini.c ├── mongoose.h ├── storage_fs.c └── shcd_http.c ├── .gitignore ├── utils ├── storage_fs_allkeys.sh ├── shcd_del.pl ├── shcd_list.pl ├── shcd_stats.pl ├── shcd_put.pl └── shcd_get.pl ├── TODO ├── .gitmodules ├── Makefile ├── test ├── shardcached_test.c ├── stringx.h ├── urlparser.h └── http-client-c.h ├── shardcached.ini.example └── README.md /docs/doxygen/.gitignore: -------------------------------------------------------------------------------- 1 | html 2 | -------------------------------------------------------------------------------- /deps/.gitignore: -------------------------------------------------------------------------------- 1 | .incs 2 | .libs 3 | -------------------------------------------------------------------------------- /storage_plugins/deps/.gitignore: -------------------------------------------------------------------------------- 1 | .libs 2 | .incs 3 | -------------------------------------------------------------------------------- /docs/arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shardcached/shardcached/HEAD/docs/arch.png -------------------------------------------------------------------------------- /docs/arch.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shardcached/shardcached/HEAD/docs/arch.graffle -------------------------------------------------------------------------------- /docs/shardcached.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shardcached/shardcached/HEAD/docs/shardcached.png -------------------------------------------------------------------------------- /docs/shardcached.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shardcached/shardcached/HEAD/docs/shardcached.graffle -------------------------------------------------------------------------------- /src/storage_fs.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int storage_fs_init(shardcache_storage_t *st, char **options); 4 | void storage_fs_destroy(void *priv); 5 | -------------------------------------------------------------------------------- /src/storage_mem.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int storage_mem_init(shardcache_storage_t *st, char **options); 4 | void storage_mem_destroy(void *priv); 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | shardcached 2 | *~ 3 | *.swp 4 | *.swo 5 | *.swn 6 | *.o 7 | *.a 8 | *.out 9 | *.so 10 | *.dylib 11 | test/*_test 12 | .DS_Store 13 | *.dSYM 14 | shardcached_access.log 15 | shardcached_error.log 16 | *.storage 17 | -------------------------------------------------------------------------------- /utils/storage_fs_allkeys.sh: -------------------------------------------------------------------------------- 1 | #/bin/sh 2 | 3 | if [ "X$1" == "X" ]; then 4 | echo "Usage : $0 " 5 | exit 6 | fi 7 | 8 | for i in `find $1 -type f`; do 9 | name=`basename $i` 10 | perl -e '$a = join("", map { chr(hex($_)) } unpack("(A2)*", "$ARGV[0]")); print "$a\n"' $name 2>/dev/null 11 | done 12 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | * more statistics on work done outside of the worker threads 2 | (like when shardcache_get()/set() is called from an http thread and 3 | it involves fetching/sending data from/to a peer) 4 | 5 | * rest storage plugin 6 | 7 | * HTTP interface: handle more received and sent HTTP headers (modification 8 | times, partial content...) 9 | -------------------------------------------------------------------------------- /storage_plugins/sqlite/Makefile: -------------------------------------------------------------------------------- 1 | TARGET=sqlite 2 | EXT=storage 3 | 4 | all: CFLAGS += -I../../deps/.incs 5 | all: LDFLAGS += -L../../deps/.libs 6 | all: $(TARGET) 7 | 8 | dynamic: $(TARGET) 9 | 10 | $(TARGET): $(TARGET).c 11 | $(CC) $(CFLAGS) $(LDFLAGS) -fPIC -shared $(TARGET).c -o $(TARGET).$(EXT) -lsqlite3 12 | 13 | clean: 14 | @rm $(TARGET).$(EXT) 15 | -------------------------------------------------------------------------------- /storage_plugins/mysql/Makefile: -------------------------------------------------------------------------------- 1 | TARGET=mysql 2 | EXT=storage 3 | 4 | all: CFLAGS += -I../../deps/.incs 5 | all: LDFLAGS += -L../../deps/.libs 6 | all: $(TARGET) 7 | 8 | dynamic: $(TARGET) 9 | 10 | $(TARGET): $(TARGET).c 11 | $(CC) $(CFLAGS) $(LDFLAGS) -fPIC -shared $(TARGET).c -o $(TARGET).$(EXT) -lmysqlclient 12 | 13 | clean: 14 | @rm $(TARGET).$(EXT) 15 | 16 | -------------------------------------------------------------------------------- /storage_plugins/Makefile: -------------------------------------------------------------------------------- 1 | PLUGINS=sqlite mysql redis 2 | 3 | all: build_deps $(PLUGINS) 4 | 5 | dynamic: 6 | @for i in $(PLUGINS); do make -C $$i dynamic; done 7 | 8 | .PHONY: $(PLUGINS) 9 | $(PLUGINS): 10 | make -C $@ all 11 | 12 | .PHONY: clean 13 | clean: 14 | -@for i in $(PLUGINS); do make -C $$i clean; done 15 | 16 | .PHONY: build_deps 17 | build_deps: 18 | @make -eC deps all 19 | -------------------------------------------------------------------------------- /storage_plugins/redis/Makefile: -------------------------------------------------------------------------------- 1 | TARGET=redis 2 | EXT=storage 3 | 4 | all: CFLAGS += -I../../deps/.incs -I../deps/.incs 5 | all: LDFLAGS += -L../../deps/.libs ../deps/.libs/libhiredis.a 6 | all: $(TARGET) 7 | 8 | dynamic: LDFLAGS += -lhiredis 9 | dynamic: $(TARGET) 10 | 11 | $(TARGET): $(TARGET).c 12 | $(CC) $(CFLAGS) -fPIC -shared $(TARGET).c $(LDFLAGS) -o $(TARGET).$(EXT) 13 | 14 | clean: 15 | @rm $(TARGET).$(EXT) 16 | -------------------------------------------------------------------------------- /storage_plugins/riak/Makefile: -------------------------------------------------------------------------------- 1 | TARGET=riak 2 | EXT=storage 3 | 4 | all: CFLAGS += -I../../deps/.incs -I../deps/.incs -g 5 | all: LDFLAGS += -L../../deps/.libs ../deps/.libs/libriak_c_client-0.5.a ../deps/.libs/libprotobuf-c.a ../deps/.libs/libprotobuf.a 6 | all: $(TARGET) 7 | 8 | dynamic: LDFLAGS += -lriak_c_client 9 | dynamic: $(TARGET) 10 | 11 | $(TARGET): $(TARGET).c 12 | $(CC) $(CFLAGS) -fPIC -shared $(TARGET).c $(LDFLAGS) -lshardcache -lhl -o $(TARGET).$(EXT) 13 | 14 | clean: 15 | @rm $(TARGET).$(EXT) 16 | -------------------------------------------------------------------------------- /docs/doxygen/Makefile: -------------------------------------------------------------------------------- 1 | docs: clean 2 | @ echo "Constructing a fake include directory hierarchy for Doxygen" 3 | @ mkdir headers 4 | @ mkdir headers/{libchash,libhl,libiomux,libshardcache} 5 | 6 | @ cp ../../deps/libshardcache/src/*h headers/libshardcache/ 7 | @ cp ../../deps/libshardcache/deps/libchash/*h headers/libchash/ 8 | @ cp ../../deps/libshardcache/deps/libiomux/src/*h headers/libiomux/ 9 | @ cp ../../deps/libshardcache/deps/libhl/src/*h headers/libhl/ 10 | 11 | @ echo "Running doxygen" 12 | @ doxygen Doxyfile 13 | @ rm -fr headers 14 | 15 | clean: 16 | @ echo "Cleaning" 17 | @ rm -fr html 18 | @ rm -fr headers 19 | 20 | -------------------------------------------------------------------------------- /utils/shcd_del.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use HTTP::Request; 5 | use LWP; 6 | use File::Slurp; 7 | use URI::Escape; 8 | 9 | my $key = shift @ARGV; 10 | my $hosts = shift @ARGV || $ENV{SHCD_HOSTS}; 11 | 12 | my @hosts_array = split(',', $hosts); 13 | my $host = $hosts_array[int(rand(scalar(@hosts_array)))]; 14 | 15 | $host =~ s/^http:\/\///; 16 | $host =~ s/\/+$//; 17 | 18 | print "Using host $host : "; 19 | 20 | my $path = uri_escape($key); 21 | my $request = HTTP::Request->new("DELETE", "http://$host/$path"); 22 | 23 | my $ua = LWP::UserAgent->new; 24 | my $response = $ua->request($request); 25 | printf "%s : %s\n", $response->code, $response->message; 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/shcd_http.h: -------------------------------------------------------------------------------- 1 | #ifndef SHCD_HTTP_H 2 | #define SHCD_HTTP_H 3 | 4 | #include 5 | #include 6 | #include "acl.h" 7 | 8 | typedef struct _shcd_http_s shcd_http_t; 9 | 10 | shcd_http_t *shcd_http_create(shardcache_t *cache, 11 | const char *me, 12 | const char *basepath, 13 | const char *adminpath, 14 | shcd_acl_t *acl, 15 | hashtable_t *mime_types, 16 | const char **options, 17 | int num_workers); 18 | 19 | void shcd_http_destroy(shcd_http_t *http); 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /docs/doxygen/template/footer.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 12 | 13 | 14 |
15 | $generatedby doxygen $doxygenversion 16 |
17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /src/storage.h: -------------------------------------------------------------------------------- 1 | #ifndef SHCD_STORAGE_H 2 | #define SHCD_STORAGE_H 3 | 4 | #include 5 | 6 | #define MAX_STORAGE_OPTIONS 256 7 | #define MAX_OPTIONS_STRING_LEN 2048 8 | 9 | // 10 | // Check /storage_plugins/README 11 | // for detaile about the Loadable Storage Modules API 12 | // 13 | 14 | typedef struct shcd_storage_s shcd_storage_t; 15 | 16 | shcd_storage_t * shcd_storage_init(char *storage_type, 17 | char *options_string, 18 | char *plugins_dir); 19 | 20 | void shcd_storage_destroy(shcd_storage_t *st); 21 | int shcd_storage_reset(shcd_storage_t *st); 22 | 23 | shardcache_storage_t *shcd_storage_get(shcd_storage_t *st); 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "deps/libshardcache"] 2 | path = deps/libshardcache 3 | url = https://github.com/xant/libshardcache.git 4 | [submodule "storage_plugins/deps/hiredis"] 5 | path = storage_plugins/deps/hiredis 6 | url = https://github.com/redis/hiredis.git 7 | [submodule "support/libut"] 8 | path = support/libut 9 | url = https://github.com/xant/libut.git 10 | [submodule "storage_plugins/deps/protobuf-c"] 11 | path = storage_plugins/deps/protobuf-c 12 | url = https://github.com/protobuf-c/protobuf-c.git 13 | [submodule "storage_plugins/deps/protobuf-cpp"] 14 | path = storage_plugins/deps/protobuf-cpp 15 | url = https://github.com/google/protobuf.git 16 | [submodule "storage_plugins/deps/riak-c-client"] 17 | path = storage_plugins/deps/riak-c-client 18 | url = https://github.com/xant/riak-c-client.git 19 | -------------------------------------------------------------------------------- /utils/shcd_list.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use HTTP::Request; 5 | use LWP; 6 | 7 | $ENV{PERL_LWP_ENV_PROXY} = 1 8 | if ($ENV{http_proxy}); 9 | 10 | my $hosts = shift @ARGV || $ENV{SHCD_HOSTS}; 11 | 12 | my @hosts_array = split(',', $hosts); 13 | 14 | foreach my $host (@hosts_array) { 15 | $host = "http://$host" if ($host !~ /^http:\/\//i); 16 | $host =~ s/\/+$//; 17 | print "** Keys on host $host\n\n"; 18 | my $request = HTTP::Request->new("GET", "$host/__index__?nohtml=1"); 19 | my $ua = LWP::UserAgent->new; 20 | my $response = $ua->request($request); 21 | my $data = $response->content; 22 | my @lines = split("\r\n", $data); 23 | foreach my $line (@lines) { 24 | my ($key, $length) = split(';', $line); 25 | printf("%-64.256s %s\n", $key, $length); 26 | } 27 | print "\n"; 28 | } 29 | 30 | -------------------------------------------------------------------------------- /utils/shcd_stats.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use HTTP::Request; 5 | use LWP; 6 | 7 | $ENV{PERL_LWP_ENV_PROXY} = 1 8 | if ($ENV{http_proxy}); 9 | 10 | my $hosts = shift @ARGV || $ENV{SHCD_HOSTS}; 11 | 12 | my @hosts_array = split(',', $hosts); 13 | 14 | foreach my $host (@hosts_array) { 15 | $host = "http://$host" if ($host !~ /^http:\/\//i); 16 | $host =~ s/\/+$//; 17 | print "** Counters from host $host\n\n"; 18 | my $request = HTTP::Request->new("GET", "$host/__stats__?nohtml=1"); 19 | my $ua = LWP::UserAgent->new; 20 | my $response = $ua->request($request); 21 | my $data = $response->content; 22 | my @lines = split("\r\n", $data); 23 | foreach my $line (@lines) { 24 | my ($key, $value) = split(';', $line); 25 | next if ($key =~ /^worker\[/ && $value == 0); 26 | printf("%-64.256s %s\n", $key, $value); 27 | } 28 | print "\n"; 29 | } 30 | 31 | -------------------------------------------------------------------------------- /deps/Makefile: -------------------------------------------------------------------------------- 1 | export DEPS_INSTALL_DIR=$(shell pwd) 2 | 3 | export SHARDCACHE_INSTALL_LIBDIR := $(DEPS_INSTALL_DIR)/.libs 4 | export SHARDCACHE_INSTALL_INCDIR := $(DEPS_INSTALL_DIR)/.incs 5 | 6 | all: 7 | @mkdir -p .libs; \ 8 | mkdir -p .incs; \ 9 | export LIBDIR="$(LIBDIR)"; \ 10 | export INCDIR="$(INCDIR)"; \ 11 | export DEPS_INSTALL_DIR=$(DEPS_INSTALL_DIR); \ 12 | export CFLAGS="$(CFLAGS)"; \ 13 | if [ ! -f libshardcache/Makefile ]; then \ 14 | cd ..;\ 15 | git submodule init;\ 16 | git submodule update;\ 17 | cd -;\ 18 | fi;\ 19 | make -eC libshardcache; \ 20 | if [ $$? -ne 0 ] ; then exit $$?; fi; \ 21 | make -eC libshardcache install; \ 22 | cp libshardcache/deps/.incs/* $(DEPS_INSTALL_DIR)/.incs/; \ 23 | cp libshardcache/deps/.libs/* $(DEPS_INSTALL_DIR)/.libs/ 24 | 25 | clean: 26 | @make -C libshardcache clean; 27 | @rm -rf $(DEPS_INSTALL_DIR)/.incs 28 | @rm -rf $(DEPS_INSTALL_DIR)/.libs 29 | 30 | -------------------------------------------------------------------------------- /utils/shcd_put.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use HTTP::Request; 5 | use LWP; 6 | use File::Slurp; 7 | use URI::Escape; 8 | 9 | $ENV{PERL_LWP_ENV_PROXY} = 1 10 | if ($ENV{http_proxy}); 11 | 12 | my $input_file = shift @ARGV; 13 | my $key = shift @ARGV; 14 | my $hosts = shift @ARGV || $ENV{SHCD_HOSTS}; 15 | 16 | die "Usage: $0 []" unless($key && $input_file && $hosts); 17 | 18 | my @hosts_array = split(',', $hosts); 19 | my $host = $hosts_array[int(rand(scalar(@hosts_array)))]; 20 | 21 | $host =~ s/^http:\/\///; 22 | $host =~ s/\/+$//; 23 | 24 | print "Using host $host : "; 25 | 26 | my $data = read_file($input_file); 27 | my $req_headers = HTTP::Headers->new( 'Content-Length' => length($data) ); 28 | my $path = uri_escape($key); 29 | my $request = HTTP::Request->new("PUT", "http://$host/$path", $req_headers, $data); 30 | 31 | my $ua = LWP::UserAgent->new; 32 | my $response = $ua->request($request); 33 | printf "%s : %s\n", $response->code, $response->message; 34 | 35 | 36 | -------------------------------------------------------------------------------- /utils/shcd_get.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use HTTP::Request; 5 | use LWP; 6 | use File::Slurp; 7 | use URI::Escape; 8 | 9 | $ENV{PERL_LWP_ENV_PROXY} = 1 10 | if ($ENV{http_proxy}); 11 | 12 | $| = 0; 13 | 14 | my $key = shift @ARGV; 15 | my $output_file = shift @ARGV; 16 | my $hosts = shift @ARGV || $ENV{SHCD_HOSTS}; 17 | 18 | $output_file = $key if !$output_file; 19 | 20 | die "Usage: $0 []" unless($key && $output_file && $hosts); 21 | 22 | my @hosts_array = split(',', $hosts); 23 | my $host = $hosts_array[int(rand(scalar(@hosts_array)))]; 24 | 25 | $host =~ s/^http:\/\///; 26 | $host =~ s/\/+$//; 27 | 28 | print "Using host $host : "; 29 | 30 | my $path = uri_escape($key); 31 | 32 | my $request = HTTP::Request->new("GET", "http://$host/$path"); 33 | 34 | my $ua = LWP::UserAgent->new; 35 | my $response = $ua->request($request); 36 | 37 | if ($response->code == 200) { 38 | my $data = $response->content; 39 | print "Saving to: $output_file\n"; 40 | write_file($output_file, $data); 41 | } 42 | print $response->code . "\n"; 43 | 44 | -------------------------------------------------------------------------------- /src/acl.h: -------------------------------------------------------------------------------- 1 | #ifndef SHCD_ACL_H 2 | #define SHCD_ACL_H 3 | 4 | #include 5 | 6 | typedef enum { 7 | SHCD_ACL_ACTION_ALLOW = 0, 8 | SHCD_ACL_ACTION_DENY = 1 9 | } shcd_acl_action_t; 10 | 11 | typedef enum { 12 | SHCD_ACL_METHOD_GET = 0x00, 13 | SHCD_ACL_METHOD_PUT = 0x01, 14 | SHCD_ACL_METHOD_DEL = 0x02, 15 | SHCD_ACL_METHOD_ANY = 0xff 16 | } shcd_acl_method_t; 17 | 18 | typedef struct _shcd_acl_item_s shcd_acl_item_t; 19 | 20 | typedef struct _shcd_acl_s shcd_acl_t; 21 | 22 | shcd_acl_t *shcd_acl_create(shcd_acl_action_t default_action); 23 | 24 | void shcd_acl_destroy(shcd_acl_t *acl); 25 | 26 | int shcd_acl_add(shcd_acl_t *acl, 27 | char *pattern, 28 | shcd_acl_action_t action, 29 | shcd_acl_method_t method, 30 | uint32_t ip, 31 | uint32_t mask); 32 | 33 | void shcd_acl_clear(shcd_acl_t *acl); 34 | 35 | shcd_acl_action_t shcd_acl_eval(shcd_acl_t *acl, 36 | shcd_acl_method_t method, 37 | char *path, 38 | uint32_t ipaddr); 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /docs/doxygen/template/header.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | $projectname: $title 9 | $title 10 | 11 | 12 | 13 | $treeview 14 | $search 15 | $mathjax 16 | 17 | $extrastylesheet 18 | 19 | 20 |
21 | 22 | 23 |
24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 37 | 38 | 39 | 40 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 |
32 |
$projectname 33 |  $projectnumber 34 |
35 |
$projectbrief
36 | 		41 |
$projectbrief
42 | 		$searchbox
53 |
54 | 55 | 56 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | UNAME := $(shell uname) 2 | 3 | DEPS += deps/.libs/libshardcache.a \ 4 | deps/.libs/libiomux.a \ 5 | deps/.libs/libhl.a \ 6 | deps/.libs/libchash.a \ 7 | deps/.libs/libsiphash.a 8 | 9 | CFLAGS += -DBUILD_INFO="$(BUILD_INFO)" 10 | LDFLAGS += -L. 11 | 12 | ifeq ($(UNAME), Linux) 13 | LDFLAGS += -pthread 14 | else 15 | LDFLAGS += 16 | CFLAGS += -Wno-deprecated-declarations 17 | endif 18 | 19 | MONGOOSE_OPTIONS="-DMONGOOSE_NO_CGI -DMONGOOSE_NO_DAV -DMONGOOSE_NO_SOCKETPAIR -DMONGOOSE_NO_DIRECTORY_LISTING" 20 | 21 | #CC = gcc 22 | TARGETS = $(patsubst %.c, %.o, $(wildcard src/*.c)) 23 | 24 | all: $(DEPS) objects shardcached 25 | 26 | .PHONY: tsan 27 | tsan: 28 | @export CC=gcc-4.8; \ 29 | export LDFLAGS="-pie -ltsan"; \ 30 | export CFLAGS="-fsanitize=thread -g -fPIC -pie"; \ 31 | make all 32 | 33 | 34 | .PHONY: build_deps 35 | build_deps: 36 | @make -eC deps all 37 | 38 | $(LIBSHARDCACHE_DIR)/libshardcache.a: 39 | make -C $(LIBSHARDCACHE_DIR) static 40 | 41 | shardcached: $(DEPS) objects 42 | $(CC) src/*.o $(LDFLAGS) $(DEPS) -o shardcached -ldl 43 | 44 | .PHONY: dynamic 45 | dynamic: objects 46 | $(CC) src/*.o $(LDFLAGS) -o shardcached -lshardcache -lhl 47 | 48 | $(DEPS): build_deps 49 | 50 | .PHONY: objects 51 | objects: CFLAGS += -fPIC $(MONGOOSE_OPTIONS) -Ideps/.incs -Isrc -Ideps/.incs -Wall -Werror -Wno-parentheses -Wno-pointer-sign -O3 -g 52 | objects: $(TARGETS) 53 | 54 | clean: 55 | rm -f src/*.o 56 | rm -f shardcached 57 | rm -f test/*_test 58 | make -C deps clean 59 | 60 | TESTS = $(patsubst %.c, %, $(wildcard test/*.c)) 61 | TEST_EXEC_ORDER = shardcached_test 62 | 63 | .PHONY: libut 64 | libut: 65 | @if [ ! -f support/libut/Makefile ]; then git submodule init; git submodule update; fi; make -C support/libut 66 | 67 | .PHONY: tests 68 | tests: CFLAGS += -Isrc -Isupport/libut/src -Wno-parentheses -Wno-pointer-sign -Wno-pointer-to-int-cast -DTHREAD_SAFE -O3 -g 69 | tests: shardcached libut 70 | @for i in $(TESTS); do\ 71 | echo "$(CC) $(CFLAGS) $$i.c -o $$i $(LDFLAGS) -lm";\ 72 | $(CC) $(CFLAGS) $$i.c -o $$i support/libut/libut.a $(LDFLAGS) -lm;\ 73 | done;\ 74 | for i in $(TEST_EXEC_ORDER); do echo; test/$$i; echo; done 75 | 76 | .PHONY: test 77 | test: tests 78 | 79 | 80 | -------------------------------------------------------------------------------- /src/ini.h: -------------------------------------------------------------------------------- 1 | /* inih -- simple .INI file parser 2 | 3 | inih is released under the New BSD license (see LICENSE.txt). Go to the project 4 | home page for more info: 5 | 6 | http://code.google.com/p/inih/ 7 | 8 | */ 9 | 10 | #ifndef SHCD_INI_H 11 | #define SHCD_INI_H 12 | 13 | /* Make this header file easier to include in C++ code */ 14 | #ifdef __cplusplus 15 | extern "C" { 16 | #endif 17 | 18 | #include 19 | 20 | /* Parse given INI-style file. May have [section]s, name=value pairs 21 | (whitespace stripped), and comments starting with ';' (semicolon). Section 22 | is "" if name=value pair parsed before any section heading. name:value 23 | pairs are also supported as a concession to Python's ConfigParser. 24 | 25 | For each name=value pair parsed, call handler function with given user 26 | pointer as well as section, name, and value (data only valid for duration 27 | of handler call). Handler should return nonzero on success, zero on error. 28 | 29 | Returns 0 on success, line number of first error on parse error (doesn't 30 | stop on first error), -1 on file open error, or -2 on memory allocation 31 | error (only when INI_USE_STACK is zero). 32 | */ 33 | int ini_parse(const char* filename, 34 | int (*handler)(void* user, const char* section, 35 | const char* name, const char* value), 36 | void* user); 37 | 38 | /* Same as ini_parse(), but takes a FILE* instead of filename. This doesn't 39 | close the file when it's finished -- the caller must do that. */ 40 | int ini_parse_file(FILE* file, 41 | int (*handler)(void* user, const char* section, 42 | const char* name, const char* value), 43 | void* user); 44 | 45 | /* Nonzero to allow multi-line value parsing, in the style of Python's 46 | ConfigParser. If allowed, ini_parse() will call the handler with the same 47 | name for each subsequent line parsed. */ 48 | #ifndef INI_ALLOW_MULTILINE 49 | #define INI_ALLOW_MULTILINE 1 50 | #endif 51 | 52 | /* Nonzero to allow a UTF-8 BOM sequence (0xEF 0xBB 0xBF) at the start of 53 | the file. See http://code.google.com/p/inih/issues/detail?id=21 */ 54 | #ifndef INI_ALLOW_BOM 55 | #define INI_ALLOW_BOM 1 56 | #endif 57 | 58 | /* Nonzero to use stack, zero to use heap (malloc/free). */ 59 | #ifndef INI_USE_STACK 60 | #define INI_USE_STACK 1 61 | #endif 62 | 63 | /* Maximum line length for any line in INI file. */ 64 | #ifndef INI_MAX_LINE 65 | #define INI_MAX_LINE 200 66 | #endif 67 | 68 | #ifdef __cplusplus 69 | } 70 | #endif 71 | 72 | #endif /* SHCD_INI_H */ 73 | -------------------------------------------------------------------------------- /test/shardcached_test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "http-client-c.h" 8 | 9 | static pid_t start_node(char * const argv[]) 10 | { 11 | pid_t pid = fork(); 12 | if (pid) { 13 | return pid; 14 | } 15 | execve("./shardcached", argv, NULL); 16 | return 0; 17 | } 18 | 19 | static pid_t stop_node(pid_t pid) 20 | { 21 | kill(pid, 2); 22 | return waitpid(pid, NULL, 0); 23 | } 24 | 25 | int main(int argc, char **argv) 26 | { 27 | ut_init("shardcached_test"); 28 | pid_t children[2]; 29 | int i; 30 | for (i = 0; i < 2; i++) { 31 | char me[32]; 32 | sprintf(me, "-mpeer%d", i+1); 33 | 34 | char listen[32]; 35 | sprintf(listen, "-llocalhost:5432%d", i+1); 36 | 37 | char * const shcd_argv[] = { 38 | "shardcached", "-f", "-npeer1:localhost:4444,peer2:localhost:4445", me, listen, NULL 39 | }; 40 | ut_testing("Start shardcached daemon %d", i+1); 41 | children[i] = start_node(shcd_argv); 42 | ut_validate_int(kill(children[i], 0), 0); 43 | } 44 | 45 | // give the daemons the time to start 46 | sleep(1); 47 | 48 | ut_testing("node1: HTTP PUT /test => TEST"); 49 | struct http_response *response = http_put("http://127.0.0.1:54321/test", NULL, "TEST", 4); 50 | ut_validate_int(response->status_code_int, 200); 51 | http_response_free(response); 52 | 53 | ut_testing("node1: HTTP GET /test == TEST"); 54 | response = http_get("http://127.0.0.1:54321/test", NULL); 55 | ut_validate_string(response->body, "TEST"); 56 | http_response_free(response); 57 | 58 | ut_testing("node2: HTTP GET /test == TEST"); 59 | response = http_get("http://127.0.0.1:54322/test", NULL); 60 | ut_validate_string(response->body, "TEST"); 61 | http_response_free(response); 62 | 63 | ut_testing("node2: HTTP DELETE /test"); 64 | response = http_delete("http://127.0.0.1:54322/test", NULL); 65 | ut_validate_int(response->status_code_int, 200); 66 | http_response_free(response); 67 | 68 | usleep(200); // give the delete command some time to propagate 69 | 70 | ut_testing("node1: HTTP GET /test == NOT FOUND"); 71 | response = http_get("http://127.0.0.1:54321/test", NULL); 72 | ut_validate_int(response->status_code_int, 404); 73 | if (response) 74 | http_response_free(response); 75 | 76 | ut_testing("node2: HTTP GET /test == NOT FOUND"); 77 | response = http_get("http://127.0.0.1:54322/test", NULL); 78 | ut_validate_int(response->status_code_int, 404); 79 | http_response_free(response); 80 | 81 | for (i = 0; i < 2; i++) { 82 | ut_testing("Stopping node %d to exit", i+1); 83 | ut_validate_int(stop_node(children[i]), children[i]); 84 | } 85 | 86 | ut_summary(); 87 | 88 | exit(ut_failed); 89 | } 90 | -------------------------------------------------------------------------------- /storage_plugins/README.md: -------------------------------------------------------------------------------- 1 | Loadable Storage Modules 2 | ------------------------ 3 | 4 | Storage plugins are expected to export only the two following symbols : 5 | 6 | - shardcache_storage_t *storage_create(const char **options) 7 | - void storage_destroy(shardcache_storage_t *storage) 8 | 9 | `storage_create()` should return a pointer to a properly initialized 10 | `shardcache_storage_t` structure. 11 | The same pointer will be provided to `storage_destroy()` in order to release 12 | all the resources used by the storage module. 13 | 14 | The `**options` argument in `storage_create()` is expcted to be a NULL-terminated 15 | array of strings containing key/value pairs. 16 | 17 | For example: 18 | 19 | char **options = { 20 | "storage_path", // key 21 | "/some/path", // value 22 | "tmp_path", // key 23 | "/some/temporary_path", // value 24 | NULL // terminator 25 | }; 26 | 27 | The storage module MUST implement the logic to check the validity of the 28 | `**options` array in `storage_create()` and return a NULL pointer if it's invalid. 29 | 30 | The `shardcache_storage_t` structure is so defined (check shardcache.h) : 31 | 32 | typedef struct __shardcache_storage_s { 33 | shardcache_fetch_item_callback_t fetch; 34 | shardcache_store_item_callback_t store; 35 | shardcache_remove_item_callback_t remove; 36 | shardcache_exist_item_callback_t exist; 37 | shardcache_get_index_callback_t index; 38 | shardcache_count_items_callback_t count; 39 | int shared; 40 | void *priv; 41 | } shardcache_storage_t; 42 | 43 | All callbacks are optional, which means that read-only storage modules are 44 | possible as well as write-only ones. 45 | 46 | The `index` callback requires the `count` callback to be present as well, 47 | since the former is used by the caller to determine the size of the index 48 | before calling the `index` callback. 49 | 50 | The `shared` member, if true, tells shardcache that the whole storage is 51 | available to all the nodes using this same storage-type. This allows 52 | the shardcache node to fallback querying the storage for a key it does not 53 | own, when the responsible peer is not available. 54 | 55 | The `priv` pointer will be provided to all callbacks and allows the storage 56 | module to save its status/context variables which might be needed in the 57 | callbacks implementation. 58 | 59 | The storage module is expected to be thread-safe since access to callbacks 60 | can happen from any shardcache worker thread and concurrent access is expected. 61 | Storage modules should try allowing concurrent (and possibly parallel) access 62 | to the storage if possible. If not feasible because of the nature of the 63 | underlying storage a global lock or any other mean to serialize access needs 64 | to be employed. 65 | 66 | Check the documentation in shardcache.h for more details about the callbacks 67 | to be exposed through the `shardcache_storage_t` structure. 68 | -------------------------------------------------------------------------------- /src/acl.c: -------------------------------------------------------------------------------- 1 | #include "acl.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | struct _shcd_acl_item_s { 10 | regex_t exp; 11 | shcd_acl_action_t action; 12 | shcd_acl_method_t method; 13 | uint32_t ipaddr; 14 | uint32_t mask; 15 | pthread_mutex_t lock; 16 | }; 17 | 18 | struct _shcd_acl_s { 19 | linked_list_t *list; 20 | shcd_acl_action_t default_action; 21 | }; 22 | 23 | void shcd_acl_item_destroy(shcd_acl_item_t *item) 24 | { 25 | regfree(&item->exp); 26 | free(item); 27 | } 28 | 29 | shcd_acl_t *shcd_acl_create(shcd_acl_action_t default_action) 30 | { 31 | shcd_acl_t *acl = calloc(1, sizeof(shcd_acl_t)); 32 | acl->list = list_create(); 33 | list_set_free_value_callback(acl->list, (free_value_callback_t)shcd_acl_item_destroy); 34 | acl->default_action = default_action; 35 | return acl; 36 | } 37 | 38 | int shcd_acl_add(shcd_acl_t *acl, 39 | char *pattern, 40 | shcd_acl_action_t action, 41 | shcd_acl_method_t method, 42 | uint32_t ipaddr, 43 | uint32_t mask) 44 | { 45 | shcd_acl_item_t *item = calloc(1, sizeof(shcd_acl_item_t)); 46 | // XXX - note it's using REG_ICASE 47 | if (regcomp(&item->exp, pattern, REG_EXTENDED|REG_ICASE) != 0) 48 | { 49 | SHC_ERROR("Bad regex pattern: %s (%s)", pattern, strerror(errno)); 50 | free(item); 51 | return -1; 52 | } 53 | item->action = action; 54 | item->method = method; 55 | item->ipaddr = ipaddr; 56 | item->mask = mask; 57 | pthread_mutex_init(&item->lock, NULL); 58 | list_push_value(acl->list, item); 59 | return 0; 60 | } 61 | 62 | shcd_acl_action_t shcd_acl_eval(shcd_acl_t *acl, 63 | shcd_acl_method_t method, 64 | char *path, 65 | uint32_t ipaddr) 66 | { 67 | shcd_acl_action_t res = acl->default_action; 68 | int i; 69 | 70 | for (i = 0; i < list_count(acl->list); i++) { 71 | shcd_acl_item_t *item = list_pick_value(acl->list, i); 72 | 73 | if (item->method != SHCD_ACL_METHOD_ANY && item->method != method) 74 | continue; 75 | 76 | if ((ipaddr & item->mask) != item->ipaddr) { 77 | continue; 78 | } 79 | 80 | pthread_mutex_lock(&item->lock); 81 | int matched = regexec(&item->exp, path, 0, NULL, 0); 82 | pthread_mutex_unlock(&item->lock); 83 | 84 | if (matched != 0) 85 | continue; 86 | 87 | res = item->action; 88 | 89 | } 90 | 91 | struct in_addr iaddr; 92 | iaddr.s_addr = ipaddr; 93 | SHC_DEBUG2("ACL result for action : %02x, key: %s, from %s == %s", 94 | method, path, inet_ntoa(iaddr), 95 | (res == SHCD_ACL_ACTION_ALLOW) ? "ALLOW" : "DENY"); 96 | return res; 97 | } 98 | 99 | void shcd_acl_destroy(shcd_acl_t *acl) 100 | { 101 | list_destroy(acl->list); 102 | free(acl); 103 | } 104 | 105 | 106 | -------------------------------------------------------------------------------- /src/storage.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "storage.h" 3 | #include "shardcache.h" 4 | 5 | #include "storage_mem.h" 6 | #include "storage_fs.h" 7 | 8 | typedef void (*shardcache_storage_destroyer_t)(void *); 9 | typedef int (*shardcache_storage_resetter_t)(void *); 10 | struct shcd_storage_s 11 | { 12 | shardcache_storage_t *storage; 13 | shardcache_storage_destroyer_t destroyer; 14 | shardcache_storage_resetter_t resetter; 15 | void *handle; 16 | int internal; 17 | }; 18 | 19 | shcd_storage_t * 20 | shcd_storage_init(char *storage_type, char *options_string, char *plugins_dir) 21 | { 22 | char *storage_options[MAX_STORAGE_OPTIONS]; 23 | 24 | shcd_storage_t *st = calloc(1, sizeof(shcd_storage_t)); 25 | 26 | int optidx = 0; 27 | char *p = options_string; 28 | char *str = p; 29 | while (*p != 0 && optidx < MAX_STORAGE_OPTIONS) { 30 | if (*p == '=' || *p == ',') { 31 | *p = 0; 32 | storage_options[optidx++] = str; 33 | str = p+1; 34 | } 35 | p++; 36 | } 37 | storage_options[optidx++] = str; 38 | storage_options[optidx] = NULL; 39 | 40 | // initialize the storage layer 41 | int initialized = -1; 42 | if (strcmp(storage_type, "mem") == 0) { 43 | // TODO - validate options 44 | st->storage = calloc(1, sizeof(shardcache_storage_t)); 45 | initialized = (storage_mem_init(st->storage, storage_options) == 0); 46 | st->destroyer = storage_mem_destroy; 47 | st->storage->version = SHARDCACHE_STORAGE_API_VERSION; 48 | st->internal = 1; 49 | } else if (strcmp(storage_type, "fs") == 0) { 50 | // TODO - validate options 51 | st->storage = calloc(1, sizeof(shardcache_storage_t)); 52 | initialized = (storage_fs_init(st->storage, storage_options) == 0); 53 | st->destroyer = storage_fs_destroy; 54 | st->storage->version = SHARDCACHE_STORAGE_API_VERSION; 55 | st->internal = 1; 56 | } else { 57 | char libname[1024]; 58 | snprintf(libname, sizeof(libname), "%s/%s.storage", 59 | plugins_dir, storage_type); 60 | st->storage = shardcache_storage_load(libname, storage_options); 61 | initialized = (st->storage != NULL); 62 | } 63 | if (!initialized) { 64 | SHC_ERROR("Can't init the storage type: %s\n", storage_type); 65 | if (st->internal && st->storage) { 66 | free(st->storage); 67 | } else if (st->storage) { 68 | shardcache_storage_dispose(st->storage); 69 | } 70 | free(st); 71 | return NULL; 72 | } 73 | return st; 74 | } 75 | 76 | shardcache_storage_t *shcd_storage_get(shcd_storage_t *st) { 77 | return st->storage; 78 | } 79 | 80 | void shcd_storage_destroy(shcd_storage_t *st) { 81 | 82 | if (st->destroyer) 83 | st->destroyer(st->storage->priv); 84 | 85 | if (st->internal) 86 | free(st->storage); 87 | else 88 | shardcache_storage_dispose(st->storage); 89 | free(st); 90 | } 91 | 92 | int shcd_storage_reset(shcd_storage_t *st) { 93 | int ret = 0; 94 | SHC_DEBUG("resetting the storage module"); 95 | if (!st->internal) 96 | ret = shardcache_storage_reset(st->storage, NULL); 97 | return ret; 98 | } 99 | -------------------------------------------------------------------------------- /storage_plugins/deps/Makefile: -------------------------------------------------------------------------------- 1 | export DEPS_INSTALL_DIR=$(shell pwd) 2 | 3 | export SHARDCACHE_INSTALL_LIBDIR := $(DEPS_INSTALL_DIR)/.libs 4 | export SHARDCACHE_INSTALL_INCDIR := $(DEPS_INSTALL_DIR)/.incs 5 | 6 | all: _prepare_env riak_deps 7 | 8 | .PHONY: _prepare_env 9 | _prepare_env: 10 | @mkdir -p .libs; \ 11 | mkdir -p .incs; \ 12 | export LIBDIR="$(LIBDIR)"; \ 13 | export INCDIR="$(INCDIR)"; \ 14 | export INSTALL_INCLUDE_PATH="$(SHARDCACHE_INSTALL_INCDIR)"; \ 15 | export INSTALL_LIBRARY_PATH="$(SHARDCACHE_INSTALL_LIBDIR)"; \ 16 | export CFLAGS="$(CFLAGS) -I$(SHARDCACHE_INSTALL_INCDIR)"; \ 17 | export CPPFLAGS="$(CPPFLAGS) -I$(SHARDCACHE_INSTALL_INCDIR)"; \ 18 | 19 | .PHONY: cunit_dep 20 | cunit_dep: 21 | @cd CUnit ;\ 22 | if [ ! -f Makefile ]; then \ 23 | libtoolize --force ;\ 24 | aclocal ;\ 25 | autoheader ;\ 26 | automake --force-missing --add-missing ;\ 27 | autoconf ;\ 28 | ./configure --datarootdir=$(SHARDCACHE_INSTALL_LIBDIR) --libdir=$(SHARDCACHE_INSTALL_LIBDIR) --includedir=$(SHARDCACHE_INSTALL_INCDIR) --bindir=$(SHARDCACHE_INSTALL_LIBDIR) --prefix=$(SHARDCACHE_INSTALL_LIBDIR);\ 29 | fi; \ 30 | cd - ;\ 31 | make -eC CUnit ; 32 | if [ $$? -ne 0 ] ; then exit $$?; fi; \ 33 | make -eC CUnit install; \ 34 | 35 | .PHONY: protobuf_dep 36 | protobuf_dep: 37 | @cd protobuf-cpp ;\ 38 | if [ ! -f Makefile ]; then \ 39 | ./autogen.sh ;\ 40 | ./configure --libdir=$(SHARDCACHE_INSTALL_LIBDIR) --includedir=$(SHARDCACHE_INSTALL_INCDIR) --bindir=$(SHARDCACHE_INSTALL_LIBDIR) --enable-shared=no --with-pic;\ 41 | fi; \ 42 | cd - ;\ 43 | make -eC protobuf-cpp ; 44 | if [ $$? -ne 0 ] ; then exit $$?; fi; \ 45 | make -eC protobuf-cpp install; 46 | 47 | .PHONY: protobuf_c_dep 48 | protobuf_c_dep: 49 | @cd protobuf-c ;\ 50 | if [ ! -f Makefile ]; then \ 51 | ./autogen.sh ;\ 52 | export PATH=$$PATH:$(SHARDCACHE_INSTALL_LIBDIR) ;\ 53 | export PROTOC=$(SHARDCACHE_INSTALL_LIBDIR)/protoc ;\ 54 | export protobuf_LIBS="-L$(SHARDCACHE_INSTALL_LIBDIR) $(SHARDCACHE_INSTALL_LIBDIR)/libprotobuf.a $(SHARDCACHE_INSTALL_LIBDIR)/libprotoc.a";\ 55 | export protobuf_CFLAGS=-I$(SHARDCACHE_INSTALL_INCDIR) ;\ 56 | ./configure --libdir=$(SHARDCACHE_INSTALL_LIBDIR) --includedir=$(SHARDCACHE_INSTALL_INCDIR) --bindir=$(SHARDCACHE_INSTALL_LIBDIR) --enable-shared=no --with-pic;\ 57 | fi; \ 58 | cd - ;\ 59 | make -eC protobuf-c ; 60 | if [ $$? -ne 0 ] ; then exit $$?; fi; \ 61 | make -eC protobuf-c install; \ 62 | 63 | .PHONY: riak_c_client_dep 64 | riak_c_client_dep: 65 | @export PROTOBUF_LIBS=$(SHARDCACHE_INSTALL_LIBDIR)/libprotobuf.a ;\ 66 | export PROTOBUF_CFLAGS=-I$(SHARDCACHE_INSTALL_INCDIR) ;\ 67 | export protobuf_LIBS="-L$(SHARDCACHE_INSTALL_LIBDIR) $(SHARDCACHE_INSTALL_LIBDIR)/libprotobuf.a $(SHARDCACHE_INSTALL_LIBDIR)/libprotoc.a";\ 68 | export protobuf_CFLAGS=-I$(SHARDCACHE_INSTALL_INCDIR) ;\ 69 | export PROTOC=$(SHARDCACHE_INSTALL_LIBDIR)/protoc ;\ 70 | export PROTOBUFC_LIBS=$(SHARDCACHE_INSTALL_LIBDIR)/libprotobuf-c.a ;\ 71 | export PROTOBUFC_CFLAGS=-I$(SHARDCACHE_INSTALL_INCDIR) ;\ 72 | export PROTOBUF_LIBS=$(SHARDCACHE_INSTALL_LIBDIR)/libprotobuf.a ;\ 73 | export PROTOBUF_CFLAGS=-I$(SHARDCACHE_INSTALL_INCDIR) ;\ 74 | export CUNIT_LIBS=$(SHARDCACHE_INSTALL_LIBDIR)/libcunit.a ;\ 75 | export CUNIT_CFLAGS=-I$(SHARDCACHE_INSTALL_INCDIR) ;\ 76 | export CFLAGS="$(CFLAGS) -I$(SHARDCACHE_INSTALL_INCDIR)"; \ 77 | export CPPFLAGS="$(CPPFLAGS) -I$(SHARDCACHE_INSTALL_INCDIR)"; \ 78 | export PATH=$$PATH:$(SHARDCACHE_INSTALL_LIBDIR) ;\ 79 | cd riak-c-client ;\ 80 | ./autogen.sh ;\ 81 | ./configure --libdir=$(SHARDCACHE_INSTALL_LIBDIR) --includedir=$(SHARDCACHE_INSTALL_INCDIR) --bindir=$(SHARDCACHE_INSTALL_LIBDIR) --enable-shared=no --with-protoc-c=$(SHARDCACHE_INSTALL_LIBDIR)/protoc-c --with-pic --with-gnu-ld;\ 82 | cd - ;\ 83 | make -eC riak-c-client ; \ 84 | if [ $$? -ne 0 ] ; then exit $$?; fi; \ 85 | make -eC riak-c-client install; 86 | 87 | .PHONY: riak_deps 88 | riak_deps: _prepare_env cunit_dep protobuf_dep protobuf_c_dep riak_c_client_dep 89 | 90 | .PHONY: redis_deps 91 | redis_deps: 92 | @make -eC hiredis; \ 93 | if [ $$? -ne 0 ] ; then exit $$?; fi; \ 94 | make -eC hiredis install; 95 | 96 | clean: 97 | @make -eC hiredis clean 98 | @make -eC protobuf-c clean 99 | @make -eC protobuf-cpp clean 100 | @rm -rf $(DEPS_INSTALL_DIR)/.incs 101 | @rm -rf $(DEPS_INSTALL_DIR)/.libs 102 | 103 | -------------------------------------------------------------------------------- /src/storage_mem.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "storage_mem.h" 6 | 7 | typedef struct { 8 | void *value; 9 | size_t size; 10 | } stored_item_t; 11 | 12 | static void 13 | free_item_cb(void *ptr) 14 | { 15 | stored_item_t *item = (stored_item_t *)ptr; 16 | if (item->value) 17 | free(item->value); 18 | free(item); 19 | } 20 | 21 | static void * 22 | copy_item_cb(void *ptr, size_t len, void *user) 23 | { 24 | stored_item_t *item = (stored_item_t *)ptr; 25 | stored_item_t *copy = malloc(sizeof(stored_item_t)); 26 | copy->value = malloc(item->size); 27 | memcpy(copy->value, item->value, item->size); 28 | copy->size = item->size; 29 | return copy; 30 | } 31 | 32 | static int 33 | st_fetch(void *key, size_t len, void **value, size_t *vlen, void *priv) 34 | { 35 | hashtable_t *storage = (hashtable_t *)priv; 36 | stored_item_t *item = ht_get_deep_copy(storage, 37 | key, 38 | len, 39 | NULL, 40 | copy_item_cb, 41 | NULL); 42 | void *v = NULL; 43 | if (item) { 44 | v = item->value; 45 | if (vlen) 46 | *vlen = item->size; 47 | free(item); 48 | } 49 | if (value) 50 | *value = v; 51 | 52 | return 0; 53 | } 54 | 55 | static int 56 | st_store(void *key, size_t len, void *value, size_t vlen, int if_not_exists, void *priv) 57 | { 58 | hashtable_t *storage = (hashtable_t *)priv; 59 | stored_item_t *new_item = malloc(sizeof(stored_item_t)); 60 | new_item->value = malloc(vlen); 61 | memcpy(new_item->value, value, vlen); 62 | new_item->size = vlen; 63 | if (if_not_exists) { 64 | if (ht_set_if_not_exists(storage, key, len, new_item, sizeof(stored_item_t)) != 0) 65 | free_item_cb(new_item); 66 | } else { 67 | ht_set(storage, key, len, new_item, sizeof(stored_item_t)); 68 | } 69 | return 0; 70 | } 71 | 72 | static int 73 | st_remove(void *key, size_t len, void *priv) 74 | { 75 | hashtable_t *storage = (hashtable_t *)priv; 76 | ht_delete(storage, key, len, NULL, NULL); 77 | return 0; 78 | } 79 | 80 | static int 81 | st_exist(void *key, size_t len, void *priv) 82 | { 83 | hashtable_t *storage = (hashtable_t *)priv; 84 | return ht_exists(storage, key, len); 85 | } 86 | 87 | typedef struct { 88 | shardcache_storage_index_item_t *index; 89 | size_t size; 90 | size_t offset; 91 | } st_pair_iterator_arg_t; 92 | 93 | static size_t 94 | st_count(void *priv) 95 | { 96 | hashtable_t *storage = (hashtable_t *)priv; 97 | return ht_count(storage); 98 | } 99 | 100 | static int 101 | st_pair_iterator(hashtable_t *table, 102 | void * key, 103 | size_t klen, 104 | void * value, 105 | size_t vlen, 106 | void * priv) 107 | { 108 | st_pair_iterator_arg_t *arg = (st_pair_iterator_arg_t *)priv; 109 | if (arg->offset < arg->size) { 110 | shardcache_storage_index_item_t *index_item; 111 | 112 | index_item = &arg->index[arg->offset++]; 113 | index_item->key = malloc(klen); 114 | memcpy(index_item->key, key, klen); 115 | index_item->klen = klen; 116 | stored_item_t *item = (stored_item_t *)value; 117 | index_item->vlen = item->size; 118 | return 1; 119 | } 120 | return 0; 121 | } 122 | 123 | static size_t 124 | st_index(shardcache_storage_index_item_t *index, size_t isize, void *priv) 125 | { 126 | hashtable_t *storage = (hashtable_t *)priv; 127 | st_pair_iterator_arg_t arg = { index, isize, 0 }; 128 | ht_foreach_pair(storage, st_pair_iterator, &arg); 129 | return arg.offset; 130 | } 131 | 132 | int 133 | storage_mem_init(shardcache_storage_t *st, char **options) 134 | { 135 | st->fetch = st_fetch; 136 | st->store = st_store; 137 | st->remove = st_remove; 138 | st->exist = st_exist; 139 | st->index = st_index; 140 | st->count = st_count; 141 | 142 | int size = 1024; 143 | int maxsize = 1 << 20; 144 | if (options) { 145 | while (*options) { 146 | char *key = (char *)*options++; 147 | 148 | if (!*key) 149 | break; 150 | 151 | char *value = NULL; 152 | if (*options) { 153 | value = (char *)*options++; 154 | } else { 155 | SHC_ERROR("Odd element in the options array"); 156 | continue; 157 | } 158 | if (key && value) { 159 | if (strcmp(key, "initial_table_size") == 0) { 160 | size = strtol(value, NULL, 10); 161 | } else if (strcmp(key, "max_table_size") == 0) { 162 | maxsize = strtol(value, NULL, 10); 163 | } else { 164 | SHC_ERROR("Unknown option name %s", key); 165 | } 166 | } 167 | } 168 | } 169 | hashtable_t *storage = ht_create(size, maxsize, free_item_cb); 170 | st->priv = storage; 171 | 172 | return 0; 173 | } 174 | 175 | void 176 | storage_mem_destroy(void *priv) 177 | { 178 | hashtable_t *storage = (hashtable_t *)priv; 179 | ht_destroy(storage); 180 | } 181 | -------------------------------------------------------------------------------- /src/ini.c: -------------------------------------------------------------------------------- 1 | /* inih -- simple .INI file parser 2 | 3 | inih is released under the New BSD license (see LICENSE.txt). Go to the project 4 | home page for more info: 5 | 6 | http://code.google.com/p/inih/ 7 | 8 | */ 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | #include "ini.h" 15 | 16 | #if !INI_USE_STACK 17 | #include 18 | #endif 19 | 20 | #define MAX_SECTION 50 21 | #define MAX_NAME 50 22 | 23 | /* Strip whitespace chars off end of given string, in place. Return s. */ 24 | static char* rstrip(char* s) 25 | { 26 | char* p = s + strlen(s); 27 | while (p > s && isspace((unsigned char)(*--p))) 28 | *p = '\0'; 29 | return s; 30 | } 31 | 32 | /* Return pointer to first non-whitespace char in given string. */ 33 | static char* lskip(const char* s) 34 | { 35 | while (*s && isspace((unsigned char)(*s))) 36 | s++; 37 | return (char*)s; 38 | } 39 | 40 | /* Return pointer to first char c or ';' comment in given string, or pointer to 41 | null at end of string if neither found. ';' must be prefixed by a whitespace 42 | character to register as a comment. */ 43 | static char* find_char_or_comment(const char* s, char c) 44 | { 45 | int was_whitespace = 0; 46 | while (*s && *s != c && !(was_whitespace && *s == ';')) { 47 | was_whitespace = isspace((unsigned char)(*s)); 48 | s++; 49 | } 50 | return (char*)s; 51 | } 52 | 53 | /* Version of strncpy that ensures dest (size bytes) is null-terminated. */ 54 | static char* strncpy0(char* dest, const char* src, size_t size) 55 | { 56 | strncpy(dest, src, size); 57 | dest[size - 1] = '\0'; 58 | return dest; 59 | } 60 | 61 | /* See documentation in header file. */ 62 | int ini_parse_file(FILE* file, 63 | int (*handler)(void*, const char*, const char*, 64 | const char*), 65 | void* user) 66 | { 67 | /* Uses a fair bit of stack (use heap instead if you need to) */ 68 | #if INI_USE_STACK 69 | char line[INI_MAX_LINE]; 70 | #else 71 | char* line; 72 | #endif 73 | char section[MAX_SECTION] = ""; 74 | char prev_name[MAX_NAME] = ""; 75 | 76 | char* start; 77 | char* end; 78 | char* name; 79 | char* value; 80 | int lineno = 0; 81 | int error = 0; 82 | 83 | #if !INI_USE_STACK 84 | line = (char*)malloc(INI_MAX_LINE); 85 | if (!line) { 86 | return -2; 87 | } 88 | #endif 89 | 90 | /* Scan through file line by line */ 91 | while (fgets(line, INI_MAX_LINE, file) != NULL) { 92 | lineno++; 93 | 94 | start = line; 95 | #if INI_ALLOW_BOM 96 | if (lineno == 1 && (unsigned char)start[0] == 0xEF && 97 | (unsigned char)start[1] == 0xBB && 98 | (unsigned char)start[2] == 0xBF) { 99 | start += 3; 100 | } 101 | #endif 102 | start = lskip(rstrip(start)); 103 | 104 | if (*start == ';' || *start == '#') { 105 | /* Per Python ConfigParser, allow '#' comments at start of line */ 106 | } 107 | #if INI_ALLOW_MULTILINE 108 | else if (*prev_name && *start && start > line) { 109 | /* Non-black line with leading whitespace, treat as continuation 110 | of previous name's value (as per Python ConfigParser). */ 111 | if (!handler(user, section, prev_name, start) && !error) 112 | error = lineno; 113 | } 114 | #endif 115 | else if (*start == '[') { 116 | /* A "[section]" line */ 117 | end = find_char_or_comment(start + 1, ']'); 118 | if (*end == ']') { 119 | *end = '\0'; 120 | strncpy0(section, start + 1, sizeof(section)); 121 | *prev_name = '\0'; 122 | } 123 | else if (!error) { 124 | /* No ']' found on section line */ 125 | error = lineno; 126 | } 127 | } 128 | else if (*start && *start != ';') { 129 | /* Not a comment, must be a name[=:]value pair */ 130 | end = find_char_or_comment(start, '='); 131 | if (*end != '=') { 132 | end = find_char_or_comment(start, ':'); 133 | } 134 | if (*end == '=' || *end == ':') { 135 | *end = '\0'; 136 | name = rstrip(start); 137 | value = lskip(end + 1); 138 | end = find_char_or_comment(value, '\0'); 139 | if (*end == ';') 140 | *end = '\0'; 141 | rstrip(value); 142 | 143 | /* Valid name[=:]value pair found, call handler */ 144 | strncpy0(prev_name, name, sizeof(prev_name)); 145 | if (!handler(user, section, name, value) && !error) 146 | error = lineno; 147 | } 148 | else if (!error) { 149 | /* No '=' or ':' found on name[=:]value line */ 150 | error = lineno; 151 | } 152 | } 153 | } 154 | 155 | #if !INI_USE_STACK 156 | free(line); 157 | #endif 158 | 159 | return error; 160 | } 161 | 162 | /* See documentation in header file. */ 163 | int ini_parse(const char* filename, 164 | int (*handler)(void*, const char*, const char*, const char*), 165 | void* user) 166 | { 167 | FILE* file; 168 | int error; 169 | 170 | file = fopen(filename, "r"); 171 | if (!file) 172 | return -1; 173 | error = ini_parse_file(file, handler, user); 174 | fclose(file); 175 | return error; 176 | } 177 | -------------------------------------------------------------------------------- /shardcached.ini.example: -------------------------------------------------------------------------------- 1 | [shardcached] 2 | stats_interval = 0 ; The interval in seconds at which output stats to stdout and/or syslog 3 | ; if '0' no stats will be reported on stdout/systlog 4 | ; (optional, defaults to '0') 5 | 6 | storage_type = fs ; The storage type (optional, defaults to 'mem') 7 | 8 | storage_options = storage_path=/home/xant/shardcache_storage,tmp_path=/tmp 9 | ; storage options (possibly optional, depend on the storage implementation) 10 | 11 | 12 | plugins_dir = ./ ; The directory where to find plugins (optional, defaults to './') 13 | loglevel = 2 ; The loglevel (optional, defaults to '0' == upto(LOG_ERR)) 14 | daemon = no ; Run as daemon or in foreground (optional, defaults to 'yes') 15 | nohttp = no ; Disable the HTTP subsystem (optional, defaults to 'no') 16 | me = peer1 ; Identifies this peer among the shardcache nodes 17 | ; which are defined in the [nodes] section 18 | ;user = username ; Assume the identity of 'username' (only when run as root) 19 | pidfile = /var/run/shardcached.pid ; File where to store the pid of the running instance (optional) 20 | 21 | ; the nodes taking part in the shardcache (required) 22 | [nodes] 23 | peer1 = my_address:4444 24 | peer2 = some_peer:4445 25 | peer3 = some_other_peer:4446 26 | 27 | [shardcache] 28 | arc_mode = strict ; ARC mode (optional, 'strict' or 'loose', defaults to 'strict') 29 | num_workers = 50 ; Number of shardcache workers (optional, defaults to '10') 30 | evict_on_delete = yes ; Evict on delete (optional, defaults to 'yes') 31 | use_persistent_connections = yes ; Use persistent connections instead of creating a new connection 32 | force_caching = no ; Always cache remote items instead of applying a 10% chance (optional, defaults to 'no') 33 | ; for each command sent to peers 34 | tcp_timeout = 0 ; Set the tcp timeout for all the outgoing connections 35 | ; (optional, a 0 value will make libshardcache use the compile-time default) 36 | ; (if set to 0 or omitted the libshardcache default timeout will be used) 37 | conn_expire_time = 0 ; Set the connection expiration time in the pool, before triggering a NOOP to check connection validity. 38 | ; (optional, a 0 value will make libshardcache use the compile-time default) 39 | ; (if set to 0 or omitted the libshardcache default timeout will be used) 40 | lazy_expiration = no ; Enable lazy expiration (optional, defaults to 'no') 41 | expire_time = 0 ; Sets the global expiration time for cached items (optional, defaults to 0, items will never expire 42 | ; and will be removed from the cache only if explicitly/naturally evicted) 43 | iomux_run_timeout_low = 0 ; Sets the low timeout (in microsecs) which will be passed to iomux_run() calls 44 | ; by both the serving workers and the async reader 45 | ; (optional, a 0 value will make libshardcache use the compile-time default) 46 | iomux_run_timeout_high = 0 ; Sets the high timeout (in microsecs) which will be passed to iomux_run() calls 47 | ; by both the listener and the expirer 48 | ; (optional, a 0 value will make libshardcache use the compile-time default) 49 | pipelining_max = 64 ; maximum number of requests to process ahead when pipelining 50 | ; (if omitted, the libshardcache compiled-in default will be used) 51 | 52 | [http] 53 | listen = *:4321 ; HTTP address:port where to listen for incoming connections (optional) 54 | num_workers = 50 ; Number of http worker threads (optional, defaults to '10') 55 | access_log = ./shardcached_access.log ; Path to the acces_log file (optional) 56 | basepath = shardcache ; Base http path (optional) 57 | baseadminpath = admin ; Base http path for administrative pages (optional, will be the same as basepath if not defined) 58 | acl_default = allow ; Default behavior for paths not matching any of those defined 59 | ; in the acl section. Possible values are : 'allow' , 'deny' 60 | ; (optional, defaults to 'allow') 61 | 62 | [acl] 63 | __(stats|index)__ = deny:*:* 64 | .* = deny:PUT:* 65 | .* = deny:DELETE:* 66 | .* = allow:*:82.173.134.166/32 67 | .* = allow:*:127.0.0.1/32 68 | 69 | 70 | [mime-types] 71 | pdf = application/pdf 72 | ps = application/postscript 73 | xml = application/xml 74 | ;js = application/javascript 75 | json = application/json 76 | gif = image/gif 77 | jpeg = image/jpeg 78 | jpg = image/jpeg 79 | png = image/png 80 | tiff = image/tiff 81 | html = text/html 82 | txt = text/plain 83 | csv = text/csv 84 | css = text/css 85 | mpg = video/mpeg 86 | mp4 = video/mp4 87 | 88 | -------------------------------------------------------------------------------- /src/mongoose.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2004-2013 Sergey Lyubka 2 | // Copyright (c) 2013-2014 Cesanta Software Limited 3 | // All rights reserved 4 | // 5 | // This software is dual-licensed: you can redistribute it and/or modify 6 | // it under the terms of the GNU General Public License version 2 as 7 | // published by the Free Software Foundation. For the terms of this 8 | // license, see . 9 | // 10 | // You are free to use this software under the terms of the GNU General 11 | // Public License, but WITHOUT ANY WARRANTY; without even the implied 12 | // warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 13 | // See the GNU General Public License for more details. 14 | // 15 | // Alternatively, you can license this software under a commercial 16 | // license, as set out in . 17 | 18 | #ifndef MONGOOSE_HEADER_INCLUDED 19 | #define MONGOOSE_HEADER_INCLUDED 20 | 21 | #define MONGOOSE_VERSION "5.6" 22 | 23 | #include // required for FILE 24 | #include // required for size_t 25 | #include // required for time_t 26 | 27 | #ifdef __cplusplus 28 | extern "C" { 29 | #endif // __cplusplus 30 | 31 | // This structure contains information about HTTP request. 32 | struct mg_connection { 33 | const char *request_method; // "GET", "POST", etc 34 | const char *uri; // URL-decoded URI 35 | const char *http_version; // E.g. "1.0", "1.1" 36 | const char *query_string; // URL part after '?', not including '?', or NULL 37 | 38 | char remote_ip[48]; // Max IPv6 string length is 45 characters 39 | char local_ip[48]; // Local IP address 40 | unsigned short remote_port; // Client's port 41 | unsigned short local_port; // Local port number 42 | 43 | int num_headers; // Number of HTTP headers 44 | struct mg_header { 45 | const char *name; // HTTP header name 46 | const char *value; // HTTP header value 47 | } http_headers[30]; 48 | 49 | char *content; // POST (or websocket message) data, or NULL 50 | size_t content_len; // Data length 51 | 52 | int is_websocket; // Connection is a websocket connection 53 | int status_code; // HTTP status code for HTTP error handler 54 | int wsbits; // First byte of the websocket frame 55 | void *server_param; // Parameter passed to mg_create_server() 56 | void *connection_param; // Placeholder for connection-specific data 57 | void *callback_param; 58 | }; 59 | 60 | struct mg_server; // Opaque structure describing server instance 61 | enum mg_result { MG_FALSE, MG_TRUE, MG_MORE }; 62 | enum mg_event { 63 | MG_POLL = 100, // Callback return value is ignored 64 | MG_CONNECT, // If callback returns MG_FALSE, connect fails 65 | MG_AUTH, // If callback returns MG_FALSE, authentication fails 66 | MG_REQUEST, // If callback returns MG_FALSE, Mongoose continues with req 67 | MG_REPLY, // If callback returns MG_FALSE, Mongoose closes connection 68 | MG_RECV, // Mongoose has received POST data chunk. 69 | // Callback should return a number of bytes to discard from 70 | // the receive buffer, or -1 to close the connection. 71 | MG_CLOSE, // Connection is closed, callback return value is ignored 72 | MG_WS_HANDSHAKE, // New websocket connection, handshake request 73 | MG_WS_CONNECT, // New websocket connection established 74 | MG_HTTP_ERROR // If callback returns MG_FALSE, Mongoose continues with err 75 | }; 76 | typedef int (*mg_handler_t)(struct mg_connection *, enum mg_event); 77 | 78 | // Websocket opcodes, from http://tools.ietf.org/html/rfc6455 79 | enum { 80 | WEBSOCKET_OPCODE_CONTINUATION = 0x0, 81 | WEBSOCKET_OPCODE_TEXT = 0x1, 82 | WEBSOCKET_OPCODE_BINARY = 0x2, 83 | WEBSOCKET_OPCODE_CONNECTION_CLOSE = 0x8, 84 | WEBSOCKET_OPCODE_PING = 0x9, 85 | WEBSOCKET_OPCODE_PONG = 0xa 86 | }; 87 | 88 | // Server management functions 89 | struct mg_server *mg_create_server(void *server_param, mg_handler_t handler); 90 | void mg_destroy_server(struct mg_server **); 91 | const char *mg_set_option(struct mg_server *, const char *opt, const char *val); 92 | time_t mg_poll_server(struct mg_server *, int milliseconds); 93 | const char **mg_get_valid_option_names(void); 94 | const char *mg_get_option(const struct mg_server *server, const char *name); 95 | void mg_copy_listeners(struct mg_server *from, struct mg_server *to); 96 | struct mg_connection *mg_next(struct mg_server *, struct mg_connection *); 97 | void mg_wakeup_server(struct mg_server *); 98 | void mg_wakeup_server_ex(struct mg_server *, mg_handler_t, const char *, ...); 99 | struct mg_connection *mg_connect(struct mg_server *, const char *); 100 | 101 | // Connection management functions 102 | void mg_send_status(struct mg_connection *, int status_code); 103 | void mg_send_header(struct mg_connection *, const char *name, const char *val); 104 | size_t mg_send_data(struct mg_connection *, const void *data, int data_len); 105 | size_t mg_printf_data(struct mg_connection *, const char *format, ...); 106 | size_t mg_write(struct mg_connection *, const void *buf, size_t len); 107 | size_t mg_printf(struct mg_connection *conn, const char *fmt, ...); 108 | 109 | size_t mg_websocket_write(struct mg_connection *, int opcode, 110 | const char *data, size_t data_len); 111 | size_t mg_websocket_printf(struct mg_connection* conn, int opcode, 112 | const char *fmt, ...); 113 | 114 | void mg_send_file(struct mg_connection *, const char *path, const char *); 115 | void mg_send_file_data(struct mg_connection *, int fd); 116 | 117 | const char *mg_get_header(const struct mg_connection *, const char *name); 118 | const char *mg_get_mime_type(const char *name, const char *default_mime_type); 119 | int mg_get_var(const struct mg_connection *conn, const char *var_name, 120 | char *buf, size_t buf_len); 121 | int mg_parse_header(const char *hdr, const char *var_name, char *buf, size_t); 122 | int mg_parse_multipart(const char *buf, int buf_len, 123 | char *var_name, int var_name_len, 124 | char *file_name, int file_name_len, 125 | const char **data, int *data_len); 126 | 127 | 128 | // Utility functions 129 | void *mg_start_thread(void *(*func)(void *), void *param); 130 | char *mg_md5(char buf[33], ...); 131 | int mg_authorize_digest(struct mg_connection *c, FILE *fp); 132 | size_t mg_url_encode(const char *src, size_t s_len, char *dst, size_t dst_len); 133 | int mg_url_decode(const char *src, size_t src_len, char *dst, size_t dst_len, int); 134 | int mg_terminate_ssl(struct mg_connection *c, const char *cert); 135 | int mg_forward(struct mg_connection *c, const char *addr); 136 | void *mg_mmap(FILE *fp, size_t size); 137 | void mg_munmap(void *p, size_t size); 138 | 139 | 140 | // Templates support 141 | struct mg_expansion { 142 | const char *keyword; 143 | void (*handler)(struct mg_connection *); 144 | }; 145 | void mg_template(struct mg_connection *, const char *text, 146 | struct mg_expansion *expansions); 147 | 148 | #ifdef __cplusplus 149 | } 150 | #endif // __cplusplus 151 | 152 | #endif // MONGOOSE_HEADER_INCLUDED 153 | -------------------------------------------------------------------------------- /test/stringx.h: -------------------------------------------------------------------------------- 1 | /* 2 | http-client-c 3 | Copyright (C) 2012-2013 Swen Kooij 4 | 5 | This file is part of http-client-c. 6 | 7 | http-client-c is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | http-client-c is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with http-client-c. If not, see . 19 | 20 | Warning: 21 | This library does not tend to work that stable nor does it fully implent the 22 | standards described by IETF. For more information on the precise implentation of the 23 | Hyper Text Transfer Protocol: 24 | 25 | http://www.ietf.org/rfc/rfc2616.txt 26 | */ 27 | 28 | #include 29 | #include 30 | #include 31 | 32 | #ifdef __cplusplus 33 | #include 34 | #endif 35 | 36 | /* 37 | Gets the offset of one string in another string 38 | */ 39 | int str_index_of(const char *a, char *b) 40 | { 41 | char *offset = (char*)strstr(a, b); 42 | return offset - a; 43 | } 44 | 45 | /* 46 | Checks if one string contains another string 47 | */ 48 | int str_contains(const char *haystack, const char *needle) 49 | { 50 | char *pos = (char*)strstr(haystack, needle); 51 | if(pos) 52 | return 1; 53 | else 54 | return 0; 55 | } 56 | 57 | /* 58 | Removes last character from string 59 | */ 60 | char* trim_end(char *string, char to_trim) 61 | { 62 | char last_char = string[strlen(string) -1]; 63 | if(last_char == to_trim) 64 | { 65 | char *new_string = string; 66 | new_string[strlen(string) - 1] = 0; 67 | return new_string; 68 | } 69 | else 70 | { 71 | return string; 72 | } 73 | } 74 | 75 | /* 76 | Concecates two strings, a wrapper for strcat from string.h, handles the resizing and copying 77 | */ 78 | char* str_cat(char *a, char *b) 79 | { 80 | char *target = (char*)malloc(strlen(a) + strlen(b) + 1); 81 | strcpy(target, a); 82 | strcat(target, b); 83 | return target; 84 | } 85 | 86 | /* 87 | Converts an integer value to its hex character 88 | */ 89 | char to_hex(char code) 90 | { 91 | static char hex[] = "0123456789abcdef"; 92 | return hex[code & 15]; 93 | } 94 | 95 | /* 96 | URL encodes a string 97 | */ 98 | char *urlencode(char *str) 99 | { 100 | char *pstr = str, *buf = (char*)malloc(strlen(str) * 3 + 1), *pbuf = buf; 101 | while (*pstr) 102 | { 103 | if (isalnum(*pstr) || *pstr == '-' || *pstr == '_' || *pstr == '.' || *pstr == '~') 104 | *pbuf++ = *pstr; 105 | else if (*pstr == ' ') 106 | *pbuf++ = '+'; 107 | else 108 | *pbuf++ = '%', *pbuf++ = to_hex(*pstr >> 4), *pbuf++ = to_hex(*pstr & 15); 109 | pstr++; 110 | } 111 | *pbuf = '\0'; 112 | return buf; 113 | } 114 | 115 | /* 116 | Replacement for the string.h strndup, fixes a bug 117 | */ 118 | char *str_ndup (const char *str, size_t max) 119 | { 120 | size_t len = strnlen (str, max); 121 | char *res = (char*)malloc (len + 1); 122 | if (res) 123 | { 124 | memcpy (res, str, len); 125 | res[len] = '\0'; 126 | } 127 | return res; 128 | } 129 | 130 | /* 131 | Replacement for the string.h strdup, fixes a bug 132 | */ 133 | char *str_dup(const char *src) 134 | { 135 | char *tmp = (char*)malloc(strlen(src) + 1); 136 | if(tmp) 137 | strcpy(tmp, src); 138 | return tmp; 139 | } 140 | 141 | /* 142 | Search and replace a string with another string , in a string 143 | */ 144 | char *str_replace(char *search , char *replace , char *subject) 145 | { 146 | char *p = NULL , *old = NULL , *new_subject = NULL ; 147 | int c = 0 , search_size; 148 | search_size = strlen(search); 149 | for(p = strstr(subject , search) ; p != NULL ; p = strstr(p + search_size , search)) 150 | { 151 | c++; 152 | } 153 | c = ( strlen(replace) - search_size )*c + strlen(subject); 154 | new_subject = (char*)malloc( c ); 155 | strcpy(new_subject , ""); 156 | old = subject; 157 | for(p = strstr(subject , search) ; p != NULL ; p = strstr(p + search_size , search)) 158 | { 159 | strncpy(new_subject + strlen(new_subject) , old , p - old); 160 | strcpy(new_subject + strlen(new_subject) , replace); 161 | old = p + search_size; 162 | } 163 | strcpy(new_subject + strlen(new_subject) , old); 164 | return new_subject; 165 | } 166 | 167 | /* 168 | Get's all characters until '*until' has been found 169 | */ 170 | char* get_until(char *haystack, char *until) 171 | { 172 | int offset = str_index_of(haystack, until); 173 | return str_ndup(haystack, offset); 174 | } 175 | 176 | 177 | /* decodeblock - decode 4 '6-bit' characters into 3 8-bit binary bytes */ 178 | void decodeblock(unsigned char in[], char *clrstr) 179 | { 180 | unsigned char out[4]; 181 | out[0] = in[0] << 2 | in[1] >> 4; 182 | out[1] = in[1] << 4 | in[2] >> 2; 183 | out[2] = in[2] << 6 | in[3] >> 0; 184 | out[3] = '\0'; 185 | strncat((char *)clrstr, (char *)out, 4); 186 | } 187 | 188 | /* 189 | Decodes a Base64 string 190 | */ 191 | char* base64_decode(char *b64src) 192 | { 193 | char *clrdst = (char*)malloc( ((strlen(b64src) - 1) / 3 ) * 4 + 4 + 50); 194 | char b64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; 195 | int c, phase, i; 196 | unsigned char in[4]; 197 | char *p; 198 | clrdst[0] = '\0'; 199 | phase = 0; i=0; 200 | while(b64src[i]) 201 | { 202 | c = (int) b64src[i]; 203 | if(c == '=') 204 | { 205 | decodeblock(in, clrdst); 206 | break; 207 | } 208 | p = strchr(b64, c); 209 | if(p) 210 | { 211 | in[phase] = p - b64; 212 | phase = (phase + 1) % 4; 213 | if(phase == 0) 214 | { 215 | decodeblock(in, clrdst); 216 | in[0]=in[1]=in[2]=in[3]=0; 217 | } 218 | } 219 | i++; 220 | } 221 | clrdst = (char*)realloc(clrdst, strlen(clrdst) + 1); 222 | return clrdst; 223 | } 224 | 225 | /* encodeblock - encode 3 8-bit binary bytes as 4 '6-bit' characters */ 226 | void encodeblock( unsigned char in[], char b64str[], int len ) 227 | { 228 | char b64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; 229 | unsigned char out[5]; 230 | out[0] = b64[ in[0] >> 2 ]; 231 | out[1] = b64[ ((in[0] & 0x03) << 4) | ((in[1] & 0xf0) >> 4) ]; 232 | out[2] = (unsigned char) (len > 1 ? b64[ ((in[1] & 0x0f) << 2) | 233 | ((in[2] & 0xc0) >> 6) ] : '='); 234 | out[3] = (unsigned char) (len > 2 ? b64[ in[2] & 0x3f ] : '='); 235 | out[4] = '\0'; 236 | strncat((char *)b64str, (char *)out, 5); 237 | } 238 | 239 | /* 240 | Encodes a string with Base64 241 | */ 242 | char* base64_encode(char *clrstr) 243 | { 244 | char *b64dst = (char*)malloc(strlen(clrstr) + 50); 245 | unsigned char in[3]; 246 | int i, len = 0; 247 | int j = 0; 248 | 249 | b64dst[0] = '\0'; 250 | while(clrstr[j]) 251 | { 252 | len = 0; 253 | for(i=0; i<3; i++) 254 | { 255 | in[i] = (unsigned char) clrstr[j]; 256 | if(clrstr[j]) 257 | { 258 | len++; j++; 259 | } 260 | else in[i] = 0; 261 | } 262 | if( len ) 263 | { 264 | encodeblock( in, b64dst, len ); 265 | } 266 | } 267 | b64dst = (char*)realloc(b64dst, strlen(b64dst) + 1); 268 | return b64dst; 269 | } 270 | -------------------------------------------------------------------------------- /storage_plugins/redis/redis.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #ifdef __MACH__ 7 | #include 8 | #define SPIN_LOCK(__mutex) OSSpinLockLock(__mutex) 9 | #define SPIN_TRYLOCK(__mutex) OSSpinLockTry(__mutex) 10 | #define SPIN_UNLOCK(__mutex) OSSpinLockUnlock(__mutex) 11 | #else 12 | #define SPIN_LOCK(__mutex) pthread_spin_lock(__mutex) 13 | #define SPIN_TRYLOCK(__mutex) pthread_spin_trylock(__mutex) 14 | #define SPIN_UNLOCK(__mutex) pthread_spin_unlock(__mutex) 15 | #endif 16 | 17 | #define REDIS_PORT_DEFAULT 6379 18 | #define REDIS_HOST_DEFAULT "localhost" 19 | #define REDIS_NUM_CONNECTIONS_DEFAULT 5 20 | 21 | int storage_version = SHARDCACHE_STORAGE_API_VERSION; 22 | 23 | typedef struct { 24 | redisContext *context; 25 | #ifdef __MACH__ 26 | OSSpinLock lock; 27 | #else 28 | pthread_spinlock_t lock; 29 | #endif 30 | int initialized; 31 | } redis_connection_t; 32 | 33 | typedef struct { 34 | char *host; 35 | int port; 36 | int num_connections; 37 | int connection_index; 38 | redis_connection_t *connections; 39 | } storage_redis_t; 40 | 41 | static void 42 | parse_options(storage_redis_t *st, const char **options) 43 | { 44 | while (options && *options) { 45 | char *key = (char *)*options++; 46 | char *value = NULL; 47 | 48 | if (!*key) 49 | break; 50 | 51 | if (*options) { 52 | value = (char *)*options++; 53 | } else { 54 | fprintf(stderr, "Odd element in the options array\n"); 55 | break; 56 | } 57 | 58 | if (key && value) { 59 | if (strcmp(key, "host") == 0) { 60 | st->host = strdup(value); 61 | } else if (strcmp(key, "port") == 0) { 62 | st->port = strtol(value, NULL, 10); 63 | } else if (strcmp(key, "num_connections") == 0) { 64 | st->num_connections = strtol(value, NULL, 10); 65 | } else { 66 | fprintf(stderr, "Unknown option name %s\n", key); 67 | } 68 | } 69 | } 70 | } 71 | 72 | static void 73 | st_clear_connection(storage_redis_t *st, redis_connection_t *c) 74 | { 75 | if (c->context) 76 | redisFree(c->context); 77 | c->context = NULL; 78 | if (c->initialized) { 79 | #ifndef __MACH__ 80 | pthread_spin_destroy(&c->lock); 81 | #endif 82 | c->initialized = 0; 83 | } 84 | } 85 | 86 | static int 87 | st_init_connection(storage_redis_t *st, redis_connection_t *c) 88 | { 89 | c->context = redisConnect(st->host, st->port); 90 | if (c->context && c->context->err) { 91 | fprintf(stderr, "Redis error: %s\n", c->context->errstr); 92 | redisFree(c->context); 93 | c->context = NULL; 94 | return -1; 95 | } 96 | if (!c->initialized) { 97 | #ifndef __MACH__ 98 | pthread_spin_init(&c->lock, 0); 99 | #endif 100 | c->initialized = 1; 101 | } 102 | return 0; 103 | } 104 | 105 | static redis_connection_t * 106 | st_get_connection(storage_redis_t *st) 107 | { 108 | int rc = 0; 109 | int index = 0; 110 | redis_connection_t *c = NULL; 111 | int retries = 0; 112 | do { 113 | index = __sync_fetch_and_add(&st->connection_index, 1)%st->num_connections; 114 | c = &st->connections[index]; 115 | rc = SPIN_TRYLOCK(&c->lock); 116 | if (retries++ == 100) { 117 | // ok .. it's too busy 118 | fprintf(stderr, "Can't acquire any connection lock\n"); 119 | return NULL; 120 | } 121 | } while (rc != 0); 122 | 123 | if (!c->context) { 124 | if (st_init_connection(st, c) != 0) { 125 | SPIN_UNLOCK(&c->lock); 126 | return NULL; 127 | } 128 | } else { 129 | redisReply *resp = redisCommand(c->context, "PING"); 130 | if (!resp || resp->type != REDIS_REPLY_STRING || strcmp(resp->str, "PONG") != 0) { 131 | // try refreshing the connection 132 | redisFree(c->context); 133 | if (st_init_connection(st, c) != 0) { 134 | SPIN_UNLOCK(&c->lock); 135 | c = NULL; 136 | } 137 | } 138 | if (resp) 139 | freeReplyObject(resp); 140 | } 141 | return c; 142 | } 143 | 144 | static int st_fetch(void *key, size_t klen, void **value, size_t *vlen, void *priv) 145 | { 146 | storage_redis_t *st = (storage_redis_t *)priv; 147 | 148 | redis_connection_t *c = st_get_connection(st); 149 | if (!c) { 150 | return -1; 151 | } 152 | redisReply *resp = redisCommand(c->context, "GET %b", key, klen); 153 | SPIN_UNLOCK(&c->lock); 154 | if (!resp || resp->type != REDIS_REPLY_STRING) { 155 | // TODO - Error messages 156 | if (resp) 157 | freeReplyObject(resp); 158 | return -1; 159 | } 160 | if (value) { 161 | *value = malloc(resp->len); 162 | memcpy(*value, resp->str, resp->len); 163 | } 164 | if (vlen) 165 | *vlen = resp->len; 166 | 167 | freeReplyObject(resp); 168 | 169 | return 0; 170 | } 171 | 172 | static int st_store(void *key, size_t klen, void *value, size_t vlen, void *priv) 173 | { 174 | storage_redis_t *st = (storage_redis_t *)priv; 175 | 176 | redis_connection_t *c = st_get_connection(st); 177 | if (!c) { 178 | return -1; 179 | } 180 | 181 | redisReply *resp = redisCommand(c->context, "SET %b %b", key, klen, value, vlen); 182 | SPIN_UNLOCK(&c->lock); 183 | if (!resp || resp->type != REDIS_REPLY_STRING || strcmp(resp->str, "OK") != 0) { 184 | // TODO - Error messages 185 | if (resp) 186 | freeReplyObject(resp); 187 | return -1; 188 | } 189 | freeReplyObject(resp); 190 | return 0; 191 | } 192 | 193 | static int st_remove(void *key, size_t klen, void *priv) 194 | { 195 | storage_redis_t *st = (storage_redis_t *)priv; 196 | 197 | redis_connection_t *c = st_get_connection(st); 198 | if (!c) { 199 | return -1; 200 | } 201 | 202 | redisReply *resp = redisCommand(c->context, "DEL %b", key, klen); 203 | SPIN_UNLOCK(&c->lock); 204 | if (!resp || resp->type != REDIS_REPLY_INTEGER || resp->integer != 1) { 205 | // TODO - Error messages 206 | if (resp) 207 | freeReplyObject(resp); 208 | return -1; 209 | } 210 | freeReplyObject(resp); 211 | return 0; 212 | } 213 | 214 | static int st_exist(void *key, size_t klen, void *priv) 215 | { 216 | storage_redis_t *st = (storage_redis_t *)priv; 217 | 218 | redis_connection_t *c = st_get_connection(st); 219 | if (!c) { 220 | return -1; 221 | } 222 | 223 | redisReply *resp = redisCommand(c->context, "DEL %b", key, klen); 224 | SPIN_UNLOCK(&c->lock); 225 | if (!resp || resp->type != REDIS_REPLY_INTEGER || resp->integer != 1) { 226 | // TODO - Error messages 227 | if (resp) 228 | freeReplyObject(resp); 229 | return -1; 230 | } 231 | freeReplyObject(resp); 232 | return 0; 233 | } 234 | 235 | static size_t st_count(void *priv) 236 | { 237 | storage_redis_t *st = (storage_redis_t *)priv; 238 | 239 | redis_connection_t *c = st_get_connection(st); 240 | if (!c) { 241 | return 0; 242 | } 243 | 244 | redisReply *resp = redisCommand(c->context, "DBSIZE"); 245 | SPIN_UNLOCK(&c->lock); 246 | if (!resp || resp->type != REDIS_REPLY_INTEGER) { 247 | // TODO - Error messages 248 | if (resp) 249 | freeReplyObject(resp); 250 | return 0; 251 | } 252 | size_t n = resp->integer; 253 | freeReplyObject(resp); 254 | return n; 255 | } 256 | 257 | static size_t st_index(shardcache_storage_index_item_t *index, size_t isize, void *priv) 258 | { 259 | storage_redis_t *st = (storage_redis_t *)priv; 260 | 261 | redis_connection_t *c = st_get_connection(st); 262 | if (!c) { 263 | return 0; 264 | } 265 | 266 | redisReply *resp = redisCommand(c->context, "KEYS *"); 267 | SPIN_UNLOCK(&c->lock); 268 | if (!resp || resp->type != REDIS_REPLY_ARRAY) { 269 | // TODO - Error messages 270 | if (resp) 271 | freeReplyObject(resp); 272 | return 0; 273 | } 274 | size_t n = resp->elements; 275 | if (n > isize) 276 | n = isize; 277 | int cnt = 0; 278 | int i; 279 | for (i = 0; i < n; i++) { 280 | redisReply *r = resp->element[i]; 281 | if (r->type == REDIS_REPLY_STRING) { 282 | shardcache_storage_index_item_t *item = &index[cnt++]; 283 | item->key = malloc(r->len); 284 | memcpy(item->key, r->str, r->len); 285 | item->klen = r->len; 286 | item->vlen = 0; // length is not returned by the redis KEYS command 287 | } else { 288 | // TODO - Error Message 289 | } 290 | } 291 | freeReplyObject(resp); 292 | return cnt; 293 | } 294 | 295 | static void 296 | storage_redis_destroy(storage_redis_t *st) 297 | { 298 | int i; 299 | for (i = 0; i < st->num_connections; i++) { 300 | redis_connection_t *c = &st->connections[i]; 301 | st_clear_connection(st, c); 302 | } 303 | free(st->connections); 304 | if (st->host) 305 | free(st->host); 306 | free(st); 307 | } 308 | 309 | void 310 | storage_destroy(void *priv) 311 | { 312 | storage_redis_t *st = (storage_redis_t *)priv; 313 | storage_redis_destroy(st); 314 | } 315 | 316 | int 317 | storage_init(shardcache_storage_t *storage, const char **options) 318 | { 319 | storage_redis_t *st = calloc(1, sizeof(storage_redis_t)); 320 | 321 | if (options) 322 | parse_options(st, options); 323 | 324 | if (!st->host) 325 | st->host = strdup(REDIS_HOST_DEFAULT); 326 | 327 | if (!st->port) 328 | st->port = REDIS_PORT_DEFAULT; 329 | 330 | if (!st->num_connections) 331 | st->num_connections = REDIS_NUM_CONNECTIONS_DEFAULT; 332 | 333 | st->connections = calloc(sizeof(redis_connection_t), st->num_connections); 334 | 335 | int i; 336 | for (i = 0; i < st->num_connections; i++) { 337 | redis_connection_t *c = &st->connections[i]; 338 | if (st_init_connection(st, c) != 0) { 339 | storage_redis_destroy(st); 340 | return -1; 341 | } 342 | } 343 | 344 | storage->fetch = st_fetch; 345 | storage->store = st_store; 346 | storage->remove = st_remove; 347 | storage->count = st_count; 348 | storage->index = st_index; 349 | storage->priv = st; 350 | 351 | return 0; 352 | } 353 | 354 | 355 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | shardcached - A distributed cache and storage system 2 | ====== 3 | 4 | C implementation of a full-featured [shardcache](http://github.com/xant/libshardcache "libshardcache") daemon 5 | 6 | shardcached implements an http frontend exposing all functionalities provided by [libshardcache](http://github.com/xant/libshardcache "libshardcache"). 7 | 8 | * the internal counters and the storage index are exposed through the 'magic' keys (urls) : `__index__` and `__stats__`. 9 | * allows to define ACLs to control which IP addresses can access which keys (including the internal keys `__index__` and `__stats__`) 10 | * supports mime-types rules to use when serving back items via the http frontend 11 | * pluggable storage backend (sqlite, mysql and redis storage plugins have been already implemented and provided as examples in the `storage_plugins/` directory) 12 | * provides builtin storage modules for both volatile (mem-based) and persistent (filesystem-based) storage 13 | * supports migrations which can be initiated by new nodes at their startup 14 | 15 | 16 | NOTE: Almost all options can be controlled/overridden via the cmdline, 17 | ACLs and mime-types rules are supported only via the configuration file. 18 | 19 | 20 | =============================================================================================================================== 21 | 22 | ``` 23 | Usage: ./shardcached [OPTION]... 24 | Version: 0.17 (libshardcache: 0.22) 25 | Possible options: 26 | -a the path where to store the access_log file (defaults to './shardcached_access.log') 27 | -c the config file to load 28 | -d the path where to look for storage plugins (defaults to './') 29 | -f run in foreground 30 | -F force caching 31 | -H disable the HTTP frontend 32 | -i change the time interval (in seconds) used to report internal stats via syslog (defaults to '0') 33 | -l ip_address:port where to listen for incoming http connections 34 | -L enable lazy expiration 35 | -E set the expiration time for cached items (defaults to: 0) 36 | -e the expiration time for a connection in the pool to trigger a NOOP (defaults to 30000) 37 | -r set the low timeout passed to iomux_run() calls (in microsecs, defaults to: 100000) 38 | -R set the high timeout pssed to iomux_run() calls (in microsecs, defaults to: 500000) 39 | -b HTTP url basepath (optional, defaults to '') 40 | -B HTTP url baseadminpath (optional, defaults to '') 41 | -n list of nodes participating in the shardcache in the form : 'label:address:port,label2:address2:port2' 42 | -N no storage subsystem, use only the internal libshardcache volatile storage 43 | -m me the label of this node, to identify it among the ones participating in the shardcache 44 | -P the maximum amount of requests to handle in parallel while still serving a response (defaults to: 64) 45 | -s cache size in bytes (defaults to : '536870912') 46 | -T tcp timeout (in milliseconds) used for connections opened by libshardcache (defaults to '5000') 47 | -t storage type (available are : 'mem' and 'fs' (defaults to 'mem') 48 | -o comma-separated list of storage options (defaults to '') 49 | -u assume the identity of (only when run as root) 50 | -v increase the log level (can be passed multiple times) 51 | -V output the version number and exit 52 | -w number of shardcache worker threads (defaults to '10') 53 | -W number of http worker threads (defaults to '10') 54 | -x new list of nodes to migrate the shardcache to. The format to use is the same as for the '-n' option 55 | 56 | Builtin storage types: 57 | * mem memory based storage 58 | Options: 59 | - initial_table_size= the initial number of slots in the internal hashtable 60 | - max_table_size= the maximum number of slots that the internal hashtable can be grown up to 61 | 62 | * fs filesystem based storage 63 | Options: 64 | - storage_path= the path where to store the keys/values on the filesystem 65 | - tmp_path= the path to a temporary directory to use while new data is being uploaded 66 | ``` 67 | 68 | =============================================================================================================================== 69 | 70 | Example configuration file : 71 | ``` 72 | [shardcached] 73 | stats_interval = 0 ; The interval in seconds at which output stats to stdout and/or syslog 74 | ; if '0' no stats will be reported on stdout/systlog 75 | ; (optional, defaults to '0') 76 | 77 | storage_type = fs ; The storage type (optional, defaults to 'mem') 78 | 79 | storage_options = storage_path=/home/xant/shardcache_storage,tmp_path=/tmp 80 | ; storage options (possibly optional, depend on the storage implementation) 81 | 82 | 83 | plugins_dir = ./ ; The directory where to find plugins (optional, defaults to './') 84 | loglevel = 2 ; The loglevel (optional, defaults to '0' == upto(LOG_ERR)) 85 | daemon = no ; Run as daemon or in foreground (optional, defaults to 'yes') 86 | nohttp = no ; Disable the HTTP subsystem (optional, defaults to 'no') 87 | me = peer1 ; Identifies this peer among the shardcache nodes 88 | ; which are defined in the [nodes] section 89 | ;user = username ; Assume the identity of 'username' (only when run as root) 90 | pidfile = /var/run/shardcached.pid ; File where to store the pid of the running instance (optional) 91 | 92 | ; the nodes taking part in the shardcache (required) 93 | [nodes] 94 | peer1 = my_address:4444 95 | peer2 = some_peer:4445 96 | peer3 = some_other_peer:4446 97 | 98 | [shardcache] 99 | num_workers = 50 ; Number of shardcache workers (optional, defaults to '10') 100 | evict_on_delete = yes ; Evict on delete (optional, defaults to 'yes') 101 | use_persistent_connections = yes ; Use persistent connections instead of creating a new connection 102 | force_caching = no ; Always cache remote items instead of applying a 10% chance (optional, defaults to 'no') 103 | ; for each command sent to peers 104 | tcp_timeout = 0 ; Set the tcp timeout for all the outgoing connections 105 | ; (optional, a 0 value will make libshardcache use the compile-time default) 106 | ; (if set to 0 or omitted the libshardcache default timeout will be used) 107 | conn_expire_time = 0 ; Set the connection expiration time in the pool, before triggering a NOOP to check connection validity. 108 | ; (optional, a 0 value will make libshardcache use the compile-time default) 109 | ; (if set to 0 or omitted the libshardcache default timeout will be used) 110 | lazy_expiration = no ; Enable lazy expiration (optional, defaults to 'no') 111 | expire_time = 0 ; Sets the global expiration time for cached items (optional, defaults to 0, items will never expire 112 | ; and will be removed from the cache only if explicitly/naturally evicted) 113 | iomux_run_timeout_low = 0 ; Sets the low timeout (in microsecs) which will be passed to iomux_run() calls 114 | ; by both the serving workers and the async reader 115 | ; (optional, a 0 value will make libshardcache use the compile-time default) 116 | iomux_run_timeout_high = 0 ; Sets the high timeout (in microsecs) which will be passed to iomux_run() calls 117 | ; by both the listener and the expirer 118 | ; (optional, a 0 value will make libshardcache use the compile-time default) 119 | pipelining_max = 64 ; maximum number of requests to process ahead when pipelining 120 | ; (if omitted, the libshardcache compiled-in default will be used) 121 | 122 | [http] 123 | listen = *:4321 ; HTTP address:port where to listen for incoming connections (optional) 124 | num_workers = 50 ; Number of http worker threads (optional, defaults to '10') 125 | access_log = ./shardcached_access.log ; Path to the acces_log file (optional) 126 | basepath = shardcache ; Base http path (optional) 127 | baseadminpath = admin ; Base http path for administrative pages (optional, will be the same as basepath if not defined) 128 | acl_default = allow ; Default behavior for paths not matching any of those defined 129 | ; in the acl section. Possible values are : 'allow' , 'deny' 130 | ; (optional, defaults to 'allow') 131 | 132 | [acl] 133 | __(stats|index)__ = deny:*:* 134 | .* = deny:PUT:* 135 | .* = deny:DELETE:* 136 | .* = allow:*:192.168.1.123/32 137 | .* = allow:*:127.0.0.1/32 138 | 139 | 140 | [mime-types] 141 | pdf = application/pdf 142 | ps = application/postscript 143 | xml = application/xml 144 | ;js = application/javascript 145 | json = application/json 146 | gif = image/gif 147 | jpeg = image/jpeg 148 | jpg = image/jpeg 149 | png = image/png 150 | tiff = image/tiff 151 | html = text/html 152 | txt = text/plain 153 | csv = text/csv 154 | css = text/css 155 | mpg = video/mpeg 156 | mp4 = video/mp4 157 | 158 | ``` 159 | -------------------------------------------------------------------------------- /test/urlparser.h: -------------------------------------------------------------------------------- 1 | /* 2 | http-client-c 3 | Copyright (C) 2012-2013 Swen Kooij 4 | 5 | This file is part of http-client-c. 6 | 7 | http-client-c is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | http-client-c is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with http-client-c. If not, see . 19 | 20 | Warning: 21 | This library does not tend to work that stable nor does it fully implent the 22 | standards described by IETF. For more information on the precise implentation of the 23 | Hyper Text Transfer Protocol: 24 | 25 | http://www.ietf.org/rfc/rfc2616.txt 26 | */ 27 | 28 | /* 29 | Represents an url 30 | */ 31 | struct parsed_url 32 | { 33 | char *uri; /* mandatory */ 34 | char *scheme; /* mandatory */ 35 | char *host; /* mandatory */ 36 | char *ip; /* mandatory */ 37 | char *port; /* optional */ 38 | char *path; /* optional */ 39 | char *query; /* optional */ 40 | char *fragment; /* optional */ 41 | char *username; /* optional */ 42 | char *password; /* optional */ 43 | }; 44 | 45 | /* 46 | Free memory of parsed url 47 | */ 48 | void parsed_url_free(struct parsed_url *purl) 49 | { 50 | if ( NULL != purl ) 51 | { 52 | if ( NULL != purl->scheme ) free(purl->scheme); 53 | if ( NULL != purl->host ) free(purl->host); 54 | if ( NULL != purl->port ) free(purl->port); 55 | if ( NULL != purl->path ) free(purl->path); 56 | if ( NULL != purl->query ) free(purl->query); 57 | if ( NULL != purl->fragment ) free(purl->fragment); 58 | if ( NULL != purl->username ) free(purl->username); 59 | if ( NULL != purl->password ) free(purl->password); 60 | free(purl); 61 | } 62 | } 63 | 64 | /* 65 | Retrieves the IP adress of a hostname 66 | */ 67 | char* hostname_to_ip(char *hostname) 68 | { 69 | struct hostent *h; 70 | if ((h=gethostbyname(hostname)) == NULL) 71 | { 72 | printf("gethostbyname"); 73 | return NULL; 74 | } 75 | return inet_ntoa(*((struct in_addr *)h->h_addr)); 76 | } 77 | 78 | /* 79 | Check whether the character is permitted in scheme string 80 | */ 81 | int is_scheme_char(int c) 82 | { 83 | return (!isalpha(c) && '+' != c && '-' != c && '.' != c) ? 0 : 1; 84 | } 85 | 86 | /* 87 | Parses a specified URL and returns the structure named 'parsed_url' 88 | Implented according to: 89 | RFC 1738 - http://www.ietf.org/rfc/rfc1738.txt 90 | RFC 3986 - http://www.ietf.org/rfc/rfc3986.txt 91 | */ 92 | struct parsed_url *parse_url(const char *url) 93 | { 94 | 95 | /* Define variable */ 96 | struct parsed_url *purl; 97 | const char *tmpstr; 98 | const char *curstr; 99 | int len; 100 | int i; 101 | int userpass_flag; 102 | int bracket_flag; 103 | 104 | /* Allocate the parsed url storage */ 105 | purl = (struct parsed_url*)malloc(sizeof(struct parsed_url)); 106 | if ( NULL == purl ) 107 | { 108 | return NULL; 109 | } 110 | purl->scheme = NULL; 111 | purl->host = NULL; 112 | purl->port = NULL; 113 | purl->path = NULL; 114 | purl->query = NULL; 115 | purl->fragment = NULL; 116 | purl->username = NULL; 117 | purl->password = NULL; 118 | curstr = url; 119 | 120 | /* 121 | * : 122 | * := [a-z\+\-\.]+ 123 | * upper case = lower case for resiliency 124 | */ 125 | /* Read scheme */ 126 | tmpstr = strchr(curstr, ':'); 127 | if ( NULL == tmpstr ) 128 | { 129 | parsed_url_free(purl); fprintf(stderr, "Error on line %d (%s)\n", __LINE__, __FILE__); 130 | 131 | return NULL; 132 | } 133 | 134 | /* Get the scheme length */ 135 | len = tmpstr - curstr; 136 | 137 | /* Check restrictions */ 138 | for ( i = 0; i < len; i++ ) 139 | { 140 | if (is_scheme_char(curstr[i]) == 0) 141 | { 142 | /* Invalid format */ 143 | parsed_url_free(purl); fprintf(stderr, "Error on line %d (%s)\n", __LINE__, __FILE__); 144 | return NULL; 145 | } 146 | } 147 | /* Copy the scheme to the storage */ 148 | purl->scheme = (char*)malloc(sizeof(char) * (len + 1)); 149 | if ( NULL == purl->scheme ) 150 | { 151 | parsed_url_free(purl); fprintf(stderr, "Error on line %d (%s)\n", __LINE__, __FILE__); 152 | 153 | return NULL; 154 | } 155 | 156 | (void)strncpy(purl->scheme, curstr, len); 157 | purl->scheme[len] = '\0'; 158 | 159 | /* Make the character to lower if it is upper case. */ 160 | for ( i = 0; i < len; i++ ) 161 | { 162 | purl->scheme[i] = tolower(purl->scheme[i]); 163 | } 164 | 165 | /* Skip ':' */ 166 | tmpstr++; 167 | curstr = tmpstr; 168 | 169 | /* 170 | * //:@:/ 171 | * Any ":", "@" and "/" must be encoded. 172 | */ 173 | /* Eat "//" */ 174 | for ( i = 0; i < 2; i++ ) 175 | { 176 | if ( '/' != *curstr ) 177 | { 178 | parsed_url_free(purl); fprintf(stderr, "Error on line %d (%s)\n", __LINE__, __FILE__); 179 | return NULL; 180 | } 181 | curstr++; 182 | } 183 | 184 | /* Check if the user (and password) are specified. */ 185 | userpass_flag = 0; 186 | tmpstr = curstr; 187 | while ( '\0' != *tmpstr ) 188 | { 189 | if ( '@' == *tmpstr ) 190 | { 191 | /* Username and password are specified */ 192 | userpass_flag = 1; 193 | break; 194 | } 195 | else if ( '/' == *tmpstr ) 196 | { 197 | /* End of : specification */ 198 | userpass_flag = 0; 199 | break; 200 | } 201 | tmpstr++; 202 | } 203 | 204 | /* User and password specification */ 205 | tmpstr = curstr; 206 | if ( userpass_flag ) 207 | { 208 | /* Read username */ 209 | while ( '\0' != *tmpstr && ':' != *tmpstr && '@' != *tmpstr ) 210 | { 211 | tmpstr++; 212 | } 213 | len = tmpstr - curstr; 214 | purl->username = (char*)malloc(sizeof(char) * (len + 1)); 215 | if ( NULL == purl->username ) 216 | { 217 | parsed_url_free(purl); fprintf(stderr, "Error on line %d (%s)\n", __LINE__, __FILE__); 218 | return NULL; 219 | } 220 | (void)strncpy(purl->username, curstr, len); 221 | purl->username[len] = '\0'; 222 | 223 | /* Proceed current pointer */ 224 | curstr = tmpstr; 225 | if ( ':' == *curstr ) 226 | { 227 | /* Skip ':' */ 228 | curstr++; 229 | 230 | /* Read password */ 231 | tmpstr = curstr; 232 | while ( '\0' != *tmpstr && '@' != *tmpstr ) 233 | { 234 | tmpstr++; 235 | } 236 | len = tmpstr - curstr; 237 | purl->password = (char*)malloc(sizeof(char) * (len + 1)); 238 | if ( NULL == purl->password ) 239 | { 240 | parsed_url_free(purl); fprintf(stderr, "Error on line %d (%s)\n", __LINE__, __FILE__); 241 | return NULL; 242 | } 243 | (void)strncpy(purl->password, curstr, len); 244 | purl->password[len] = '\0'; 245 | curstr = tmpstr; 246 | } 247 | /* Skip '@' */ 248 | if ( '@' != *curstr ) 249 | { 250 | parsed_url_free(purl); fprintf(stderr, "Error on line %d (%s)\n", __LINE__, __FILE__); 251 | return NULL; 252 | } 253 | curstr++; 254 | } 255 | 256 | if ( '[' == *curstr ) 257 | { 258 | bracket_flag = 1; 259 | } 260 | else 261 | { 262 | bracket_flag = 0; 263 | } 264 | /* Proceed on by delimiters with reading host */ 265 | tmpstr = curstr; 266 | while ( '\0' != *tmpstr ) { 267 | if ( bracket_flag && ']' == *tmpstr ) 268 | { 269 | /* End of IPv6 address. */ 270 | tmpstr++; 271 | break; 272 | } 273 | else if ( !bracket_flag && (':' == *tmpstr || '/' == *tmpstr) ) 274 | { 275 | /* Port number is specified. */ 276 | break; 277 | } 278 | tmpstr++; 279 | } 280 | len = tmpstr - curstr; 281 | purl->host = (char*)malloc(sizeof(char) * (len + 1)); 282 | if ( NULL == purl->host || len <= 0 ) 283 | { 284 | parsed_url_free(purl); fprintf(stderr, "Error on line %d (%s)\n", __LINE__, __FILE__); 285 | return NULL; 286 | } 287 | (void)strncpy(purl->host, curstr, len); 288 | purl->host[len] = '\0'; 289 | curstr = tmpstr; 290 | 291 | /* Is port number specified? */ 292 | if ( ':' == *curstr ) 293 | { 294 | curstr++; 295 | /* Read port number */ 296 | tmpstr = curstr; 297 | while ( '\0' != *tmpstr && '/' != *tmpstr ) 298 | { 299 | tmpstr++; 300 | } 301 | len = tmpstr - curstr; 302 | purl->port = (char*)malloc(sizeof(char) * (len + 1)); 303 | if ( NULL == purl->port ) 304 | { 305 | parsed_url_free(purl); fprintf(stderr, "Error on line %d (%s)\n", __LINE__, __FILE__); 306 | return NULL; 307 | } 308 | (void)strncpy(purl->port, curstr, len); 309 | purl->port[len] = '\0'; 310 | curstr = tmpstr; 311 | } 312 | else 313 | { 314 | purl->port = "80"; 315 | } 316 | 317 | /* Get ip */ 318 | char *ip = hostname_to_ip(purl->host); 319 | purl->ip = ip; 320 | 321 | /* Set uri */ 322 | purl->uri = (char*)url; 323 | 324 | /* End of the string */ 325 | if ( '\0' == *curstr ) 326 | { 327 | return purl; 328 | } 329 | 330 | /* Skip '/' */ 331 | if ( '/' != *curstr ) 332 | { 333 | parsed_url_free(purl); fprintf(stderr, "Error on line %d (%s)\n", __LINE__, __FILE__); 334 | return NULL; 335 | } 336 | curstr++; 337 | 338 | /* Parse path */ 339 | tmpstr = curstr; 340 | while ( '\0' != *tmpstr && '#' != *tmpstr && '?' != *tmpstr ) 341 | { 342 | tmpstr++; 343 | } 344 | len = tmpstr - curstr; 345 | purl->path = (char*)malloc(sizeof(char) * (len + 1)); 346 | if ( NULL == purl->path ) 347 | { 348 | parsed_url_free(purl); fprintf(stderr, "Error on line %d (%s)\n", __LINE__, __FILE__); 349 | return NULL; 350 | } 351 | (void)strncpy(purl->path, curstr, len); 352 | purl->path[len] = '\0'; 353 | curstr = tmpstr; 354 | 355 | /* Is query specified? */ 356 | if ( '?' == *curstr ) 357 | { 358 | /* Skip '?' */ 359 | curstr++; 360 | /* Read query */ 361 | tmpstr = curstr; 362 | while ( '\0' != *tmpstr && '#' != *tmpstr ) 363 | { 364 | tmpstr++; 365 | } 366 | len = tmpstr - curstr; 367 | purl->query = (char*)malloc(sizeof(char) * (len + 1)); 368 | if ( NULL == purl->query ) 369 | { 370 | parsed_url_free(purl); fprintf(stderr, "Error on line %d (%s)\n", __LINE__, __FILE__); 371 | return NULL; 372 | } 373 | (void)strncpy(purl->query, curstr, len); 374 | purl->query[len] = '\0'; 375 | curstr = tmpstr; 376 | } 377 | 378 | /* Is fragment specified? */ 379 | if ( '#' == *curstr ) 380 | { 381 | /* Skip '#' */ 382 | curstr++; 383 | /* Read fragment */ 384 | tmpstr = curstr; 385 | while ( '\0' != *tmpstr ) 386 | { 387 | tmpstr++; 388 | } 389 | len = tmpstr - curstr; 390 | purl->fragment = (char*)malloc(sizeof(char) * (len + 1)); 391 | if ( NULL == purl->fragment ) 392 | { 393 | parsed_url_free(purl); fprintf(stderr, "Error on line %d (%s)\n", __LINE__, __FILE__); 394 | return NULL; 395 | } 396 | (void)strncpy(purl->fragment, curstr, len); 397 | purl->fragment[len] = '\0'; 398 | curstr = tmpstr; 399 | } 400 | return purl; 401 | } 402 | -------------------------------------------------------------------------------- /storage_plugins/riak/riak.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #define ST_HOST_DEFAULT "localhost" 10 | #define ST_PORT_DEFAULT "8087" 11 | #define ST_NUM_CONNECTIONS_DEFAULT 5 12 | 13 | int storage_version = SHARDCACHE_STORAGE_API_VERSION; 14 | 15 | typedef struct { 16 | char *hostname; 17 | char *port; 18 | int num_connections; 19 | unsigned int connect_timeout; 20 | unsigned int read_timeout; 21 | queue_t *connections; 22 | riak_config *riak_cfg; 23 | } storage_riak_t; 24 | 25 | static void 26 | storage_riak_set_defaults(storage_riak_t *st) 27 | { 28 | if (!st->hostname) 29 | st->hostname = strdup(ST_HOST_DEFAULT); 30 | 31 | if (!st->port) 32 | st->port = strdup(ST_PORT_DEFAULT); 33 | 34 | if (!st->num_connections) 35 | st->num_connections = ST_NUM_CONNECTIONS_DEFAULT; 36 | 37 | if (!st->riak_cfg) { 38 | riak_error err = riak_config_new_default(&st->riak_cfg); 39 | if (err) { 40 | SHC_ERROR("Can't create a new riak config : %s", riak_strerror(err)); 41 | } 42 | } 43 | 44 | 45 | } 46 | 47 | static void 48 | storage_riak_config(storage_riak_t *st, const char **options) 49 | { 50 | if (options) { 51 | while (*options) { 52 | char *key = (char *)*options++; 53 | if (!*key) 54 | break; 55 | char *value = NULL; 56 | if (*options) { 57 | value = (char *)*options++; 58 | } else { 59 | SHC_ERROR("Odd element in the options array"); 60 | continue; 61 | } 62 | if (key && value) { 63 | if (strcmp(key, "hostname") == 0) { 64 | st->hostname = strdup(value); 65 | } else if (strcmp(key, "port") == 0) { 66 | st->port = strdup(value); 67 | } else if (strcmp(key, "num_connections") == 0) { 68 | st->num_connections = strtol(value, NULL, 10); 69 | } else if (strcmp(key, "connect_timeout") == 0) { 70 | st->connect_timeout = strtol(value, NULL, 10); 71 | } else if (strcmp(key, "read_timeout") == 0) { 72 | st->read_timeout = strtol(value, NULL, 10); 73 | } else { 74 | SHC_ERROR("Unknown option name %s", key); 75 | } 76 | } 77 | } 78 | } 79 | storage_riak_set_defaults(st); 80 | } 81 | 82 | static inline riak_connection * 83 | st_init_connection(storage_riak_t *st) 84 | { 85 | riak_connection *conn; 86 | SHC_DEBUG("Connecting to riak server %s:%s", st->hostname, st->port); 87 | riak_error err = riak_connection_new(st->riak_cfg, &conn, st->hostname, st->port, NULL); 88 | if (err) 89 | SHC_ERROR("Can't create a new riak connection : %s", riak_strerror(err)); 90 | 91 | return conn; 92 | } 93 | 94 | static inline riak_connection * 95 | st_get_connection(storage_riak_t *st) 96 | { 97 | int rc = 0; 98 | int index = 0; 99 | int retries = 0; 100 | 101 | riak_connection *riak = queue_pop_left(st->connections); 102 | if (riak) { 103 | // we got a cached connection 104 | // let's check if it's still alive 105 | riak_error err = riak_ping(riak); 106 | if (err == ERIAK_OK) 107 | return riak; 108 | 109 | riak_connection_free(&riak); 110 | riak = NULL; 111 | } 112 | 113 | return st_init_connection(st); 114 | } 115 | 116 | static void 117 | st_release_connection(storage_riak_t *st, riak_connection *riak) 118 | { 119 | if (queue_count(st->connections) >= st->num_connections) 120 | riak_connection_free(&riak); 121 | else 122 | queue_push_right(st->connections, riak); 123 | } 124 | 125 | static int 126 | st_remove(void *key, size_t klen, void *priv) 127 | { 128 | storage_riak_t *st = (storage_riak_t *)priv; 129 | 130 | char *keystr = calloc(1, klen+1); 131 | memcpy(keystr, key, klen); 132 | char *tofree = keystr; 133 | 134 | char *bucket = strsep(&keystr, ","); 135 | char *keybin = keystr; 136 | 137 | if (!bucket || !keybin) { 138 | char k[klen+1]; 139 | snprintf(k, sizeof(k), "%s", key); 140 | SHC_WARNING("Unsupported key : %s", k); 141 | free(tofree); 142 | return -1; 143 | } 144 | 145 | riak_connection *riak = st_get_connection(st); 146 | if (!riak) { 147 | free(tofree); 148 | return -1; 149 | } 150 | 151 | riak_delete_options *delete_options = riak_delete_options_new(st->riak_cfg); 152 | if (delete_options == NULL) { 153 | SHC_ERROR("Could not allocate a Riak Delete Options"); 154 | riak_connection_free(&riak); 155 | free(tofree); 156 | return -1; 157 | } 158 | riak_delete_options_set_w(delete_options, 1); 159 | riak_delete_options_set_dw(delete_options, 1); 160 | 161 | riak_binary *bucket_bin = riak_binary_copy_from_string(st->riak_cfg, bucket); 162 | riak_binary *key_bin = riak_binary_copy_from_string(st->riak_cfg, keybin); 163 | 164 | int rc = -1; 165 | riak_error err = riak_delete(riak, NULL, bucket_bin, key_bin, delete_options); 166 | riak_delete_options_free(st->riak_cfg, &delete_options); 167 | if (err) { 168 | SHC_ERROR("Delete Problems [%s]", riak_strerror(err)); 169 | riak_connection_free(&riak); 170 | } else { 171 | rc = 0; 172 | st_release_connection(st, riak); 173 | } 174 | 175 | riak_binary_free(st->riak_cfg, &bucket_bin); 176 | riak_binary_free(st->riak_cfg, &key_bin); 177 | free(tofree); 178 | 179 | return 0; 180 | } 181 | 182 | static int 183 | st_store(void *key, size_t klen, void *value, size_t vlen, void *priv) 184 | { 185 | storage_riak_t *st = (storage_riak_t *)priv; 186 | char *keystr = calloc(1, klen+1); 187 | memcpy(keystr, key, klen); 188 | char *tofree = keystr; 189 | 190 | char *bucket = strsep(&keystr, ","); 191 | char *keybin = keystr; 192 | 193 | if (!bucket || !keybin) { 194 | char k[klen+1]; 195 | snprintf(k, sizeof(k), "%s", key); 196 | SHC_WARNING("Unsupported key : %s", k); 197 | free(tofree); 198 | return -1; 199 | } 200 | 201 | riak_object *obj = riak_object_new(st->riak_cfg); 202 | if (obj == NULL) { 203 | SHC_ERROR("Could not allocate a Riak Object"); 204 | free(tofree); 205 | return -1; 206 | } 207 | 208 | riak_binary *value_bin = riak_binary_new(st->riak_cfg, vlen, (riak_uint8_t *)value); 209 | riak_binary *bucket_bin = riak_binary_copy_from_string(st->riak_cfg, bucket); 210 | riak_binary *key_bin = riak_binary_copy_from_string(st->riak_cfg, keybin); 211 | 212 | riak_object_set_bucket(st->riak_cfg, obj, bucket_bin); 213 | riak_object_set_key(st->riak_cfg, obj, key_bin); 214 | riak_object_set_value(st->riak_cfg, obj, value_bin); 215 | 216 | riak_binary_free(st->riak_cfg, &bucket_bin); 217 | riak_binary_free(st->riak_cfg, &key_bin); 218 | riak_binary_free(st->riak_cfg, &value_bin); 219 | 220 | if (riak_object_get_bucket(obj) == NULL || 221 | riak_object_get_value(obj) == NULL) { 222 | SHC_ERROR("Could not allocate bucket/value"); 223 | riak_free(st->riak_cfg, &obj); 224 | free(tofree); 225 | return -1; 226 | } 227 | riak_put_options *put_options = riak_put_options_new(st->riak_cfg); 228 | if (put_options == NULL) { 229 | SHC_ERROR("Could not allocate a Riak Put Options"); 230 | free(tofree); 231 | return -1; 232 | } 233 | 234 | riak_put_options_set_return_head(put_options, RIAK_FALSE); 235 | riak_put_options_set_return_body(put_options, RIAK_FALSE); 236 | 237 | int rc = -1; 238 | 239 | riak_connection *riak = st_get_connection(st); 240 | if (riak) { 241 | riak_put_response *put_response = NULL; 242 | riak_error err = riak_put(riak, obj, put_options, &put_response); 243 | if (err == ERIAK_OK) 244 | rc = 0; 245 | else 246 | SHC_ERROR("Put Problems [%s]", riak_strerror(err)); 247 | 248 | if (put_response) 249 | riak_put_response_free(st->riak_cfg, &put_response); 250 | 251 | st_release_connection(st, riak); 252 | } 253 | 254 | riak_object_free(st->riak_cfg, &obj); 255 | riak_put_options_free(st->riak_cfg, &put_options); 256 | 257 | free(tofree); 258 | 259 | 260 | return rc; 261 | } 262 | 263 | static int 264 | st_fetch(void *key, size_t klen, void **value, size_t *vlen, void *priv) 265 | { 266 | storage_riak_t *st = (storage_riak_t *)priv; 267 | 268 | char *keystr = calloc(1, klen+1); 269 | memcpy(keystr, key, klen); 270 | char *tofree = keystr; 271 | 272 | char *bucket = strsep(&keystr, ","); 273 | char *keybin = keystr; 274 | 275 | if (!bucket || !keybin) { 276 | char k[klen+1]; 277 | snprintf(k, sizeof(k), "%s", key); 278 | SHC_WARNING("Unsupported key : %s", k); 279 | free(tofree); 280 | return -1; 281 | } 282 | 283 | riak_connection *riak = st_get_connection(st); 284 | if (!riak) { 285 | free(tofree); 286 | return -1; 287 | } 288 | 289 | riak_get_options *get_options = riak_get_options_new(st->riak_cfg); 290 | if (get_options == NULL) { 291 | SHC_ERROR("Could not allocate a Riak Get Options"); 292 | riak_connection_free(&riak); 293 | free(tofree); 294 | return -1; 295 | } 296 | riak_get_options_set_basic_quorum(get_options, RIAK_TRUE); 297 | riak_get_options_set_r(get_options, 1); 298 | riak_get_response *get_response = NULL; 299 | 300 | riak_binary *bucket_bin = riak_binary_copy_from_string(st->riak_cfg, bucket); 301 | riak_binary *key_bin = riak_binary_copy_from_string(st->riak_cfg, keybin); 302 | 303 | riak_error err = riak_get(riak, NULL, bucket_bin, key_bin, get_options, &get_response); 304 | if (err == ERIAK_OK) { 305 | if (get_response && riak_get_is_found(get_response)) { 306 | riak_object **objects = riak_get_get_content(get_response); 307 | riak_binary *bin = riak_object_get_value(objects[0]); // XXX - HC to access the first object 308 | riak_size_t size = riak_binary_len(bin); 309 | riak_uint8_t *data = riak_binary_data(bin); 310 | if (size && data) { 311 | if (vlen) 312 | *vlen = size; 313 | if (value) { 314 | *value = malloc(size); 315 | memcpy(*value, data, size); 316 | } 317 | 318 | } 319 | } 320 | } else { 321 | SHC_ERROR("Get Problems [%s]\n", riak_strerror(err)); 322 | riak_connection_free(&riak); 323 | free(tofree); 324 | riak_get_options_free(st->riak_cfg, &get_options); 325 | riak_binary_free(st->riak_cfg, &bucket_bin); 326 | riak_binary_free(st->riak_cfg, &key_bin); 327 | return -1; 328 | } 329 | 330 | if (get_response) 331 | riak_get_response_free(st->riak_cfg, &get_response); 332 | 333 | if (get_options) 334 | riak_get_options_free(st->riak_cfg, &get_options); 335 | 336 | st_release_connection(st, riak); 337 | 338 | free(tofree); 339 | riak_binary_free(st->riak_cfg, &bucket_bin); 340 | riak_binary_free(st->riak_cfg, &key_bin); 341 | 342 | return 0; 343 | } 344 | 345 | static void 346 | storage_riak_destroy(storage_riak_t *st) 347 | { 348 | int i; 349 | if (st->connections) 350 | queue_destroy(st->connections); 351 | 352 | if (st->hostname) 353 | free(st->hostname); 354 | 355 | if (st->port) 356 | free(st->port); 357 | 358 | if (st->riak_cfg) 359 | riak_config_free(&st->riak_cfg); 360 | 361 | free(st); 362 | } 363 | 364 | void 365 | storage_destroy(void *priv) 366 | { 367 | storage_riak_t *st = (storage_riak_t *)priv; 368 | if (st) 369 | storage_riak_destroy(st); 370 | 371 | } 372 | 373 | static void 374 | riak_connection_free_wrapper(riak_connection *riak) 375 | { 376 | riak_connection_free(&riak); 377 | } 378 | 379 | int 380 | storage_init(shardcache_storage_t *storage, const char **options) 381 | { 382 | storage_riak_t *st = calloc(1, sizeof(storage_riak_t)); 383 | 384 | storage_riak_config(st, options); 385 | 386 | st->connections = queue_create(); 387 | queue_set_free_value_callback(st->connections, 388 | (queue_free_value_callback_t)riak_connection_free_wrapper); 389 | 390 | int i; 391 | for (i = 0; i < st->num_connections; i++) { 392 | riak_connection *riak = st_init_connection(st); 393 | if (!riak) { 394 | storage_riak_destroy(st); 395 | return -1; 396 | } 397 | queue_push_right(st->connections, riak); 398 | } 399 | 400 | storage->fetch = st_fetch; 401 | storage->store = st_store; 402 | storage->remove = st_remove; 403 | storage->shared = 1; 404 | storage->global = 1; 405 | storage->priv = st; 406 | 407 | return 0; 408 | } 409 | 410 | -------------------------------------------------------------------------------- /src/storage_fs.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "storage_fs.h" 14 | 15 | typedef struct { 16 | char *path; 17 | char *tmp; 18 | hashtable_t *index; 19 | } storage_fs_t; 20 | 21 | static char * 22 | st_fs_filename(char *basepath, void *key, size_t klen, char **intermediate_path) 23 | { 24 | int i; 25 | struct stat st; 26 | 27 | if (!klen) 28 | return NULL; 29 | 30 | if (stat(basepath, &st) != 0) { 31 | if (mkdir(basepath, S_IRWXU) != 0 && errno != EEXIST) { 32 | fprintf(stderr, "Can't create directory %s: %s\n", 33 | basepath, strerror(errno)); 34 | return NULL; 35 | } 36 | } 37 | 38 | char fname[(klen*2)+1]; 39 | char *p = &fname[0]; 40 | for (i = 0; i < klen; i++) { 41 | snprintf(p, 3, "%02x", ((char *)key)[i]); 42 | p+=2; 43 | } 44 | 45 | char dname[5]; 46 | if (klen >= 2) { 47 | snprintf(dname, 5, "%02x%02x", ((char *)key)[0], ((char *)key)[klen-1]); 48 | } else { 49 | snprintf(dname, 5, "%02x00", ((char *)key)[0]); 50 | } 51 | dname[4] = 0; 52 | 53 | int dirnamelen = strlen(basepath)+6; 54 | char dirname[dirnamelen]; 55 | snprintf(dirname, dirnamelen, "%s/%s", basepath, dname); 56 | if (stat(dirname, &st) != 0) { 57 | if (mkdir(dirname, S_IRWXU) != 0 && errno != EEXIST) { 58 | fprintf(stderr, "Can't create directory %s: %s\n", 59 | dirname, strerror(errno)); 60 | return NULL; 61 | } 62 | } 63 | 64 | size_t fullpath_len = strlen(dirname) + strlen(fname) + 2; 65 | char *fullpath = malloc(fullpath_len); 66 | 67 | snprintf(fullpath, fullpath_len, "%s/%s", dirname, fname); 68 | 69 | if (intermediate_path) { 70 | *intermediate_path = strdup(dirname); 71 | } 72 | return fullpath; 73 | } 74 | 75 | static int 76 | st_fetch(void *key, size_t klen, void **value, size_t *vlen, void *priv) 77 | { 78 | storage_fs_t *storage = (storage_fs_t *)priv; 79 | char *fullpath = st_fs_filename(storage->path, key, klen, NULL); 80 | 81 | if (!fullpath) 82 | return -1; 83 | 84 | int fd = open(fullpath, O_RDONLY); 85 | if (fd >=0) { 86 | flock(fd, LOCK_SH); 87 | fbuf_t buf = FBUF_STATIC_INITIALIZER; 88 | int rb = fbuf_read(&buf, fd, 1024); 89 | while (rb != -1) { 90 | rb = fbuf_read(&buf, fd, 1024); 91 | if (rb == 0) 92 | break; 93 | } 94 | flock(fd, LOCK_UN); 95 | close(fd); 96 | if (fbuf_used(&buf)) { 97 | if (vlen) 98 | *vlen = fbuf_used(&buf); 99 | if (value) 100 | *value = fbuf_data(&buf); 101 | else 102 | fbuf_destroy(&buf); 103 | free(fullpath); 104 | return 0; 105 | } 106 | } else if (errno == ENOENT) { 107 | if (vlen) 108 | *vlen = 0; 109 | if (value) 110 | *value = NULL; 111 | free(fullpath); 112 | return 0; 113 | } 114 | 115 | free(fullpath); 116 | return -1; 117 | } 118 | 119 | static int 120 | st_store(void *key, size_t klen, void *value, size_t vlen, int if_not_exists, void *priv) 121 | { 122 | storage_fs_t *storage = (storage_fs_t *)priv; 123 | 124 | int ret = -1; 125 | long r = random(); 126 | int i; 127 | 128 | char dname[9]; 129 | char *p = &dname[0]; 130 | for (i = 0; i < 4; i++) { 131 | snprintf(p, 3, "%02x", ((char *)&r)[i]); 132 | p += 2; 133 | } 134 | 135 | size_t tmpdir_len = strlen(storage->tmp)+9; 136 | char tmpdir[tmpdir_len]; 137 | char *tmp_intermediate = NULL; 138 | char *intermediate_dir = NULL; 139 | 140 | snprintf(tmpdir, tmpdir_len, "%s/%s", storage->tmp, dname); 141 | 142 | char *fullpath = st_fs_filename(storage->path, 143 | key, 144 | klen, 145 | &intermediate_dir); 146 | 147 | if (if_not_exists) { 148 | struct stat st; 149 | if (stat(fullpath, &st) == 0) { 150 | free(fullpath); 151 | return 1; 152 | } 153 | } 154 | 155 | char *tmppath = st_fs_filename(tmpdir, key, klen, &tmp_intermediate); 156 | 157 | int fd = open(tmppath, O_WRONLY|O_TRUNC|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); 158 | if (fd >=0) { 159 | int ofx = 0; 160 | while (ofx != vlen) { 161 | int wb = write(fd, value+ofx, vlen - ofx); 162 | if (wb > 0) 163 | { 164 | ofx += wb; 165 | } else if (wb == 0 || 166 | (wb == -1 && errno != EINTR && errno != EAGAIN)) 167 | { 168 | // TODO - Error messages 169 | break; 170 | } 171 | } 172 | if (ofx == vlen) { 173 | ret = rename(tmppath, fullpath); 174 | if (ret != 0) 175 | SHC_ERROR("Can't store data on file %s : %s\n", 176 | fullpath, strerror(errno)); 177 | 178 | } 179 | close(fd); 180 | } 181 | 182 | if (ret != 0) 183 | unlink(tmppath); 184 | 185 | free(fullpath); 186 | rmdir(tmp_intermediate); 187 | rmdir(tmpdir); 188 | free(tmppath); 189 | free(tmp_intermediate); 190 | if (intermediate_dir) 191 | free(intermediate_dir); 192 | size_t *sizep = malloc(sizeof(size_t)); 193 | *sizep = vlen; 194 | ht_set(storage->index, key, klen, sizep, sizeof(size_t)); 195 | return ret; 196 | } 197 | 198 | static int 199 | st_remove(void *key, size_t klen, void *priv) 200 | { 201 | storage_fs_t *storage = (storage_fs_t *)priv; 202 | char *intermediate_dir = NULL; 203 | char *fullpath = st_fs_filename(storage->path, 204 | key, 205 | klen, 206 | &intermediate_dir); 207 | int ret = unlink(fullpath); 208 | rmdir(intermediate_dir); 209 | free(fullpath); 210 | free(intermediate_dir); 211 | ht_delete(storage->index, key, klen, NULL, NULL); 212 | return ret; 213 | } 214 | 215 | static int 216 | st_exist(void *key, size_t klen, void *priv) 217 | { 218 | storage_fs_t *storage = (storage_fs_t *)priv; 219 | char *intermediate_dir = NULL; 220 | char *fullpath = st_fs_filename(storage->path, 221 | key, 222 | klen, 223 | &intermediate_dir); 224 | 225 | 226 | struct stat st; 227 | int rc = stat(fullpath, &st); 228 | 229 | free(fullpath); 230 | free(intermediate_dir); 231 | return (rc == 0); 232 | } 233 | 234 | 235 | static size_t 236 | st_count(void *priv) 237 | { 238 | storage_fs_t *storage = (storage_fs_t *)priv; 239 | return ht_count(storage->index); 240 | } 241 | 242 | typedef struct { 243 | shardcache_storage_index_item_t *index; 244 | size_t size; 245 | size_t offset; 246 | } st_pair_iterator_arg_t; 247 | 248 | static int 249 | st_pair_iterator(hashtable_t *table, 250 | void * key, 251 | size_t klen, 252 | void * value, 253 | size_t vlen, 254 | void * priv) 255 | { 256 | st_pair_iterator_arg_t *arg = (st_pair_iterator_arg_t *)priv; 257 | if (arg->offset < arg->size) { 258 | shardcache_storage_index_item_t *index_item; 259 | 260 | index_item = &arg->index[arg->offset++]; 261 | index_item->key = malloc(klen); 262 | memcpy(index_item->key, key, klen); 263 | size_t *size = (size_t *)value; 264 | index_item->klen = klen; 265 | index_item->vlen = *size; 266 | 267 | return 1; 268 | } 269 | return 0; 270 | } 271 | 272 | static size_t 273 | st_index(shardcache_storage_index_item_t *index, size_t isize, void *priv) 274 | { 275 | storage_fs_t *storage = (storage_fs_t *)priv; 276 | st_pair_iterator_arg_t arg = { index, isize, 0 }; 277 | ht_foreach_pair(storage->index, st_pair_iterator, &arg); 278 | return arg.offset; 279 | } 280 | 281 | 282 | static void 283 | storage_fs_walk_and_fill_index(char *path, hashtable_t *index) 284 | { 285 | DIR *dirp = opendir(path); 286 | if (dirp) { 287 | struct dirent *dirent = readdir(dirp); 288 | while(dirent) { 289 | if (dirent->d_name[0] == '.') { 290 | dirent = readdir(dirp); 291 | continue; 292 | } 293 | 294 | size_t fpath_size = strlen(path) + dirent->d_reclen + 3; 295 | char *fpath = malloc(fpath_size); 296 | snprintf(fpath, fpath_size, "%s/%s", path, dirent->d_name); 297 | 298 | switch (dirent->d_type) { 299 | 300 | case DT_DIR: 301 | { 302 | storage_fs_walk_and_fill_index(fpath, index); 303 | break; 304 | } 305 | case DT_REG: 306 | { 307 | size_t namelen = strlen(dirent->d_name); 308 | size_t keylen = namelen/2; 309 | char *keyname = malloc(keylen); 310 | char *p = keyname; 311 | int i; 312 | for (i = 0; i < namelen; i+=2) { 313 | uint8_t c; 314 | sscanf(&dirent->d_name[i], "%02hhx", &c); 315 | *p++ = c; 316 | } 317 | struct stat st; 318 | if (stat(fpath, &st) != 0) { 319 | // TODO - Error messages 320 | } 321 | size_t *sizep = malloc(sizeof(size_t)); 322 | *sizep = st.st_size; 323 | ht_set(index, keyname, keylen, sizep, sizeof(size_t)); 324 | free(keyname); 325 | break; 326 | } 327 | default: 328 | break; 329 | } 330 | free(fpath); 331 | dirent = readdir(dirp); 332 | } 333 | closedir(dirp); 334 | } else { 335 | fprintf(stderr, "Can't open dir %s : %s\n", path, strerror(errno)); 336 | } 337 | } 338 | 339 | int 340 | storage_fs_init(shardcache_storage_t *st, char **options) 341 | { 342 | st->fetch = st_fetch; 343 | st->store = st_store; 344 | st->remove = st_remove; 345 | st->exist = st_exist; 346 | st->index = st_index; 347 | st->count = st_count; 348 | 349 | storage_fs_t *storage = NULL; 350 | char *storage_path = NULL; 351 | char *tmp_path = NULL; 352 | if (options) { 353 | while (*options) { 354 | char *key = (char *)*options++; 355 | if (!*key) 356 | break; 357 | char *value = NULL; 358 | if (*options) { 359 | value = (char *)*options++; 360 | } else { 361 | SHC_ERROR("Odd element in the options array"); 362 | continue; 363 | } 364 | if (key && value) { 365 | if (strcmp(key, "storage_path") == 0) { 366 | storage_path = strdup(value); 367 | } else if (strcmp(key, "tmp_path") == 0) { 368 | tmp_path = strdup(value); 369 | }else { 370 | SHC_ERROR("Unknown option name %s", key); 371 | } 372 | } 373 | } 374 | } 375 | 376 | if (storage_path) { 377 | struct stat s; 378 | if (stat(storage_path, &s) != 0) { 379 | if (mkdir(storage_path, S_IRWXU) != 0) { 380 | SHC_ERROR("Can't create storage path %s: %s", 381 | storage_path, strerror(errno)); 382 | free(storage_path); 383 | if (tmp_path) 384 | free(tmp_path); 385 | return -1; 386 | } 387 | SHC_NOTICE("Created storage path: %s", storage_path); 388 | } 389 | 390 | int check = access(storage_path, R_OK|W_OK); 391 | if (check != 0) { 392 | SHC_ERROR("Can't access the storage path %s : %s", 393 | storage_path, strerror(errno)); 394 | free(storage_path); 395 | if (tmp_path) 396 | free(tmp_path); 397 | return -1; 398 | } 399 | 400 | storage = calloc(1, sizeof(storage_fs_t)); 401 | storage->path = storage_path; 402 | if (!tmp_path) { 403 | tmp_path = strdup("/tmp"); 404 | } 405 | storage->tmp = tmp_path; 406 | if (storage->tmp) { 407 | check = access(storage->tmp, R_OK|W_OK); 408 | if (check != 0) { 409 | SHC_ERROR("Can't access the temporary path %s : %s", 410 | storage->tmp, strerror(errno)); 411 | free(storage); 412 | free(storage_path); 413 | free(tmp_path); 414 | return -1; 415 | } 416 | } 417 | } else { 418 | SHC_ERROR("No storage path defined"); 419 | if (tmp_path) 420 | free(tmp_path); 421 | return -1; 422 | } 423 | storage->index = ht_create(1<<16, 0, free); 424 | storage_fs_walk_and_fill_index(storage->path, storage->index); 425 | 426 | st->priv = storage; 427 | return 0; 428 | } 429 | 430 | void 431 | storage_fs_destroy(void *priv) 432 | { 433 | storage_fs_t *storage = (storage_fs_t *)priv; 434 | free(storage->path); 435 | free(storage->tmp); 436 | ht_destroy(storage->index); 437 | free(storage); 438 | } 439 | -------------------------------------------------------------------------------- /storage_plugins/sqlite/sqlite.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #define ST_KEYFIELD_DEFAULT "key" 9 | #define ST_KEYBYTESFIELD_DEFAULT "keybytes" 10 | #define ST_KEYSIZEFIELD_DEFAULT "keysize" 11 | #define ST_VALUEFIELD_DEFAULT "value" 12 | #define ST_VALUESIZEFIELD_DEFAULT "valuesize" 13 | #define ST_DBNAME_DEFAULT "shardcache" 14 | #define ST_TABLE_DEFAULT "storage" 15 | 16 | int storage_version = SHARDCACHE_STORAGE_API_VERSION; 17 | 18 | typedef struct { 19 | char *dbfile; 20 | char *dbname; 21 | char *table; 22 | char *keyfield; 23 | char *keybytesfield; 24 | char *keysizefield; 25 | char *valuefield; 26 | char *valuesizefield; 27 | int external_blobs; 28 | char *storage_path; 29 | sqlite3 *dbh; 30 | sqlite3_stmt *select_stmt; 31 | sqlite3_stmt *insert_stmt; 32 | sqlite3_stmt *delete_stmt; 33 | sqlite3_stmt *exist_stmt; 34 | sqlite3_stmt *count_stmt; 35 | sqlite3_stmt *index_stmt; 36 | pthread_mutex_t lock; 37 | } storage_sqlite_t; 38 | 39 | static void 40 | parse_options(storage_sqlite_t *st, const char **options) 41 | { 42 | while (options && *options) { 43 | char *key = (char *)*options++; 44 | if (!*key) 45 | break; 46 | char *value = NULL; 47 | if (*options) { 48 | value = (char *)*options++; 49 | } else { 50 | fprintf(stderr, "Odd element in the options array"); 51 | continue; 52 | } 53 | if (key && value) { 54 | if (strcmp(key, "dbfile") == 0) { 55 | st->dbfile = strdup(value); 56 | } else if (strcmp(key, "dbname") == 0) { 57 | st->dbname = strdup(value); 58 | } else if (strcmp(key, "table") == 0) { 59 | st->table = strdup(value); 60 | } else if (strcmp(key, "keyfield") == 0) { 61 | st->keyfield = strdup(value); 62 | } else if (strcmp(key, "keybytesfield") == 0) { 63 | st->keybytesfield = strdup(value); 64 | } else if (strcmp(key, "keysizefield") == 0) { 65 | st->keysizefield = strdup(value); 66 | } else if (strcmp(key, "valuefield") == 0) { 67 | st->valuefield = strdup(value); 68 | } else if (strcmp(key, "valuesizefield") == 0) { 69 | st->valuesizefield = strdup(value); 70 | } else if (strcmp(key, "external_blobs") == 0) { 71 | if (strcmp("value", "yes") == 0 || 72 | strcmp("value", "true") == 0 || 73 | strcmp("value", "1") == 0) 74 | { 75 | st->external_blobs = 1; 76 | } 77 | } else if (strcmp(key, "storage_path") == 0) { 78 | st->storage_path = strdup(value); 79 | } else { 80 | fprintf(stderr, "Unknown option name %s\n", key); 81 | } 82 | } 83 | } 84 | } 85 | 86 | static int 87 | st_fetch(void *key, size_t klen, void **value, size_t *vlen, void *priv) 88 | { 89 | storage_sqlite_t *st = (storage_sqlite_t *)priv; 90 | 91 | char *keystr = malloc((klen*2)+ 1); 92 | char *p = (char *)key; 93 | char *o = (char *)keystr; 94 | int i; 95 | for (i = 0; i < klen; i++) { 96 | snprintf(o, 3, "%02x", p[i]); 97 | o += 2; 98 | } 99 | *o = 0; 100 | 101 | void *data = NULL; 102 | 103 | pthread_mutex_lock(&st->lock); 104 | int rc = sqlite3_reset(st->select_stmt); 105 | 106 | rc = sqlite3_bind_text(st->select_stmt, 1, keystr, strlen(keystr), SQLITE_STATIC); 107 | 108 | int cnt = 0; 109 | rc = sqlite3_step(st->select_stmt); 110 | while (rc == SQLITE_ROW) { 111 | if (cnt++ > 0) { 112 | fprintf(stderr, "Multiple rows found for key %s (%d)\n", keystr, rc); 113 | continue; 114 | } 115 | int bytes; 116 | const void *sqlite_data; 117 | bytes = sqlite3_column_int(st->select_stmt, 3); 118 | sqlite_data = sqlite3_column_blob (st->select_stmt, 4); 119 | if (bytes && sqlite_data) { 120 | if (value) { 121 | *value = malloc(bytes); 122 | memcpy(*value, sqlite_data, bytes); 123 | } 124 | 125 | if (vlen) 126 | *vlen = bytes; 127 | } else { 128 | if (vlen) 129 | *vlen = 0; 130 | if (value) 131 | *value = NULL; 132 | } 133 | rc = sqlite3_step(st->select_stmt); 134 | } 135 | 136 | pthread_mutex_unlock(&st->lock); 137 | 138 | 139 | free(keystr); 140 | 141 | return 0; 142 | } 143 | 144 | static int 145 | st_store(void *key, size_t klen, void *value, size_t vlen, void *priv) 146 | { 147 | storage_sqlite_t *st = (storage_sqlite_t *)priv; 148 | char* errorMessage; 149 | 150 | char *keystr = malloc((klen*2)+ 1); 151 | char *p = (char *)key; 152 | char *o = (char *)keystr; 153 | int i; 154 | for (i = 0; i < klen; i++) { 155 | snprintf(o, 3, "%02x", p[i]); 156 | o += 2; 157 | } 158 | *o = 0; 159 | 160 | pthread_mutex_lock(&st->lock); 161 | sqlite3_reset(st->insert_stmt); 162 | 163 | sqlite3_bind_text(st->insert_stmt, 1, keystr, strlen(keystr), SQLITE_STATIC); 164 | sqlite3_bind_int(st->insert_stmt, 2, klen); 165 | sqlite3_bind_blob(st->insert_stmt, 3, key, klen, SQLITE_STATIC); 166 | sqlite3_bind_int(st->insert_stmt, 4, vlen); 167 | sqlite3_bind_blob(st->insert_stmt, 5, value, vlen, SQLITE_STATIC); 168 | 169 | int rc = sqlite3_step(st->insert_stmt); 170 | if (rc != SQLITE_DONE) { 171 | fprintf(stderr, "Insert Failed! %d\n", rc); 172 | pthread_mutex_unlock(&st->lock); 173 | free(keystr); 174 | return -1; 175 | } 176 | 177 | pthread_mutex_unlock(&st->lock); 178 | free(keystr); 179 | return 0; 180 | } 181 | 182 | static int 183 | st_remove(void *key, size_t klen, void *priv) 184 | { 185 | 186 | storage_sqlite_t *st = (storage_sqlite_t *)priv; 187 | char* errorMessage; 188 | 189 | char *keystr = malloc((klen*2)+ 1); 190 | char *p = (char *)key; 191 | char *o = (char *)keystr; 192 | int i; 193 | for (i = 0; i < klen; i++) { 194 | snprintf(o, 3, "%02x", p[i]); 195 | o += 2; 196 | } 197 | *o = 0; 198 | 199 | pthread_mutex_lock(&st->lock); 200 | sqlite3_reset(st->delete_stmt); 201 | sqlite3_bind_text(st->delete_stmt, 1, keystr, strlen(keystr), SQLITE_STATIC); 202 | int rc = sqlite3_step(st->delete_stmt); 203 | 204 | if (rc != SQLITE_DONE) { 205 | fprintf(stderr, "Delete Failed! %d\n", rc); 206 | pthread_mutex_unlock(&st->lock); 207 | free(keystr); 208 | return -1; 209 | } 210 | 211 | pthread_mutex_unlock(&st->lock); 212 | free(keystr); 213 | return 0; 214 | } 215 | 216 | static int 217 | st_exist(void *key, size_t klen, void *priv) { 218 | storage_sqlite_t *st = (storage_sqlite_t *)priv; 219 | char* errorMessage; 220 | 221 | char *keystr = malloc((klen*2)+ 1); 222 | char *p = (char *)key; 223 | char *o = (char *)keystr; 224 | int i; 225 | for (i = 0; i < klen; i++) { 226 | snprintf(o, 3, "%02x", p[i]); 227 | o += 2; 228 | } 229 | *o = 0; 230 | 231 | pthread_mutex_lock(&st->lock); 232 | sqlite3_reset(st->exist_stmt); 233 | sqlite3_bind_text(st->exist_stmt, 1, keystr, strlen(keystr), SQLITE_STATIC); 234 | int rc = sqlite3_step(st->exist_stmt); 235 | 236 | int count; 237 | if (rc == SQLITE_ROW) { 238 | count = sqlite3_column_int(st->count_stmt, 0); 239 | } 240 | 241 | rc = sqlite3_step(st->exist_stmt); 242 | if (rc != SQLITE_DONE) { 243 | fprintf(stderr, "Exist Failed! %d\n", rc); 244 | pthread_mutex_unlock(&st->lock); 245 | free(keystr); 246 | return -1; 247 | } 248 | 249 | pthread_mutex_unlock(&st->lock); 250 | free(keystr); 251 | return (count == 1); 252 | } 253 | 254 | static size_t 255 | st_count(void *priv) 256 | { 257 | storage_sqlite_t *st = (storage_sqlite_t *)priv; 258 | 259 | size_t count = 0; 260 | 261 | pthread_mutex_lock(&st->lock); 262 | 263 | int rc = sqlite3_reset(st->count_stmt); 264 | 265 | rc = sqlite3_step(st->count_stmt); 266 | if (rc == SQLITE_ROW) { 267 | count = sqlite3_column_int(st->count_stmt, 0); 268 | } 269 | pthread_mutex_unlock(&st->lock); 270 | 271 | return count; 272 | } 273 | 274 | static size_t 275 | st_index(shardcache_storage_index_item_t *index, size_t isize, void *priv) 276 | { 277 | storage_sqlite_t *st = (storage_sqlite_t *)priv; 278 | 279 | pthread_mutex_lock(&st->lock); 280 | 281 | int rc = sqlite3_reset(st->index_stmt); 282 | 283 | size_t i = 0; 284 | rc = sqlite3_step(st->index_stmt); 285 | while (rc == SQLITE_ROW && i < isize) { 286 | int vlen; 287 | int klen; 288 | const void *key; 289 | 290 | key = sqlite3_column_blob(st->index_stmt, 0); 291 | klen = sqlite3_column_int(st->index_stmt, 1); 292 | vlen = sqlite3_column_int(st->index_stmt, 2); 293 | 294 | shardcache_storage_index_item_t *item = &index[i++]; 295 | item->key = malloc(klen); 296 | memcpy(item->key, key, klen); 297 | item->klen = klen; 298 | item->vlen = vlen; 299 | 300 | rc = sqlite3_step(st->index_stmt); 301 | } 302 | pthread_mutex_unlock(&st->lock); 303 | 304 | return i; 305 | } 306 | 307 | void 308 | storage_destroy(void *priv) 309 | { 310 | storage_sqlite_t *st = (storage_sqlite_t *)priv; 311 | sqlite3_finalize(st->select_stmt); 312 | sqlite3_finalize(st->insert_stmt); 313 | sqlite3_finalize(st->delete_stmt); 314 | sqlite3_finalize(st->exist_stmt); 315 | sqlite3_finalize(st->count_stmt); 316 | sqlite3_finalize(st->index_stmt); 317 | sqlite3_close(st->dbh); 318 | free(st->dbname); 319 | free(st->table); 320 | free(st->keyfield); 321 | free(st->keybytesfield); 322 | free(st->keysizefield); 323 | free(st->valuefield); 324 | free(st->valuesizefield); 325 | free(st); 326 | } 327 | 328 | int 329 | storage_init(shardcache_storage_t *storage, const char **options) 330 | { 331 | storage_sqlite_t *st = calloc(1, sizeof(storage_sqlite_t)); 332 | 333 | if (options) 334 | parse_options(st, options); 335 | 336 | if (!st->dbfile) { 337 | fprintf(stderr, "the dbfile option is mandatory!\n"); 338 | free(st); 339 | return -1; 340 | } 341 | 342 | if (!st->dbname) 343 | st->dbname = strdup(ST_DBNAME_DEFAULT); 344 | 345 | if (!st->table) 346 | st->table = strdup(ST_TABLE_DEFAULT); 347 | 348 | if (!st->keyfield) 349 | st->keyfield = strdup(ST_KEYFIELD_DEFAULT); 350 | 351 | if (!st->keybytesfield) 352 | st->keybytesfield = strdup(ST_KEYBYTESFIELD_DEFAULT); 353 | 354 | if (!st->keysizefield) 355 | st->keysizefield = strdup(ST_KEYSIZEFIELD_DEFAULT); 356 | 357 | if (!st->valuefield) 358 | st->valuefield = strdup(ST_VALUEFIELD_DEFAULT); 359 | 360 | if (!st->valuesizefield) 361 | st->valuesizefield = strdup(ST_VALUESIZEFIELD_DEFAULT); 362 | 363 | int rc = sqlite3_open(st->dbfile, &st->dbh); 364 | if (rc != SQLITE_OK) { 365 | free(st->dbname); 366 | free(st->table); 367 | free(st->keyfield); 368 | free(st->keybytesfield); 369 | free(st->keysizefield); 370 | free(st->valuefield); 371 | free(st->valuesizefield); 372 | free(st); 373 | return -1; 374 | } 375 | 376 | char create_table_sql[2048]; 377 | snprintf(create_table_sql, sizeof(create_table_sql), 378 | "CREATE TABLE IF NOT EXISTS %s (%s TEXT PRIMARY KEY, %s INTEGER, %s BLOB, %s INTEGER, %s BLOB)", 379 | st->table, st->keyfield, st->keysizefield, st->keybytesfield, st->valuesizefield, st->valuefield); 380 | 381 | rc = sqlite3_exec(st->dbh, create_table_sql, NULL, NULL, NULL); 382 | 383 | const char *tail = NULL; 384 | char sql[2048]; 385 | snprintf(sql, sizeof(sql), "SELECT * FROM %s WHERE %s = ?", st->table, st->keyfield); 386 | rc = sqlite3_prepare_v2(st->dbh, sql, 2048, &st->select_stmt, &tail); 387 | if (rc != SQLITE_OK) { 388 | // TODO - Errors 389 | } 390 | 391 | snprintf(sql, sizeof(sql), "INSERT OR REPLACE INTO %s VALUES(?, ?, ?, ?, ?)", st->table); 392 | rc = sqlite3_prepare_v2(st->dbh, sql, 2048, &st->insert_stmt, &tail); 393 | if (rc != SQLITE_OK) { 394 | // TODO - Errors 395 | } 396 | 397 | snprintf(sql, sizeof(sql), "DELETE FROM %s WHERE %s = ?", st->table, st->keyfield); 398 | rc = sqlite3_prepare_v2(st->dbh, sql, 2048, &st->delete_stmt, &tail); 399 | if (rc != SQLITE_OK) { 400 | // TODO - Errors 401 | } 402 | 403 | snprintf(sql, sizeof(sql), "SELECT COUNT(*) FROM %s WHERE %s = ?", st->table, st->keyfield); 404 | rc = sqlite3_prepare_v2(st->dbh, sql, 2048, &st->exist_stmt, &tail); 405 | if (rc != SQLITE_OK) { 406 | // TODO - Errors 407 | } 408 | 409 | snprintf(sql, sizeof(sql), "SELECT COUNT(*) FROM %s", st->table); 410 | rc = sqlite3_prepare_v2(st->dbh, sql, 2048, &st->count_stmt, &tail); 411 | if (rc != SQLITE_OK) { 412 | // TODO - Errors 413 | } 414 | 415 | snprintf(sql, sizeof(sql), "SELECT %s, %s, %s FROM %s", 416 | st->keybytesfield, st->keysizefield, st->valuesizefield, st->table); 417 | rc = sqlite3_prepare_v2(st->dbh, sql, 2048, &st->index_stmt, &tail); 418 | if (rc != SQLITE_OK) { 419 | // TODO - Errors 420 | } 421 | 422 | storage->fetch = st_fetch; 423 | storage->store = st_store; 424 | storage->remove = st_remove; 425 | storage->count = st_count; 426 | storage->index = st_index; 427 | storage->priv = st; 428 | 429 | pthread_mutex_init(&st->lock, NULL); 430 | return 0; 431 | } 432 | 433 | 434 | -------------------------------------------------------------------------------- /storage_plugins/mysql/mysql.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #ifdef __MACH__ 9 | #include 10 | #define SPIN_LOCK(__mutex) OSSpinLockLock(__mutex) 11 | #define SPIN_TRYLOCK(__mutex) OSSpinLockTry(__mutex) 12 | #define SPIN_UNLOCK(__mutex) OSSpinLockUnlock(__mutex) 13 | #else 14 | #define SPIN_LOCK(__mutex) pthread_spin_lock(__mutex) 15 | #define SPIN_TRYLOCK(__mutex) pthread_spin_trylock(__mutex) 16 | #define SPIN_UNLOCK(__mutex) pthread_spin_unlock(__mutex) 17 | #endif 18 | 19 | 20 | #define ST_KEYFIELD_DEFAULT "key" 21 | #define ST_KEYBYTESFIELD_DEFAULT "keybytes" 22 | #define ST_VALUEFIELD_DEFAULT "value" 23 | #define ST_VALUESIZEFIELD_DEFAULT "valuesize" 24 | #define ST_DBNAME_DEFAULT "shardcache" 25 | #define ST_TABLE_DEFAULT "storage" 26 | #define ST_NUM_CONNECTIONS_DEFAULT 5 27 | 28 | int storage_version = SHARDCACHE_STORAGE_API_VERSION; 29 | 30 | typedef struct { 31 | MYSQL dbh; 32 | MYSQL_STMT *select_stmt; 33 | MYSQL_STMT *insert_stmt; 34 | MYSQL_STMT *delete_stmt; 35 | MYSQL_STMT *exist_stmt; 36 | MYSQL_STMT *count_stmt; 37 | MYSQL_STMT *index_stmt; 38 | #ifdef __MACH__ 39 | OSSpinLock lock; 40 | #else 41 | pthread_spinlock_t lock; 42 | #endif 43 | int initialized; 44 | } db_connection_t; 45 | 46 | typedef struct { 47 | char *dbname; 48 | char *dbhost; 49 | int dbport; 50 | char *dbuser; 51 | char *dbpasswd; 52 | char *unix_socket; 53 | char *table; 54 | char *keyfield; 55 | char *keybytesfield; 56 | char *valuefield; 57 | char *valuesizefield; 58 | int external_blobs; 59 | char *storage_path; 60 | int num_connections; 61 | int connection_index; 62 | db_connection_t *dbconnections; 63 | int table_checked; 64 | } storage_mysql_t; 65 | 66 | static void 67 | parse_options(storage_mysql_t *st, const char **options) 68 | { 69 | while (options && *options) { 70 | char *key = (char *)*options++; 71 | if (!*key) 72 | break; 73 | char *value = NULL; 74 | if (*options) { 75 | value = (char *)*options++; 76 | } else { 77 | fprintf(stderr, "Odd element in the options array"); 78 | break; 79 | } 80 | if (key && value) { 81 | if (strcmp(key, "dbname") == 0) { 82 | st->dbname = strdup(value); 83 | } else if (strcmp(key, "dbhost") == 0) { 84 | st->dbhost = strdup(value); 85 | } else if (strcmp(key, "dbport") == 0) { 86 | st->dbport = strtol(value, NULL, 10); 87 | } else if (strcmp(key, "dbuser") == 0) { 88 | st->dbuser = strdup(value); 89 | } else if (strcmp(key, "dbpasswd") == 0) { 90 | st->dbpasswd = strdup(value); 91 | } else if (strcmp(key, "unix_socket") == 0) { 92 | st->unix_socket = strdup(value); 93 | } else if (strcmp(key, "table") == 0) { 94 | st->table = strdup(value); 95 | } else if (strcmp(key, "keyfield") == 0) { 96 | st->keyfield = strdup(value); 97 | } else if (strcmp(key, "keybytesfield") == 0) { 98 | st->keybytesfield = strdup(value); 99 | } else if (strcmp(key, "valuefield") == 0) { 100 | st->valuefield = strdup(value); 101 | } else if (strcmp(key, "valuesizefield") == 0) { 102 | st->valuesizefield = strdup(value); 103 | } else if (strcmp(key, "external_blobs") == 0) { 104 | if (strcmp("value", "yes") == 0 || 105 | strcmp("value", "true") == 0 || 106 | strcmp("value", "1") == 0) 107 | { 108 | st->external_blobs = 1; 109 | } 110 | } else if (strcmp(key, "storage_path") == 0) { 111 | st->storage_path = strdup(value); 112 | } else if (strcmp(key, "num_connections") == 0) { 113 | st->num_connections = strtol(value, NULL, 10); 114 | } else { 115 | fprintf(stderr, "Unknown option name %s\n", key); 116 | } 117 | } 118 | } 119 | } 120 | 121 | static void 122 | st_clear_dbconnection(storage_mysql_t *st, db_connection_t *dbc) 123 | { 124 | if (dbc->select_stmt) { 125 | mysql_stmt_close(dbc->select_stmt); 126 | dbc->select_stmt = NULL; 127 | } 128 | 129 | if (dbc->insert_stmt) { 130 | mysql_stmt_close(dbc->insert_stmt); 131 | dbc->insert_stmt = NULL; 132 | } 133 | 134 | if (dbc->delete_stmt) { 135 | mysql_stmt_close(dbc->delete_stmt); 136 | dbc->delete_stmt = NULL; 137 | } 138 | 139 | if (dbc->exist_stmt) { 140 | mysql_stmt_close(dbc->exist_stmt); 141 | dbc->exist_stmt = NULL; 142 | } 143 | 144 | if (dbc->count_stmt) { 145 | mysql_stmt_close(dbc->count_stmt); 146 | dbc->count_stmt = NULL; 147 | } 148 | 149 | if (dbc->index_stmt) { 150 | mysql_stmt_close(dbc->index_stmt); 151 | dbc->index_stmt = NULL; 152 | } 153 | 154 | if (dbc->initialized) { 155 | mysql_close(&dbc->dbh); 156 | #ifndef __MACH__ 157 | pthread_spin_destroy(&dbc->lock); 158 | #endif 159 | dbc->initialized = 0; 160 | } 161 | } 162 | 163 | static int 164 | st_init_dbconnection(storage_mysql_t *st, db_connection_t *dbc) 165 | { 166 | MYSQL *mysql = mysql_init(&dbc->dbh); 167 | if (!mysql) { 168 | fprintf(stderr, "Can't initialize the mysql handler\n"); 169 | return -1; 170 | } 171 | #ifndef __MACH__ 172 | pthread_spin_init(&dbc->lock, 0); 173 | #endif 174 | dbc->initialized = 1; 175 | 176 | my_bool b = 1; 177 | 178 | if (!mysql_real_connect(&dbc->dbh, 179 | st->dbhost, 180 | st->dbuser, 181 | st->dbpasswd, 182 | st->dbname, 183 | st->dbport, 184 | st->unix_socket, 185 | 0)) 186 | { 187 | fprintf(stderr, "Can't connect to mysql database: %s\n", 188 | mysql_error(&dbc->dbh)); 189 | return -1; 190 | } 191 | 192 | if (!st->table_checked) { 193 | char create_table_sql[2048]; 194 | snprintf(create_table_sql, sizeof(create_table_sql), 195 | "CREATE TABLE IF NOT EXISTS `%s` (`%s` char(255) primary key, `%s` blob, `%s` int, `%s` longblob)", 196 | st->table, st->keyfield, st->keybytesfield, st->valuesizefield, st->valuefield); 197 | mysql_query(&dbc->dbh, create_table_sql); 198 | st->table_checked = 1; 199 | } 200 | 201 | char sql[2048]; 202 | snprintf(sql, sizeof(sql), "SELECT `%s` FROM `%s` WHERE `%s` = ?", st->valuefield, st->table, st->keyfield); 203 | dbc->select_stmt = mysql_stmt_init(mysql); 204 | int rc = mysql_stmt_prepare(dbc->select_stmt, sql, strlen(sql)); 205 | if (rc != 0) { 206 | fprintf(stderr, "Can't prepare the select_stmt '%s' : %s\n", 207 | sql, mysql_stmt_error(dbc->select_stmt)); 208 | return -1; 209 | } 210 | 211 | snprintf(sql, sizeof(sql), "REPLACE INTO `%s` VALUES(?, ?, ?, ?)", st->table); 212 | dbc->insert_stmt = mysql_stmt_init(mysql); 213 | rc = mysql_stmt_prepare(dbc->insert_stmt, sql, strlen(sql)); 214 | if (rc != 0) { 215 | fprintf(stderr, "Can't prepare the insert_stmt '%s' : %s\n", 216 | sql, mysql_stmt_error(dbc->insert_stmt)); 217 | return -1; 218 | } 219 | 220 | snprintf(sql, sizeof(sql), "DELETE FROM `%s` WHERE `%s` = ?", st->table, st->keyfield); 221 | dbc->delete_stmt = mysql_stmt_init(mysql); 222 | rc = mysql_stmt_prepare(dbc->delete_stmt, sql, strlen(sql)); 223 | if (rc != 0) { 224 | fprintf(stderr, "Can't prepare the delete_stmt '%s' : %s\n", 225 | sql, mysql_stmt_error(dbc->delete_stmt)); 226 | return -1; 227 | } 228 | 229 | snprintf(sql, sizeof(sql), "SELECT COUNT(*) FROM `%s` WHERE `%s` = ?", st->table, st->keyfield); 230 | dbc->exist_stmt = mysql_stmt_init(mysql); 231 | rc = mysql_stmt_prepare(dbc->exist_stmt, sql, strlen(sql)); 232 | if (rc != 0) { 233 | fprintf(stderr, "Can't prepare the exist_stmt '%s' : %s\n", 234 | sql, mysql_stmt_error(dbc->exist_stmt)); 235 | return -1; 236 | } 237 | 238 | snprintf(sql, sizeof(sql), "SELECT COUNT(*) FROM `%s`", st->table); 239 | dbc->count_stmt = mysql_stmt_init(mysql); 240 | rc = mysql_stmt_prepare(dbc->count_stmt, sql, strlen(sql)); 241 | if (rc != 0) { 242 | fprintf(stderr, "Can't prepare the count_stmt '%s' : %s\n", 243 | sql, mysql_stmt_error(dbc->count_stmt)); 244 | return -1; 245 | } 246 | 247 | snprintf(sql, sizeof(sql), "SELECT `%s`, `%s` FROM `%s`", 248 | st->keybytesfield, st->valuesizefield, st->table); 249 | dbc->index_stmt = mysql_stmt_init(mysql); 250 | rc = mysql_stmt_prepare(dbc->index_stmt, sql, strlen(sql)); 251 | if (rc != 0) { 252 | fprintf(stderr, "Can't prepare the index_stmt '%s' : %s\n", 253 | sql, mysql_stmt_error(dbc->index_stmt)); 254 | return -1; 255 | } 256 | 257 | return 0; 258 | } 259 | 260 | static db_connection_t * 261 | st_get_dbconnection(storage_mysql_t *st) 262 | { 263 | int rc = 0; 264 | int index = 0; 265 | db_connection_t *dbc = NULL; 266 | int retries = 0; 267 | do { 268 | index = __sync_fetch_and_add(&st->connection_index, 1)%st->num_connections; 269 | dbc = &st->dbconnections[index]; 270 | rc = SPIN_TRYLOCK(&dbc->lock); 271 | if (retries++ == 100) { 272 | // ok .. it's too busy 273 | fprintf(stderr, "Can't acquire any connection lock\n"); 274 | return NULL; 275 | } 276 | } while (rc != 0); 277 | 278 | // we acquired the lock for a connection 279 | // let's check if it's still alive 280 | if (mysql_ping(&dbc->dbh) != 0) { 281 | st_clear_dbconnection(st, dbc); 282 | if (st_init_dbconnection(st, dbc) != 0) 283 | return NULL; 284 | } 285 | // TODO - we might try again using a different connection, but if connect failed 286 | // most likely the database is really unreachable 287 | return dbc; 288 | } 289 | 290 | 291 | 292 | static int st_fetch(void *key, size_t klen, void **value, size_t *vlen, void *priv) 293 | { 294 | storage_mysql_t *st = (storage_mysql_t *)priv; 295 | 296 | char *keystr = malloc((klen*2)+ 1); 297 | char *p = (char *)key; 298 | char *o = (char *)keystr; 299 | int i; 300 | for (i = 0; i < klen; i++) { 301 | snprintf(o, 3, "%02x", p[i]); 302 | o += 2; 303 | } 304 | *o = 0; 305 | 306 | 307 | db_connection_t *dbc = st_get_dbconnection(st); 308 | if (!dbc) { 309 | free(keystr); 310 | return -1; 311 | } 312 | 313 | MYSQL_BIND bnd = { 314 | .buffer_type = MYSQL_TYPE_STRING, 315 | .buffer = keystr, 316 | .buffer_length = strlen(keystr) 317 | }; 318 | 319 | if (mysql_stmt_bind_param(dbc->select_stmt, &bnd) != 0) { 320 | // TODO - error messages 321 | SPIN_UNLOCK(&dbc->lock); 322 | free(keystr); 323 | return -1; 324 | } 325 | 326 | if (mysql_stmt_execute(dbc->select_stmt) != 0) { 327 | // TODO - error messages 328 | fprintf(stderr, "Can't execute fetch statement : %s\n", mysql_stmt_error(dbc->select_stmt)); 329 | SPIN_UNLOCK(&dbc->lock); 330 | free(keystr); 331 | return -1; 332 | } 333 | 334 | size_t size = 256; 335 | void *data = malloc(size); 336 | my_bool error = 0; 337 | 338 | MYSQL_BIND obnd = { 339 | .buffer_type = MYSQL_TYPE_LONG_BLOB, 340 | .buffer = data, 341 | .buffer_length = size, 342 | .length = &size, 343 | .error = &error 344 | }; 345 | 346 | mysql_stmt_bind_result(dbc->select_stmt, &obnd); 347 | 348 | int rc = mysql_stmt_fetch(dbc->select_stmt); 349 | 350 | if (error == 1) { 351 | data = realloc(data, size); 352 | obnd.buffer = data; 353 | obnd.buffer_length = size; 354 | error = 0; 355 | mysql_stmt_bind_result(dbc->select_stmt, &obnd); 356 | mysql_stmt_fetch(dbc->select_stmt); 357 | } 358 | 359 | if (rc != 0 || obnd.is_null) { 360 | free(data); 361 | data = NULL; 362 | } else { 363 | 364 | if (value) { 365 | *value = data; 366 | } else { 367 | free(data); 368 | } 369 | 370 | if (vlen) { 371 | *vlen = size; 372 | } 373 | } 374 | 375 | mysql_stmt_free_result(dbc->select_stmt); 376 | mysql_stmt_reset(dbc->select_stmt); 377 | 378 | SPIN_UNLOCK(&dbc->lock); 379 | 380 | free(keystr); 381 | return 0; 382 | } 383 | 384 | static int st_store(void *key, size_t klen, void *value, size_t vlen, void *priv) 385 | { 386 | storage_mysql_t *st = (storage_mysql_t *)priv; 387 | char* errorMessage; 388 | 389 | char *keystr = malloc((klen*2)+ 1); 390 | char *p = (char *)key; 391 | char *o = (char *)keystr; 392 | int i; 393 | for (i = 0; i < klen; i++) { 394 | snprintf(o, 3, "%02x", p[i]); 395 | o += 2; 396 | } 397 | *o = 0; 398 | 399 | db_connection_t *dbc = st_get_dbconnection(st); 400 | if (!dbc) { 401 | free(keystr); 402 | return -1; 403 | } 404 | 405 | MYSQL_BIND bnd[5] = { 406 | { 407 | .buffer_type = MYSQL_TYPE_STRING, 408 | .buffer = keystr, 409 | .buffer_length = strlen(keystr), 410 | .length = 0, 411 | .is_null = 0 412 | }, 413 | { 414 | .buffer_type = MYSQL_TYPE_LONG_BLOB, 415 | .buffer = key, 416 | .buffer_length = klen, 417 | .length = 0, 418 | .is_null = 0 419 | }, 420 | { 421 | .buffer_type = MYSQL_TYPE_LONG, 422 | .buffer = &vlen, 423 | .length = 0, 424 | .is_null = 0 425 | }, 426 | { 427 | .buffer_type = MYSQL_TYPE_LONG_BLOB, 428 | .buffer = value, 429 | .buffer_length = vlen, 430 | .length = 0, 431 | .is_null = 0 432 | } 433 | }; 434 | 435 | if (mysql_stmt_bind_param(dbc->insert_stmt, bnd) != 0) { 436 | // TODO - error messages 437 | SPIN_UNLOCK(&dbc->lock); 438 | fprintf(stderr, "Can't bind params to the insert statement: %s\n", mysql_stmt_error(dbc->insert_stmt)); 439 | free(keystr); 440 | return -1; 441 | } 442 | 443 | if (mysql_stmt_execute(dbc->insert_stmt) != 0) { 444 | // TODO - error messages 445 | SPIN_UNLOCK(&dbc->lock); 446 | free(keystr); 447 | return -1; 448 | } 449 | 450 | mysql_stmt_free_result(dbc->insert_stmt); 451 | 452 | SPIN_UNLOCK(&dbc->lock); 453 | free(keystr); 454 | return 0; 455 | } 456 | 457 | static int st_remove(void *key, size_t klen, void *priv) 458 | { 459 | 460 | storage_mysql_t *st = (storage_mysql_t *)priv; 461 | char* errorMessage; 462 | 463 | char *keystr = malloc((klen*2)+ 1); 464 | char *p = (char *)key; 465 | char *o = (char *)keystr; 466 | int i; 467 | for (i = 0; i < klen; i++) { 468 | snprintf(o, 3, "%02x", p[i]); 469 | o += 2; 470 | } 471 | *o = 0; 472 | 473 | db_connection_t *dbc = st_get_dbconnection(st); 474 | if (!dbc) { 475 | free(keystr); 476 | return -1; 477 | } 478 | 479 | MYSQL_BIND bnd = { 480 | .buffer_type = MYSQL_TYPE_STRING, 481 | .buffer = keystr, 482 | .buffer_length = strlen(keystr) 483 | }; 484 | 485 | if (mysql_stmt_bind_param(dbc->delete_stmt, &bnd) != 0) { 486 | // TODO - error messages 487 | SPIN_UNLOCK(&dbc->lock); 488 | free(keystr); 489 | return -1; 490 | } 491 | 492 | if (mysql_stmt_execute(dbc->delete_stmt) != 0) { 493 | // TODO - error messages 494 | SPIN_UNLOCK(&dbc->lock); 495 | free(keystr); 496 | return -1; 497 | } 498 | 499 | mysql_stmt_free_result(dbc->delete_stmt); 500 | 501 | SPIN_UNLOCK(&dbc->lock); 502 | free(keystr); 503 | return 0; 504 | } 505 | 506 | static int st_exist(void *key, size_t klen, void *priv) { 507 | storage_mysql_t *st = (storage_mysql_t *)priv; 508 | char* errorMessage; 509 | 510 | char *keystr = malloc((klen*2)+ 1); 511 | char *p = (char *)key; 512 | char *o = (char *)keystr; 513 | int i; 514 | for (i = 0; i < klen; i++) { 515 | snprintf(o, 3, "%02x", p[i]); 516 | o += 2; 517 | } 518 | *o = 0; 519 | 520 | db_connection_t *dbc = st_get_dbconnection(st); 521 | if (!dbc) { 522 | free(keystr); 523 | return 0; 524 | } 525 | 526 | MYSQL_BIND bnd = { 527 | .buffer_type = MYSQL_TYPE_STRING, 528 | .buffer = keystr, 529 | .buffer_length = strlen(keystr) 530 | }; 531 | 532 | if (mysql_stmt_bind_param(dbc->exist_stmt, &bnd) != 0) { 533 | // TODO - error messages 534 | SPIN_UNLOCK(&dbc->lock); 535 | free(keystr); 536 | return 0; 537 | } 538 | 539 | if (mysql_stmt_execute(dbc->exist_stmt) != 0) { 540 | // TODO - error messages 541 | SPIN_UNLOCK(&dbc->lock); 542 | free(keystr); 543 | return 0; 544 | } 545 | 546 | mysql_stmt_fetch(dbc->select_stmt); 547 | 548 | int count = 0; 549 | MYSQL_BIND obnd = { 550 | .buffer_type = MYSQL_TYPE_LONG, 551 | .buffer = &count 552 | }; 553 | 554 | if (mysql_stmt_fetch_column(dbc->exist_stmt, &obnd, 0, 0) != 0) { 555 | // TODO - error messages 556 | mysql_stmt_free_result(dbc->exist_stmt); 557 | SPIN_UNLOCK(&dbc->lock); 558 | free(keystr); 559 | return 0; 560 | } 561 | 562 | mysql_stmt_free_result(dbc->exist_stmt); 563 | 564 | if (count > 1) { 565 | // TODO - error messages 566 | } 567 | 568 | SPIN_UNLOCK(&dbc->lock); 569 | free(keystr); 570 | return (count == 1); 571 | } 572 | 573 | static size_t st_count(void *priv) 574 | { 575 | storage_mysql_t *st = (storage_mysql_t *)priv; 576 | 577 | db_connection_t *dbc = st_get_dbconnection(st); 578 | if (!dbc) { 579 | return 0; 580 | } 581 | 582 | if (mysql_stmt_execute(dbc->count_stmt) != 0) { 583 | // TODO - error messages 584 | SPIN_UNLOCK(&dbc->lock); 585 | return 0; 586 | } 587 | 588 | 589 | size_t count = 0; 590 | MYSQL_BIND obnd = { 591 | .buffer_type = MYSQL_TYPE_LONG, 592 | .buffer = &count 593 | }; 594 | 595 | mysql_stmt_bind_result(dbc->count_stmt, &obnd); 596 | 597 | if (mysql_stmt_fetch(dbc->count_stmt) != 0) { 598 | // TODO - error messages 599 | mysql_stmt_free_result(dbc->count_stmt); 600 | SPIN_UNLOCK(&dbc->lock); 601 | return 0; 602 | } 603 | 604 | mysql_stmt_free_result(dbc->count_stmt); 605 | 606 | SPIN_UNLOCK(&dbc->lock); 607 | 608 | return count; 609 | } 610 | 611 | static size_t st_index(shardcache_storage_index_item_t *index, size_t isize, void *priv) 612 | { 613 | storage_mysql_t *st = (storage_mysql_t *)priv; 614 | 615 | db_connection_t *dbc = st_get_dbconnection(st); 616 | if (!dbc) { 617 | return 0; 618 | } 619 | 620 | if (mysql_stmt_execute(dbc->index_stmt) != 0) { 621 | // TODO - error messages 622 | SPIN_UNLOCK(&dbc->lock); 623 | return 0; 624 | } 625 | 626 | size_t klen, vlen = 0; 627 | my_bool error = 0; 628 | MYSQL_BIND obnd[2] = { 629 | { 630 | .buffer_type = MYSQL_TYPE_BLOB, 631 | .buffer_length = klen, 632 | .length = &klen, 633 | .error = &error 634 | }, 635 | { 636 | .buffer_type = MYSQL_TYPE_LONG, 637 | .buffer_length = sizeof(vlen), 638 | .buffer = &vlen 639 | } 640 | }; 641 | int rc = 0; 642 | size_t cnt = 0; 643 | while (cnt < isize) { 644 | klen = 256; 645 | void *key = malloc(klen); 646 | obnd[0].buffer = key; 647 | obnd[0].buffer_length = klen; 648 | 649 | rc = mysql_stmt_bind_result(dbc->index_stmt, obnd); 650 | rc = mysql_stmt_fetch(dbc->index_stmt); 651 | if (rc != 0) { 652 | free(key); 653 | break; 654 | } 655 | 656 | if (error == 1) { 657 | key = realloc(key, klen); 658 | obnd[0].buffer = key; 659 | obnd[0].buffer_length = klen; 660 | error = 0; 661 | mysql_stmt_bind_result(dbc->select_stmt, obnd); 662 | mysql_stmt_fetch(dbc->select_stmt); 663 | } 664 | 665 | shardcache_storage_index_item_t *item = &index[cnt++]; 666 | item->key = key; 667 | item->klen = klen; 668 | item->vlen = vlen; 669 | } 670 | 671 | if (rc != MYSQL_NO_DATA) { 672 | // TODO - Error messages 673 | } 674 | 675 | mysql_stmt_free_result(dbc->index_stmt); 676 | mysql_stmt_reset(dbc->index_stmt); 677 | 678 | SPIN_UNLOCK(&dbc->lock); 679 | 680 | return cnt; 681 | } 682 | 683 | static void 684 | storage_mysql_destroy(storage_mysql_t *st) 685 | { 686 | int i; 687 | for (i = 0; i < st->num_connections; i++) { 688 | db_connection_t *dbc = &st->dbconnections[i]; 689 | st_clear_dbconnection(st, dbc); 690 | } 691 | free(st->dbconnections); 692 | mysql_server_end(); 693 | 694 | free(st->dbhost); 695 | free(st->dbname); 696 | if (st->dbuser) 697 | free(st->dbuser); 698 | if (st->dbpasswd) 699 | free(st->dbpasswd); 700 | if (st->unix_socket) 701 | free(st->unix_socket); 702 | free(st->table); 703 | free(st->keyfield); 704 | free(st->keybytesfield); 705 | free(st->valuefield); 706 | free(st->valuesizefield); 707 | if (st->storage_path) 708 | free(st->storage_path); 709 | free(st); 710 | } 711 | 712 | void 713 | storage_destroy(void *priv) 714 | { 715 | storage_mysql_t *st = (storage_mysql_t *)priv; 716 | storage_mysql_destroy(st); 717 | } 718 | 719 | static void 720 | st_thread_start(void *priv) 721 | { 722 | mysql_thread_init(); 723 | } 724 | 725 | static void 726 | st_thread_exit(void *priv) 727 | { 728 | mysql_thread_end(); 729 | } 730 | 731 | int 732 | storage_init(shardcache_storage_t *storage, const char **options) 733 | { 734 | storage_mysql_t *st = calloc(1, sizeof(storage_mysql_t)); 735 | 736 | if (options) 737 | parse_options(st, options); 738 | 739 | if (!st->dbname) 740 | st->dbname = strdup(ST_DBNAME_DEFAULT); 741 | 742 | if (!st->table) 743 | st->table = strdup(ST_TABLE_DEFAULT); 744 | 745 | if (!st->keyfield) 746 | st->keyfield = strdup(ST_KEYFIELD_DEFAULT); 747 | 748 | if (!st->keybytesfield) 749 | st->keybytesfield = strdup(ST_KEYBYTESFIELD_DEFAULT); 750 | 751 | if (!st->valuefield) 752 | st->valuefield = strdup(ST_VALUEFIELD_DEFAULT); 753 | 754 | if (!st->valuesizefield) 755 | st->valuesizefield = strdup(ST_VALUESIZEFIELD_DEFAULT); 756 | 757 | if (!st->num_connections) 758 | st->num_connections = ST_NUM_CONNECTIONS_DEFAULT; 759 | 760 | 761 | st->dbconnections = calloc(sizeof(db_connection_t), st->num_connections); 762 | 763 | int i; 764 | for (i = 0; i < st->num_connections; i++) { 765 | db_connection_t *dbc = &st->dbconnections[i]; 766 | if (st_init_dbconnection(st, dbc) != 0) { 767 | storage_mysql_destroy(st); 768 | return -1; 769 | } 770 | } 771 | 772 | storage->fetch = st_fetch; 773 | storage->store = st_store; 774 | storage->remove = st_remove; 775 | storage->count = st_count; 776 | storage->index = st_index; 777 | storage->thread_start = st_thread_start; 778 | storage->thread_exit = st_thread_exit; 779 | storage->priv = st; 780 | 781 | return 0; 782 | } 783 | 784 | 785 | -------------------------------------------------------------------------------- /src/shcd_http.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "mongoose.h" 15 | 16 | #include "shcd_http.h" 17 | 18 | #define HTTP_HEADERS_BASE "HTTP/1.0 200 OK\r\n" \ 19 | "Content-Type: %s\r\n" \ 20 | "Server: shardcached\r\n" \ 21 | "Connection: Close\r\n" 22 | 23 | #define HTTP_HEADERS HTTP_HEADERS_BASE "Content-Length: %d\r\n\r\n" 24 | 25 | #define HTTP_HEADERS_WITH_TIME HTTP_HEADERS_BASE "Last-Modified: %s\r\n\r\n" 26 | 27 | #define ATOMIC_INCREMENT(_v) (void)__sync_add_and_fetch(&(_v), 1) 28 | #define ATOMIC_DECREMENT(_v) (void)__sync_sub_and_fetch(&(_v), 1) 29 | 30 | #define ATOMIC_READ(_v) __sync_fetch_and_add(&(_v), 0) 31 | 32 | #define HTTP_CONFIGURE_COMMAND "__configure__" 33 | #define HTTP_STATS_COMMAND "__stats__" 34 | #define HTTP_INDEX_COMMAND "__index__" 35 | #define HTTP_HEALTH_COMMAND "__health__" 36 | 37 | #define HTTP_MAX_KEYLEN 2048 38 | 39 | static inline int 40 | is_admin_command(char *key, char **extra) 41 | { 42 | static __thread char buf[HTTP_MAX_KEYLEN]; 43 | 44 | int i = 0; 45 | while (key[i] && key[i] != '/') { 46 | buf[i] = key[i]; 47 | i++; 48 | } 49 | buf[i] = 0; 50 | 51 | if (strcmp(buf, HTTP_STATS_COMMAND) == 0 || 52 | strcmp(buf, HTTP_INDEX_COMMAND) == 0 || 53 | strcmp(buf, HTTP_HEALTH_COMMAND) == 0 || 54 | strcmp(buf, HTTP_CONFIGURE_COMMAND) == 0) 55 | { 56 | key[i++] = 0; 57 | if (extra) 58 | *extra = key + i; 59 | 60 | return 1; 61 | } 62 | return 0; 63 | } 64 | 65 | typedef struct _http_worker_s { 66 | TAILQ_ENTRY(_http_worker_s) next; 67 | pthread_t th; 68 | struct mg_server *server; 69 | const char *me; 70 | const char *basepath; 71 | const char *adminpath; 72 | shardcache_t *cache; 73 | shcd_acl_t *acl; 74 | hashtable_t *mime_types; 75 | int leave; 76 | } http_worker_t; 77 | 78 | struct _shcd_http_s { 79 | int num_workers; 80 | TAILQ_HEAD(, _http_worker_s) workers; 81 | }; 82 | 83 | typedef struct _http_job_s { 84 | 85 | } http_job_t; 86 | 87 | static int shcd_active_requests = 0; 88 | 89 | static void 90 | shardcached_build_index_response(fbuf_t *buf, int do_html, shardcache_t *cache) 91 | { 92 | int i; 93 | 94 | shardcache_storage_index_t *index = shardcache_get_index(cache); 95 | 96 | if (do_html) { 97 | fbuf_printf(buf, 98 | "" 99 | "" 102 | "" 103 | "" 104 | "" 105 | ""); 106 | } 107 | for (i = 0; i < index->size; i++) { 108 | size_t klen = index->items[i].klen; 109 | char keystr[klen * 5 + 1]; 110 | char *t = keystr; 111 | char c; 112 | int p; 113 | for (p = 0 ; p < klen ; ++p) { 114 | c = ((char*)index->items[i].key)[p]; 115 | if (c == '<') 116 | t = stpcpy(t, "<"); 117 | else if (c == '>') 118 | t = stpcpy(t, ">"); 119 | else if (c == '&') 120 | t = stpcpy(t, "&"); 121 | else if (c < ' ') { 122 | sprintf(t, "\\x%2x", (int)c); 123 | t += 4; 124 | } 125 | else 126 | *t++ = c; 127 | } 128 | *t = 0; 129 | if (do_html) 130 | fbuf_printf(buf, 131 | "" 132 | "", 133 | keystr, 134 | index->items[i].vlen); 135 | else 136 | fbuf_printf(buf, 137 | "%s;%d\r\n", 138 | keystr, 139 | index->items[i].vlen); 140 | } 141 | 142 | if (do_html) 143 | fbuf_printf(buf, "
KeyValue size
%s(%d)
"); 144 | 145 | shardcache_free_index(index); 146 | } 147 | 148 | static void 149 | shardcached_build_stats_response(fbuf_t *buf, int do_html, http_worker_t *wrk) 150 | { 151 | int i; 152 | int num_nodes = 0; 153 | shardcache_node_t **nodes = shardcache_get_nodes(wrk->cache, &num_nodes); 154 | if (do_html) { 155 | fbuf_printf(buf, 156 | "" 157 | "

%s

" 158 | "" 161 | "" 162 | "" 163 | "" 164 | "" 165 | "" 166 | "" 167 | "" 168 | "" 169 | "" 170 | "" 171 | "" 172 | "", 173 | wrk->me, 174 | ATOMIC_READ(shcd_active_requests), 175 | num_nodes); 176 | 177 | for (i = 0; i < num_nodes; i++) { 178 | fbuf_printf(buf, 179 | "" 180 | "" 181 | "", 182 | shardcache_node_get_label(nodes[i]), shardcache_node_get_address_at_index(nodes[i], 0)); 183 | } 184 | } else { 185 | fbuf_printf(buf, 186 | "active_http_requests;%d\r\nnum_nodes;%d\r\n", 187 | ATOMIC_READ(shcd_active_requests), 188 | num_nodes); 189 | for (i = 0; i < num_nodes; i++) { 190 | fbuf_printf(buf, "node::%s;%s\r\n", shardcache_node_get_label(nodes[i]), shardcache_node_get_address(nodes[i])); 191 | } 192 | } 193 | 194 | if (nodes) 195 | shardcache_free_nodes(nodes, num_nodes); 196 | 197 | shardcache_counter_t *counters; 198 | int ncounters = shardcache_get_counters(wrk->cache, &counters); 199 | 200 | for (i = 0; i < ncounters; i++) { 201 | if (do_html) 202 | fbuf_printf(buf, 203 | "", 204 | counters[i].name, 205 | counters[i].value); 206 | else 207 | fbuf_printf(buf, 208 | "%s;%llu\r\n", 209 | counters[i].name, 210 | counters[i].value); 211 | } 212 | if (do_html) 213 | fbuf_printf(buf, "
CounterValue
active_http_requests%d
num_nodes%d
node::%s%s
%s%u
"); 214 | free(counters); 215 | } 216 | 217 | /* 218 | static int 219 | shardcached_parse_querystring( 220 | */ 221 | 222 | static void 223 | shardcached_handle_admin_request(http_worker_t *wrk, 224 | struct mg_connection *conn, 225 | char *key, 226 | char *extra, 227 | int is_head) 228 | { 229 | if (wrk->acl) { 230 | shcd_acl_method_t method = SHCD_ACL_METHOD_GET; 231 | struct in_addr remote_addr; 232 | inet_aton(conn->remote_ip, &remote_addr); 233 | if (shcd_acl_eval(wrk->acl, method, key, remote_addr.s_addr) != SHCD_ACL_ACTION_ALLOW) { 234 | mg_printf(conn, "HTTP/1.0 403 Forbidden\r\nContent-Length: 9\r\n\r\nForbidden"); 235 | return; 236 | } 237 | } 238 | 239 | if (strcmp(key, HTTP_CONFIGURE_COMMAND) == 0) { 240 | 241 | int do_html = (!conn->query_string || 242 | !strstr(conn->query_string, "nohtml=1")); 243 | 244 | char *resp = do_html ? "OK" : "OK"; 245 | 246 | mg_printf(conn, HTTP_HEADERS, 247 | do_html ? "text/html" : "text/plain", 248 | (int)strlen(resp)); 249 | 250 | if (!is_head) 251 | mg_printf(conn, "%s", resp); 252 | 253 | } else if (strcmp(key, HTTP_STATS_COMMAND) == 0) { 254 | int do_html = (!conn->query_string || 255 | !strstr(conn->query_string, "nohtml=1")); 256 | 257 | fbuf_t buf = FBUF_STATIC_INITIALIZER; 258 | shardcached_build_stats_response(&buf, do_html, wrk); 259 | 260 | mg_printf(conn, HTTP_HEADERS, 261 | do_html ? "text/html" : "text/plain", 262 | fbuf_used(&buf)); 263 | 264 | if (!is_head) 265 | mg_printf(conn, "%s", fbuf_data(&buf)); 266 | 267 | fbuf_destroy(&buf); 268 | 269 | } else if (strcmp(key, HTTP_INDEX_COMMAND) == 0) { 270 | fbuf_t buf = FBUF_STATIC_INITIALIZER; 271 | int do_html = (!conn->query_string || 272 | !strstr(conn->query_string, "nohtml=1")); 273 | 274 | shardcached_build_index_response(&buf, do_html, wrk->cache); 275 | 276 | mg_printf(conn, HTTP_HEADERS, 277 | do_html ? "text/html" : "text/plain", 278 | fbuf_used(&buf)); 279 | 280 | if (!is_head) 281 | mg_printf(conn, "%s", fbuf_data(&buf)); 282 | 283 | fbuf_destroy(&buf); 284 | } else if (strcmp(key, HTTP_HEALTH_COMMAND) == 0) { 285 | int do_html = (!conn->query_string || 286 | !strstr(conn->query_string, "nohtml=1")); 287 | 288 | char *resp = do_html ? "OK" : "OK"; 289 | 290 | mg_printf(conn, HTTP_HEADERS, 291 | do_html ? "text/html" : "text/plain", 292 | (int)strlen(resp)); 293 | 294 | if (!is_head) 295 | mg_printf(conn, "%s", resp); 296 | } else { 297 | mg_printf(conn, "HTTP/1.0 404 Not Found\r\nContent-Length: 9\r\n\r\nNot Found"); 298 | } 299 | } 300 | 301 | #define MG_REQUEST_PROCESSED 1 302 | 303 | typedef struct { 304 | http_worker_t *wrk; 305 | struct mg_connection *conn; 306 | char *mtype; 307 | int req_status; 308 | int found; 309 | int eof; 310 | pthread_mutex_t slock; 311 | fbuf_t *sbuf; 312 | } connection_status; 313 | 314 | static int 315 | shardcache_get_async_callback(void *key, 316 | size_t klen, 317 | void *data, 318 | size_t dlen, 319 | size_t total_size, 320 | struct timeval *timestamp, 321 | void *priv) 322 | { 323 | connection_status *st = (connection_status *)priv; 324 | 325 | pthread_mutex_lock(&st->slock); 326 | 327 | if (st->eof) { // the connection has been closed prematurely 328 | pthread_mutex_unlock(&st->slock); 329 | fbuf_free(st->sbuf); 330 | pthread_mutex_destroy(&st->slock); 331 | free(st); 332 | return -1; 333 | } 334 | 335 | if (!dlen && !total_size) { 336 | fbuf_printf(st->sbuf, "HTTP/1.0 404 Not Found\r\nContent-Length: 9\r\n\r\nNot Found"); 337 | st->req_status = MG_REQUEST_PROCESSED; 338 | pthread_mutex_unlock(&st->slock); 339 | return 0; 340 | } 341 | 342 | if (!st->found) { 343 | fbuf_printf(st->sbuf, HTTP_HEADERS, st->mtype, total_size); 344 | st->found = 1; 345 | } 346 | 347 | if (dlen) 348 | fbuf_add_binary(st->sbuf, data, dlen); 349 | 350 | if (total_size && timestamp) 351 | st->req_status = MG_REQUEST_PROCESSED; 352 | 353 | pthread_mutex_unlock(&st->slock); 354 | return 0; 355 | } 356 | 357 | static int 358 | shardcached_handle_get_request(http_worker_t *wrk, struct mg_connection *conn, char *key, int is_head) 359 | { 360 | if (wrk->acl) { 361 | shcd_acl_method_t method = SHCD_ACL_METHOD_GET; 362 | struct in_addr remote_addr; 363 | inet_aton(conn->remote_ip, &remote_addr); 364 | if (shcd_acl_eval(wrk->acl, method, key, remote_addr.s_addr) != SHCD_ACL_ACTION_ALLOW) { 365 | mg_printf(conn, "HTTP/1.0 403 Forbidden\r\nContent-Length: 9\r\n\r\nForbidden"); 366 | return MG_TRUE; 367 | } 368 | } 369 | 370 | char *mtype = NULL; 371 | if (wrk->mime_types) { 372 | char *p = key; 373 | while (*p && *p != '.') 374 | p++; 375 | if (*p && *(p+1)) { 376 | p++; 377 | mtype = (char *)ht_get(wrk->mime_types, p, strlen(p), NULL); 378 | if (!mtype) 379 | mtype = (char *)mg_get_mime_type(key, "application/octet-stream"); 380 | } 381 | } else { 382 | mtype = (char *)mg_get_mime_type(key, "application/octet-stream"); 383 | } 384 | 385 | 386 | size_t vlen = 0; 387 | struct timeval ts = { 0, 0 }; 388 | void *value = NULL; 389 | if (is_head) { 390 | vlen = shardcache_head(wrk->cache, key, strlen(key), NULL, 0, &ts); 391 | if (vlen) { 392 | int i; 393 | for (i = 0; i < conn->num_headers; i++) { 394 | struct tm tm; 395 | const char *hdr_name = conn->http_headers[i].name; 396 | const char *hdr_value = conn->http_headers[i].value; 397 | if (strcasecmp(hdr_name, "If-Modified-Since") == 0) { 398 | if (strptime(hdr_value, "%a, %d %b %Y %T %z", &tm) != NULL) { 399 | time_t time = mktime(&tm); 400 | if (ts.tv_sec < time) { 401 | mg_printf(conn, "HTTP/1.0 304 Not Modified\r\nContent-Length: 12\r\n\r\nNot Modified"); 402 | if (value) 403 | free(value); 404 | return MG_TRUE; 405 | } 406 | } 407 | } else if (strcasecmp(hdr_name, "If-Unmodified-Since") == 0) { 408 | if (strptime(hdr_value, "%a, %d %b %Y %T %z", &tm) != NULL) { 409 | time_t time = mktime(&tm); 410 | if (ts.tv_sec > time) { 411 | mg_printf(conn, "HTTP/1.0 412 Precondition Failed\r\nContent-Length: 19\r\n\r\nPrecondition Failed"); 412 | if (value) 413 | free(value); 414 | return MG_TRUE; 415 | } 416 | 417 | } 418 | } 419 | } 420 | 421 | char timestamp[256]; 422 | struct tm gmts; 423 | strftime(timestamp, sizeof(timestamp), "%a, %d %b %Y %T %z", gmtime_r(&ts.tv_sec, &gmts)); 424 | mg_printf(conn, HTTP_HEADERS_WITH_TIME, mtype, (int)vlen, timestamp); 425 | 426 | if (!is_head && value) 427 | mg_write(conn, value, vlen); 428 | 429 | if (value) 430 | free(value); 431 | } else { 432 | mg_printf(conn, "HTTP/1.0 404 Not Found\r\nContent-Length: 9\r\n\r\nNot Found"); 433 | } 434 | } else { 435 | connection_status *st = calloc(1, sizeof(connection_status)); 436 | 437 | st->wrk = wrk; 438 | st->conn = conn; 439 | st->mtype = mtype; 440 | pthread_mutex_init(&st->slock, NULL); 441 | st->sbuf = fbuf_create(0); 442 | 443 | int rc = shardcache_get(wrk->cache, key, strlen(key), shardcache_get_async_callback, st); 444 | if (rc != 0) { 445 | mg_printf(conn, "HTTP/1.0 404 Not Found\r\nContent-Length: 9\r\n\r\nNot Found"); 446 | pthread_mutex_destroy(&st->slock); 447 | fbuf_free(st->sbuf); 448 | free(st); 449 | return MG_TRUE; 450 | } 451 | 452 | conn->connection_param = st; 453 | 454 | return MG_MORE; 455 | } 456 | 457 | return MG_TRUE; 458 | } 459 | 460 | static void 461 | shardcached_handle_delete_request(http_worker_t *wrk, struct mg_connection *conn, char *key) 462 | { 463 | if (wrk->acl) { 464 | shcd_acl_method_t method = SHCD_ACL_METHOD_DEL; 465 | struct in_addr remote_addr; 466 | inet_aton(conn->remote_ip, &remote_addr); 467 | if (shcd_acl_eval(wrk->acl, method, key, remote_addr.s_addr) != SHCD_ACL_ACTION_ALLOW) { 468 | mg_printf(conn, "HTTP/1.0 403 Forbidden\r\nContent-Length: 9\r\n\r\nForbidden"); 469 | return; 470 | } 471 | } 472 | 473 | int rc = shardcache_del(wrk->cache, key, strlen(key), NULL, NULL); 474 | mg_printf(conn, "HTTP/1.0 %s\r\n" 475 | "Content-Length: 0\r\n\r\n", 476 | rc == 0 ? "200 OK" : "500 ERR"); 477 | 478 | } 479 | 480 | static void 481 | shardcached_handle_put_request(http_worker_t *wrk, struct mg_connection *conn, char *key) 482 | { 483 | if (wrk->acl) { 484 | shcd_acl_method_t method = SHCD_ACL_METHOD_PUT; 485 | struct in_addr remote_addr; 486 | inet_aton(conn->remote_ip, &remote_addr); 487 | if (shcd_acl_eval(wrk->acl, method, key, remote_addr.s_addr) != SHCD_ACL_ACTION_ALLOW) { 488 | mg_printf(conn, "HTTP/1.0 403 Forbidden\r\nContent-Length: 9\r\n\r\nForbidden"); 489 | return; 490 | } 491 | } 492 | 493 | int clen = 0; 494 | const char *clen_hdr = mg_get_header(conn, "Content-Length"); 495 | if (clen_hdr) { 496 | clen = strtol(clen_hdr, NULL, 10); 497 | } 498 | 499 | if (!clen) { 500 | mg_printf(conn, "HTTP/1.0 400 Bad Request\r\nContent-Length: 0\r\n\r\n"); 501 | return; 502 | } 503 | 504 | shardcache_set(wrk->cache, key, strlen(key), conn->content, conn->content_len, 0, 0, 0, NULL, NULL); 505 | 506 | mg_printf(conn, "HTTP/1.0 200 OK\r\nContent-Length: 0\r\n\r\n"); 507 | } 508 | 509 | static int 510 | shardcached_http_close_handler(struct mg_connection *conn) 511 | { 512 | if (conn->connection_param) { 513 | connection_status *st = (connection_status *)conn->connection_param; 514 | pthread_mutex_lock(&st->slock); 515 | if (st->req_status == MG_REQUEST_PROCESSED) { 516 | fbuf_free(st->sbuf); 517 | pthread_mutex_destroy(&st->slock); 518 | free(st); 519 | } else { 520 | st->eof = 1; 521 | pthread_mutex_unlock(&st->slock); 522 | } 523 | conn->connection_param = NULL; 524 | } 525 | return 0; 526 | } 527 | 528 | static inline int 529 | shardcached_parse_request(const char *uri, 530 | const char *basepath, 531 | const char *adminpath, 532 | char **key, 533 | char **extra, 534 | int *is_admin) 535 | { 536 | char *k = (char *)uri; 537 | 538 | int basepath_len = strlen(basepath); 539 | int baseadminpath_len = strlen(adminpath); 540 | int basepaths_differ = (basepath_len != baseadminpath_len || strcmp(basepath, adminpath) != 0); 541 | 542 | while (*k == '/' && *k) 543 | k++; 544 | 545 | if (basepath_len || baseadminpath_len) { 546 | if (basepath_len && strncmp(k, basepath, basepath_len) == 0 && 547 | strlen(k) > basepath_len && k[basepath_len] == '/') 548 | { 549 | k += basepath_len + 1; 550 | while (*k == '/' && *k) 551 | k++; 552 | } 553 | else if (basepaths_differ && baseadminpath_len && 554 | strncmp(k, adminpath, baseadminpath_len) == 0 && 555 | strlen(k) > baseadminpath_len && k[baseadminpath_len] == '/') 556 | { 557 | k += baseadminpath_len + 1; 558 | while (*k == '/' && *k) 559 | k++; 560 | } 561 | else 562 | { 563 | SHC_DEBUG("Out-of-scope uri : %s", uri); 564 | return 0; 565 | } 566 | } 567 | 568 | if (*k) 569 | { 570 | if (key) 571 | *key = k; 572 | 573 | char *e = NULL; 574 | int is_admin_url = ((!baseadminpath_len || !basepaths_differ) && is_admin_command(k, &e)); 575 | if (is_admin) 576 | *is_admin = is_admin_url; 577 | 578 | if (extra) 579 | *extra = e; 580 | 581 | return 1; 582 | } 583 | 584 | return 0; 585 | } 586 | 587 | static int 588 | shardcached_request_handler(struct mg_connection *conn, enum mg_event event) 589 | { 590 | switch(event) 591 | { 592 | case MG_CLOSE: 593 | if (conn->connection_param) 594 | return shardcached_http_close_handler(conn); 595 | break; 596 | case MG_RECV: 597 | // ignore MG_RECV events 598 | // NOTE : mongoose API expects the number of consumed bytes 599 | // as return value from MG_RECV events 600 | return 0; 601 | case MG_POLL: 602 | { 603 | connection_status *st = (connection_status *)conn->connection_param; 604 | if (st) { 605 | int status = st->req_status; 606 | int len = fbuf_used(st->sbuf); 607 | if (len) { 608 | mg_write(conn, fbuf_data(st->sbuf), len); 609 | fbuf_remove(st->sbuf, len); 610 | } 611 | if (status == MG_REQUEST_PROCESSED) { 612 | conn->connection_param = NULL; 613 | fbuf_free(st->sbuf); 614 | pthread_mutex_destroy(&st->slock); 615 | free(st); 616 | return MG_TRUE; 617 | } 618 | } 619 | return MG_MORE; 620 | } 621 | case MG_REQUEST: 622 | { 623 | http_worker_t *wrk = conn->server_param; 624 | 625 | char *key = NULL; 626 | int is_admin = 0; 627 | char *extra = NULL; 628 | 629 | if (!shardcached_parse_request(conn->uri, wrk->basepath, wrk->adminpath, &key, &extra, &is_admin)) 630 | { 631 | mg_printf(conn, "HTTP/1.0 404 Not Found\r\nContent-Length: 9\r\n\r\nNot Found"); 632 | ATOMIC_DECREMENT(shcd_active_requests); 633 | break; 634 | } 635 | 636 | ATOMIC_INCREMENT(shcd_active_requests); 637 | 638 | // if baseadminpath is not defined or it's the same as basepath, 639 | // we need to check for the "special" admin keys and handle them differently 640 | // (in such cases the labels HTTP_STATS_COMMAND, HTTP_INDEX_COMMAND and __health__ become reserved 641 | // and can't be used as keys from the http interface) 642 | if (is_admin) 643 | { 644 | if (strncasecmp(conn->request_method, "GET", 3) == 0) 645 | shardcached_handle_admin_request(wrk, conn, key, extra, 0); 646 | else 647 | mg_printf(conn, "HTTP/1.0 403 Forbidden\r\nContent-Length: 9\r\n\r\nForbidden"); 648 | ATOMIC_DECREMENT(shcd_active_requests); 649 | break; 650 | } 651 | 652 | // handle the actual GET/PUT/DELETE request 653 | if (strncasecmp(conn->request_method, "GET", 3) == 0) 654 | return shardcached_handle_get_request(wrk, conn, key, 0); 655 | else if (strncasecmp(conn->request_method, "HEAD", 4) == 0) 656 | return shardcached_handle_get_request(wrk, conn, key, 1); 657 | else if (strncasecmp(conn->request_method, "DELETE", 6) == 0) 658 | shardcached_handle_delete_request(wrk, conn, key); 659 | else if (strncasecmp(conn->request_method, "PUT", 3) == 0) 660 | shardcached_handle_put_request(wrk, conn, key); 661 | else 662 | mg_printf(conn, "HTTP/1.0 405 Method Not Allowed\r\nContent-Length: 11\r\n\r\nNot Allowed"); 663 | 664 | 665 | ATOMIC_DECREMENT(shcd_active_requests); 666 | break; 667 | } 668 | default: 669 | break; 670 | } 671 | return MG_TRUE; 672 | } 673 | 674 | 675 | void * 676 | shcd_http_run(void *priv) 677 | { 678 | http_worker_t *wrk = (http_worker_t *)priv; 679 | shardcache_thread_init(wrk->cache); 680 | while (!ATOMIC_READ(wrk->leave)) { 681 | mg_poll_server(wrk->server, 1000); 682 | } 683 | shardcache_thread_end(wrk->cache); 684 | return NULL; 685 | } 686 | 687 | shcd_http_t * 688 | shcd_http_create(shardcache_t *cache, 689 | const char *me, 690 | const char *basepath, 691 | const char *adminpath, 692 | shcd_acl_t *acl, 693 | hashtable_t *mime_types, 694 | const char **options, 695 | int num_workers) 696 | { 697 | int i, n; 698 | if (num_workers < 0) 699 | return NULL; 700 | 701 | shcd_http_t *http = calloc(1, sizeof(shcd_http_t)); 702 | 703 | http->num_workers = num_workers; 704 | 705 | TAILQ_INIT(&http->workers); 706 | for (i = 0; i < num_workers; i++) { 707 | 708 | http_worker_t *wrk = calloc(1, sizeof(http_worker_t)); 709 | 710 | wrk->server = mg_create_server(wrk, shardcached_request_handler); 711 | if (!wrk->server) { 712 | SHC_ERROR("Can't start mongoose server"); 713 | shcd_http_destroy(http); 714 | return NULL; 715 | } 716 | 717 | wrk->cache = cache; 718 | wrk->me = me; 719 | wrk->basepath = basepath; 720 | wrk->adminpath = adminpath; 721 | wrk->acl = acl; 722 | wrk->mime_types = mime_types; 723 | 724 | for (n = 0; options[n]; n += 2) { 725 | const char *option = options[n]; 726 | const char *value = options[n+1]; 727 | if (!option || !value) { 728 | SHC_ERROR("Bad mongoose options"); 729 | shcd_http_destroy(http); 730 | return NULL; 731 | 732 | } 733 | if (strcmp(option, "listening_port") == 0 && i > 0) { 734 | //mg_set_listening_socket(wrk->server, mg_get_listening_socket(TAILQ_FIRST(&http->workers)->server)); 735 | mg_copy_listeners(TAILQ_FIRST(&http->workers)->server, wrk->server); 736 | } else { 737 | const char *msg = mg_set_option(wrk->server, option, value); 738 | if (msg != NULL) { 739 | SHC_ERROR("Failed to set mongoose option [%s=%s]: %s", 740 | option, value, msg); 741 | shcd_http_destroy(http); 742 | return NULL; 743 | } 744 | } 745 | } 746 | 747 | TAILQ_INSERT_TAIL(&http->workers, wrk, next); 748 | if (pthread_create(&wrk->th, NULL, shcd_http_run, wrk) != 0) { 749 | SHC_ERROR("Failed to start an http worker thread: %s", 750 | strerror(errno)); 751 | shcd_http_destroy(http); 752 | return NULL; 753 | } 754 | } 755 | return http; 756 | }; 757 | 758 | void 759 | shcd_http_destroy(shcd_http_t *http) 760 | { 761 | http_worker_t *worker, *tmp; 762 | TAILQ_FOREACH_SAFE(worker, &http->workers, next, tmp) { 763 | TAILQ_REMOVE(&http->workers, worker, next); 764 | ATOMIC_INCREMENT(worker->leave); 765 | //pthread_cancel(worker->th); 766 | pthread_join(worker->th, NULL); 767 | mg_destroy_server(&worker->server); 768 | free(worker); 769 | } 770 | free(http); 771 | } 772 | -------------------------------------------------------------------------------- /test/http-client-c.h: -------------------------------------------------------------------------------- 1 | /* 2 | http-client-c 3 | Copyright (C) 2012-2013 Swen Kooij 4 | 5 | This file is part of http-client-c. 6 | 7 | http-client-c is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | http-client-c is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with http-client-c. If not, see . 19 | 20 | Warning: 21 | This library does not tend to work that stable nor does it fully implent the 22 | standards described by IETF. For more information on the precise implentation of the 23 | Hyper Text Transfer Protocol: 24 | 25 | http://www.ietf.org/rfc/rfc2616.txt 26 | */ 27 | 28 | #pragma GCC diagnostic ignored "-Wwrite-strings" 29 | #include 30 | #include 31 | #include 32 | #include 33 | #ifdef _WIN32 34 | #include 35 | #include 36 | #include 37 | #pragma comment(lib, "Ws2_32.lib") 38 | #elif _LINUX 39 | #include 40 | #elif __FreeBSD__ 41 | #include 42 | #include 43 | #include 44 | #include 45 | #elif __MACH__ 46 | #include 47 | #include 48 | #include 49 | #include 50 | #else 51 | #include 52 | #include 53 | #include 54 | #include 55 | #endif 56 | 57 | #include 58 | #include "stringx.h" 59 | #include "urlparser.h" 60 | 61 | /* 62 | Prototype functions 63 | */ 64 | struct http_response* http_req(char *http_headers, int http_len, struct parsed_url *purl); 65 | struct http_response* http_get(char *url, char *custom_headers); 66 | struct http_response* http_head(char *url, char *custom_headers); 67 | struct http_response* http_post(char *url, char *custom_headers, char *post_data); 68 | struct http_response* http_put(char *url, char *custom_headers, char *post_data, size_t put_size); 69 | struct http_response* http_delete(char *url, char *custom_headers); 70 | 71 | /* 72 | Represents an HTTP html response 73 | */ 74 | struct http_response 75 | { 76 | struct parsed_url *request_uri; 77 | char *body; 78 | char *status_code; 79 | int status_code_int; 80 | char *status_text; 81 | char *request_headers; 82 | char *response_headers; 83 | }; 84 | 85 | /* 86 | Handles redirect if needed for get requests 87 | */ 88 | struct http_response* handle_redirect_get(struct http_response* hresp, char* custom_headers) 89 | { 90 | if(hresp->status_code_int > 300 && hresp->status_code_int < 399) 91 | { 92 | char *token = strtok(hresp->response_headers, "\r\n"); 93 | while(token != NULL) 94 | { 95 | if(str_contains(token, "Location:")) 96 | { 97 | /* Extract url */ 98 | char *location = str_replace("Location: ", "", token); 99 | return http_get(location, custom_headers); 100 | } 101 | token = strtok(NULL, "\r\n"); 102 | } 103 | } 104 | /* We're not dealing with a redirect, just return the same structure */ 105 | return hresp; 106 | } 107 | 108 | /* 109 | Handles redirect if needed for get requests 110 | */ 111 | struct http_response* handle_redirect_delete(struct http_response* hresp, char* custom_headers) 112 | { 113 | if(hresp->status_code_int > 300 && hresp->status_code_int < 399) 114 | { 115 | char *token = strtok(hresp->response_headers, "\r\n"); 116 | while(token != NULL) 117 | { 118 | if(str_contains(token, "Location:")) 119 | { 120 | /* Extract url */ 121 | char *location = str_replace("Location: ", "", token); 122 | return http_delete(location, custom_headers); 123 | } 124 | token = strtok(NULL, "\r\n"); 125 | } 126 | } 127 | /* We're not dealing with a redirect, just return the same structure */ 128 | return hresp; 129 | } 130 | /* 131 | Handles redirect if needed for head requests 132 | */ 133 | struct http_response* handle_redirect_head(struct http_response* hresp, char* custom_headers) 134 | { 135 | if(hresp->status_code_int > 300 && hresp->status_code_int < 399) 136 | { 137 | char *token = strtok(hresp->response_headers, "\r\n"); 138 | while(token != NULL) 139 | { 140 | if(str_contains(token, "Location:")) 141 | { 142 | /* Extract url */ 143 | char *location = str_replace("Location: ", "", token); 144 | return http_head(location, custom_headers); 145 | } 146 | token = strtok(NULL, "\r\n"); 147 | } 148 | } 149 | /* We're not dealing with a redirect, just return the same structure */ 150 | return hresp; 151 | } 152 | 153 | /* 154 | Handles redirect if needed for post requests 155 | */ 156 | struct http_response* handle_redirect_post(struct http_response* hresp, char* custom_headers, char *post_data) 157 | { 158 | if(hresp->status_code_int > 300 && hresp->status_code_int < 399) 159 | { 160 | char *token = strtok(hresp->response_headers, "\r\n"); 161 | while(token != NULL) 162 | { 163 | if(str_contains(token, "Location:")) 164 | { 165 | /* Extract url */ 166 | char *location = str_replace("Location: ", "", token); 167 | return http_post(location, custom_headers, post_data); 168 | } 169 | token = strtok(NULL, "\r\n"); 170 | } 171 | } 172 | /* We're not dealing with a redirect, just return the same structure */ 173 | return hresp; 174 | } 175 | 176 | /* 177 | Handles redirect if needed for post requests 178 | */ 179 | struct http_response* handle_redirect_put(struct http_response* hresp, char* custom_headers, char *put_data, size_t put_size) 180 | { 181 | if(hresp->status_code_int > 300 && hresp->status_code_int < 399) 182 | { 183 | char *token = strtok(hresp->response_headers, "\r\n"); 184 | while(token != NULL) 185 | { 186 | if(str_contains(token, "Location:")) 187 | { 188 | /* Extract url */ 189 | char *location = str_replace("Location: ", "", token); 190 | return http_put(location, custom_headers, put_data, put_size); 191 | } 192 | token = strtok(NULL, "\r\n"); 193 | } 194 | } 195 | /* We're not dealing with a redirect, just return the same structure */ 196 | return hresp; 197 | } 198 | 199 | /* 200 | Makes a HTTP request and returns the response 201 | */ 202 | struct http_response* http_req(char *http_headers, int http_len, struct parsed_url *purl) 203 | { 204 | /* Parse url */ 205 | if(purl == NULL) 206 | { 207 | printf("Unable to parse url"); 208 | return NULL; 209 | } 210 | 211 | /* Declare variable */ 212 | int sock; 213 | int tmpres; 214 | struct sockaddr_in *remote; 215 | 216 | /* Allocate memeory for htmlcontent */ 217 | struct http_response *hresp = (struct http_response*)malloc(sizeof(struct http_response)); 218 | if(hresp == NULL) 219 | { 220 | printf("Unable to allocate memory for htmlcontent."); 221 | return NULL; 222 | } 223 | hresp->body = NULL; 224 | hresp->request_headers = NULL; 225 | hresp->response_headers = NULL; 226 | hresp->status_code = NULL; 227 | hresp->status_text = NULL; 228 | 229 | /* Create TCP socket */ 230 | if((sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) 231 | { 232 | printf("Can't create TCP socket"); 233 | return NULL; 234 | } 235 | 236 | /* Set remote->sin_addr.s_addr */ 237 | remote = (struct sockaddr_in *)malloc(sizeof(struct sockaddr_in *)); 238 | remote->sin_family = AF_INET; 239 | tmpres = inet_pton(AF_INET, purl->ip, (void *)(&(remote->sin_addr.s_addr))); 240 | if( tmpres < 0) 241 | { 242 | printf("Can't set remote->sin_addr.s_addr"); 243 | return NULL; 244 | } 245 | else if(tmpres == 0) 246 | { 247 | printf("Not a valid IP"); 248 | return NULL; 249 | } 250 | remote->sin_port = htons(atoi(purl->port)); 251 | 252 | /* Connect */ 253 | if(connect(sock, (struct sockaddr *)remote, sizeof(struct sockaddr)) < 0) 254 | { 255 | printf("Could not connect"); 256 | return NULL; 257 | } 258 | 259 | /* Send headers to server */ 260 | int sent = 0; 261 | while(sent < http_len) 262 | { 263 | tmpres = send(sock, http_headers+sent, http_len-sent, 0); 264 | if(tmpres == -1) 265 | { 266 | printf("Can't send headers"); 267 | return NULL; 268 | } 269 | sent += tmpres; 270 | } 271 | 272 | /* Recieve into response*/ 273 | char *response = NULL; 274 | char BUF[BUFSIZ]; 275 | int received_len = 0; 276 | int offset = 0; 277 | while((received_len = recv(sock, BUF, BUFSIZ-1, 0)) > 0) 278 | { 279 | BUF[received_len] = '\0'; 280 | response = (char*)realloc(response, offset + received_len + 1); 281 | memcpy(response + offset, BUF, received_len + 1); 282 | offset += received_len; 283 | } 284 | if (received_len < 0) 285 | { 286 | free(http_headers); 287 | #ifdef _WIN32 288 | closesocket(sock); 289 | #else 290 | close(sock); 291 | #endif 292 | printf("Unabel to recieve"); 293 | return NULL; 294 | } 295 | 296 | /* Close socket */ 297 | #ifdef _WIN32 298 | closesocket(sock); 299 | #else 300 | close(sock); 301 | #endif 302 | 303 | if (!response) 304 | return hresp; 305 | 306 | /* Parse status code and text */ 307 | char *status_line = get_until(response, "\r\n"); 308 | if (strstr(status_line, "HTTP/1.1")) 309 | status_line = str_replace("HTTP/1.1 ", "", status_line); 310 | else 311 | status_line = str_replace("HTTP/1.0 ", "", status_line); 312 | 313 | char *status_code = str_ndup(status_line, 4); 314 | status_code = str_replace(" ", "", status_code); 315 | char *status_text = str_replace(status_code, "", status_line); 316 | status_text = str_replace(" ", "", status_text); 317 | hresp->status_code = status_code; 318 | hresp->status_code_int = atoi(status_code); 319 | hresp->status_text = status_text; 320 | 321 | /* Parse response headers */ 322 | char *headers = get_until(response, "\r\n\r\n"); 323 | hresp->response_headers = headers; 324 | 325 | /* Assign request headers */ 326 | hresp->request_headers = http_headers; 327 | 328 | /* Assign request url */ 329 | hresp->request_uri = purl; 330 | 331 | /* Parse body */ 332 | char *body = strstr(response, "\r\n\r\n"); 333 | body = str_replace("\r\n\r\n", "", body); 334 | hresp->body = body; 335 | 336 | /* Return response */ 337 | return hresp; 338 | } 339 | 340 | /* 341 | Makes a HTTP GET request to the given url 342 | */ 343 | struct http_response* http_delete(char *url, char *custom_headers) 344 | { 345 | /* Parse url */ 346 | struct parsed_url *purl = parse_url(url); 347 | if(purl == NULL) 348 | { 349 | printf("Unable to parse url"); 350 | return NULL; 351 | } 352 | 353 | /* Declare variable */ 354 | char *http_headers = (char*)malloc(1024); 355 | 356 | /* Build query/headers */ 357 | if(purl->path != NULL) 358 | { 359 | if(purl->query != NULL) 360 | { 361 | sprintf(http_headers, "DELETE /%s?%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\n", purl->path, purl->query, purl->host); 362 | } 363 | else 364 | { 365 | sprintf(http_headers, "DELETE /%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\n", purl->path, purl->host); 366 | } 367 | } 368 | else 369 | { 370 | if(purl->query != NULL) 371 | { 372 | sprintf(http_headers, "DELETE /?%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\n", purl->query, purl->host); 373 | } 374 | else 375 | { 376 | sprintf(http_headers, "DELETE / HTTP/1.1\r\nHost:%s\r\nConnection:close\r\n", purl->host); 377 | } 378 | } 379 | 380 | /* Handle authorisation if needed */ 381 | if(purl->username != NULL) 382 | { 383 | /* Format username:password pair */ 384 | char *upwd = (char*)malloc(1024); 385 | sprintf(upwd, "%s:%s", purl->username, purl->password); 386 | upwd = (char*)realloc(upwd, strlen(upwd) + 1); 387 | 388 | /* Base64 encode */ 389 | char *base64 = base64_encode(upwd); 390 | 391 | /* Form header */ 392 | char *auth_header = (char*)malloc(1024); 393 | sprintf(auth_header, "Authorization: Basic %s\r\n", base64); 394 | auth_header = (char*)realloc(auth_header, strlen(auth_header) + 1); 395 | 396 | /* Add to header */ 397 | http_headers = (char*)realloc(http_headers, strlen(http_headers) + strlen(auth_header) + 2); 398 | sprintf(http_headers, "%s%s", http_headers, auth_header); 399 | } 400 | 401 | /* Add custom headers, and close */ 402 | if(custom_headers != NULL) 403 | { 404 | sprintf(http_headers, "%s%s\r\n", http_headers, custom_headers); 405 | } 406 | else 407 | { 408 | sprintf(http_headers, "%s\r\n", http_headers); 409 | } 410 | http_headers = (char*)realloc(http_headers, strlen(http_headers) + 1); 411 | 412 | /* Make request and return response */ 413 | struct http_response *hresp = http_req(http_headers, strlen(http_headers), purl); 414 | 415 | /* Handle redirect */ 416 | return handle_redirect_delete(hresp, custom_headers); 417 | } 418 | 419 | 420 | /* 421 | Makes a HTTP GET request to the given url 422 | */ 423 | struct http_response* http_get(char *url, char *custom_headers) 424 | { 425 | /* Parse url */ 426 | struct parsed_url *purl = parse_url(url); 427 | if(purl == NULL) 428 | { 429 | printf("Unable to parse url"); 430 | return NULL; 431 | } 432 | 433 | /* Declare variable */ 434 | char *http_headers = (char*)malloc(1024); 435 | 436 | /* Build query/headers */ 437 | if(purl->path != NULL) 438 | { 439 | if(purl->query != NULL) 440 | { 441 | sprintf(http_headers, "GET /%s?%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\n", purl->path, purl->query, purl->host); 442 | } 443 | else 444 | { 445 | sprintf(http_headers, "GET /%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\n", purl->path, purl->host); 446 | } 447 | } 448 | else 449 | { 450 | if(purl->query != NULL) 451 | { 452 | sprintf(http_headers, "GET /?%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\n", purl->query, purl->host); 453 | } 454 | else 455 | { 456 | sprintf(http_headers, "GET / HTTP/1.1\r\nHost:%s\r\nConnection:close\r\n", purl->host); 457 | } 458 | } 459 | 460 | /* Handle authorisation if needed */ 461 | if(purl->username != NULL) 462 | { 463 | /* Format username:password pair */ 464 | char *upwd = (char*)malloc(1024); 465 | sprintf(upwd, "%s:%s", purl->username, purl->password); 466 | upwd = (char*)realloc(upwd, strlen(upwd) + 1); 467 | 468 | /* Base64 encode */ 469 | char *base64 = base64_encode(upwd); 470 | 471 | /* Form header */ 472 | char *auth_header = (char*)malloc(1024); 473 | sprintf(auth_header, "Authorization: Basic %s\r\n", base64); 474 | auth_header = (char*)realloc(auth_header, strlen(auth_header) + 1); 475 | 476 | /* Add to header */ 477 | http_headers = (char*)realloc(http_headers, strlen(http_headers) + strlen(auth_header) + 2); 478 | sprintf(http_headers, "%s%s", http_headers, auth_header); 479 | } 480 | 481 | /* Add custom headers, and close */ 482 | if(custom_headers != NULL) 483 | { 484 | sprintf(http_headers, "%s%s\r\n", http_headers, custom_headers); 485 | } 486 | else 487 | { 488 | sprintf(http_headers, "%s\r\n", http_headers); 489 | } 490 | http_headers = (char*)realloc(http_headers, strlen(http_headers) + 1); 491 | 492 | /* Make request and return response */ 493 | struct http_response *hresp = http_req(http_headers, strlen(http_headers), purl); 494 | 495 | /* Handle redirect */ 496 | return handle_redirect_get(hresp, custom_headers); 497 | } 498 | 499 | /* 500 | Makes a HTTP PUTT request to the given url 501 | */ 502 | struct http_response* http_put(char *url, char *custom_headers, char *put_data, size_t put_size) 503 | { 504 | /* Parse url */ 505 | struct parsed_url *purl = parse_url(url); 506 | if(purl == NULL) 507 | { 508 | printf("Unable to parse url"); 509 | return NULL; 510 | } 511 | 512 | /* Declare variable */ 513 | char *http_headers = (char*)malloc(1024); 514 | 515 | /* Build query/headers */ 516 | if(purl->path != NULL) 517 | { 518 | if(purl->query != NULL) 519 | { 520 | sprintf(http_headers, "PUT /%s?%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\nContent-Length:%zu\r\n", purl->path, purl->query, purl->host, put_size); 521 | } 522 | else 523 | { 524 | sprintf(http_headers, "PUT /%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\nContent-Length:%zu\r\n", purl->path, purl->host, put_size); 525 | } 526 | } 527 | else 528 | { 529 | if(purl->query != NULL) 530 | { 531 | sprintf(http_headers, "PUT /?%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\nContent-Length:%zu\r\n", purl->query, purl->host, put_size); 532 | } 533 | else 534 | { 535 | sprintf(http_headers, "PUT / HTTP/1.1\r\nHost:%s\r\nConnection:close\r\nContent-Length:%zu\r\n", purl->host, put_size); 536 | } 537 | } 538 | 539 | /* Handle authorisation if needed */ 540 | if(purl->username != NULL) 541 | { 542 | /* Format username:password pair */ 543 | char *upwd = (char*)malloc(1024); 544 | sprintf(upwd, "%s:%s", purl->username, purl->password); 545 | upwd = (char*)realloc(upwd, strlen(upwd) + 1); 546 | 547 | /* Base64 encode */ 548 | char *base64 = base64_encode(upwd); 549 | 550 | /* Form header */ 551 | char *auth_header = (char*)malloc(1024); 552 | sprintf(auth_header, "Authorization: Basic %s\r\n", base64); 553 | auth_header = (char*)realloc(auth_header, strlen(auth_header) + 1); 554 | 555 | /* Add to header */ 556 | http_headers = (char*)realloc(http_headers, strlen(http_headers) + strlen(auth_header) + 2); 557 | sprintf(http_headers, "%s%s", http_headers, auth_header); 558 | } 559 | 560 | int hlen = 0; 561 | if(custom_headers != NULL) 562 | { 563 | sprintf(http_headers, "%s%s\r\n\r\n", http_headers, custom_headers); 564 | hlen = strlen(http_headers); 565 | http_headers = (char*)realloc(http_headers, hlen+put_size + 1); 566 | memcpy(http_headers + hlen, put_data, put_size); 567 | } 568 | else 569 | { 570 | sprintf(http_headers, "%s\r\n", http_headers); 571 | hlen = strlen(http_headers); 572 | http_headers = (char*)realloc(http_headers, hlen+put_size + 1); 573 | memcpy(http_headers + hlen, put_data, put_size); 574 | } 575 | 576 | /* Make request and return response */ 577 | struct http_response *hresp = http_req(http_headers, hlen + put_size, purl); 578 | 579 | /* Handle redirect */ 580 | return handle_redirect_put(hresp, custom_headers, put_data, put_size); 581 | } 582 | 583 | 584 | /* 585 | Makes a HTTP POST request to the given url 586 | */ 587 | struct http_response* http_post(char *url, char *custom_headers, char *post_data) 588 | { 589 | /* Parse url */ 590 | struct parsed_url *purl = parse_url(url); 591 | if(purl == NULL) 592 | { 593 | printf("Unable to parse url"); 594 | return NULL; 595 | } 596 | 597 | /* Declare variable */ 598 | char *http_headers = (char*)malloc(1024); 599 | 600 | /* Build query/headers */ 601 | if(purl->path != NULL) 602 | { 603 | if(purl->query != NULL) 604 | { 605 | sprintf(http_headers, "POST /%s?%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\nContent-Length:%zu\r\nContent-Type:application/x-www-form-urlencoded\r\n", purl->path, purl->query, purl->host, strlen(post_data)); 606 | } 607 | else 608 | { 609 | sprintf(http_headers, "POST /%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\nContent-Length:%zu\r\nContent-Type:application/x-www-form-urlencoded\r\n", purl->path, purl->host, strlen(post_data)); 610 | } 611 | } 612 | else 613 | { 614 | if(purl->query != NULL) 615 | { 616 | sprintf(http_headers, "POST /?%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\nContent-Length:%zu\r\nContent-Type:application/x-www-form-urlencoded\r\n", purl->query, purl->host, strlen(post_data)); 617 | } 618 | else 619 | { 620 | sprintf(http_headers, "POST / HTTP/1.1\r\nHost:%s\r\nConnection:close\r\nContent-Length:%zu\r\nContent-Type:application/x-www-form-urlencoded\r\n", purl->host, strlen(post_data)); 621 | } 622 | } 623 | 624 | /* Handle authorisation if needed */ 625 | if(purl->username != NULL) 626 | { 627 | /* Format username:password pair */ 628 | char *upwd = (char*)malloc(1024); 629 | sprintf(upwd, "%s:%s", purl->username, purl->password); 630 | upwd = (char*)realloc(upwd, strlen(upwd) + 1); 631 | 632 | /* Base64 encode */ 633 | char *base64 = base64_encode(upwd); 634 | 635 | /* Form header */ 636 | char *auth_header = (char*)malloc(1024); 637 | sprintf(auth_header, "Authorization: Basic %s\r\n", base64); 638 | auth_header = (char*)realloc(auth_header, strlen(auth_header) + 1); 639 | 640 | /* Add to header */ 641 | http_headers = (char*)realloc(http_headers, strlen(http_headers) + strlen(auth_header) + 2); 642 | sprintf(http_headers, "%s%s", http_headers, auth_header); 643 | } 644 | 645 | if(custom_headers != NULL) 646 | { 647 | sprintf(http_headers, "%s%s\r\n", http_headers, custom_headers); 648 | sprintf(http_headers, "%s\r\n%s", http_headers, post_data); 649 | } 650 | else 651 | { 652 | sprintf(http_headers, "%s\r\n%s", http_headers, post_data); 653 | } 654 | http_headers = (char*)realloc(http_headers, strlen(http_headers) + 1); 655 | 656 | /* Make request and return response */ 657 | struct http_response *hresp = http_req(http_headers, strlen(http_headers), purl); 658 | 659 | /* Handle redirect */ 660 | return handle_redirect_post(hresp, custom_headers, post_data); 661 | } 662 | 663 | /* 664 | Makes a HTTP HEAD request to the given url 665 | */ 666 | struct http_response* http_head(char *url, char *custom_headers) 667 | { 668 | /* Parse url */ 669 | struct parsed_url *purl = parse_url(url); 670 | if(purl == NULL) 671 | { 672 | printf("Unable to parse url"); 673 | return NULL; 674 | } 675 | 676 | /* Declare variable */ 677 | char *http_headers = (char*)malloc(1024); 678 | 679 | /* Build query/headers */ 680 | if(purl->path != NULL) 681 | { 682 | if(purl->query != NULL) 683 | { 684 | sprintf(http_headers, "HEAD /%s?%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\n", purl->path, purl->query, purl->host); 685 | } 686 | else 687 | { 688 | sprintf(http_headers, "HEAD /%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\n", purl->path, purl->host); 689 | } 690 | } 691 | else 692 | { 693 | if(purl->query != NULL) 694 | { 695 | sprintf(http_headers, "HEAD/?%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\n", purl->query, purl->host); 696 | } 697 | else 698 | { 699 | sprintf(http_headers, "HEAD / HTTP/1.1\r\nHost:%s\r\nConnection:close\r\n", purl->host); 700 | } 701 | } 702 | 703 | /* Handle authorisation if needed */ 704 | if(purl->username != NULL) 705 | { 706 | /* Format username:password pair */ 707 | char *upwd = (char*)malloc(1024); 708 | sprintf(upwd, "%s:%s", purl->username, purl->password); 709 | upwd = (char*)realloc(upwd, strlen(upwd) + 1); 710 | 711 | /* Base64 encode */ 712 | char *base64 = base64_encode(upwd); 713 | 714 | /* Form header */ 715 | char *auth_header = (char*)malloc(1024); 716 | sprintf(auth_header, "Authorization: Basic %s\r\n", base64); 717 | auth_header = (char*)realloc(auth_header, strlen(auth_header) + 1); 718 | 719 | /* Add to header */ 720 | http_headers = (char*)realloc(http_headers, strlen(http_headers) + strlen(auth_header) + 2); 721 | sprintf(http_headers, "%s%s", http_headers, auth_header); 722 | } 723 | 724 | if(custom_headers != NULL) 725 | { 726 | sprintf(http_headers, "%s%s\r\n", http_headers, custom_headers); 727 | } 728 | else 729 | { 730 | sprintf(http_headers, "%s\r\n", http_headers); 731 | } 732 | http_headers = (char*)realloc(http_headers, strlen(http_headers) + 1); 733 | 734 | /* Make request and return response */ 735 | struct http_response *hresp = http_req(http_headers, strlen(http_headers), purl); 736 | 737 | /* Handle redirect */ 738 | return handle_redirect_head(hresp, custom_headers); 739 | } 740 | 741 | /* 742 | Do HTTP OPTIONs requests 743 | */ 744 | struct http_response* http_options(char *url) 745 | { 746 | /* Parse url */ 747 | struct parsed_url *purl = parse_url(url); 748 | if(purl == NULL) 749 | { 750 | printf("Unable to parse url"); 751 | return NULL; 752 | } 753 | 754 | /* Declare variable */ 755 | char *http_headers = (char*)malloc(1024); 756 | 757 | /* Build query/headers */ 758 | if(purl->path != NULL) 759 | { 760 | if(purl->query != NULL) 761 | { 762 | sprintf(http_headers, "OPTIONS /%s?%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\n", purl->path, purl->query, purl->host); 763 | } 764 | else 765 | { 766 | sprintf(http_headers, "OPTIONS /%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\n", purl->path, purl->host); 767 | } 768 | } 769 | else 770 | { 771 | if(purl->query != NULL) 772 | { 773 | sprintf(http_headers, "OPTIONS/?%s HTTP/1.1\r\nHost:%s\r\nConnection:close\r\n", purl->query, purl->host); 774 | } 775 | else 776 | { 777 | sprintf(http_headers, "OPTIONS / HTTP/1.1\r\nHost:%s\r\nConnection:close\r\n", purl->host); 778 | } 779 | } 780 | 781 | /* Handle authorisation if needed */ 782 | if(purl->username != NULL) 783 | { 784 | /* Format username:password pair */ 785 | char *upwd = (char*)malloc(1024); 786 | sprintf(upwd, "%s:%s", purl->username, purl->password); 787 | upwd = (char*)realloc(upwd, strlen(upwd) + 1); 788 | 789 | /* Base64 encode */ 790 | char *base64 = base64_encode(upwd); 791 | 792 | /* Form header */ 793 | char *auth_header = (char*)malloc(1024); 794 | sprintf(auth_header, "Authorization: Basic %s\r\n", base64); 795 | auth_header = (char*)realloc(auth_header, strlen(auth_header) + 1); 796 | 797 | /* Add to header */ 798 | http_headers = (char*)realloc(http_headers, strlen(http_headers) + strlen(auth_header) + 2); 799 | sprintf(http_headers, "%s%s", http_headers, auth_header); 800 | } 801 | 802 | /* Build headers */ 803 | sprintf(http_headers, "%s\r\n", http_headers); 804 | http_headers = (char*)realloc(http_headers, strlen(http_headers) + 1); 805 | 806 | /* Make request and return response */ 807 | struct http_response *hresp = http_req(http_headers, strlen(http_headers), purl); 808 | 809 | /* Handle redirect */ 810 | return hresp; 811 | } 812 | 813 | /* 814 | Free memory of http_response 815 | */ 816 | void http_response_free(struct http_response *hresp) 817 | { 818 | if(hresp != NULL) 819 | { 820 | if(hresp->request_uri != NULL) parsed_url_free(hresp->request_uri); 821 | if(hresp->body != NULL) free(hresp->body); 822 | if(hresp->status_code != NULL) free(hresp->status_code); 823 | if(hresp->status_text != NULL) free(hresp->status_text); 824 | if(hresp->request_headers != NULL) free(hresp->request_headers); 825 | if(hresp->response_headers != NULL) free(hresp->response_headers); 826 | free(hresp); 827 | } 828 | } 829 | --------------------------------------------------------------------------------