├── .cproject ├── .gitignore ├── .project ├── .settings └── org.eclipse.core.resources.prefs ├── Makefile ├── README ├── example ├── Makefile ├── ares_dns.cpp ├── client.cpp ├── client2.cpp └── server.cpp ├── lib ├── c-ares-1.10.0.tar.gz ├── c-ares-1.10.0_auxten.patch ├── jemalloc-3.0.0.tar.bz2 ├── libev-4.11.tar.gz ├── libev_auxten.patch ├── patch_build_all.sh ├── patch_build_c-ares.sh ├── patch_build_jemalloc.sh └── patch_build_libev.sh └── src ├── AUTHORS ├── COPYING ├── ChangeLog ├── Makefile.am ├── NEWS ├── README ├── TODO ├── async_clnt.cpp ├── async_conn.cpp ├── async_dns.cpp ├── async_pool.h ├── async_threads.cpp ├── autogen.sh ├── configure.in ├── dict.c ├── dict.h ├── gingko.h ├── gingko_base.cpp ├── gko.h ├── gko_errno.h ├── hash ├── gko_zip.cpp ├── gko_zip.h ├── lz4.cpp ├── lz4.h ├── md5.cpp ├── md5.h ├── xor_hash.cpp └── xor_hash.h ├── hermes_errno.h ├── libgko.pc.in ├── limit.cpp ├── limit.h ├── log.cpp ├── log.h ├── memory.cpp ├── memory.h ├── path.cpp ├── path.h ├── socket.cpp └── socket.h /.cproject: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 36 | 41 | 42 | 43 | 44 | 53 | 58 | 59 | 60 | 61 | 67 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | configure 2 | example/ares_dns 3 | lib/jemalloc 4 | lib/jemalloc-3.0.0/ 5 | lib/jemalloc-3.0.0built/ 6 | lib/libjemalloc/ 7 | lib/c-ares-1.10.0.patched/ 8 | lib/c-ares-1.10.0 9 | lib/c-ares-1.10.0built 10 | lib/c-ares-1.8.0.patched/ 11 | lib/c-ares 12 | lib/c-ares-1.8.0 13 | lib/c-ares-1.8.0built 14 | lib/libgko/ 15 | src/libgko.pc 16 | example/async_dns 17 | example/client2 18 | *.dSYM 19 | example/client 20 | example/log 21 | example/server 22 | lib/libev-4.04fixed 23 | lib/libev 24 | lib/libev-4.04 25 | lib/libev-4.11/ 26 | lib/libev-4.11built/ 27 | lib/libzdb 28 | lib/libzdb-2.10.3/ 29 | lib/libzdb-2.10.3built/ 30 | svn-commit.* 31 | .deps/ 32 | clnt_unittest 33 | serv_unittest 34 | config.h 35 | config.log 36 | config.status 37 | *.tar.gz 38 | src/hash/Makefile 39 | src/Makefile 40 | *.dmg 41 | .DS_Store 42 | autom4te* 43 | gingko_clnt 44 | gingko_serv 45 | stamp-h1 46 | *.orig 47 | kfp_docs* 48 | server.log 49 | client.log 50 | *.[ao] 51 | build/ 52 | *~ 53 | .svn/ 54 | test 55 | output* 56 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | gko_pool 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.cdt.managedbuilder.core.genmakebuilder 10 | clean,full,incremental, 11 | 12 | 13 | ?name? 14 | 15 | 16 | 17 | org.eclipse.cdt.make.core.append_environment 18 | true 19 | 20 | 21 | org.eclipse.cdt.make.core.autoBuildTarget 22 | all 23 | 24 | 25 | org.eclipse.cdt.make.core.buildArguments 26 | 27 | 28 | 29 | org.eclipse.cdt.make.core.buildCommand 30 | make 31 | 32 | 33 | org.eclipse.cdt.make.core.cleanBuildTarget 34 | clean 35 | 36 | 37 | org.eclipse.cdt.make.core.contents 38 | org.eclipse.cdt.make.core.activeConfigSettings 39 | 40 | 41 | org.eclipse.cdt.make.core.enableAutoBuild 42 | false 43 | 44 | 45 | org.eclipse.cdt.make.core.enableCleanBuild 46 | true 47 | 48 | 49 | org.eclipse.cdt.make.core.enableFullBuild 50 | true 51 | 52 | 53 | org.eclipse.cdt.make.core.fullBuildTarget 54 | all 55 | 56 | 57 | org.eclipse.cdt.make.core.stopOnError 58 | true 59 | 60 | 61 | org.eclipse.cdt.make.core.useDefaultBuildCmd 62 | true 63 | 64 | 65 | 66 | 67 | org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder 68 | full,incremental, 69 | 70 | 71 | 72 | 73 | 74 | org.eclipse.cdt.core.cnature 75 | org.eclipse.cdt.core.ccnature 76 | org.eclipse.cdt.managedbuilder.core.managedBuildNature 77 | org.eclipse.cdt.managedbuilder.core.ScannerConfigNature 78 | 79 | 80 | -------------------------------------------------------------------------------- /.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | encoding//src/async_clnt.cpp=UTF-8 3 | encoding/=UTF-8 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | -include version_scmpf.env 2 | 3 | .PHONY: all makelib check_configure compile rm_config clean move check 4 | all: check_configure makelib compile 5 | 6 | check_configure: 7 | if [ ! -x src/configure ];then ( cd src && ./autogen.sh );fi 8 | 9 | makelib: 10 | cd lib && bash -x patch_build_all.sh && cd .. 11 | 12 | compile: 13 | cd src && ./autogen.sh && ./configure --enable-debug --prefix=$(shell pwd)/lib/libgko CXXFLAGS='-DNDEBUG -ggdb -D_GKO_VERSION=\"$(subst VERSION:,,$(VERSION_SCMPF))\"' && make clean ;sleep 1 &&\ 14 | make -j 4 && make install && cd .. 15 | 16 | rm_config: 17 | rm ./src/config.h 18 | rm -rf ./lib/libev/include 19 | 20 | clean: 21 | pwd 22 | #cd src && make clean && cd .. && rm -rf output 23 | #find ./../../../../../../.. -type f -name "event.h" 24 | 25 | move: 26 | if [ ! -d output ];then mkdir output;fi 27 | cd output && if [ ! -d bin ];then mkdir bin; fi && if [ ! -d testbin ];then mkdir testbin; fi 28 | cp ./src/gingko_serv ./output/bin/gkod 29 | cp ./src/gingko_clnt ./output/bin/gkocp 30 | cp ./src/serv_unittest ./output/testbin/ 31 | cp ./src/clnt_unittest ./output/testbin/ 32 | cp -r ./output/testbin ./test 33 | #cp ./bin/* ./output/bin/ 34 | #cp ./src/erase_job.py ./output/bin/ 35 | #cp ./src/run2.sh ./output/bin/gkod_ctl 36 | cp deploy ./output/ && chmod +x ./output/deploy 37 | cd output && md5sum deploy bin/* > md5sum 38 | #cd output/bin && cp gkocp{,.new} 39 | #cd output/bin && cp gkod{,.new} 40 | #cd output/bin && cp gkod_ctl{,.new} 41 | chmod +x ./output/bin/* 42 | cd output && tar czvf gingko.tgz bin md5sum deploy 43 | 44 | check: 45 | cd test && ./clnt_unittest && ./serv_unittest 46 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | _ _ 2 | __ _| | _____ _ __ ___ ___ | | 3 | / _` | |/ / _ \ | '_ \ / _ \ / _ \| | 4 | | (_| | < (_) | | |_) | (_) | (_) | | 5 | \__, |_|\_\___/____| .__/ \___/ \___/|_| 6 | |___/ |_____|_| 7 | 8 | # gko_pool: 9 | * an async multithread server client framework 10 | 11 | # author: 12 | * auxten 13 | 14 | # site: 15 | * http://blog.51reboot.com/gko_pool 16 | -------------------------------------------------------------------------------- /example/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean all client server ares_dns stress_confAgent 2 | 3 | LIBS= $(shell PKG_CONFIG_PATH=../lib/libgko/lib/pkgconfig pkg-config libgko --libs) 4 | CFLAGS= -g -O2 $(shell PKG_CONFIG_PATH=../lib/libgko/lib/pkgconfig pkg-config libgko --cflags) 5 | BINS = server client client2 ares_dns stress_confAgent 6 | all: $(BINS) 7 | 8 | server: 9 | g++ $@.cpp -g -O0 $(CFLAGS) $(LIBS) -o $@ 10 | 11 | client: 12 | g++ $@.cpp -g -O0 $(CFLAGS) $(LIBS) -o $@ 13 | 14 | client2: 15 | g++ $@.cpp -g -O0 $(CFLAGS) $(LIBS) -o $@ 16 | 17 | ares_dns: 18 | g++ $@.cpp -g -O0 $(CFLAGS) $(LIBS) -o $@ 19 | 20 | stress_confAgent: 21 | g++ $@.cpp -g -O0 $(CFLAGS) $(LIBS) -o $@ 22 | 23 | clean: 24 | -rm -rf *.dSYM 25 | -rm -f *.o 26 | -rm -f $(BINS) 27 | -------------------------------------------------------------------------------- /example/ares_dns.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * async_dns.cpp 3 | * 4 | * Created on: May 22, 2012 5 | * Author: auxten 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "event.h" 14 | #include "ares.h" 15 | #include "gko.h" 16 | 17 | struct event_base* ev_base; 18 | struct event ev_dns; 19 | 20 | void dns_callback(void* arg, int status, int timeouts, struct hostent* host) 21 | { 22 | if (status == ARES_SUCCESS) 23 | { 24 | u_int32_t addr = *(in_addr_t *) host->h_addr; 25 | printf("%u.%u.%u.%u\n", (addr) % 256, (addr >> 8) % 256, (addr >> 16) % 256, (addr >> 24) % 256); 26 | } 27 | else 28 | { 29 | std::cout << "lookup failed: " << status << '\n'; 30 | } 31 | event_del(&ev_dns); 32 | // event_base_loopexit(ev_base, &t); 33 | } 34 | 35 | void dns_ev_callback(int fd, short ev, void *arg) 36 | { 37 | if (ev & EV_READ) 38 | ares_process_fd((ares_channel) arg, fd, ARES_SOCKET_BAD); 39 | else if (ev & EV_WRITE) 40 | ares_process_fd((ares_channel) arg, ARES_SOCKET_BAD, fd); 41 | } 42 | 43 | void main_loop(ares_channel channel) 44 | { 45 | int nfds, count; 46 | fd_set readers, writers; 47 | ev_base = (struct event_base*) event_init(); 48 | 49 | ares_socket_t *read_fds; 50 | ares_socket_t *write_fds; 51 | // FD_ZERO(&readers); 52 | // FD_ZERO(&writers); 53 | //// ares_ 54 | // nfds = ares_fds(channel, &readers, &writers); 55 | // for (int i = 0; i < readers.fd_count; i++) 56 | // { 57 | // if (FD_ISSET(readers.fd_array[i], readers)) 58 | // event_set(&ev_dns, 1, EV_READ, ev_callback, 59 | // (void *) (&channel)); 60 | // } 61 | // for (int i = 0; i < writers.fd_count; i++) 62 | // { 63 | // if (FD_ISSET(writers.fd_array[i], writers)) 64 | // event_set(&ev_dns, 1, EV_WRITE, ev_callback, 65 | // (void *) (&channel)); 66 | // } 67 | 68 | //////////////////// 69 | // struct server_state *server; 70 | // 71 | // /* Are there any active queries? */ 72 | // int active_queries = !ares__is_list_empty(&(channel->all_queries)); 73 | // 74 | // for (int i = 0; i < channel->nservers; i++) 75 | // { 76 | // server = &channel->servers[i]; 77 | // /* We only need to register interest in UDP sockets if we have 78 | // * outstanding queries. 79 | // */ 80 | // if (active_queries && server->udp_socket != ARES_SOCKET_BAD) 81 | // { 82 | //// FD_SET(server->udp_socket, read_fds); 83 | // event_set(&ev_dns, server->udp_socket, EV_READ, ev_callback, 84 | // (void *) (&channel)); 85 | // } 86 | // /* We always register for TCP events, because we want to know 87 | // * when the other side closes the connection, so we don't waste 88 | // * time trying to use a broken connection. 89 | // */ 90 | // if (server->tcp_socket != ARES_SOCKET_BAD) 91 | // { 92 | //// FD_SET(server->tcp_socket, read_fds); 93 | // event_set(&ev_dns, server->tcp_socket, EV_READ, ev_callback, 94 | // (void *) (&channel)); 95 | // if (server->qhead) 96 | //// FD_SET(server->tcp_socket, write_fds); 97 | // event_set(&ev_dns, server->tcp_socket, EV_WRITE, ev_callback, 98 | // (void *) (&channel)); 99 | // } 100 | // } 101 | //////////////////// 102 | ares_fds_array(channel, &read_fds, &write_fds); 103 | 104 | for (int i = 0; *(read_fds + i) != ARES_SOCKET_BAD; i++) 105 | { 106 | // printf( "read %d\n", *(read_fds + i)); 107 | event_set(&ev_dns, *(read_fds + i), EV_READ | EV_PERSIST, dns_ev_callback, 108 | (void *) (channel)); 109 | } 110 | for (int i = 0; *(write_fds + i) != ARES_SOCKET_BAD; i++) 111 | { 112 | // printf("write %d\n", *(write_fds + i)); 113 | event_set(&ev_dns, *(write_fds + i), EV_WRITE | EV_PERSIST, dns_ev_callback, 114 | (void *) (channel)); 115 | } 116 | 117 | free(read_fds); 118 | free(write_fds); 119 | event_base_set(ev_base, &ev_dns); 120 | event_add(&ev_dns, 0); 121 | event_base_loop(ev_base, 0); 122 | 123 | // timeval tv, *tvp; 124 | // while (1) 125 | // { 126 | // FD_ZERO(&readers); 127 | // FD_ZERO(&writers); 128 | //// ares_ 129 | // nfds = ares_fds(channel, &readers, &writers); 130 | // if (nfds == 0) 131 | // break; 132 | // tvp = ares_timeout(channel, NULL, &tv); 133 | // count = select(nfds, &readers, &writers, NULL, tvp); 134 | // ares_process(channel, &readers, &writers); 135 | // } 136 | 137 | } 138 | int main(int argc, char **argv) 139 | { 140 | struct in_addr ip; 141 | int res; 142 | if (argc < 2) 143 | { 144 | std::cout << "usage: " << argv[0] << " ip.address\n"; 145 | return 1; 146 | } 147 | // inet_aton(argv[1], &ip); 148 | ares_library_init(ARES_LIB_INIT_ALL); 149 | ares_channel channel; 150 | if ((res = ares_init(&channel)) != ARES_SUCCESS) 151 | { 152 | std::cout << "ares feiled: " << res << '\n'; 153 | return 1; 154 | } 155 | 156 | ares_gethostbyname(channel, argv[1], AF_INET, dns_callback, NULL); 157 | main_loop(channel); 158 | return 0; 159 | } 160 | 161 | -------------------------------------------------------------------------------- /example/client.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * server.cpp 3 | * 4 | * Created on: May 18, 2012 5 | * Author: auxten 6 | */ 7 | #include "gko.h" 8 | #include 9 | /// gingko global stuff 10 | s_gingko_global_t gko; 11 | 12 | pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; 13 | int cnt = 3; 14 | int counter = 0; 15 | void report_result(void * c, const char * msg) 16 | { 17 | GKOLOG(NOTICE, "%s", msg); 18 | pthread_mutex_lock(&lock); 19 | if (++counter == cnt) 20 | { 21 | printf("finished\n"); 22 | exit(0); 23 | } 24 | pthread_mutex_unlock(&lock); 25 | } 26 | 27 | int main(int argc, char** argv) 28 | { 29 | char cmd[2048]; 30 | 31 | gko.opt.to_debug = 0; 32 | gko.ready_to_serv = 1; 33 | gko.sig_flag = 0; 34 | gko.opt.worker_thread = 8; 35 | gko.opt.connlimit = SERV_POOL_SIZE; 36 | // gko.opt.to_debug = 1; 37 | gko_pool * gingko = gko_pool::getInstance(); 38 | gingko->setPort(-1); 39 | gingko->setOption(&gko.opt); 40 | // gingko->setProcessHandler(conn_send_data); 41 | gingko->setReportHandler(report_result); 42 | gingko->gko_run(); 43 | 44 | char ua[56][256] = 45 | { 46 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:15.0) Gecko/20100101 Firefox/15.0.1", 47 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4", 48 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4", 49 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14", 50 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1", 51 | "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)", 52 | "Mozilla/5.0 (Windows NT 5.1; rv:15.0) Gecko/20100101 Firefox/15.0.1", 53 | "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20100101 Firefox/15.0.1", 54 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4", 55 | "Mozilla/5.0 (Linux; U; Android 2.2; fr-fr; Desire_A8181 Build/FRF91) App3leWebKit/53.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", 56 | "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4", 57 | "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4", 58 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:5.0) Gecko/20100101 Firefox/5.0", 59 | "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1", 60 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:16.0) Gecko/20100101 Firefox/16.0", 61 | "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.0) Opera 7.02 Bork-edition [en]", 62 | "Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.02", 63 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1", 64 | "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4", 65 | "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; .NET CLR 3.5.30729)", 66 | "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322)", 67 | "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; FunWebProducts; .NET CLR 1.1.4322; PeoplePal 6.2)", 68 | "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; MRA 5.8 (build 4157); .NET CLR 2.0.50727; AskTbPTV/5.11.3.15590)", 69 | "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)", 70 | "Mozilla/5.0 (Windows NT 6.1; rv:2.0b7pre) Gecko/20100921 Firefox/4.0b7pre", 71 | "Mozilla/5.0 (Windows NT 5.1; rv:5.0.1) Gecko/20100101 Firefox/5.0.1", 72 | "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)", 73 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4", 74 | "Opera/9.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.01", 75 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:15.0) Gecko/20100101 Firefox/15.0.1", 76 | "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A403 Safari/8536.25", 77 | "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1", 78 | "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)", 79 | "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4", 80 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:15.0) Gecko/20100101 Firefox/15.0.1", 81 | "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1", 82 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2", 83 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4", 84 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14", 85 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4", 86 | "Mozilla/5.0 (Windows NT 5.1; rv:13.0) Gecko/20100101 Firefox/13.0.1", 87 | "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1", 88 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6", 89 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1", 90 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:15.0) Gecko/20100101 Firefox/15.0.1", 91 | "Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A403 Safari/8536.25", 92 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/536.25 (KHTML, like Gecko) Version/6.0 Safari/536.25", 93 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4", 94 | "Mozilla/5.0 (Windows NT 5.1; rv:16.0) Gecko/20100101 Firefox/16.0", 95 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4", 96 | "Mozilla/5.0 (iPad; CPU OS 5_1_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9B206 Safari/7534.48.3", 97 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4", 98 | "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:15.0) Gecko/20100101 Firefox/15.0.1", 99 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b", 100 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:14.0) Gecko/20100101 Firefox/14.0.1", 101 | "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:14.0) Gecko/20100101 Firefox/14.0.1", 102 | }; 103 | 104 | char srv[3][32] = 105 | { 106 | "202.108.43.189", 107 | "202.106.182.237", 108 | "home" 109 | }; 110 | 111 | int i = cnt; 112 | while (i--) 113 | { 114 | struct timeval tv; 115 | gettimeofday(&tv, NULL); 116 | long t = tv.tv_usec; 117 | sprintf(cmd, 118 | "GET /Poll.php?project_id=5168&id=49 HTTP/1.%ld\r\n" 119 | // "GET /Poll.php?project_id=5168&id=80 HTTP/1.%ld\r\n" 120 | "Host: hi.video.sina.com.cn\r\n" 121 | "User-Agent: %s\r\n" 122 | "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" 123 | "Accept-Language: zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3\r\n" 124 | "DNT: 1\r\n" 125 | "Client-IP: %ld.%ld.%ld.%ld\r\n" 126 | "Connection: keep-alive\r\n" 127 | "Referer: http://hi.video.sina.com.cn/you/2012banjiti/ph.php?dpc=1\r\n\r\n" 128 | , t % 2,ua[t % 56], 122 + t % 20, 121 + (i + t) % 128, 24 + (i + t) % 135, 24 + (i + t) % 213); 129 | gingko->make_active_connect(srv[i%3], 80, strlen(cmd), cmd, -1, -1, 0, 10); 130 | // gingko->make_active_connect("61.30.127.2", 80, strlen(cmd), cmd, -1, -1, 0, 10); 131 | } 132 | sleep(10); 133 | return 0; 134 | } 135 | 136 | 137 | -------------------------------------------------------------------------------- /example/client2.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * client.cpp 3 | * 4 | * Created on: Feb 27, 2012 5 | * Author: auxten 6 | */ 7 | 8 | #include "gko.h" 9 | 10 | /// gingko global stuff 11 | s_gingko_global_t gko; 12 | const int T_NUM = 100; 13 | const int CMD_CNT = 100; 14 | 15 | const s_host_t server = 16 | { 17 | "127.0.0.1", 18 | 2120 19 | }; 20 | 21 | void * send_test(void *) 22 | { 23 | char msg[MSG_LEN] = 24 | "TEST\tdddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"; 25 | for (int i = 0; i < CMD_CNT; i++) 26 | { 27 | if (chat_with_host(&server, msg, 2, 2) < 0) 28 | { 29 | GKOLOG(FATAL, "sending test message failed"); 30 | } 31 | } 32 | // GKOLOG(DEBUG, "%s", msg); 33 | pthread_exit(NULL); 34 | } 35 | 36 | void * send_scmd(void *) 37 | { 38 | char msg[MSG_LEN] = 39 | "SCMD\tlocalhost\tdf -h"; 40 | for (int i = 0; i < CMD_CNT; i++) 41 | { 42 | if (chat_with_host(&server, msg, 2, 2) < 0) 43 | { 44 | GKOLOG(FATAL, "sending test message failed"); 45 | } 46 | } 47 | // GKOLOG(DEBUG, "%s", msg); 48 | pthread_exit(NULL); 49 | } 50 | 51 | int main(int argc, char** argv) 52 | { 53 | gko.opt.to_debug = 1; 54 | pthread_attr_t g_attr; 55 | pthread_t vnode_pthread[T_NUM]; 56 | void *status; 57 | unsigned short proto_ver; 58 | int msg_len; 59 | 60 | // char buf[20] = 61 | // { '\0' }; 62 | // fill_cmd_head(buf, INT32_MAX); 63 | // parse_cmd_head(buf, &proto_ver, &msg_len); 64 | // GKOLOG(DEBUG, "%s %d", buf, msg_len); 65 | // std::vector conn_vec; 66 | // std::vector srv_vec(1000, server); 67 | // connect_hosts(srv_vec, &conn_vec); 68 | // sleep(10); 69 | // disconnect_hosts(conn_vec); 70 | 71 | // char * array[5] = {NULL, NULL, NULL, NULL, NULL}; 72 | // char t1[100] = "aa\t\t\tddd\tbbb\t\tccc\trrr\tyyy"; 73 | // sep_arg(t1, array, 5); 74 | // GKOLOG(DEBUG, "%s, %s, %s, %s, %s", array[0], array[1], array[2], array[3], array[4]); 75 | // GKOLOG(DEBUG, "for test %s %d", "ddd", 10); 76 | // return 0; 77 | 78 | if (pthread_attr_init(&g_attr) != 0) 79 | { 80 | GKOLOG(FATAL, FLF("pthread_mutex_destroy error")); 81 | return -1; 82 | } 83 | if (pthread_attr_setdetachstate(&g_attr, PTHREAD_CREATE_JOINABLE) != 0) 84 | { 85 | GKOLOG(FATAL, FLF("pthread_mutex_destroy error")); 86 | return -1; 87 | } 88 | 89 | for (int i = 0; i < T_NUM; i++) 90 | { 91 | if (pthread_create(&vnode_pthread[i], &g_attr, send_scmd, NULL)) 92 | { 93 | GKOLOG(FATAL, "download thread %d create error", i); 94 | return -1; 95 | } 96 | } 97 | for (int i = 0; i < T_NUM; i++) 98 | { 99 | if (pthread_join(vnode_pthread[i], &status)) 100 | { 101 | GKOLOG(FATAL, "download thread %d join error", i); 102 | return -1; 103 | } 104 | if (status != (void *) 0) 105 | { 106 | GKOLOG(FATAL, "thread %d joined with error num %lld", i, 107 | (long long) status); 108 | return -1; 109 | } 110 | } 111 | 112 | return 0; 113 | } 114 | 115 | -------------------------------------------------------------------------------- /example/server.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * server.cpp 3 | * 4 | * Created on: May 18, 2012 5 | * Author: auxten 6 | */ 7 | #include "gko.h" 8 | /// gingko global stuff 9 | s_gingko_global_t gko; 10 | 11 | 12 | void * conn_send_data(void * arg) 13 | { 14 | int process_ret; 15 | struct conn_client *client = (struct conn_client *) arg; 16 | char * p = ((char *) client->read_buffer) + CMD_PREFIX_BYTE; 17 | GKOLOG(DEBUG, "conn_send_data %s", p); 18 | 19 | client->need_write = snprintf(client->write_buffer, 20 | client->wbuf_size, 21 | "{\"errno\":\"%d\",\"message\":\"%s\"}", 22 | 0, 23 | "Hello World"); 24 | 25 | return NULL; 26 | } 27 | 28 | int main(int argc, char** argv) 29 | { 30 | gko.opt.to_debug = 1; 31 | gko.ready_to_serv = 1; 32 | gko.sig_flag = 0; 33 | gko.opt.worker_thread = 8; 34 | gko.opt.connlimit = SERV_POOL_SIZE; 35 | gko.opt.bind_ip = htons(INADDR_ANY); 36 | // gko.opt.to_debug = 1; 37 | gko_pool * gingko = gko_pool::getInstance(); 38 | gingko->setPort(2121); 39 | gingko->setOption(&gko.opt); 40 | gingko->setProcessHandler(conn_send_data); 41 | // gingko->setReportHandler(report_result); 42 | 43 | 44 | GKOLOG(DEBUG, "Debug mode start, i will print tons of log :p!"); 45 | 46 | return gingko->gko_run(); 47 | } 48 | 49 | -------------------------------------------------------------------------------- /lib/c-ares-1.10.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auxten/gko_pool/aa23c545a1f1e9a8dc1c2be18d0912fa77864d61/lib/c-ares-1.10.0.tar.gz -------------------------------------------------------------------------------- /lib/c-ares-1.10.0_auxten.patch: -------------------------------------------------------------------------------- 1 | diff -u c-ares-1.10.0/ares.h c-ares-1.10.0.patched/ares.h 2 | --- c-ares-1.10.0/ares.h 2013-04-09 04:16:48.000000000 +0800 3 | +++ c-ares-1.10.0.patched/ares.h 2013-06-26 15:35:37.000000000 +0800 4 | @@ -386,6 +386,10 @@ 5 | fd_set *read_fds, 6 | fd_set *write_fds); 7 | 8 | +CARES_EXTERN int ares_fds_array(ares_channel channel, 9 | + ares_socket_t **read_fds, 10 | + ares_socket_t **write_fds); 11 | + 12 | CARES_EXTERN int ares_getsock(ares_channel channel, 13 | ares_socket_t *socks, 14 | int numsocks); 15 | diff -u c-ares-1.10.0/ares_fds.c c-ares-1.10.0.patched/ares_fds.c 16 | --- c-ares-1.10.0/ares_fds.c 2013-02-13 18:01:50.000000000 +0800 17 | +++ c-ares-1.10.0.patched/ares_fds.c 2013-06-26 15:36:32.000000000 +0800 18 | @@ -53,7 +53,64 @@ 19 | FD_SET(server->tcp_socket, write_fds); 20 | if (server->tcp_socket >= nfds) 21 | nfds = server->tcp_socket + 1; 22 | - } 23 | + } 24 | } 25 | return (int)nfds; 26 | } 27 | + 28 | +int ares_fds_array(ares_channel channel, ares_socket_t **read_fds, ares_socket_t **write_fds) 29 | +{ 30 | + struct server_state *server; 31 | + int i; 32 | + int r_cnt = 0; 33 | + int w_cnt = 0; 34 | + int r_size = 128; 35 | + int w_size = 128; 36 | + 37 | + *read_fds = (ares_socket_t *)malloc(r_size * sizeof(ares_socket_t)); 38 | + *write_fds = (ares_socket_t *)malloc(w_size * sizeof(ares_socket_t)); 39 | + 40 | + /* Are there any active queries? */ 41 | + int active_queries = !ares__is_list_empty(&(channel->all_queries)); 42 | + 43 | + for (i = 0; i < channel->nservers; i++) 44 | + { 45 | + server = &channel->servers[i]; 46 | + /* We only need to register interest in UDP sockets if we have 47 | + * outstanding queries. 48 | + */ 49 | + if (active_queries && server->udp_socket != ARES_SOCKET_BAD) 50 | + { 51 | + *(*read_fds + r_cnt++) = server->udp_socket; 52 | + } 53 | + /* We always register for TCP events, because we want to know 54 | + * when the other side closes the connection, so we don't waste 55 | + * time trying to use a broken connection. 56 | + */ 57 | + if (server->tcp_socket != ARES_SOCKET_BAD) 58 | + { 59 | + *(*read_fds + r_cnt++) = server->tcp_socket; 60 | + if (server->qhead) 61 | + { 62 | + *(*write_fds + w_cnt++) = server->tcp_socket; 63 | + } 64 | + } 65 | + if (r_cnt >= r_size - 2) 66 | + { 67 | + r_size *= 2; 68 | + *read_fds = (ares_socket_t *)realloc(*read_fds, r_size * sizeof(ares_socket_t)); 69 | + } 70 | + 71 | + if (w_cnt >= w_size - 2) 72 | + { 73 | + w_size *= 2; 74 | + *write_fds = (ares_socket_t *)realloc(*write_fds, w_size * sizeof(ares_socket_t)); 75 | + } 76 | + 77 | + } 78 | + 79 | + *(*read_fds + r_cnt) = ARES_SOCKET_BAD; 80 | + *(*write_fds + w_cnt) = ARES_SOCKET_BAD; 81 | + 82 | + return r_cnt + w_cnt; 83 | +} 84 | Common subdirectories: c-ares-1.10.0/m4 and c-ares-1.10.0.patched/m4 85 | Common subdirectories: c-ares-1.10.0/vc and c-ares-1.10.0.patched/vc 86 | -------------------------------------------------------------------------------- /lib/jemalloc-3.0.0.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auxten/gko_pool/aa23c545a1f1e9a8dc1c2be18d0912fa77864d61/lib/jemalloc-3.0.0.tar.bz2 -------------------------------------------------------------------------------- /lib/libev-4.11.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auxten/gko_pool/aa23c545a1f1e9a8dc1c2be18d0912fa77864d61/lib/libev-4.11.tar.gz -------------------------------------------------------------------------------- /lib/libev_auxten.patch: -------------------------------------------------------------------------------- 1 | diff -upNr libev-4.04/ev.c libev-4.04-fixed/ev.c 2 | --- libev-4.04/ev.c 2011-02-09 07:17:37.000000000 +0800 3 | +++ libev-4.04-fixed/ev.c 2011-09-07 15:29:55.000000000 +0800 4 | @@ -948,6 +948,8 @@ fd_event_nocheck (EV_P_ int fd, int reve 5 | 6 | if (ev) 7 | ev_feed_event (EV_A_ (W)w, ev); 8 | + if (w == (ev_io *)((WL)w)->next) 9 | + break; 10 | } 11 | } 12 | 13 | @@ -1017,7 +1019,11 @@ fd_reify (EV_P) 14 | anfd->events = 0; 15 | 16 | for (w = (ev_io *)anfd->head; w; w = (ev_io *)((WL)w)->next) 17 | - anfd->events |= (unsigned char)w->events; 18 | + { 19 | + anfd->events |= (unsigned char)w->events; 20 | + if (w == (ev_io *)((WL)w)->next) 21 | + break; 22 | + } 23 | 24 | if (o_events != anfd->events) 25 | o_reify = EV__IOFDSET; /* actually |= */ 26 | @@ -1495,6 +1501,8 @@ child_reap (EV_P_ int chain, int pid, in 27 | w->rstatus = status; 28 | ev_feed_event (EV_A_ (W)w, EV_CHILD); 29 | } 30 | + if (w == (ev_child *)((WL)w)->next) 31 | + break; 32 | } 33 | } 34 | 35 | -------------------------------------------------------------------------------- /lib/patch_build_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | sh patch_build_libev.sh 4 | sh patch_build_c-ares.sh 5 | #sh patch_build_jemalloc.sh 6 | 7 | #which mysql_config || echo "can't find mysql_config" 8 | #which mysql_config || exit 1 9 | #sh patch_build_libzdb.sh 10 | 11 | 12 | echo "done!" 13 | 14 | 15 | -------------------------------------------------------------------------------- /lib/patch_build_c-ares.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | NAME='c-ares' 4 | VER='1.10.0' 5 | TYPE='tar.gz' 6 | CONFIGURE_FLAGS='' 7 | 8 | #specify where to install by PREFIX or COMMAND-LINE ARG1 9 | if [ -n "$1" ]; then 10 | PREFIX=$1 11 | else 12 | PREFIX="$PWD/${NAME}-${VER}built" 13 | fi 14 | SRCDIR=${NAME}-${VER} 15 | LIBNAME="${NAME}-${VER}" 16 | if [ -d "${SRCDIR}" ]; then 17 | rm -rf ${SRCDIR} 18 | fi 19 | FILE=${LIBNAME}.${TYPE} 20 | if [ ! -f "${FILE}" ]; then 21 | echo "error: no ${FILE}" 22 | exit 1 23 | fi 24 | tar xvzf ${FILE} 25 | if [ ! -d "${SRCDIR}" ]; then 26 | echo "error: no ${SRCDIR}" 27 | exit 1 28 | fi 29 | 30 | 31 | cp ${LIBNAME}_auxten.patch ${SRCDIR}/ 32 | cd ${SRCDIR} 33 | patch -p1 < ${LIBNAME}_auxten.patch 34 | 35 | ./configure --prefix=${PREFIX} --enable-shared=no --enable-static ${CONFIGURE_FLAGS} 36 | 37 | make; make install 38 | #cp *.h ../c-ares/include/ 39 | cd .. 40 | 41 | rm -rf ${NAME} 42 | ln -s ${NAME}-${VER}built ${NAME} 43 | #rm -rf ${SRCDIR} 44 | 45 | echo "done!" 46 | 47 | 48 | -------------------------------------------------------------------------------- /lib/patch_build_jemalloc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | NAME='jemalloc' 4 | VER='3.0.0' 5 | TYPE='tar.bz2' 6 | CONFIGURE_FLAGS='' 7 | 8 | #specify where to install by PREFIX or COMMAND-LINE ARG1 9 | if [ -n "$1" ]; then 10 | PREFIX=$1 11 | else 12 | PREFIX="$PWD/${NAME}-${VER}built" 13 | fi 14 | SRCDIR=${NAME}-${VER} 15 | LIBNAME="${NAME}-${VER}" 16 | if [ -d "${SRCDIR}" ]; then 17 | rm -rf ${SRCDIR} 18 | fi 19 | FILE=${LIBNAME}.${TYPE} 20 | if [ ! -f "${FILE}" ]; then 21 | echo "error: no ${FILE}" 22 | exit 1 23 | fi 24 | tar xvjf ${FILE} 25 | if [ ! -d "${SRCDIR}" ]; then 26 | echo "error: no ${SRCDIR}" 27 | exit 1 28 | fi 29 | 30 | #cp ${NAME}_auxten.patch ${SRCDIR}/${NAME}_auxten.patch 31 | cd ${SRCDIR} 32 | #patch -p1 < ${NAME}_auxten.patch 33 | 34 | ./configure --prefix=${PREFIX} --enable-shared=no --enable-static ${CONFIGURE_FLAGS} 35 | 36 | make; make install 37 | cd .. 38 | 39 | rm -rf ${NAME} 40 | ln -s ${NAME}-${VER}built ${NAME} 41 | #rm -rf ${SRCDIR} 42 | 43 | echo "done!" 44 | 45 | 46 | -------------------------------------------------------------------------------- /lib/patch_build_libev.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | NAME='libev' 4 | VER='4.11' 5 | TYPE='tar.gz' 6 | CONFIGURE_FLAGS='' 7 | 8 | #specify where to install by PREFIX or COMMAND-LINE ARG1 9 | if [ -n "$1" ]; then 10 | PREFIX=$1 11 | else 12 | PREFIX="$PWD/${NAME}-${VER}built" 13 | fi 14 | SRCDIR=${NAME}-${VER} 15 | LIBNAME="${NAME}-${VER}" 16 | if [ -d "${SRCDIR}" ]; then 17 | rm -rf ${SRCDIR} 18 | fi 19 | FILE=${LIBNAME}.${TYPE} 20 | if [ ! -f "${FILE}" ]; then 21 | echo "error: no ${FILE}" 22 | exit 1 23 | fi 24 | tar xvzf ${FILE} 25 | if [ ! -d "${SRCDIR}" ]; then 26 | echo "error: no ${SRCDIR}" 27 | exit 1 28 | fi 29 | 30 | #cp ${NAME}_auxten.patch ${SRCDIR}/${NAME}_auxten.patch 31 | cd ${SRCDIR} 32 | #patch -p1 < ${NAME}_auxten.patch 33 | 34 | ./configure --prefix=${PREFIX} --enable-shared=no --enable-static ${CONFIGURE_FLAGS} 35 | 36 | make; make install 37 | cd .. 38 | 39 | rm -rf ${NAME} 40 | ln -s ${NAME}-${VER}built ${NAME} 41 | #rm -rf ${SRCDIR} 42 | 43 | echo "done!" 44 | 45 | 46 | -------------------------------------------------------------------------------- /src/AUTHORS: -------------------------------------------------------------------------------- 1 | Wang Pengcheng 2 | -------------------------------------------------------------------------------- /src/ChangeLog: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auxten/gko_pool/aa23c545a1f1e9a8dc1c2be18d0912fa77864d61/src/ChangeLog -------------------------------------------------------------------------------- /src/Makefile.am: -------------------------------------------------------------------------------- 1 | INCLUDES = \ 2 | -I$(srcdir)/src\ 3 | -include config.h 4 | # -I$(srcdir)/../lib/libev/include \ 5 | # -I$(srcdir)/../lib/c-ares/include 6 | # -I$(srcdir)/../../../../../../third-64/libev/include 7 | # -I$(srcdir)/../../../../../../../third-64/libev/include 8 | export INCLUDES 9 | 10 | #noinst_PROGRAMS=gingko_serv gingko_clnt serv_unittest clnt_unittest 11 | lib_LIBRARIES=libgko.a 12 | 13 | libgko_a_includedir=$(includedir)/ 14 | libgko_a_include_HEADERS = \ 15 | $(srcdir)/gko.h \ 16 | $(srcdir)/gingko.h \ 17 | $(srcdir)/async_pool.h \ 18 | $(srcdir)/log.h \ 19 | $(srcdir)/socket.h \ 20 | $(srcdir)/memory.h \ 21 | $(srcdir)/dict.h \ 22 | $(srcdir)/path.h \ 23 | $(srcdir)/hash/gko_zip.h \ 24 | $(srcdir)/hash/lz4.h \ 25 | $(srcdir)/hash/md5.h \ 26 | $(srcdir)/hash/xor_hash.h \ 27 | $(srcdir)/gko_errno.h 28 | 29 | pkgconfigdir = $(libdir)/pkgconfig 30 | pkgconfig_DATA = libgko.pc 31 | 32 | 33 | DEFS+=-D_GNU_SOURCE -ggdb -funsigned-char -fPIC -DPIC -pipe 34 | libgko_a_SOURCES=async_threads.cpp async_conn.cpp async_clnt.cpp async_dns.cpp \ 35 | gingko_base.cpp log.cpp socket.cpp limit.cpp memory.cpp dict.c path.cpp \ 36 | hash/gko_zip.cpp hash/lz4.cpp hash/md5.cpp hash/xor_hash.cpp 37 | #libgko_a_LDFLAGS=-D_GNU_SOURCE -ggdb -funsigned-char -------------------------------------------------------------------------------- /src/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auxten/gko_pool/aa23c545a1f1e9a8dc1c2be18d0912fa77864d61/src/NEWS -------------------------------------------------------------------------------- /src/README: -------------------------------------------------------------------------------- 1 | currently p2p algorithm is complete, could be used for data transfer 2 | usage see --help 3 | ____ _ _ 4 | / ___(_)_ __ __ _| | _____ 5 | | | _| | '_ \ / _` | |/ / _ \ 6 | | |_| | | | | | (_| | < (_) | 7 | \____|_|_| |_|\__, |_|\_\___/ 8 | |___/ 9 | 10 | # Start project gingko 11 | # Gingko is a point to point rsync rolling checksum proto- 12 | # col implemented file synchronize application. Instead 13 | # of original rsync builds, we want to make it able to 14 | # sync files from a large mesh computer network so-called 15 | # the cloud and pushing files from computers to computers, 16 | # the P2P protocol will make it more efficient. 17 | # Hope it success! 18 | 19 | #define Rolling_checksum_algorithm http://samba.anu.edu.au/rsync/tech_report/node3.html 20 | 21 | BUILD: 22 | make 23 | 24 | NOTE： 25 | For reusing some code between clnt and serv, I write some funcs in gingko_common.h 26 | with some MACRO, I know this is ugly, but the bins compiled out work well...so, let it 27 | be. 28 | 29 | DEPS: 30 | libevent 31 | gtest (when ./configure --enable-unittest) 32 | 33 | auxtenwpc@gmail.com 34 | -EOF- 35 | 36 | -------------------------------------------------------------------------------- /src/TODO: -------------------------------------------------------------------------------- 1 | #connection timeout 2 | #connection error update_state() 3 | #r/w transform update time 4 | #nonblocking DNS resolve 5 | make a replacement of my_getaddr 6 | long connection support 7 | mem pool 8 | -------------------------------------------------------------------------------- /src/async_clnt.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * async_clnt.cpp 3 | * 4 | * Created on: May 4, 2012 5 | * Author: auxten 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "config.h" 14 | #include "gingko.h" 15 | #include "log.h" 16 | #include "socket.h" 17 | #ifdef __APPLE__ 18 | #include 19 | #else 20 | #include 21 | #endif /** __APPLE__ **/ 22 | 23 | #include "async_pool.h" 24 | #include "socket.h" 25 | 26 | /** 27 | * @brief non-blocking version connect 28 | * 29 | * @see 30 | * @note 31 | * @author auxten 32 | * @date Apr 22, 2012 33 | **/ 34 | int gko_pool::nb_connect(struct conn_client* conn) 35 | { 36 | int sock = -1; 37 | struct sockaddr_in channel; 38 | 39 | sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); 40 | if (FAIL_CHECK(sock < 0)) 41 | { 42 | GKOLOG(FATAL, "get socket error"); 43 | goto NB_CONNECT_END; 44 | } 45 | 46 | memset(&channel, 0, sizeof(channel)); 47 | channel.sin_family = AF_INET; 48 | channel.sin_addr.s_addr = conn->client_addr; 49 | channel.sin_port = htons(conn->client_port); 50 | 51 | /** set the connect non-blocking then blocking for add timeout on connect **/ 52 | if (FAIL_CHECK(setnonblock(sock) < 0)) 53 | { 54 | GKOLOG(FATAL, "set socket non-blocking error"); 55 | goto NB_CONNECT_END; 56 | } 57 | 58 | // GKOLOG(DEBUG, "before connect"); 59 | char ip[16]; 60 | /** connect and send the msg **/ 61 | if (FAIL_CHECK(connect(sock, (struct sockaddr *) &channel, sizeof(channel)) && 62 | errno != EINPROGRESS)) 63 | { 64 | GKOLOG(WARNING, "connect error %s:%d", addr_itoa(conn->client_addr, ip), conn->client_port); 65 | } 66 | // GKOLOG(DEBUG, "after connect"); 67 | 68 | NB_CONNECT_END: 69 | 70 | conn->client_fd = sock; 71 | /// 72 | // ret = sock; 73 | // if (ret < 0 && sock >= 0) 74 | // { 75 | // close_socket(sock); 76 | // } 77 | return sock; 78 | } 79 | 80 | int gko_pool::make_active_connect(const char * host, const int port, int len, const char * cmd, 81 | const long task_id, const long sub_task_id, const u_int8_t flag, const int wrote) 82 | { 83 | struct conn_client * conn; 84 | 85 | conn = add_new_conn_client(FD_BEFORE_CONNECT); 86 | if (!conn) 87 | { 88 | ///close socket and further receives will be disallowed 89 | GKOLOG(FATAL, "Server limited: I cannot serve more clients"); 90 | return SERVER_INTERNAL_ERROR; 91 | } 92 | 93 | int worker_id; 94 | worker_id = thread_list_find_next(); 95 | if (worker_id < 0) 96 | { 97 | GKOLOG(WARNING, "can't find available thread"); 98 | return FAIL; 99 | } 100 | else 101 | { 102 | conn->worker_id = worker_id; 103 | } 104 | 105 | // GKOLOG(DEBUG, "conn_buffer_init"); 106 | conn_buffer_init(conn); 107 | 108 | strncpy(conn->client_hostname, host, sizeof(conn->client_hostname) - 1); 109 | conn->client_hostname[sizeof(conn->client_hostname) - 1] = '\0'; 110 | conn->client_port = port; 111 | 112 | conn->type = active_conn; 113 | conn->task_id = task_id; 114 | conn->sub_task_id = sub_task_id; 115 | 116 | conn->need_write = len; 117 | conn->have_write = wrote; 118 | memcpy(conn->write_buffer, cmd, len); 119 | if (wrote) 120 | { 121 | fill_cmd_head(conn->__write_buffer, len); 122 | // conn->need_read = 1500; 123 | } 124 | thread_worker_dispatch(conn->id, conn->worker_id); 125 | return SUCC; 126 | } 127 | 128 | 129 | -------------------------------------------------------------------------------- /src/async_conn.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * async_conn.cpp 3 | * gingko 4 | * 5 | * Created on: Mar 9, 2012 6 | * Author: auxten 7 | * 8 | **/ 9 | 10 | #include "gingko.h" 11 | #include "async_pool.h" 12 | #include "log.h" 13 | #include "socket.h" 14 | #include "memory.h" 15 | 16 | gko_pool * gko_pool::_instance = NULL; 17 | /// init global lock 18 | pthread_mutex_t gko_pool::instance_lock = PTHREAD_MUTEX_INITIALIZER; 19 | /// init global connecting list lock 20 | pthread_mutex_t gko_pool::conn_list_lock = PTHREAD_MUTEX_INITIALIZER; 21 | /// init global thread list lock 22 | pthread_mutex_t gko_pool::thread_list_lock = PTHREAD_MUTEX_INITIALIZER; 23 | /// init global thread list lock 24 | pthread_mutex_t gko_pool::DNS_cache_lock = PTHREAD_MUTEX_INITIALIZER; 25 | /// init gko_pool::gko_serv 26 | s_host_t gko_pool::gko_serv = {"\0", 0}; 27 | 28 | /** 29 | * @brief Initialization of client list 30 | * 31 | * @see 32 | * @note 33 | * @author wangpengcheng01 34 | * @date 2011-8-1 35 | **/ 36 | int gko_pool::conn_client_list_init() 37 | { 38 | g_client_list = new conn_client *[option->connlimit]; 39 | memset(g_client_list, 0, option->connlimit * sizeof(struct conn_client *)); 40 | g_total_clients = 0; 41 | g_total_connect = 0; 42 | 43 | GKOLOG(NOTICE, "Client pool initialized as %d", option->connlimit); 44 | 45 | return option->connlimit; 46 | } 47 | 48 | /** 49 | * @brief Init for gingko_clnt 50 | * 51 | * @see 52 | * @note 53 | * @author auxten 54 | * @date 2011-8-1 55 | **/ 56 | int gko_pool::gko_async_server_base_init() 57 | { 58 | g_server = new conn_server; 59 | memset(g_server, 0, sizeof(struct conn_server)); 60 | 61 | g_server->srv_addr = option->bind_ip; 62 | if (this->port < 0) 63 | { 64 | unsigned int randseed = (unsigned int)getpid(); 65 | int pt = MAX_LIS_PORT - rand_r(&randseed) % 500; 66 | g_server->srv_port = pt; 67 | } 68 | else 69 | { 70 | g_server->srv_port = this->port; 71 | } 72 | g_server->listen_queue_length = REQ_QUE_LEN; 73 | g_server->nonblock = 1; 74 | g_server->tcp_nodelay = 1; 75 | g_server->tcp_reuse = 1; 76 | g_server->tcp_send_buffer_size = TCP_BUF_SZ; 77 | g_server->tcp_recv_buffer_size = TCP_BUF_SZ; 78 | g_server->send_timeout = SND_TIMEOUT; 79 | g_server->on_data_callback = pHandler; 80 | g_server->is_server = this->port < 0 ? 0 : 1; 81 | /// A new TCP server 82 | int ret; 83 | while ((ret = conn_tcp_server(g_server)) < 0) 84 | { 85 | if (ret == BIND_FAIL) 86 | { 87 | if (g_server->srv_port < MIN_PORT || this->port >= 0) 88 | { 89 | GKOLOG(FATAL, "bind port failed, last try is %d", port); 90 | return -1; 91 | } 92 | g_server->srv_port --; 93 | usleep(BIND_INTERVAL); 94 | } 95 | else 96 | { 97 | GKOLOG(FATAL, "conn_tcp_server start error"); 98 | return -1; 99 | } 100 | } 101 | 102 | set_sig(int_handler); 103 | 104 | ares_library_init(ARES_LIB_INIT_ALL);/// not thread safe 105 | 106 | gko_serv.port = g_server->srv_port; 107 | return g_server->srv_port; 108 | } 109 | 110 | /** 111 | * @brief Generate a TCP server by given struct 112 | * 113 | * @see 114 | * @note 115 | * @author auxten 116 | * @date 2011-8-1 117 | **/ 118 | int gko_pool::conn_tcp_server(struct conn_server *c) 119 | { 120 | int flags; 121 | 122 | if (g_server->srv_port > MAX_PORT) 123 | { 124 | GKOLOG(FATAL, "serv port %d > %d", g_server->srv_port, MAX_PORT); 125 | return -1; 126 | } 127 | 128 | /// If port number below 1024, root privilege needed 129 | if (g_server->srv_port < MIN_PORT) 130 | { 131 | /// CHECK ROOT PRIVILEGE 132 | if (ROOT != geteuid()) 133 | { 134 | GKOLOG(FATAL, "Port %d number below 1024, root privilege needed", 135 | g_server->srv_port); 136 | return -1; 137 | } 138 | } 139 | 140 | /// Create new socket 141 | g_server->listen_fd = socket(AF_INET, SOCK_STREAM, 0); 142 | if (g_server->listen_fd < 0) 143 | { 144 | GKOLOG(WARNING, "Socket creation failed"); 145 | return -1; 146 | } 147 | 148 | if ((flags = fcntl(g_server->listen_fd, F_GETFD)) != -1) 149 | { 150 | fcntl(g_server->listen_fd, F_SETFD, flags | FD_CLOEXEC); 151 | } 152 | 153 | g_server->listen_addr.sin_family = AF_INET; 154 | g_server->listen_addr.sin_addr.s_addr = g_server->srv_addr; 155 | g_server->listen_addr.sin_port = htons(g_server->srv_port); 156 | 157 | setsockopt(g_server->listen_fd, SOL_SOCKET, SO_REUSEADDR, &g_server->tcp_reuse, 158 | sizeof(g_server->tcp_reuse)); 159 | 160 | /// Bind socket 161 | if (bind(g_server->listen_fd, (struct sockaddr *) &g_server->listen_addr, 162 | sizeof(g_server->listen_addr)) < 0) 163 | { 164 | if (g_server->is_server) 165 | { 166 | GKOLOG(FATAL, "Socket bind failed on port"); 167 | return -1; 168 | } 169 | else 170 | { 171 | return BIND_FAIL; 172 | } 173 | } 174 | GKOLOG(NOTICE, "Upload port bind on port %d", g_server->srv_port); 175 | 176 | /// Listen socket 177 | if (listen(g_server->listen_fd, g_server->listen_queue_length) < 0) 178 | { 179 | GKOLOG(FATAL, "Socket listen failed"); 180 | return -1; 181 | } 182 | 183 | /// Set socket options 184 | struct timeval send_timeout; 185 | send_timeout.tv_sec = g_server->send_timeout; 186 | send_timeout.tv_usec = 0; 187 | 188 | setsockopt(g_server->listen_fd, SOL_SOCKET, SO_SNDTIMEO, 189 | (char *) &send_timeout, sizeof(struct timeval)); 190 | setsockopt(g_server->listen_fd, SOL_SOCKET, SO_SNDBUF, 191 | &g_server->tcp_send_buffer_size, sizeof(g_server->tcp_send_buffer_size)); 192 | setsockopt(g_server->listen_fd, SOL_SOCKET, SO_RCVBUF, 193 | &g_server->tcp_recv_buffer_size, sizeof(g_server->tcp_recv_buffer_size)); 194 | setsockopt(g_server->listen_fd, IPPROTO_TCP, TCP_NODELAY, 195 | (char *) &g_server->tcp_nodelay, sizeof(g_server->tcp_nodelay)); 196 | 197 | /// Set socket non-blocking 198 | if (g_server->nonblock && setnonblock(g_server->listen_fd) < 0) 199 | { 200 | GKOLOG(WARNING, "Socket set non-blocking failed"); 201 | return -1; 202 | } 203 | 204 | g_server->start_time = time(NULL); 205 | 206 | ///GKOLOG(WARNING, "Socket server created on port %d", server->srv_port); 207 | ///g_ev_base = event_init(); 208 | /// Add data handler 209 | event_set(&g_server->ev_accept, g_server->listen_fd, EV_READ | EV_PERSIST, 210 | conn_tcp_server_accept, (void *) c); 211 | event_base_set(g_ev_base, &g_server->ev_accept); 212 | event_add(&g_server->ev_accept, NULL); 213 | ///event_base_loop(g_ev_base, 0); 214 | return g_server->listen_fd; 215 | } 216 | 217 | /** 218 | * @brief Accept new connection 219 | * 220 | * @see 221 | * @note 222 | * @author auxten 223 | * @date 2011-8-1 224 | **/ 225 | void gko_pool::conn_tcp_server_accept(int fd, short ev, void *arg) 226 | { 227 | int client_fd; 228 | struct sockaddr_in client_addr; 229 | socklen_t client_len = sizeof(client_addr); 230 | struct conn_client *client; 231 | char ip[16]; 232 | ///struct conn_server *server = (struct conn_server *) arg; 233 | /// Accept new connection 234 | client_fd = accept(fd, (struct sockaddr *) &client_addr, &client_len); 235 | if (-1 == client_fd) 236 | { 237 | GKOLOG(WARNING, "Accept error"); 238 | return; 239 | } 240 | /// Add connection to event queue 241 | client = gko_pool::getInstance()->add_new_conn_client(client_fd); 242 | if (!client) 243 | { 244 | ///close socket and further receives will be disallowed 245 | // shutdown(client_fd, SHUT_RD); 246 | close(client_fd); 247 | GKOLOG(WARNING, "Server limited: I cannot serve more clients"); 248 | return; 249 | } 250 | 251 | /// set blocking 252 | ///fcntl(fd, F_SETFL, fcntl(fd, F_GETFL)& ~O_NONBLOCK); 253 | 254 | /// Try to set non-blocking 255 | if (setnonblock(client_fd) < 0) 256 | { 257 | gko_pool::getInstance()->conn_client_free(client); 258 | GKOLOG(FATAL, "Client socket set non-blocking error"); 259 | return; 260 | } 261 | 262 | /// Client initialize 263 | client->client_addr = inet_addr(inet_ntoa(client_addr.sin_addr)); 264 | client->client_port = ntohs(client_addr.sin_port); 265 | client->type = coming_conn; 266 | 267 | GKOLOG(DEBUG, "coming conn %s:%d", 268 | addr_itoa(client->client_addr, ip), client->client_port); 269 | 270 | gko_pool::getInstance()->thread_worker_dispatch(client->id); 271 | 272 | return; 273 | } 274 | 275 | /** 276 | * @brief ADD New connection client 277 | * 278 | * @see 279 | * @note 280 | * @author auxten 281 | * @date 2011-8-1 282 | **/ 283 | struct conn_client * gko_pool::add_new_conn_client(int client_fd) 284 | { 285 | int id; 286 | struct conn_client *tmp = (struct conn_client *) NULL; 287 | /// Find a free slot 288 | id = conn_client_list_find_free(); 289 | // GKOLOG(DEBUG, "add_new_conn_client id %d",id);///test 290 | if (id >= 0) 291 | { 292 | tmp = conn_client_list_get(id); 293 | } 294 | else 295 | { 296 | /// FIXME 297 | /// Client list pool full, if you want to enlarge it, 298 | /// modify async_pool.h source please 299 | GKOLOG(FATAL, "Client list full"); 300 | return tmp; 301 | } 302 | 303 | if (tmp) 304 | { 305 | tmp->id = id; 306 | tmp->client_fd = client_fd; 307 | g_client_list[id] = tmp; 308 | if (client_fd == FD_BEFORE_CONNECT) 309 | { 310 | g_total_connect++; 311 | } 312 | else 313 | { 314 | g_total_clients++; 315 | } 316 | } 317 | return tmp; 318 | } 319 | 320 | /** 321 | * @brief Find a free slot from client pool 322 | * 323 | * @see 324 | * @note 325 | * @author auxten 326 | * @date 2011-8-1 327 | **/ 328 | int gko_pool::conn_client_list_find_free() 329 | { 330 | int i; 331 | int tmp = -1; 332 | 333 | pthread_mutex_lock(&conn_list_lock); 334 | for (i = 0; i < option->connlimit; i++) 335 | { 336 | tmp = (i + g_curr_conn + 1) % option->connlimit; 337 | if (!g_client_list[tmp] || 0 == g_client_list[tmp]->conn_time) 338 | { 339 | if (!g_client_list[tmp]) 340 | { 341 | g_client_list[tmp] = new conn_client; 342 | memset(g_client_list[tmp], 0, sizeof(conn_client)); 343 | } 344 | 345 | conn_client_clear(g_client_list[tmp]); 346 | g_client_list[tmp]->conn_time = time(NULL); 347 | 348 | g_curr_conn = tmp; 349 | break; 350 | } 351 | } 352 | pthread_mutex_unlock(&conn_list_lock); 353 | 354 | if (i == option->connlimit) 355 | tmp = -1; 356 | return tmp; 357 | } 358 | 359 | /** 360 | * @brief Close a client and free all data 361 | * 362 | * @see 363 | * @note 364 | * @author auxten 365 | * @date 2011-8-1 366 | **/ 367 | int gko_pool::conn_client_free(struct conn_client *client) 368 | { 369 | if (!client || !client->client_fd) 370 | { 371 | return -1; 372 | } 373 | ///close socket and further receives will be disallowed 374 | // shutdown(client->client_fd, SHUT_RD); 375 | close(client->client_fd); 376 | conn_client_clear(client); 377 | g_total_clients--; 378 | 379 | return 0; 380 | } 381 | 382 | void gko_pool::conn_buffer_init(conn_client *client) 383 | { 384 | gko_pool * Pool = gko_pool::getInstance(); 385 | thread_worker * worker = *(Pool->g_worker_list + client->worker_id); 386 | 387 | client->err_no = INVILID; 388 | #if defined (USE_GKO_MEMORY_POOL) 389 | /// todo calloc every connection comes? 390 | client->r_buf_arena_id = worker->mem.get_block(); 391 | client->w_buf_arena_id = worker->mem.get_block(); 392 | assert(client->r_buf_arena_id >= 0); 393 | assert(client->w_buf_arena_id >= 0); 394 | #endif 395 | 396 | #if defined (USE_GKO_MEMORY_POOL) 397 | client->read_buffer = (char *)worker->mem.id2addr(client->r_buf_arena_id); 398 | #else 399 | client->read_buffer = (char *)malloc(RBUF_SZ); 400 | #endif 401 | client->rbuf_size = RBUF_SZ; 402 | client->have_read = 0; 403 | client->need_read = CMD_PREFIX_BYTE; 404 | 405 | /// todo calloc every connection comes? 406 | #if defined (USE_GKO_MEMORY_POOL) 407 | client->__write_buffer = (char *)worker->mem.id2addr(client->w_buf_arena_id); 408 | #else 409 | client->__write_buffer = (char *)malloc(WBUF_SZ); 410 | #endif 411 | client->write_buffer = client->__write_buffer + CMD_PREFIX_BYTE; 412 | client->wbuf_size = WBUF_SZ - CMD_PREFIX_BYTE; 413 | client->have_write = 0; 414 | client->__need_write = CMD_PREFIX_BYTE; 415 | client->need_write = 0; 416 | } 417 | 418 | /** 419 | * @brief Empty client struct data 420 | * 421 | * @see 422 | * @note 423 | * @author auxten 424 | * @date 2011-8-1 425 | **/ 426 | int gko_pool::conn_client_clear(struct conn_client *client) 427 | { 428 | if (client) 429 | { 430 | /// Clear all data 431 | client->client_fd = 0; 432 | client->client_addr = 0; 433 | client->client_port = 0; 434 | client->err_no = SUCC; 435 | thread_worker * worker = *(g_worker_list + client->worker_id); 436 | 437 | /// todo free every connection comes? 438 | if (client->read_buffer) 439 | { 440 | #if defined (USE_GKO_MEMORY_POOL) 441 | if(client->r_buf_arena_id >= 0) 442 | { 443 | worker->mem.free_block(client->r_buf_arena_id); 444 | client->r_buf_arena_id = INVILID_BLOCK; 445 | client->read_buffer = NULL; 446 | } 447 | else 448 | { 449 | #endif 450 | 451 | free(client->read_buffer); 452 | client->read_buffer = NULL; 453 | #if defined (USE_GKO_MEMORY_POOL) 454 | } 455 | #endif 456 | } 457 | client->rbuf_size = RBUF_SZ; 458 | client->have_read = 0; 459 | client->need_read = CMD_PREFIX_BYTE; 460 | 461 | /// todo free every connection comes? 462 | if (client->__write_buffer) 463 | { 464 | #if defined (USE_GKO_MEMORY_POOL) 465 | if(client->w_buf_arena_id >= 0) 466 | { 467 | worker->mem.free_block(client->w_buf_arena_id); 468 | client->w_buf_arena_id = INVILID_BLOCK; 469 | client->__write_buffer = NULL; 470 | } 471 | else 472 | { 473 | #endif 474 | free(client->__write_buffer); 475 | client->__write_buffer = NULL; 476 | #if defined (USE_GKO_MEMORY_POOL) 477 | } 478 | #endif 479 | } 480 | client->wbuf_size = WBUF_SZ - CMD_PREFIX_BYTE; 481 | client->have_write = 0; 482 | client->__need_write = CMD_PREFIX_BYTE; 483 | client->need_write = 0; 484 | 485 | /// Delete event 486 | event_del(&client->event); 487 | 488 | /// Delete DNS event 489 | del_dns_event(client); 490 | /** 491 | * this is the flag of client usage, 492 | * we must put it in the last place 493 | */ 494 | client->conn_time = 0; 495 | return 0; 496 | } 497 | return -1; 498 | } 499 | 500 | int gko_pool::conn_renew(struct conn_client *client) 501 | { 502 | if (client) 503 | { 504 | /** 505 | * this is the flag of client usage, 506 | * we must put it in the last place 507 | */ 508 | client->conn_time = time(NULL); 509 | /// Clear all data 510 | client->err_no = SUCC; 511 | client->task_id = -1; 512 | client->sub_task_id = -1; 513 | thread_worker * worker = *(g_worker_list + client->worker_id); 514 | 515 | if (client->read_buffer) 516 | { 517 | #if defined (USE_GKO_MEMORY_POOL) 518 | if(client->r_buf_arena_id >= 0) 519 | { 520 | worker->mem.free_block(client->r_buf_arena_id); 521 | client->r_buf_arena_id = INVILID_BLOCK; 522 | client->read_buffer = NULL; 523 | } 524 | else 525 | { 526 | #endif 527 | free(client->read_buffer); 528 | client->read_buffer = NULL; 529 | #if defined (USE_GKO_MEMORY_POOL) 530 | } 531 | #endif 532 | } 533 | 534 | #if defined (USE_GKO_MEMORY_POOL) 535 | client->r_buf_arena_id = worker->mem.get_block(); 536 | client->read_buffer = (char *)worker->mem.id2addr(client->r_buf_arena_id); 537 | #else 538 | client->read_buffer = (char *)malloc(RBUF_SZ); 539 | #endif 540 | 541 | client->rbuf_size = RBUF_SZ; 542 | client->have_read = 0; 543 | client->need_read = CMD_PREFIX_BYTE; 544 | 545 | /// todo free every connection comes? 546 | if (client->__write_buffer) 547 | { 548 | #if defined (USE_GKO_MEMORY_POOL) 549 | if(client->w_buf_arena_id >= 0) 550 | { 551 | worker->mem.free_block(client->w_buf_arena_id); 552 | client->w_buf_arena_id = INVILID_BLOCK; 553 | client->__write_buffer = NULL; 554 | } 555 | else 556 | { 557 | #endif 558 | free(client->__write_buffer); 559 | client->__write_buffer = NULL; 560 | #if defined (USE_GKO_MEMORY_POOL) 561 | } 562 | #endif 563 | } 564 | 565 | #if defined (USE_GKO_MEMORY_POOL) 566 | client->w_buf_arena_id = worker->mem.get_block(); 567 | client->__write_buffer = (char *)worker->mem.id2addr(client->w_buf_arena_id); 568 | #else 569 | client->__write_buffer = (char *)malloc(WBUF_SZ); 570 | #endif 571 | client->write_buffer = client->__write_buffer + CMD_PREFIX_BYTE; 572 | 573 | client->wbuf_size = WBUF_SZ - CMD_PREFIX_BYTE; 574 | client->have_write = 0; 575 | client->__need_write = CMD_PREFIX_BYTE; 576 | client->need_write = 0; 577 | 578 | 579 | return 0; 580 | } 581 | 582 | return -1; 583 | } 584 | /** 585 | * @brief Get client object from pool by given client_id 586 | * 587 | * @see 588 | * @note 589 | * @author auxten 590 | * @date 2011-8-1 591 | **/ 592 | struct conn_client * gko_pool::conn_client_list_get(int id) 593 | { 594 | return g_client_list[id]; 595 | } 596 | 597 | 598 | gko_pool::gko_pool(const int pt) 599 | : 600 | g_curr_thread(0), 601 | g_curr_conn(0), 602 | g_server(NULL), 603 | port(pt), 604 | pHandler(NULL), 605 | reportHandler(NULL), 606 | HMTRHandler(defaultHMTRHandler) 607 | { 608 | DNSDict = init_dns_cache(); 609 | g_ev_base = (struct event_base*)event_init(); 610 | if (!g_ev_base) 611 | { 612 | GKOLOG(FATAL, "event init failed"); 613 | } 614 | } 615 | 616 | gko_pool::gko_pool() 617 | : 618 | g_curr_thread(0), 619 | g_curr_conn(0), 620 | g_server(NULL), 621 | port(-1), 622 | pHandler(NULL), 623 | reportHandler(NULL), 624 | HMTRHandler(defaultHMTRHandler) 625 | { 626 | DNSDict = init_dns_cache(); 627 | g_ev_base = (struct event_base*)event_init(); 628 | if (!g_ev_base) 629 | { 630 | GKOLOG(FATAL, "event init failed"); 631 | } 632 | } 633 | 634 | int gko_pool::getPort() const 635 | { 636 | return port; 637 | } 638 | 639 | void gko_pool::setPort(int port) 640 | { 641 | this->port = port; 642 | } 643 | 644 | s_option_t *gko_pool::getOption() const 645 | { 646 | return option; 647 | } 648 | 649 | void gko_pool::setOption(s_option_t *option) 650 | { 651 | this->option = option; 652 | } 653 | 654 | void gko_pool::setProcessHandler(ProcessHandler_t process_func) 655 | { 656 | this->pHandler = process_func; 657 | } 658 | 659 | void gko_pool::setReportHandler(ReportHandler_t report_func) 660 | { 661 | this->reportHandler = report_func; 662 | } 663 | 664 | /** 665 | * @brief 666 | * set "How Many To Read" handler 667 | * 668 | * @see 669 | * @note 670 | * the handler prototype is as: 671 | * int (*HMTRHandler_t)(void *, const char *, const int); 672 | * 673 | * @ret: 674 | * n == 0 --> need read more 675 | * n > 0 --> need read n byte(s) in total, NOT n byte(s) more!! 676 | * n < 0 --> an error in the message 677 | * 678 | * @args: 679 | * void * --> for the specified connection 680 | * const char * --> the data already read 681 | * const int --> the data count already read, in byte 682 | * 683 | * @author auxten 684 | * @date 2012-9-29 685 | **/ 686 | void gko_pool::setHMTRHandler(HMTRHandler_t HMTR_func) 687 | { 688 | this->HMTRHandler = HMTR_func; 689 | } 690 | 691 | gko_pool *gko_pool::getInstance() 692 | { if (! _instance) 693 | { 694 | pthread_mutex_lock(&instance_lock); 695 | if (!_instance) 696 | { 697 | _instance = new gko_pool(); 698 | } 699 | pthread_mutex_unlock(&instance_lock); 700 | } 701 | return _instance; 702 | } 703 | 704 | int gko_pool::gko_run() 705 | { 706 | 707 | if (port >= 0 && gko_async_server_base_init() < 0) 708 | { 709 | GKOLOG(FATAL, "gko_async_server_base_init failed"); 710 | return -2; 711 | } 712 | 713 | if (conn_client_list_init() < 1) 714 | { 715 | GKOLOG(FATAL, "conn_client_list_init failed"); 716 | return -3; 717 | } 718 | if (thread_init() != 0) 719 | { 720 | GKOLOG(FATAL, FLF("thread_init failed")); 721 | return -4; 722 | } 723 | 724 | if (sig_watcher(int_worker) != 0) 725 | { 726 | GKOLOG(FATAL, "signal watcher start error");//todo 727 | gko_quit(1); 728 | } 729 | 730 | return event_base_loop(g_ev_base, 0); 731 | } 732 | -------------------------------------------------------------------------------- /src/async_dns.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * async_dns.cpp 3 | * 4 | * Created on: May 22, 2012 5 | * Author: auxten 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | 14 | #include "event.h" 15 | #include "ares.h" 16 | #include "dict.h" 17 | #include "gko.h" 18 | 19 | static const time_t DNS_EXPIRE_TIME = -1; 20 | 21 | typedef struct DNSCacheVal 22 | { 23 | time_t expire_time; 24 | in_addr_t addr; 25 | } DNSCacheVal; 26 | 27 | /* And a case insensitive str hash function (based on dictGenCaseHashFunction hash) */ 28 | unsigned int dictGenStrCaseHashFunction(const void *p) 29 | { 30 | unsigned char *buf = (unsigned char *) p; 31 | unsigned int hash = (unsigned int) DICT_HASH_FUNCTION_SEED; 32 | 33 | while (*buf != '\0') 34 | hash = ((hash << 5) + hash) + (tolower(*buf++)); /* hash * 33 + c */ 35 | return hash; 36 | } 37 | 38 | int dictStrCaseKeyCompare(void *privdata, const void *key1, const void *key2) 39 | { 40 | DICT_NOTUSED(privdata); 41 | return strcasecmp((const char *) key1, (const char *) key2) == 0; 42 | } 43 | 44 | void * strKeyDup(void *privdata, const void *key) 45 | { 46 | DICT_NOTUSED(privdata); 47 | return (void *) strdup((const char *) key); 48 | } 49 | 50 | void strKeyDestructor(void *privdata, void *key) 51 | { 52 | DICT_NOTUSED(privdata); 53 | free(key); 54 | } 55 | 56 | void strValDestructor(void *privdata, void *key) 57 | { 58 | DICT_NOTUSED(privdata); 59 | free(key); 60 | } 61 | 62 | dictType DNSDictType = 63 | { 64 | dictGenStrCaseHashFunction, /* hash function */ 65 | strKeyDup, /* key dup */ 66 | NULL, /* val dup */ 67 | dictStrCaseKeyCompare, /* key compare */ 68 | strKeyDestructor, /* key destructor */ 69 | strValDestructor /* val destructor */ 70 | }; 71 | 72 | dict * gko_pool::init_dns_cache(void) 73 | { 74 | return dictCreate(&DNSDictType, NULL); 75 | } 76 | 77 | /** 78 | * try if hostname hit the cache, so we save a query 79 | * @param conn_client * 80 | * @return if hit cache return 0, elsewise -1 81 | */ 82 | int gko_pool::try_dns_cache(conn_client *c) 83 | { 84 | int ret; 85 | if (DNS_EXPIRE_TIME <= 0) 86 | { 87 | ret = -1; 88 | return ret; 89 | } 90 | 91 | DNSCacheVal *val; 92 | time_t now = time(NULL); 93 | 94 | pthread_mutex_lock(&DNS_cache_lock); 95 | val = (DNSCacheVal *)dictFetchValue(DNSDict, c->client_hostname); 96 | if (val != NULL) 97 | { 98 | /// hit ! 99 | if (val->expire_time < now) 100 | { 101 | /// TTL expire 102 | dictDelete(DNSDict, c->client_hostname); 103 | ret = -1; 104 | } 105 | else 106 | { 107 | /// within TTL 108 | c->client_addr = val->addr; 109 | ret = 0; 110 | } 111 | } 112 | else 113 | { 114 | /// not hit 115 | ret = -1; 116 | } 117 | pthread_mutex_unlock(&DNS_cache_lock); 118 | return ret; 119 | } 120 | 121 | void gko_pool::update_dns_cache(conn_client *c, in_addr_t addr) 122 | { 123 | if (DNS_EXPIRE_TIME <= 0) 124 | { 125 | return; 126 | } 127 | pthread_mutex_lock(&DNS_cache_lock); 128 | DNSCacheVal * val = (DNSCacheVal *)malloc(sizeof (DNSCacheVal)); 129 | val->addr = addr; 130 | val->expire_time = time(NULL) + DNS_EXPIRE_TIME; /// 131 | dictReplace(DNSDict, c->client_hostname, val); 132 | pthread_mutex_unlock(&DNS_cache_lock); 133 | } 134 | 135 | int gko_pool::del_dns_event(conn_client *c) 136 | { 137 | for (std::vector::iterator it = c->ev_dns_vec.begin(); 138 | it != c->ev_dns_vec.end(); 139 | ++it) 140 | { 141 | event_del(*it); 142 | free(*it); 143 | } 144 | c->ev_dns_vec.clear(); 145 | return 0; 146 | } 147 | 148 | void gko_pool::dns_callback(void* arg, int status, int timeouts, struct hostent* host) 149 | { 150 | conn_client *c = (conn_client *) arg; 151 | 152 | if (status == ARES_SUCCESS) 153 | { 154 | u_int32_t addr = *(in_addr_t *) host->h_addr; 155 | c->client_addr = addr; 156 | gko_pool::getInstance()->update_dns_cache(c, addr); 157 | conn_set_state(c, conn_connecting); 158 | GKOLOG(DEBUG, "DNS OK %s ==> %u.%u.%u.%u", 159 | host->h_name, 160 | (addr) % 256, (addr >> 8) % 256, (addr >> 16) % 256, (addr >> 24) % 256); 161 | } 162 | else 163 | { 164 | c->err_no = DNS_RESOLVE_FAIL; 165 | conn_set_state(c, conn_closing); 166 | GKOLOG(WARNING, "lookup failed: %s", ares_strerror(status)); 167 | } 168 | del_dns_event(c); 169 | 170 | /// call my lovly state_machine~~~ mua~ 171 | state_machine(c); 172 | } 173 | 174 | void gko_pool::dns_ev_callback(int fd, short ev, void *arg) 175 | { 176 | gko_pool * Pool = gko_pool::getInstance(); 177 | conn_client *c = (conn_client *) arg; 178 | thread_worker * w = *(Pool->g_worker_list + c->worker_id); 179 | 180 | if (c->state == conn_closing) 181 | { 182 | GKOLOG(FATAL, "closing socket have DNS event"); 183 | del_dns_event(c); 184 | state_machine(c); 185 | return; 186 | } 187 | ares_channel ch = w->dns_channel; 188 | if (ev & EV_READ) 189 | ares_process_fd(ch, fd, ARES_SOCKET_BAD); 190 | if (ev & EV_WRITE) 191 | ares_process_fd(ch, ARES_SOCKET_BAD, fd); 192 | if (ev & EV_TIMEOUT) 193 | { 194 | ares_process_fd(ch, fd, fd); 195 | c->err_no = DNS_RESOLVE_FAIL; 196 | conn_set_state(c, conn_closing); 197 | GKOLOG(FATAL, "dns timeout"); 198 | del_dns_event(c); 199 | state_machine(c); 200 | } 201 | } 202 | 203 | void gko_pool::nb_gethostbyname(conn_client *c) 204 | { 205 | ares_socket_t *read_fds; 206 | ares_socket_t *write_fds; 207 | // struct timeval timeout = {5, 0}; 208 | 209 | // gko_pool * Pool = gko_pool::getInstance(); 210 | thread_worker * worker = *(g_worker_list + c->worker_id); 211 | 212 | ares_gethostbyname(worker->dns_channel, c->client_hostname, AF_INET, dns_callback, (void *) c); 213 | ares_fds_array(worker->dns_channel, &read_fds, &write_fds); 214 | 215 | if ((*read_fds == *write_fds) && (*write_fds == ARES_SOCKET_BAD)) 216 | { 217 | // GKOLOG(DEBUG, "error no fd"); 218 | del_dns_event(c); 219 | free(read_fds); 220 | free(write_fds); 221 | return; 222 | } 223 | 224 | for (int i = 0; *(read_fds + i) != ARES_SOCKET_BAD; i++) 225 | { 226 | struct event * ev_tmp = (struct event *) malloc(sizeof(struct event)); 227 | event_set(ev_tmp, *(read_fds + i), EV_READ | EV_PERSIST, dns_ev_callback, 228 | (void *) (c)); 229 | event_base_set(worker->ev_base, ev_tmp); 230 | event_add(ev_tmp, NULL); 231 | 232 | c->ev_dns_vec.push_back(ev_tmp); 233 | } 234 | for (int i = 0; *(write_fds + i) != ARES_SOCKET_BAD; i++) 235 | { 236 | struct event * ev_tmp = (struct event *) malloc(sizeof(struct event)); 237 | event_set(ev_tmp, *(write_fds + i), EV_WRITE | EV_PERSIST, dns_ev_callback, 238 | (void *) (c)); 239 | event_base_set(worker->ev_base, ev_tmp); 240 | event_add(ev_tmp, NULL); 241 | 242 | c->ev_dns_vec.push_back(ev_tmp); 243 | } 244 | 245 | free(read_fds); 246 | free(write_fds); 247 | } 248 | -------------------------------------------------------------------------------- /src/async_pool.h: -------------------------------------------------------------------------------- 1 | /** 2 | * async_pool.h 3 | * gingko 4 | * 5 | * Created on: Mar 9, 2012 6 | * Author: auxten 7 | * 8 | **/ 9 | #ifndef ASYNC_POOL_H_ 10 | #define ASYNC_POOL_H_ 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #include 29 | 30 | #include "ares.h" 31 | #include "event.h" 32 | #include "memory.h" 33 | #include "dict.h" 34 | 35 | #include "gko_errno.h" 36 | 37 | static const int RBUF_SZ = SLOT_SIZE; 38 | static const int WBUF_SZ = SLOT_SIZE; 39 | static const u_int8_t F_KEEPALIVE = 0x01; 40 | static const u_int8_t F_CONN_REUSE = 0x02; 41 | 42 | /// Thread worker 43 | class thread_worker 44 | { 45 | public: 46 | int id; 47 | pthread_t tid; 48 | struct event_base *ev_base; 49 | struct event ev_notify; 50 | struct event ev_cleantimeout; 51 | int notify_recv_fd; 52 | int notify_send_fd; 53 | std::set conn_set; 54 | ares_channel dns_channel; 55 | gkoAlloc mem; 56 | void * userData; 57 | 58 | /// put conn into current thread conn_set 59 | void add_conn(int c_id); 60 | /// del conn from current thread conn_set 61 | void del_conn(int c_id); 62 | }; 63 | 64 | enum conn_states { 65 | conn_nouse = 0, 66 | conn_listening, /**< now the main schedule thread do the accept */ 67 | conn_waiting, /**< waiting for a readable socket */ 68 | conn_read, /**< reading the cmd header */ 69 | conn_nread, /**< reading in a fixed number of bytes */ 70 | conn_parse_header, /**< try to parse the cmd header */ 71 | conn_parse_cmd, /**< try to parse a command from the input buffer */ 72 | conn_write, /**< writing out a simple response */ 73 | conn_mwrite, /**< writing out many items sequentially */ 74 | conn_state_renew, /**< reset conn for following requests(long connection) */ 75 | conn_closing, /**< closing this connection */ 76 | 77 | /// for client side 78 | conn_connecting = 101, 79 | conn_dns_cache, /**< try internal DNS cache */ 80 | conn_resolving, /**< resoving DNS */ 81 | 82 | conn_max_state /**< Max state value (used for assertion) */ 83 | }; 84 | 85 | enum conn_type { 86 | coming_conn = 0, /// this is default 87 | active_conn = 1 88 | }; 89 | 90 | /// Connection client 91 | struct conn_client 92 | { 93 | int id; 94 | int worker_id; 95 | int client_fd; 96 | long task_id; 97 | long sub_task_id; 98 | in_addr_t client_addr; 99 | int client_port; 100 | time_t conn_time; 101 | func_t handle_client; 102 | struct event event; 103 | std::vector ev_dns_vec; 104 | enum conn_states state; 105 | enum error_no err_no; 106 | enum conn_type type; 107 | int ev_flags; 108 | 109 | void * userData; 110 | 111 | int r_buf_arena_id; 112 | char *read_buffer; 113 | unsigned int rbuf_size; 114 | unsigned int need_read; 115 | unsigned int have_read; 116 | 117 | int w_buf_arena_id; 118 | char *__write_buffer; 119 | char *write_buffer; 120 | unsigned int wbuf_size; 121 | unsigned int __need_write; 122 | unsigned int need_write; 123 | unsigned int have_write; 124 | char client_hostname[MAX_HOST_NAME + 1]; 125 | 126 | }; 127 | 128 | /// Connection server struct 129 | struct conn_server 130 | { 131 | char is_server; 132 | int listen_fd; 133 | struct sockaddr_in listen_addr; 134 | in_addr_t srv_addr; 135 | int srv_port; 136 | unsigned int start_time; 137 | int nonblock; 138 | int listen_queue_length; 139 | int tcp_send_buffer_size; 140 | int tcp_recv_buffer_size; 141 | int send_timeout; 142 | int tcp_reuse; 143 | int tcp_nodelay; 144 | struct event ev_accept; 145 | //void *(* on_data_callback)(void *); 146 | ProcessHandler_t on_data_callback; 147 | }; 148 | 149 | enum aread_result { 150 | READ_DATA_RECEIVED, 151 | READ_HEADER_RECEIVED, 152 | READ_NEED_MORE, 153 | READ_NO_DATA_RECEIVED, 154 | READ_ERROR, /** an error occured (on the socket) (or client closed connection) */ 155 | READ_MEMORY_ERROR /** failed to allocate more memory */ 156 | }; 157 | 158 | enum awrite_result { 159 | WRITE_DATA_SENT, 160 | WRITE_HEADER_SENT, 161 | WRITE_SENT_MORE, 162 | WRITE_NO_DATA_SENT, 163 | WRITE_ERROR /** an error occured (on the socket) (or client closed connection) */ 164 | }; 165 | 166 | /// default HMTR handler for "0000000005HELLO" style msg 167 | int defaultHMTRHandler(void * p, const char * buf, const int len); 168 | 169 | 170 | class gko_pool 171 | { 172 | private: 173 | static gko_pool * _instance; 174 | /// global lock 175 | static pthread_mutex_t instance_lock; 176 | static pthread_mutex_t conn_list_lock; 177 | static pthread_mutex_t thread_list_lock; 178 | static pthread_mutex_t DNS_cache_lock; 179 | 180 | int g_curr_thread; 181 | int g_curr_conn; 182 | 183 | thread_worker ** g_worker_list; 184 | struct event_base *g_ev_base; 185 | int g_total_clients; 186 | int g_total_connect; 187 | struct conn_client **g_client_list; 188 | struct conn_server *g_server; 189 | int port; 190 | s_option_t * option; 191 | ProcessHandler_t pHandler; 192 | ReportHandler_t reportHandler; 193 | HMTRHandler_t HMTRHandler; 194 | dict * DNSDict; 195 | 196 | static void conn_send_data(void *c); 197 | /// Accept new connection 198 | static void conn_tcp_server_accept(int fd, short ev, void *arg); 199 | /// Close conn, shutdown && close 200 | static void * thread_worker_init(void *arg); 201 | /// Close conn, shutdown && close 202 | static void thread_worker_process(int fd, short ev, void *arg); 203 | /// Connection buffer init 204 | static void conn_buffer_init(conn_client *client); 205 | /// ReAdd the event 206 | static bool update_event(conn_client *c, const int new_flags); 207 | /// State updater 208 | static void conn_set_state(conn_client *c, enum conn_states state); 209 | /// Event handler for worker thread 210 | static void worker_event_handler(const int fd, const short which, void *arg); 211 | /// Event handler for clean up timeout conn 212 | static void clean_handler(const int fd, const short which, void *arg); 213 | /// Async read 214 | static enum aread_result aread(conn_client *c); 215 | /// Async write 216 | static enum awrite_result awrite(conn_client *c); 217 | /// The drive machine of memcached 218 | static void state_machine(conn_client *c); 219 | 220 | /// non-blocking version connect 221 | int nb_connect(struct conn_client* conn); 222 | int connect_hosts(const std::vector & host_vec, 223 | std::vector * conn_vec); 224 | int disconnect_hosts(std::vector & conn_vec); 225 | 226 | /// DNS cache 227 | dict * init_dns_cache(void); 228 | int try_dns_cache(conn_client *c); 229 | void update_dns_cache(conn_client *c, in_addr_t addr); 230 | 231 | 232 | /// non-blocking DNS 233 | static int del_dns_event(conn_client *c); 234 | static void dns_callback(void* arg, int status, int timeouts, struct hostent* host); 235 | static void dns_ev_callback(int fd, short ev, void *arg); 236 | void nb_gethostbyname(conn_client *c); 237 | 238 | int clean_conn_timeout(thread_worker *worker, time_t now); 239 | int thread_worker_new(int id); 240 | int thread_list_find_next(void); 241 | int conn_client_list_init(void); 242 | int gko_async_server_base_init(void); 243 | /// Accept new connection, start listen etc. 244 | int conn_tcp_server(struct conn_server *c); 245 | /// Accept new connection 246 | struct conn_client * add_new_conn_client(int client_fd); 247 | /// Event on data from client 248 | int conn_client_list_find_free(); 249 | /// reset conn for following requests(long connection) 250 | int conn_renew(struct conn_client *client); 251 | /// clear client struct 252 | int conn_client_clear(struct conn_client *client); 253 | /// clear the "session" 254 | int conn_client_free(struct conn_client *client); 255 | /// Get client object from pool by given client_id 256 | struct conn_client * conn_client_list_get(int id); 257 | /// Dispatch to worker to a thread 258 | void thread_worker_dispatch(int c_id); 259 | /// Dispatch to worker to the thread 260 | void thread_worker_dispatch(int c_id, int worker_id); 261 | /// init the whole thread pool 262 | int thread_init(); 263 | /// construct func 264 | gko_pool(const int pt); 265 | /// another construct func 266 | gko_pool(); 267 | 268 | public: 269 | static s_host_t gko_serv; 270 | 271 | static gko_pool *getInstance(); 272 | 273 | void setProcessHandler(ProcessHandler_t func_list); 274 | void setReportHandler(ReportHandler_t report_func); 275 | void setHMTRHandler(HMTRHandler_t HMTR_func); 276 | int getPort() const; 277 | void setPort(int port); 278 | s_option_t *getOption() const; 279 | void setOption(s_option_t *option); 280 | 281 | thread_worker * getWorker(const struct conn_client *); 282 | /// global run func 283 | int gko_run(); 284 | int gko_loopexit(int timeout); 285 | 286 | int make_active_connect(const char * host, const int port, int len, const char * cmd, 287 | const long task_id = -1, const long sub_task_id = -1, 288 | const u_int8_t flag = 0, const int wrote = 0); 289 | 290 | }; 291 | 292 | #endif /** ASYNC_POOL_H_ **/ 293 | -------------------------------------------------------------------------------- /src/autogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | if [ -x "`which autoreconf 2>/dev/null`" ] ; then 3 | exec autoreconf --verbose --force -i 4 | fi 5 | 6 | LIBTOOLIZE=libtoolize 7 | SYSNAME=`uname` 8 | if [ "x$SYSNAME" = "xDarwin" ] ; then 9 | LIBTOOLIZE=glibtoolize 10 | fi 11 | aclocal && \ 12 | autoheader && \ 13 | autoconf && \ 14 | automake --add-missing --force-missing --copy 15 | -------------------------------------------------------------------------------- /src/configure.in: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([1.68]) 5 | AC_INIT([libgko.a], [3.14], [blog.optool.net]) 6 | AC_CONFIG_SRCDIR([config.h.in]) 7 | AM_CONFIG_HEADER([config.h]) 8 | #AC_CONFIG_HEADERS([config.h]) 9 | AM_INIT_AUTOMAKE([libgko.a],3.14) 10 | 11 | ########################################################################## 12 | # debug compilation support 13 | ########################################################################## 14 | 15 | AC_MSG_CHECKING([whether to build with debug information]) 16 | AC_ARG_ENABLE([debug], 17 | [AS_HELP_STRING([--enable-debug], 18 | [enable debug data generation (def=no)])], 19 | [debugit="$enableval"], 20 | [debugit=no]) 21 | AC_MSG_RESULT([$debugit]) 22 | 23 | if test x"$debugit" = x"yes"; then 24 | AC_DEFINE([GKO_DEBUG],[],[Debug Mode]) 25 | AM_CXXFLAGS="-ggdb -Wall -O0" 26 | CXXFLAGS="-ggdb -Wall -O0" 27 | else 28 | AC_DEFINE([NGKO_DEBUG],[],[No-debug Mode]) 29 | AM_CXXFLAGS="-ggdb -Wall -O2 -march=nocona" 30 | CXXFLAGS="-ggdb -Wall -O2 -march=nocona" 31 | fi 32 | 33 | AC_SUBST([AM_CXXFLAGS]) 34 | ########################################################################## 35 | 36 | ########################################################################## 37 | # unittest compilation support 38 | ########################################################################## 39 | 40 | AC_MSG_CHECKING([whether to build unittest version]) 41 | AC_ARG_ENABLE([unittest], 42 | [AS_HELP_STRING([--enable-unittest], 43 | [enable unittest code generation (def=no)])], 44 | [unittestit="$enableval"], 45 | [unittestit=no]) 46 | AC_MSG_RESULT([$unittestit]) 47 | 48 | if test x"$unittestit" = x"yes"; then 49 | AC_DEFINE([UNITTEST],[],[Unittest Mode]) 50 | AM_CXXFLAGS="-ggdb -Wall" 51 | CXXFLAGS="-ggdb -Wall" 52 | else 53 | AC_DEFINE([NUNITTEST],[],[No-unittest Mode]) 54 | AM_CXXFLAGS="$AM_CXXFLAGS" 55 | fi 56 | 57 | AC_SUBST([AM_CXXFLAGS]) 58 | ########################################################################## 59 | 60 | ########################################################################## 61 | # profile compilation support 62 | ########################################################################## 63 | 64 | AC_MSG_CHECKING([whether to build with profile information]) 65 | AC_ARG_ENABLE([profile], 66 | [AS_HELP_STRING([--enable-profile], 67 | [enable profile code generation (def=no)])], 68 | [profileit="$enableval"], 69 | [profileit=no]) 70 | AC_MSG_RESULT([$profileit]) 71 | 72 | if test x"$profileit" = x"yes"; then 73 | AC_DEFINE([PROFILE],[],[Profile Mode]) 74 | AM_CXXFLAGS="-ggdb -Wall -lprofiler" 75 | else 76 | AC_DEFINE([NPROFILE],[],[No-profile Mode]) 77 | AM_CXXFLAGS="$AM_CXXFLAGS" 78 | fi 79 | 80 | AC_SUBST([AM_CXXFLAGS]) 81 | ########################################################################## 82 | 83 | ########################################################################## 84 | # gprofile compilation support 85 | ########################################################################## 86 | 87 | AC_MSG_CHECKING([whether to build with gprofile information]) 88 | AC_ARG_ENABLE([gprofile], 89 | [AS_HELP_STRING([--enable-gprofile], 90 | [enable gprofile code generation (def=no)])], 91 | [gprofileit="$enableval"], 92 | [gprofileit=no]) 93 | AC_MSG_RESULT([$gprofileit]) 94 | 95 | if test x"$gprofileit" = x"yes"; then 96 | AC_DEFINE([GPROFILE],[],[Profile Mode]) 97 | AM_CXXFLAGS="-ggdb -Wall -pg" 98 | else 99 | AC_DEFINE([NGPROFILE],[],[No-gprofile Mode]) 100 | AM_CXXFLAGS="$AM_CXXFLAGS" 101 | fi 102 | 103 | AC_SUBST([AM_CXXFLAGS]) 104 | ########################################################################## 105 | 106 | ########################################################################## 107 | # O3 optimize compilation support 108 | ########################################################################## 109 | 110 | AC_MSG_CHECKING([whether to build with O3 optimization]) 111 | AC_ARG_ENABLE([o3], 112 | [AS_HELP_STRING([--enable-o3], 113 | [enable O3 code generation (def=no)])], 114 | [o3it="$enableval"], 115 | [o3it=no]) 116 | AC_MSG_RESULT([$o3it]) 117 | 118 | if test x"$o3it" = x"yes"; then 119 | AC_DEFINE([O3],[],[O3 Mode]) 120 | AM_CXXFLAGS="-ggdb -Wall -O3 -march=nocona" 121 | CXXFLAGS="-ggdb -Wall -O3 -march=nocona" 122 | else 123 | AC_DEFINE([NO3],[],[No-O3 Mode]) 124 | AM_CXXFLAGS="$AM_CXXFLAGS" 125 | fi 126 | 127 | AC_SUBST([AM_CXXFLAGS]) 128 | ########################################################################## 129 | 130 | # Checks for programs. 131 | AC_PROG_CXX 132 | AC_PROG_CC 133 | AC_PROG_MAKE_SET 134 | 135 | # Checks for libraries. 136 | AC_SEARCH_LIBS([pthread_rwlock_init], [pthread]) 137 | AC_SEARCH_LIBS([clock_gettime], [rt]) # macos do not hava librt? 138 | AC_PROG_RANLIB 139 | 140 | # Checks for header files. 141 | AC_CHECK_HEADERS([arpa/inet.h fcntl.h netdb.h netinet/in.h stdlib.h string.h sys/socket.h unistd.h]) 142 | 143 | # Checks for typedefs, structures, and compiler characteristics. 144 | AC_TYPE_MODE_T 145 | AC_TYPE_SIZE_T 146 | 147 | # Checks for library functions. 148 | AC_FUNC_FORK 149 | AC_CHECK_FUNCS([gethostbyname memset select socket strerror gettimeofday inet_ntoa poll]) 150 | 151 | ########################################################################### 152 | ## Use jemalloc on Linux 153 | ########################################################################### 154 | #JEMALLOC_SUBDIR= 155 | #JEMALLOC_LDADD= 156 | #AC_ARG_WITH([jemalloc], 157 | # [AS_HELP_STRING([--with-jemalloc], 158 | # [use jemalloc memory allocator. Default is yes on Linux, no elsewhere])], 159 | # [], 160 | # [with_jemalloc=check]) 161 | # 162 | #case $target in 163 | # *-*-linux*) 164 | # if test "x$with_jemalloc" != xno; then 165 | # AC_CHECK_LIB([jemalloc], [malloc_conf], 166 | # [JEMALLOC_LDADD=" -Wl,-ljemalloc"], 167 | # [AC_MSG_NOTICE([No system jemalloc found, using bundled version]) 168 | # JEMALLOC_SUBDIR=libjemalloc 169 | # JEMALLOC_LDADD=" -Wl,`pwd`/../lib/libjemalloc/libjemalloc_mt.a"]) 170 | # fi 171 | # ;; 172 | #esac 173 | #venderlibsinc=${JEMALLOC_LDADD}" "${venderlibsinc} 174 | #AC_SUBST(JEMALLOC_SUBDIR) 175 | #AC_SUBST(JEMALLOC_LDADD) 176 | ########################################################################### 177 | 178 | ########################################################################## 179 | # check if libev exist 180 | ########################################################################## 181 | libev="yes" 182 | libevprefix="`pwd`/../lib/libev" 183 | AC_ARG_WITH(libev,[ --with-libev Compile with given libev library.],[libevprefix=$withval libev="yes"],[]) 184 | # if there is no value given, it appears that libevprefix is set to "yes" 185 | if test $libev = 'yes'; then 186 | libevincs=" -I"${libevprefix}"/include" 187 | venderlibsinc=${venderlibsinc}" -Wl,-L"${libevprefix}"/lib -Wl,-lev" 188 | venderincs=${venderincs}" "${libevincs} 189 | AC_CHECK_HEADERS([${libevprefix}/include/event.h],,[AC_MSG_ERROR([Cannot find event.h, We NEED libev or libevent])]) 190 | else 191 | AC_CHECK_HEADERS([event.h],,[AC_MSG_ERROR([Cannot find event.h])]) 192 | fi 193 | ########################################################################## 194 | 195 | ########################################################################## 196 | # check if c-ares exist 197 | ########################################################################## 198 | libcares="yes" 199 | libcaresprefix="`pwd`/../lib/c-ares" 200 | AC_ARG_WITH(libcares,[ --with-libcares Compile with given libcares library for thread safe and async DNS. BETTER NOT],[libcaresprefix=$withval libcares="yes"],[]) 201 | # if there is no value given, it appears that libcaresprefix is set to "yes" 202 | if test $libcares = 'yes'; then 203 | libcaresincs=" -I"${libcaresprefix}"/include" 204 | venderlibsinc=${venderlibsinc}" -Wl,-L"${libcaresprefix}"/lib -Wl,-lcares" 205 | venderincs=${venderincs}" "${libcaresincs} 206 | AC_CHECK_HEADERS([${libcaresprefix}/include/ares.h],,[AC_MSG_ERROR([Cannot find ares.h, We NEED libcares for async DNS])]) 207 | else 208 | AC_CHECK_HEADERS([ares.h],,[AC_MSG_ERROR([Cannot find ares.h])]) 209 | fi 210 | 211 | AC_SEARCH_LIBS([clock_gettime], [rt], [venderlibsinc=${venderlibsinc}" -Wl,-lrt"]) 212 | ########################################################################## 213 | 214 | AC_SUBST(venderlibsinc) 215 | AC_SUBST(venderincs) 216 | 217 | AM_CXXFLAGS=${libevincs}" "${libcaresincs}" "${AM_CXXFLAGS} 218 | #AC_CONFIG_SUBDIRS([hash]) 219 | #AC_CONFIG_FILES([Makefile hash/Makefile]) 220 | AC_CONFIG_FILES([Makefile libgko.pc]) 221 | AC_OUTPUT 222 | -------------------------------------------------------------------------------- /src/dict.h: -------------------------------------------------------------------------------- 1 | /* Hash Tables Implementation. 2 | * 3 | * This file implements in-memory hash tables with insert/del/replace/find/ 4 | * get-random-element operations. Hash tables will auto-resize if needed 5 | * tables of power of two in size are used, collisions are handled by 6 | * chaining. See the source code for more information... :) 7 | * 8 | * Copyright (c) 2006-2012, Salvatore Sanfilippo 9 | * All rights reserved. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions are met: 13 | * 14 | * * Redistributions of source code must retain the above copyright notice, 15 | * this list of conditions and the following disclaimer. 16 | * * Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * * Neither the name of Redis nor the names of its contributors may be used 20 | * to endorse or promote products derived from this software without 21 | * specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 24 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 27 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33 | * POSSIBILITY OF SUCH DAMAGE. 34 | */ 35 | 36 | #include 37 | 38 | #ifndef __DICT_H 39 | #define __DICT_H 40 | 41 | #ifdef __cplusplus 42 | extern "C" { 43 | #endif 44 | 45 | #define DICT_OK 0 46 | #define DICT_ERR 1 47 | 48 | /* Unused arguments generate annoying warnings... */ 49 | #define DICT_NOTUSED(V) ((void) V) 50 | 51 | #define DICT_HASH_FUNCTION_SEED 5381; 52 | 53 | typedef struct dictEntry 54 | { 55 | void *key; 56 | union 57 | { 58 | void *val; 59 | uint64_t u64; 60 | int64_t s64; 61 | } v; 62 | struct dictEntry *next; 63 | } dictEntry; 64 | 65 | typedef struct dictType 66 | { 67 | unsigned int (*hashFunction)(const void *key); 68 | void *(*keyDup)(void *privdata, const void *key); 69 | void *(*valDup)(void *privdata, const void *obj); 70 | int (*keyCompare)(void *privdata, const void *key1, const void *key2); 71 | void (*keyDestructor)(void *privdata, void *key); 72 | void (*valDestructor)(void *privdata, void *obj); 73 | } dictType; 74 | 75 | /* This is our hash table structure. Every dictionary has two of this as we 76 | * implement incremental rehashing, for the old to the new table. */ 77 | typedef struct dictht 78 | { 79 | dictEntry **table; 80 | unsigned long size; 81 | unsigned long sizemask; 82 | unsigned long used; 83 | } dictht; 84 | 85 | typedef struct dict 86 | { 87 | dictType *type; 88 | void *privdata; 89 | dictht ht[2]; 90 | int rehashidx; /* rehashing not in progress if rehashidx == -1 */ 91 | int iterators; /* number of iterators currently running */ 92 | } dict; 93 | 94 | /* If safe is set to 1 this is a safe iterator, that means, you can call 95 | * dictAdd, dictFind, and other functions against the dictionary even while 96 | * iterating. Otherwise it is a non safe iterator, and only dictNext() 97 | * should be called while iterating. */ 98 | typedef struct dictIterator 99 | { 100 | dict *d; 101 | int table, index, safe; 102 | dictEntry *entry, *nextEntry; 103 | } dictIterator; 104 | 105 | /* This is the initial size of every hash table */ 106 | #define DICT_HT_INITIAL_SIZE 4 107 | 108 | /* ------------------------------- Macros ------------------------------------*/ 109 | #define dictFreeVal(d, entry) \ 110 | if ((d)->type->valDestructor) \ 111 | (d)->type->valDestructor((d)->privdata, (entry)->v.val) 112 | 113 | #define dictSetVal(d, entry, _val_) do { \ 114 | if ((d)->type->valDup) \ 115 | entry->v.val = (d)->type->valDup((d)->privdata, _val_); \ 116 | else \ 117 | entry->v.val = (_val_); \ 118 | } while(0) 119 | 120 | #define dictSetSignedIntegerVal(entry, _val_) \ 121 | do { entry->v.s64 = _val_; } while(0) 122 | 123 | #define dictSetUnsignedIntegerVal(entry, _val_) \ 124 | do { entry->v.u64 = _val_; } while(0) 125 | 126 | #define dictFreeKey(d, entry) \ 127 | if ((d)->type->keyDestructor) \ 128 | (d)->type->keyDestructor((d)->privdata, (entry)->key) 129 | 130 | #define dictSetKey(d, entry, _key_) do { \ 131 | if ((d)->type->keyDup) \ 132 | entry->key = (d)->type->keyDup((d)->privdata, _key_); \ 133 | else \ 134 | entry->key = (_key_); \ 135 | } while(0) 136 | 137 | #define dictCompareKeys(d, key1, key2) \ 138 | (((d)->type->keyCompare) ? \ 139 | (d)->type->keyCompare((d)->privdata, key1, key2) : \ 140 | (key1) == (key2)) 141 | 142 | #define dictHashKey(d, key) (d)->type->hashFunction(key) 143 | #define dictGetKey(he) ((he)->key) 144 | #define dictGetVal(he) ((he)->v.val) 145 | #define dictGetSignedIntegerVal(he) ((he)->v.s64) 146 | #define dictGetUnsignedIntegerVal(he) ((he)->v.u64) 147 | #define dictSlots(d) ((d)->ht[0].size+(d)->ht[1].size) 148 | #define dictSize(d) ((d)->ht[0].used+(d)->ht[1].used) 149 | #define dictIsRehashing(ht) ((ht)->rehashidx != -1) 150 | 151 | /* API */ 152 | dict *dictCreate(dictType *type, void *privDataPtr); 153 | int dictExpand(dict *d, unsigned long size); 154 | int dictAdd(dict *d, void *key, void *val); 155 | dictEntry *dictAddRaw(dict *d, void *key); 156 | int dictReplace(dict *d, void *key, void *val); 157 | dictEntry *dictReplaceRaw(dict *d, void *key); 158 | int dictDelete(dict *d, const void *key); 159 | int dictDeleteNoFree(dict *d, const void *key); 160 | void dictRelease(dict *d); 161 | dictEntry * dictFind(dict *d, const void *key); 162 | void *dictFetchValue(dict *d, const void *key); 163 | int dictResize(dict *d); 164 | dictIterator *dictGetIterator(dict *d); 165 | dictIterator *dictGetSafeIterator(dict *d); 166 | dictEntry *dictNext(dictIterator *iter); 167 | void dictReleaseIterator(dictIterator *iter); 168 | dictEntry *dictGetRandomKey(dict *d); 169 | void dictPrintStats(dict *d); 170 | unsigned int dictGenHashFunction(const void *key, int len); 171 | unsigned int dictGenCaseHashFunction(const unsigned char *buf, int len); 172 | void dictEmpty(dict *d); 173 | void dictEnableResize(void); 174 | void dictDisableResize(void); 175 | int dictRehash(dict *d, int n); 176 | int dictRehashMilliseconds(dict *d, int ms); 177 | void dictSetHashFunctionSeed(unsigned int initval); 178 | unsigned int dictGetHashFunctionSeed(void); 179 | 180 | /* Hash table types */ 181 | extern dictType dictTypeHeapStringCopyKey; 182 | extern dictType dictTypeHeapStrings; 183 | extern dictType dictTypeHeapStringCopyKeyValue; 184 | 185 | #ifdef __cplusplus 186 | } 187 | # endif 188 | 189 | #endif /* __DICT_H */ 190 | -------------------------------------------------------------------------------- /src/gko.h: -------------------------------------------------------------------------------- 1 | /* 2 | * gko.h 3 | * 4 | * Created on: Feb 27, 2012 5 | * Author: auxten 6 | */ 7 | 8 | #ifndef GKO_H_ 9 | #define GKO_H_ 10 | 11 | #include "gingko.h" 12 | #include "async_pool.h" 13 | #include "log.h" 14 | #include "socket.h" 15 | 16 | 17 | 18 | #endif /* GKO_H_ */ 19 | -------------------------------------------------------------------------------- /src/gko_errno.h: -------------------------------------------------------------------------------- 1 | hermes_errno.h -------------------------------------------------------------------------------- /src/hash/gko_zip.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * gko_zip.cpp 3 | * 4 | * Created on: Jan 11, 2012 5 | * Author: auxten 6 | */ 7 | 8 | 9 | #include "lz4.h" 10 | #include "gko_zip.h" 11 | 12 | int gko_zip(char* source, char* dest, int isize) 13 | { 14 | return LZ4_compress(source, dest, isize); 15 | } 16 | 17 | int gko_unzip(char* source, char* dest, int osize) 18 | { 19 | return LZ4_uncompress(source, dest, osize); 20 | } 21 | -------------------------------------------------------------------------------- /src/hash/gko_zip.h: -------------------------------------------------------------------------------- 1 | /* 2 | * gko_zip.h 3 | * 4 | * Created on: Jan 11, 2012 5 | * Author: auxten 6 | */ 7 | 8 | #ifndef GKO_ZIP_H_ 9 | #define GKO_ZIP_H_ 10 | 11 | int gko_zip(char* source, char* dest, int isize); 12 | int gko_unzip(char* source, char* dest, int osize); 13 | 14 | #endif /* GKO_ZIP_H_ */ 15 | -------------------------------------------------------------------------------- /src/hash/lz4.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 - Fast LZ compression algorithm 3 | Copyright (C) 2011-2012, Yann Collet. 4 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are 8 | met: 9 | 10 | * Redistributions of source code must retain the above copyright 11 | notice, this list of conditions and the following disclaimer. 12 | * Redistributions in binary form must reproduce the above 13 | copyright notice, this list of conditions and the following disclaimer 14 | in the documentation and/or other materials provided with the 15 | distribution. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | //************************************** 31 | // Compilation Directives 32 | //************************************** 33 | #if __STDC_VERSION__ >= 199901L 34 | /* "restrict" is a known keyword */ 35 | #else 36 | #define restrict // Disable restrict 37 | #endif 38 | 39 | #ifdef _MSC_VER 40 | #define inline __forceinline 41 | #endif 42 | 43 | #ifdef __GNUC__ 44 | #define _PACKED __attribute__ ((packed)) 45 | #else 46 | #define _PACKED 47 | #endif 48 | 49 | #define ARCH64 (__x86_64__ || __ppc64__ || _WIN64 || __LP64__) // Detect 64 bits mode 50 | 51 | 52 | //************************************** 53 | // Includes 54 | //************************************** 55 | #include // for malloc 56 | #include // for memset 57 | #include "lz4.h" 58 | 59 | 60 | //************************************** 61 | // Performance parameter 62 | //************************************** 63 | // Increasing this value improves compression ratio 64 | // Lowering this value reduces memory usage 65 | // Lowering may also improve speed, typically on reaching cache size limits (L1 32KB for Intel, 64KB for AMD) 66 | // Memory usage formula for 32 bits systems : N->2^(N+2) Bytes (examples : 17 -> 512KB ; 12 -> 16KB) 67 | #define HASH_LOG 12 68 | 69 | //#define _FORCE_SW_BITCOUNT // Uncomment for better performance if target platform has no hardware support for LowBitCount 70 | 71 | 72 | //************************************** 73 | // Basic Types 74 | //************************************** 75 | #if defined(_MSC_VER) // Visual Studio does not support 'stdint' natively 76 | #define BYTE unsigned __int8 77 | #define U16 unsigned __int16 78 | #define U32 unsigned __int32 79 | #define S32 __int32 80 | #define U64 unsigned __int64 81 | #else 82 | #include 83 | #define BYTE uint8_t 84 | #define U16 uint16_t 85 | #define U32 uint32_t 86 | #define S32 int32_t 87 | #define U64 uint64_t 88 | #endif 89 | 90 | 91 | //************************************** 92 | // Constants 93 | //************************************** 94 | #define MINMATCH 4 95 | #define SKIPSTRENGTH 6 96 | #define STACKLIMIT 13 97 | #define HEAPMODE (HASH_LOG>STACKLIMIT) // Defines if memory is allocated into the stack (local variable), or into the heap (malloc()). 98 | #define COPYLENGTH 8 99 | #define LASTLITERALS 5 100 | #define MFLIMIT (COPYLENGTH+MINMATCH) 101 | #define MINLENGTH (MFLIMIT+1) 102 | 103 | #define MAXD_LOG 16 104 | #define MAX_DISTANCE ((1 << MAXD_LOG) - 1) 105 | 106 | #define HASHTABLESIZE (1 << HASH_LOG) 107 | #define HASH_MASK (HASHTABLESIZE - 1) 108 | 109 | #define ML_BITS 4 110 | #define ML_MASK ((1U<v) 139 | #define A32(x) (((U32_S *)(x))->v) 140 | #define A16(x) (((U16_S *)(x))->v) 141 | 142 | 143 | //************************************** 144 | // Architecture-specific macros 145 | //************************************** 146 | #if ARCH64 // 64-bit 147 | #define COPYSTEP 8 148 | #define LZ4_COPYSTEP(s,d) A64(d) = A64(s); d+=8; s+=8; 149 | #define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d) 150 | #else // 32-bit 151 | #define COPYSTEP 4 152 | #define LZ4_COPYSTEP(s,d) A32(d) = A32(s); d+=4; s+=4; 153 | #define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d); LZ4_COPYSTEP(s,d); 154 | #endif 155 | 156 | #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 157 | #define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = s - A16(p); } 158 | #define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; } 159 | #define LZ4_NbCommonBytes LZ4_NbCommonBytes_LittleEndian 160 | #else // Big Endian 161 | #define LZ4_READ_LITTLEENDIAN_16(d,s,p) { int delta = p[0]; delta += p[1] << 8; d = s - delta; } 162 | #define LZ4_WRITE_LITTLEENDIAN_16(p,v) { int delta = v; *p++ = delta; *op++ = delta>>8; } 163 | #define LZ4_NbCommonBytes LZ4_NbCommonBytes_BigEndian 164 | #endif 165 | 166 | 167 | //************************************** 168 | // Macros 169 | //************************************** 170 | #define LZ4_HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG)) 171 | #define LZ4_HASH_VALUE(p) LZ4_HASH_FUNCTION(A32(p)) 172 | #define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (d>3); 185 | #elif defined(__GNUC__) && !defined(_FORCE_SW_BITCOUNT) 186 | return (__builtin_ctz(val) >> 3); 187 | #else 188 | static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; 189 | return DeBruijnBytePos[((U32)((val & -val) * 0x077CB531U)) >> 27]; 190 | #endif 191 | } 192 | 193 | inline static int LZ4_NbCommonBytes_BigEndian (register U32 val) 194 | { 195 | #if defined(_MSC_VER) && !defined(_FORCE_SW_BITCOUNT) 196 | unsigned long r = 0; 197 | _BitScanReverse( &r, val ); 198 | return (int)(r>>3); 199 | #elif defined(__GNUC__) && !defined(_FORCE_SW_BITCOUNT) 200 | return (__builtin_clz(val) >> 3); 201 | #else 202 | int r; 203 | if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } 204 | r += (!val); 205 | return r; 206 | #endif 207 | } 208 | 209 | 210 | //****************************** 211 | // Public Compression functions 212 | //****************************** 213 | int LZ4_compressCtx(void** ctx, 214 | char* source, 215 | char* dest, 216 | int isize) 217 | { 218 | #if HEAPMODE 219 | struct refTables *srt = (struct refTables *) (*ctx); 220 | const BYTE** HashTable; 221 | #else 222 | const BYTE* HashTable[HASHTABLESIZE] = {0}; 223 | #endif 224 | 225 | const BYTE* ip = (BYTE*) source; 226 | const BYTE* anchor = ip; 227 | const BYTE* const iend = ip + isize; 228 | const BYTE* const mflimit = iend - MFLIMIT; 229 | #define matchlimit (iend - LASTLITERALS) 230 | 231 | BYTE* op = (BYTE*) dest; 232 | 233 | int len, length; 234 | const int skipStrength = SKIPSTRENGTH; 235 | U32 forwardH; 236 | 237 | 238 | // Init 239 | if (isizehashTable; 247 | memset((void*)HashTable, 0, sizeof(srt->hashTable)); 248 | #else 249 | (void) ctx; 250 | #endif 251 | 252 | 253 | // First Byte 254 | HashTable[LZ4_HASH_VALUE(ip)] = ip; 255 | ip++; forwardH = LZ4_HASH_VALUE(ip); 256 | 257 | // Main Loop 258 | for ( ; ; ) 259 | { 260 | int findMatchAttempts = (1U << skipStrength) + 3; 261 | const BYTE* forwardIp = ip; 262 | const BYTE* ref; 263 | BYTE* token; 264 | 265 | // Find a match 266 | do { 267 | U32 h = forwardH; 268 | int step = findMatchAttempts++ >> skipStrength; 269 | ip = forwardIp; 270 | forwardIp = ip + step; 271 | 272 | if (forwardIp > mflimit) { goto _last_literals; } 273 | 274 | forwardH = LZ4_HASH_VALUE(forwardIp); 275 | ref = HashTable[h]; 276 | HashTable[h] = ip; 277 | 278 | } while ((ref < ip - MAX_DISTANCE) || (A32(ref) != A32(ip))); 279 | 280 | // Catch up 281 | while ((ip>anchor) && (ref>(BYTE*)source) && (ip[-1]==ref[-1])) { ip--; ref--; } 282 | 283 | // Encode Literal length 284 | length = ip - anchor; 285 | token = op++; 286 | if (length>=(int)RUN_MASK) { *token=(RUN_MASK< 254 ; len-=255) *op++ = 255; *op++ = (BYTE)len; } 287 | else *token = (length<=(int)ML_MASK) { *token+=ML_MASK; len-=ML_MASK; for(; len > 509 ; len-=510) { *op++ = 255; *op++ = 255; } if (len > 254) { len-=255; *op++ = 255; } *op++ = (BYTE)len; } 314 | else *token += len; 315 | 316 | // Test end of chunk 317 | if (ip > mflimit) { anchor = ip; break; } 318 | 319 | // Fill table 320 | HashTable[LZ4_HASH_VALUE(ip-2)] = ip-2; 321 | 322 | // Test next position 323 | ref = HashTable[LZ4_HASH_VALUE(ip)]; 324 | HashTable[LZ4_HASH_VALUE(ip)] = ip; 325 | if ((ref > ip - (MAX_DISTANCE + 1)) && (A32(ref) == A32(ip))) { token = op++; *token=0; goto _next_match; } 326 | 327 | // Prepare next loop 328 | anchor = ip++; 329 | forwardH = LZ4_HASH_VALUE(ip); 330 | } 331 | 332 | _last_literals: 333 | // Encode Last Literals 334 | { 335 | int lastRun = iend - anchor; 336 | if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } 337 | else *op++ = (lastRun<> ((MINMATCH*8)-HASHLOG64K)) 352 | #define LZ4_HASH64K_VALUE(p) LZ4_HASH64K_FUNCTION(A32(p)) 353 | int LZ4_compress64kCtx(void** ctx, 354 | char* source, 355 | char* dest, 356 | int isize) 357 | { 358 | #if HEAPMODE 359 | struct refTables *srt = (struct refTables *) (*ctx); 360 | U16* HashTable; 361 | #else 362 | U16 HashTable[HASHTABLESIZE<<1] = {0}; 363 | #endif 364 | 365 | const BYTE* ip = (BYTE*) source; 366 | const BYTE* anchor = ip; 367 | const BYTE* const base = ip; 368 | const BYTE* const iend = ip + isize; 369 | const BYTE* const mflimit = iend - MFLIMIT; 370 | #define matchlimit (iend - LASTLITERALS) 371 | 372 | BYTE* op = (BYTE*) dest; 373 | 374 | int len, length; 375 | const int skipStrength = SKIPSTRENGTH; 376 | U32 forwardH; 377 | 378 | 379 | // Init 380 | if (isizehashTable); 388 | memset((void*)HashTable, 0, sizeof(srt->hashTable)); 389 | #else 390 | (void) ctx; 391 | #endif 392 | 393 | 394 | // First Byte 395 | ip++; forwardH = LZ4_HASH64K_VALUE(ip); 396 | 397 | // Main Loop 398 | for ( ; ; ) 399 | { 400 | int findMatchAttempts = (1U << skipStrength) + 3; 401 | const BYTE* forwardIp = ip; 402 | const BYTE* ref; 403 | BYTE* token; 404 | 405 | // Find a match 406 | do { 407 | U32 h = forwardH; 408 | int step = findMatchAttempts++ >> skipStrength; 409 | ip = forwardIp; 410 | forwardIp = ip + step; 411 | 412 | if (forwardIp > mflimit) { goto _last_literals; } 413 | 414 | forwardH = LZ4_HASH64K_VALUE(forwardIp); 415 | ref = base + HashTable[h]; 416 | HashTable[h] = ip - base; 417 | 418 | } while (A32(ref) != A32(ip)); 419 | 420 | // Catch up 421 | while ((ip>anchor) && (ref>(BYTE*)source) && (ip[-1]==ref[-1])) { ip--; ref--; } 422 | 423 | // Encode Literal length 424 | length = ip - anchor; 425 | token = op++; 426 | if (length>=(int)RUN_MASK) { *token=(RUN_MASK< 254 ; len-=255) *op++ = 255; *op++ = (BYTE)len; } 427 | else *token = (length<=(int)ML_MASK) { *token+=ML_MASK; len-=ML_MASK; for(; len > 509 ; len-=510) { *op++ = 255; *op++ = 255; } if (len > 254) { len-=255; *op++ = 255; } *op++ = (BYTE)len; } 454 | else *token += len; 455 | 456 | // Test end of chunk 457 | if (ip > mflimit) { anchor = ip; break; } 458 | 459 | // Fill table 460 | HashTable[LZ4_HASH64K_VALUE(ip-2)] = ip - 2 - base; 461 | 462 | // Test next position 463 | ref = base + HashTable[LZ4_HASH64K_VALUE(ip)]; 464 | HashTable[LZ4_HASH64K_VALUE(ip)] = ip - base; 465 | if (A32(ref) == A32(ip)) { token = op++; *token=0; goto _next_match; } 466 | 467 | // Prepare next loop 468 | anchor = ip++; 469 | forwardH = LZ4_HASH64K_VALUE(ip); 470 | } 471 | 472 | _last_literals: 473 | // Encode Last Literals 474 | { 475 | int lastRun = iend - anchor; 476 | if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } 477 | else *op++ = (lastRun<>ML_BITS)) == RUN_MASK) { for (;(len=*ip++)==255;length+=255){} length += len; } 542 | 543 | // copy literals 544 | cpy = op+length; 545 | if (cpy>oend-COPYLENGTH) 546 | { 547 | if (cpy > oend) goto _output_error; 548 | memcpy(op, ip, length); 549 | ip += length; 550 | break; // Necessarily EOF 551 | } 552 | LZ4_WILDCOPY(ip, op, cpy); ip -= (op-cpy); op = cpy; 553 | 554 | // get offset 555 | LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2; 556 | if (ref < (BYTE* const)dest) goto _output_error; 557 | 558 | // get matchlength 559 | if ((length=(token&ML_MASK)) == ML_MASK) { for (;*ip==255;length+=255) {ip++;} length += *ip++; } 560 | 561 | // copy repeated sequence 562 | if (op-refoend-COPYLENGTH) 580 | { 581 | if (cpy > oend) goto _output_error; 582 | LZ4_WILDCOPY(ref, op, (oend-COPYLENGTH)); 583 | while(op>ML_BITS)) == RUN_MASK) { for (;(len=*ip++)==255;length+=255){} length += len; } 628 | 629 | // copy literals 630 | cpy = op+length; 631 | if (cpy>oend-COPYLENGTH) 632 | { 633 | if (cpy > oend) goto _output_error; 634 | memcpy(op, ip, length); 635 | op += length; 636 | break; // Necessarily EOF 637 | } 638 | LZ4_WILDCOPY(ip, op, cpy); ip -= (op-cpy); op = cpy; 639 | if (ip>=iend) break; // check EOF 640 | 641 | // get offset 642 | LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2; 643 | if (ref < (BYTE* const)dest) goto _output_error; 644 | 645 | // get matchlength 646 | if ((length=(token&ML_MASK)) == ML_MASK) { for (;(len=*ip++)==255;length+=255){} length += len; } 647 | 648 | // copy repeated sequence 649 | if (op-refoend-COPYLENGTH) 667 | { 668 | if (cpy > oend) goto _output_error; 669 | LZ4_WILDCOPY(ref, op, (oend-COPYLENGTH)); 670 | while(op 8 | * 9 | * All rights reserved. 10 | * 11 | * This program is free software; you can redistribute it and/or modify 12 | * it under the terms of the GNU General Public License as published by 13 | * the Free Software Foundation; either version 2 of the License, or 14 | * (at your option) any later version. 15 | * 16 | * This program is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | * GNU General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU General Public License along 22 | * with this program; if not, write to the Free Software Foundation, Inc., 23 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 24 | */ 25 | /* 26 | * The MD5 algorithm was designed by Ron Rivest in 1991. 27 | * 28 | * http://www.ietf.org/rfc/rfc1321.txt 29 | */ 30 | 31 | #define POLARSSL_MD5_C 32 | #if defined(POLARSSL_MD5_C) 33 | 34 | #include "md5.h" 35 | 36 | #if defined(POLARSSL_FS_IO) || defined(POLARSSL_SELF_TEST) 37 | #include 38 | #endif 39 | 40 | /* 41 | * 32-bit integer manipulation macros (little endian) 42 | */ 43 | #ifndef GET_ULONG_LE 44 | #define GET_ULONG_LE(n,b,i) \ 45 | { \ 46 | (n) = ( (unsigned long) (b)[(i) ] ) \ 47 | | ( (unsigned long) (b)[(i) + 1] << 8 ) \ 48 | | ( (unsigned long) (b)[(i) + 2] << 16 ) \ 49 | | ( (unsigned long) (b)[(i) + 3] << 24 ); \ 50 | } 51 | #endif 52 | 53 | #ifndef PUT_ULONG_LE 54 | #define PUT_ULONG_LE(n,b,i) \ 55 | { \ 56 | (b)[(i) ] = (unsigned char) ( (n) ); \ 57 | (b)[(i) + 1] = (unsigned char) ( (n) >> 8 ); \ 58 | (b)[(i) + 2] = (unsigned char) ( (n) >> 16 ); \ 59 | (b)[(i) + 3] = (unsigned char) ( (n) >> 24 ); \ 60 | } 61 | #endif 62 | 63 | /* 64 | * MD5 context setup 65 | */ 66 | void md5_starts( md5_context *ctx ) 67 | { 68 | ctx->total[0] = 0; 69 | ctx->total[1] = 0; 70 | 71 | ctx->state[0] = 0x67452301; 72 | ctx->state[1] = 0xEFCDAB89; 73 | ctx->state[2] = 0x98BADCFE; 74 | ctx->state[3] = 0x10325476; 75 | } 76 | 77 | static void md5_process( md5_context *ctx, const unsigned char data[64] ) 78 | { 79 | unsigned long X[16], A, B, C, D; 80 | 81 | GET_ULONG_LE( X[ 0], data, 0 ); 82 | GET_ULONG_LE( X[ 1], data, 4 ); 83 | GET_ULONG_LE( X[ 2], data, 8 ); 84 | GET_ULONG_LE( X[ 3], data, 12 ); 85 | GET_ULONG_LE( X[ 4], data, 16 ); 86 | GET_ULONG_LE( X[ 5], data, 20 ); 87 | GET_ULONG_LE( X[ 6], data, 24 ); 88 | GET_ULONG_LE( X[ 7], data, 28 ); 89 | GET_ULONG_LE( X[ 8], data, 32 ); 90 | GET_ULONG_LE( X[ 9], data, 36 ); 91 | GET_ULONG_LE( X[10], data, 40 ); 92 | GET_ULONG_LE( X[11], data, 44 ); 93 | GET_ULONG_LE( X[12], data, 48 ); 94 | GET_ULONG_LE( X[13], data, 52 ); 95 | GET_ULONG_LE( X[14], data, 56 ); 96 | GET_ULONG_LE( X[15], data, 60 ); 97 | 98 | #define S(x,n) ((x << n) | ((x & 0xFFFFFFFF) >> (32 - n))) 99 | 100 | #define P(a,b,c,d,k,s,t) \ 101 | { \ 102 | a += F(b,c,d) + X[k] + t; a = S(a,s) + b; \ 103 | } 104 | 105 | A = ctx->state[0]; 106 | B = ctx->state[1]; 107 | C = ctx->state[2]; 108 | D = ctx->state[3]; 109 | 110 | #define F(x,y,z) (z ^ (x & (y ^ z))) 111 | 112 | P( A, B, C, D, 0, 7, 0xD76AA478 ); 113 | P( D, A, B, C, 1, 12, 0xE8C7B756 ); 114 | P( C, D, A, B, 2, 17, 0x242070DB ); 115 | P( B, C, D, A, 3, 22, 0xC1BDCEEE ); 116 | P( A, B, C, D, 4, 7, 0xF57C0FAF ); 117 | P( D, A, B, C, 5, 12, 0x4787C62A ); 118 | P( C, D, A, B, 6, 17, 0xA8304613 ); 119 | P( B, C, D, A, 7, 22, 0xFD469501 ); 120 | P( A, B, C, D, 8, 7, 0x698098D8 ); 121 | P( D, A, B, C, 9, 12, 0x8B44F7AF ); 122 | P( C, D, A, B, 10, 17, 0xFFFF5BB1 ); 123 | P( B, C, D, A, 11, 22, 0x895CD7BE ); 124 | P( A, B, C, D, 12, 7, 0x6B901122 ); 125 | P( D, A, B, C, 13, 12, 0xFD987193 ); 126 | P( C, D, A, B, 14, 17, 0xA679438E ); 127 | P( B, C, D, A, 15, 22, 0x49B40821 ); 128 | 129 | #undef F 130 | 131 | #define F(x,y,z) (y ^ (z & (x ^ y))) 132 | 133 | P( A, B, C, D, 1, 5, 0xF61E2562 ); 134 | P( D, A, B, C, 6, 9, 0xC040B340 ); 135 | P( C, D, A, B, 11, 14, 0x265E5A51 ); 136 | P( B, C, D, A, 0, 20, 0xE9B6C7AA ); 137 | P( A, B, C, D, 5, 5, 0xD62F105D ); 138 | P( D, A, B, C, 10, 9, 0x02441453 ); 139 | P( C, D, A, B, 15, 14, 0xD8A1E681 ); 140 | P( B, C, D, A, 4, 20, 0xE7D3FBC8 ); 141 | P( A, B, C, D, 9, 5, 0x21E1CDE6 ); 142 | P( D, A, B, C, 14, 9, 0xC33707D6 ); 143 | P( C, D, A, B, 3, 14, 0xF4D50D87 ); 144 | P( B, C, D, A, 8, 20, 0x455A14ED ); 145 | P( A, B, C, D, 13, 5, 0xA9E3E905 ); 146 | P( D, A, B, C, 2, 9, 0xFCEFA3F8 ); 147 | P( C, D, A, B, 7, 14, 0x676F02D9 ); 148 | P( B, C, D, A, 12, 20, 0x8D2A4C8A ); 149 | 150 | #undef F 151 | 152 | #define F(x,y,z) (x ^ y ^ z) 153 | 154 | P( A, B, C, D, 5, 4, 0xFFFA3942 ); 155 | P( D, A, B, C, 8, 11, 0x8771F681 ); 156 | P( C, D, A, B, 11, 16, 0x6D9D6122 ); 157 | P( B, C, D, A, 14, 23, 0xFDE5380C ); 158 | P( A, B, C, D, 1, 4, 0xA4BEEA44 ); 159 | P( D, A, B, C, 4, 11, 0x4BDECFA9 ); 160 | P( C, D, A, B, 7, 16, 0xF6BB4B60 ); 161 | P( B, C, D, A, 10, 23, 0xBEBFBC70 ); 162 | P( A, B, C, D, 13, 4, 0x289B7EC6 ); 163 | P( D, A, B, C, 0, 11, 0xEAA127FA ); 164 | P( C, D, A, B, 3, 16, 0xD4EF3085 ); 165 | P( B, C, D, A, 6, 23, 0x04881D05 ); 166 | P( A, B, C, D, 9, 4, 0xD9D4D039 ); 167 | P( D, A, B, C, 12, 11, 0xE6DB99E5 ); 168 | P( C, D, A, B, 15, 16, 0x1FA27CF8 ); 169 | P( B, C, D, A, 2, 23, 0xC4AC5665 ); 170 | 171 | #undef F 172 | 173 | #define F(x,y,z) (y ^ (x | ~z)) 174 | 175 | P( A, B, C, D, 0, 6, 0xF4292244 ); 176 | P( D, A, B, C, 7, 10, 0x432AFF97 ); 177 | P( C, D, A, B, 14, 15, 0xAB9423A7 ); 178 | P( B, C, D, A, 5, 21, 0xFC93A039 ); 179 | P( A, B, C, D, 12, 6, 0x655B59C3 ); 180 | P( D, A, B, C, 3, 10, 0x8F0CCC92 ); 181 | P( C, D, A, B, 10, 15, 0xFFEFF47D ); 182 | P( B, C, D, A, 1, 21, 0x85845DD1 ); 183 | P( A, B, C, D, 8, 6, 0x6FA87E4F ); 184 | P( D, A, B, C, 15, 10, 0xFE2CE6E0 ); 185 | P( C, D, A, B, 6, 15, 0xA3014314 ); 186 | P( B, C, D, A, 13, 21, 0x4E0811A1 ); 187 | P( A, B, C, D, 4, 6, 0xF7537E82 ); 188 | P( D, A, B, C, 11, 10, 0xBD3AF235 ); 189 | P( C, D, A, B, 2, 15, 0x2AD7D2BB ); 190 | P( B, C, D, A, 9, 21, 0xEB86D391 ); 191 | 192 | #undef F 193 | 194 | ctx->state[0] += A; 195 | ctx->state[1] += B; 196 | ctx->state[2] += C; 197 | ctx->state[3] += D; 198 | } 199 | 200 | /* 201 | * MD5 process buffer 202 | */ 203 | void md5_update( md5_context *ctx, const unsigned char *input, size_t ilen ) 204 | { 205 | size_t fill; 206 | unsigned long left; 207 | 208 | if( ilen <= 0 ) 209 | return; 210 | 211 | left = ctx->total[0] & 0x3F; 212 | fill = 64 - left; 213 | 214 | ctx->total[0] += (unsigned long) ilen; 215 | ctx->total[0] &= 0xFFFFFFFF; 216 | 217 | if( ctx->total[0] < (unsigned long) ilen ) 218 | ctx->total[1]++; 219 | 220 | if( left && ilen >= fill ) 221 | { 222 | memcpy( (void *) (ctx->buffer + left), 223 | (void *) input, fill ); 224 | md5_process( ctx, ctx->buffer ); 225 | input += fill; 226 | ilen -= fill; 227 | left = 0; 228 | } 229 | 230 | while( ilen >= 64 ) 231 | { 232 | md5_process( ctx, input ); 233 | input += 64; 234 | ilen -= 64; 235 | } 236 | 237 | if( ilen > 0 ) 238 | { 239 | memcpy( (void *) (ctx->buffer + left), 240 | (void *) input, ilen ); 241 | } 242 | } 243 | 244 | static const unsigned char md5_padding[64] = 245 | { 246 | 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 250 | }; 251 | 252 | /* 253 | * MD5 final digest 254 | */ 255 | void md5_finish( md5_context *ctx, unsigned char output[16] ) 256 | { 257 | unsigned long last, padn; 258 | unsigned long high, low; 259 | unsigned char msglen[8]; 260 | 261 | high = ( ctx->total[0] >> 29 ) 262 | | ( ctx->total[1] << 3 ); 263 | low = ( ctx->total[0] << 3 ); 264 | 265 | PUT_ULONG_LE( low, msglen, 0 ); 266 | PUT_ULONG_LE( high, msglen, 4 ); 267 | 268 | last = ctx->total[0] & 0x3F; 269 | padn = ( last < 56 ) ? ( 56 - last ) : ( 120 - last ); 270 | 271 | md5_update( ctx, (unsigned char *) md5_padding, padn ); 272 | md5_update( ctx, msglen, 8 ); 273 | 274 | PUT_ULONG_LE( ctx->state[0], output, 0 ); 275 | PUT_ULONG_LE( ctx->state[1], output, 4 ); 276 | PUT_ULONG_LE( ctx->state[2], output, 8 ); 277 | PUT_ULONG_LE( ctx->state[3], output, 12 ); 278 | } 279 | 280 | /* 281 | * output = MD5( input buffer ) 282 | */ 283 | void md5( const unsigned char *input, size_t ilen, unsigned char output[16] ) 284 | { 285 | md5_context ctx; 286 | 287 | md5_starts( &ctx ); 288 | md5_update( &ctx, input, ilen ); 289 | md5_finish( &ctx, output ); 290 | 291 | memset( &ctx, 0, sizeof( md5_context ) ); 292 | } 293 | 294 | #if defined(POLARSSL_FS_IO) 295 | /* 296 | * output = MD5( file contents ) 297 | */ 298 | int md5_file( const char *path, unsigned char output[16] ) 299 | { 300 | FILE *f; 301 | size_t n; 302 | md5_context ctx; 303 | unsigned char buf[1024]; 304 | 305 | if( ( f = fopen( path, "rb" ) ) == NULL ) 306 | return( 1 ); 307 | 308 | md5_starts( &ctx ); 309 | 310 | while( ( n = fread( buf, 1, sizeof( buf ), f ) ) > 0 ) 311 | md5_update( &ctx, buf, n ); 312 | 313 | md5_finish( &ctx, output ); 314 | 315 | memset( &ctx, 0, sizeof( md5_context ) ); 316 | 317 | if( ferror( f ) != 0 ) 318 | { 319 | fclose( f ); 320 | return( 2 ); 321 | } 322 | 323 | fclose( f ); 324 | return( 0 ); 325 | } 326 | #endif /* POLARSSL_FS_IO */ 327 | 328 | /* 329 | * MD5 HMAC context setup 330 | */ 331 | void md5_hmac_starts( md5_context *ctx, const unsigned char *key, size_t keylen ) 332 | { 333 | size_t i; 334 | unsigned char sum[16]; 335 | 336 | if( keylen > 64 ) 337 | { 338 | md5( key, keylen, sum ); 339 | keylen = 16; 340 | key = sum; 341 | } 342 | 343 | memset( ctx->ipad, 0x36, 64 ); 344 | memset( ctx->opad, 0x5C, 64 ); 345 | 346 | for( i = 0; i < keylen; i++ ) 347 | { 348 | ctx->ipad[i] = (unsigned char)( ctx->ipad[i] ^ key[i] ); 349 | ctx->opad[i] = (unsigned char)( ctx->opad[i] ^ key[i] ); 350 | } 351 | 352 | md5_starts( ctx ); 353 | md5_update( ctx, ctx->ipad, 64 ); 354 | 355 | memset( sum, 0, sizeof( sum ) ); 356 | } 357 | 358 | /* 359 | * MD5 HMAC process buffer 360 | */ 361 | void md5_hmac_update( md5_context *ctx, const unsigned char *input, size_t ilen ) 362 | { 363 | md5_update( ctx, input, ilen ); 364 | } 365 | 366 | /* 367 | * MD5 HMAC final digest 368 | */ 369 | void md5_hmac_finish( md5_context *ctx, unsigned char output[16] ) 370 | { 371 | unsigned char tmpbuf[16]; 372 | 373 | md5_finish( ctx, tmpbuf ); 374 | md5_starts( ctx ); 375 | md5_update( ctx, ctx->opad, 64 ); 376 | md5_update( ctx, tmpbuf, 16 ); 377 | md5_finish( ctx, output ); 378 | 379 | memset( tmpbuf, 0, sizeof( tmpbuf ) ); 380 | } 381 | 382 | /* 383 | * MD5 HMAC context reset 384 | */ 385 | void md5_hmac_reset( md5_context *ctx ) 386 | { 387 | md5_starts( ctx ); 388 | md5_update( ctx, ctx->ipad, 64 ); 389 | } 390 | 391 | /* 392 | * output = HMAC-MD5( hmac key, input buffer ) 393 | */ 394 | void md5_hmac( const unsigned char *key, size_t keylen, 395 | const unsigned char *input, size_t ilen, 396 | unsigned char output[16] ) 397 | { 398 | md5_context ctx; 399 | 400 | md5_hmac_starts( &ctx, key, keylen ); 401 | md5_hmac_update( &ctx, input, ilen ); 402 | md5_hmac_finish( &ctx, output ); 403 | 404 | memset( &ctx, 0, sizeof( md5_context ) ); 405 | } 406 | 407 | #if defined(POLARSSL_SELF_TEST) 408 | /* 409 | * RFC 1321 test vectors 410 | */ 411 | static unsigned char md5_test_buf[7][81] = 412 | { 413 | { "" }, 414 | { "a" }, 415 | { "abc" }, 416 | { "message digest" }, 417 | { "abcdefghijklmnopqrstuvwxyz" }, 418 | { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" }, 419 | { "12345678901234567890123456789012345678901234567890123456789012" \ 420 | "345678901234567890" } 421 | }; 422 | 423 | static const int md5_test_buflen[7] = 424 | { 425 | 0, 1, 3, 14, 26, 62, 80 426 | }; 427 | 428 | static const unsigned char md5_test_sum[7][16] = 429 | { 430 | { 0xD4, 0x1D, 0x8C, 0xD9, 0x8F, 0x00, 0xB2, 0x04, 431 | 0xE9, 0x80, 0x09, 0x98, 0xEC, 0xF8, 0x42, 0x7E }, 432 | { 0x0C, 0xC1, 0x75, 0xB9, 0xC0, 0xF1, 0xB6, 0xA8, 433 | 0x31, 0xC3, 0x99, 0xE2, 0x69, 0x77, 0x26, 0x61 }, 434 | { 0x90, 0x01, 0x50, 0x98, 0x3C, 0xD2, 0x4F, 0xB0, 435 | 0xD6, 0x96, 0x3F, 0x7D, 0x28, 0xE1, 0x7F, 0x72 }, 436 | { 0xF9, 0x6B, 0x69, 0x7D, 0x7C, 0xB7, 0x93, 0x8D, 437 | 0x52, 0x5A, 0x2F, 0x31, 0xAA, 0xF1, 0x61, 0xD0 }, 438 | { 0xC3, 0xFC, 0xD3, 0xD7, 0x61, 0x92, 0xE4, 0x00, 439 | 0x7D, 0xFB, 0x49, 0x6C, 0xCA, 0x67, 0xE1, 0x3B }, 440 | { 0xD1, 0x74, 0xAB, 0x98, 0xD2, 0x77, 0xD9, 0xF5, 441 | 0xA5, 0x61, 0x1C, 0x2C, 0x9F, 0x41, 0x9D, 0x9F }, 442 | { 0x57, 0xED, 0xF4, 0xA2, 0x2B, 0xE3, 0xC9, 0x55, 443 | 0xAC, 0x49, 0xDA, 0x2E, 0x21, 0x07, 0xB6, 0x7A } 444 | }; 445 | 446 | /* 447 | * RFC 2202 test vectors 448 | */ 449 | static unsigned char md5_hmac_test_key[7][26] = 450 | { 451 | { "\x0B\x0B\x0B\x0B\x0B\x0B\x0B\x0B\x0B\x0B\x0B\x0B\x0B\x0B\x0B\x0B" }, 452 | { "Jefe" }, 453 | { "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA" }, 454 | { "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10" 455 | "\x11\x12\x13\x14\x15\x16\x17\x18\x19" }, 456 | { "\x0C\x0C\x0C\x0C\x0C\x0C\x0C\x0C\x0C\x0C\x0C\x0C\x0C\x0C\x0C\x0C" }, 457 | { "" }, /* 0xAA 80 times */ 458 | { "" } 459 | }; 460 | 461 | static const int md5_hmac_test_keylen[7] = 462 | { 463 | 16, 4, 16, 25, 16, 80, 80 464 | }; 465 | 466 | static unsigned char md5_hmac_test_buf[7][74] = 467 | { 468 | { "Hi There" }, 469 | { "what do ya want for nothing?" }, 470 | { "\xDD\xDD\xDD\xDD\xDD\xDD\xDD\xDD\xDD\xDD" 471 | "\xDD\xDD\xDD\xDD\xDD\xDD\xDD\xDD\xDD\xDD" 472 | "\xDD\xDD\xDD\xDD\xDD\xDD\xDD\xDD\xDD\xDD" 473 | "\xDD\xDD\xDD\xDD\xDD\xDD\xDD\xDD\xDD\xDD" 474 | "\xDD\xDD\xDD\xDD\xDD\xDD\xDD\xDD\xDD\xDD" }, 475 | { "\xCD\xCD\xCD\xCD\xCD\xCD\xCD\xCD\xCD\xCD" 476 | "\xCD\xCD\xCD\xCD\xCD\xCD\xCD\xCD\xCD\xCD" 477 | "\xCD\xCD\xCD\xCD\xCD\xCD\xCD\xCD\xCD\xCD" 478 | "\xCD\xCD\xCD\xCD\xCD\xCD\xCD\xCD\xCD\xCD" 479 | "\xCD\xCD\xCD\xCD\xCD\xCD\xCD\xCD\xCD\xCD" }, 480 | { "Test With Truncation" }, 481 | { "Test Using Larger Than Block-Size Key - Hash Key First" }, 482 | { "Test Using Larger Than Block-Size Key and Larger" 483 | " Than One Block-Size Data" } 484 | }; 485 | 486 | static const int md5_hmac_test_buflen[7] = 487 | { 488 | 8, 28, 50, 50, 20, 54, 73 489 | }; 490 | 491 | static const unsigned char md5_hmac_test_sum[7][16] = 492 | { 493 | { 0x92, 0x94, 0x72, 0x7A, 0x36, 0x38, 0xBB, 0x1C, 494 | 0x13, 0xF4, 0x8E, 0xF8, 0x15, 0x8B, 0xFC, 0x9D }, 495 | { 0x75, 0x0C, 0x78, 0x3E, 0x6A, 0xB0, 0xB5, 0x03, 496 | 0xEA, 0xA8, 0x6E, 0x31, 0x0A, 0x5D, 0xB7, 0x38 }, 497 | { 0x56, 0xBE, 0x34, 0x52, 0x1D, 0x14, 0x4C, 0x88, 498 | 0xDB, 0xB8, 0xC7, 0x33, 0xF0, 0xE8, 0xB3, 0xF6 }, 499 | { 0x69, 0x7E, 0xAF, 0x0A, 0xCA, 0x3A, 0x3A, 0xEA, 500 | 0x3A, 0x75, 0x16, 0x47, 0x46, 0xFF, 0xAA, 0x79 }, 501 | { 0x56, 0x46, 0x1E, 0xF2, 0x34, 0x2E, 0xDC, 0x00, 502 | 0xF9, 0xBA, 0xB9, 0x95 }, 503 | { 0x6B, 0x1A, 0xB7, 0xFE, 0x4B, 0xD7, 0xBF, 0x8F, 504 | 0x0B, 0x62, 0xE6, 0xCE, 0x61, 0xB9, 0xD0, 0xCD }, 505 | { 0x6F, 0x63, 0x0F, 0xAD, 0x67, 0xCD, 0xA0, 0xEE, 506 | 0x1F, 0xB1, 0xF5, 0x62, 0xDB, 0x3A, 0xA5, 0x3E } 507 | }; 508 | 509 | /* 510 | * Checkup routine 511 | */ 512 | int md5_self_test( int verbose ) 513 | { 514 | int i, buflen; 515 | unsigned char buf[1024]; 516 | unsigned char md5sum[16]; 517 | md5_context ctx; 518 | 519 | for( i = 0; i < 7; i++ ) 520 | { 521 | if( verbose != 0 ) 522 | printf( " MD5 test #%d: ", i + 1 ); 523 | 524 | md5( md5_test_buf[i], md5_test_buflen[i], md5sum ); 525 | 526 | if( memcmp( md5sum, md5_test_sum[i], 16 ) != 0 ) 527 | { 528 | if( verbose != 0 ) 529 | printf( "failed\n" ); 530 | 531 | return( 1 ); 532 | } 533 | 534 | if( verbose != 0 ) 535 | printf( "passed\n" ); 536 | } 537 | 538 | if( verbose != 0 ) 539 | printf( "\n" ); 540 | 541 | for( i = 0; i < 7; i++ ) 542 | { 543 | if( verbose != 0 ) 544 | printf( " HMAC-MD5 test #%d: ", i + 1 ); 545 | 546 | if( i == 5 || i == 6 ) 547 | { 548 | memset( buf, '\xAA', buflen = 80 ); 549 | md5_hmac_starts( &ctx, buf, buflen ); 550 | } 551 | else 552 | md5_hmac_starts( &ctx, md5_hmac_test_key[i], 553 | md5_hmac_test_keylen[i] ); 554 | 555 | md5_hmac_update( &ctx, md5_hmac_test_buf[i], 556 | md5_hmac_test_buflen[i] ); 557 | 558 | md5_hmac_finish( &ctx, md5sum ); 559 | 560 | buflen = ( i == 4 ) ? 12 : 16; 561 | 562 | if( memcmp( md5sum, md5_hmac_test_sum[i], buflen ) != 0 ) 563 | { 564 | if( verbose != 0 ) 565 | printf( "failed\n" ); 566 | 567 | return( 1 ); 568 | } 569 | 570 | if( verbose != 0 ) 571 | printf( "passed\n" ); 572 | } 573 | 574 | if( verbose != 0 ) 575 | printf( "\n" ); 576 | 577 | return( 0 ); 578 | } 579 | 580 | #endif 581 | 582 | #endif 583 | -------------------------------------------------------------------------------- /src/hash/md5.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file md5.h 3 | * 4 | * \brief MD5 message digest algorithm (hash function) 5 | * 6 | * Copyright (C) 2006-2010, Brainspark B.V. 7 | * 8 | * This file is part of PolarSSL (http://www.polarssl.org) 9 | * Lead Maintainer: Paul Bakker 10 | * 11 | * All rights reserved. 12 | * 13 | * This program is free software; you can redistribute it and/or modify 14 | * it under the terms of the GNU General Public License as published by 15 | * the Free Software Foundation; either version 2 of the License, or 16 | * (at your option) any later version. 17 | * 18 | * This program is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 | * GNU General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU General Public License along 24 | * with this program; if not, write to the Free Software Foundation, Inc., 25 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 26 | */ 27 | #ifndef POLARSSL_MD5_H 28 | #define POLARSSL_MD5_H 29 | 30 | #include 31 | 32 | /** 33 | * \brief MD5 context structure 34 | */ 35 | typedef struct 36 | { 37 | unsigned long total[2]; /*!< number of bytes processed */ 38 | unsigned long state[4]; /*!< intermediate digest state */ 39 | unsigned char buffer[64]; /*!< data block being processed */ 40 | 41 | unsigned char ipad[64]; /*!< HMAC: inner padding */ 42 | unsigned char opad[64]; /*!< HMAC: outer padding */ 43 | } 44 | md5_context; 45 | 46 | #ifdef __cplusplus 47 | extern "C" { 48 | #endif 49 | 50 | /** 51 | * \brief MD5 context setup 52 | * 53 | * \param ctx context to be initialized 54 | */ 55 | void md5_starts( md5_context *ctx ); 56 | 57 | /** 58 | * \brief MD5 process buffer 59 | * 60 | * \param ctx MD5 context 61 | * \param input buffer holding the data 62 | * \param ilen length of the input data 63 | */ 64 | void md5_update( md5_context *ctx, const unsigned char *input, size_t ilen ); 65 | 66 | /** 67 | * \brief MD5 final digest 68 | * 69 | * \param ctx MD5 context 70 | * \param output MD5 checksum result 71 | */ 72 | void md5_finish( md5_context *ctx, unsigned char output[16] ); 73 | 74 | /** 75 | * \brief Output = MD5( input buffer ) 76 | * 77 | * \param input buffer holding the data 78 | * \param ilen length of the input data 79 | * \param output MD5 checksum result 80 | */ 81 | void md5( const unsigned char *input, size_t ilen, unsigned char output[16] ); 82 | 83 | /** 84 | * \brief Output = MD5( file contents ) 85 | * 86 | * \param path input file name 87 | * \param output MD5 checksum result 88 | * 89 | * \return 0 if successful, 1 if fopen failed, 90 | * or 2 if fread failed 91 | */ 92 | int md5_file( const char *path, unsigned char output[16] ); 93 | 94 | /** 95 | * \brief MD5 HMAC context setup 96 | * 97 | * \param ctx HMAC context to be initialized 98 | * \param key HMAC secret key 99 | * \param keylen length of the HMAC key 100 | */ 101 | void md5_hmac_starts( md5_context *ctx, 102 | const unsigned char *key, size_t keylen ); 103 | 104 | /** 105 | * \brief MD5 HMAC process buffer 106 | * 107 | * \param ctx HMAC context 108 | * \param input buffer holding the data 109 | * \param ilen length of the input data 110 | */ 111 | void md5_hmac_update( md5_context *ctx, 112 | const unsigned char *input, size_t ilen ); 113 | 114 | /** 115 | * \brief MD5 HMAC final digest 116 | * 117 | * \param ctx HMAC context 118 | * \param output MD5 HMAC checksum result 119 | */ 120 | void md5_hmac_finish( md5_context *ctx, unsigned char output[16] ); 121 | 122 | /** 123 | * \brief MD5 HMAC context reset 124 | * 125 | * \param ctx HMAC context to be reset 126 | */ 127 | void md5_hmac_reset( md5_context *ctx ); 128 | 129 | /** 130 | * \brief Output = HMAC-MD5( hmac key, input buffer ) 131 | * 132 | * \param key HMAC secret key 133 | * \param keylen length of the HMAC key 134 | * \param input buffer holding the data 135 | * \param ilen length of the input data 136 | * \param output HMAC-MD5 result 137 | */ 138 | void md5_hmac( const unsigned char *key, size_t keylen, 139 | const unsigned char *input, size_t ilen, 140 | unsigned char output[16] ); 141 | 142 | /** 143 | * \brief Checkup routine 144 | * 145 | * \return 0 if successful, or 1 if the test failed 146 | */ 147 | int md5_self_test( int verbose ); 148 | 149 | #ifdef __cplusplus 150 | } 151 | #endif 152 | 153 | #endif /* md5.h */ 154 | -------------------------------------------------------------------------------- /src/hash/xor_hash.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * xor_hash.cpp 3 | * 4 | * Created on: 2011-5-9 5 | * Author: auxten 6 | **/ 7 | #include 8 | #include "xor_hash.h" 9 | #include "../gingko.h" 10 | #include "../log.h" 11 | 12 | /** 13 | * @brief xor hash a given length buf 14 | * 15 | * @see 16 | * @note 17 | * if hval is not 0, use it as the init hash value 18 | * @author auxten 19 | * @date 2011-8-1 20 | **/ 21 | unsigned xor_hash(const void *key, int len, unsigned hval) 22 | { 23 | #if defined(ROT_XOR_HASH) 24 | u_char *p = (u_char *) key; 25 | hval = hval ? hval : 2166136261; 26 | #if defined(HASH_BYTE_NUM_ONCE) 27 | for (int i = 0; i <= len - HASH_BYTE_NUM_ONCE; i += HASH_BYTE_NUM_ONCE) 28 | { 29 | hval = ROLL(hval) ^ p[i]; 30 | hval = ROLL(hval) ^ p[i + 1]; 31 | hval = ROLL(hval) ^ p[i + 2]; 32 | hval = ROLL(hval) ^ p[i + 3]; 33 | #if HASH_BYTE_NUM_ONCE == 8 34 | hval = ROLL(hval) ^ p[i + 4]; 35 | hval = ROLL(hval) ^ p[i + 5]; 36 | hval = ROLL(hval) ^ p[i + 6]; 37 | hval = ROLL(hval) ^ p[i + 7]; 38 | #endif /** HASH_BYTE_NUM_ONCE == 8 **/ 39 | } 40 | /** 41 | * hash the remained bytes 42 | **/ 43 | for (int i = len - len % HASH_BYTE_NUM_ONCE; i < len; i++) 44 | { 45 | hval = ROLL(hval) ^ p[i]; 46 | } 47 | #else 48 | for (int i = 0; i < len; i++) 49 | { 50 | hval = ROLL(hval) ^ p[i]; 51 | } 52 | #endif 53 | 54 | #elif defined(FNV_XOR_HASH) 55 | u_char *p = (u_char *) key; 56 | hval = hval ? hval : 2166136261; 57 | 58 | for (int i = 0; i < len; i++) 59 | { 60 | #if defined(NO_SO_CALLED_FNV_OPTIMIZE) 61 | hval = (hval * 16777619) ^ p[i]; 62 | #else 63 | hval += (hval << 1) + (hval << 4) + (hval << 7) + (hval << 8) + (hval 64 | << 24); 65 | hval ^= p[i]; 66 | #endif 67 | } 68 | 69 | return hval; 70 | #endif /** ROT_XOR_HASH **/ 71 | return hval; 72 | } 73 | 74 | /** 75 | * @brief check if the fnv check sum is OK 76 | * 77 | * @see 78 | * @note 79 | * @author auxten 80 | * @date 2011-8-1 81 | **/ 82 | char digest_ok(void * buf, s_block_t * b) 83 | { 84 | return (xor_hash(buf, b->size, 0) == b->digest); 85 | } 86 | 87 | /** 88 | * @brief xor hash specified block 89 | * 90 | * @see 91 | * @note 92 | * @author auxten 93 | * @date 2011-8-1 94 | **/ 95 | unsigned xor_hash_block(s_job_t * jo, GKO_INT64 block_id, u_char * buf) 96 | { 97 | s_file_t * files = jo->files; 98 | s_block_t * blocks = jo->blocks; 99 | GKO_INT64 read_counter = 0; 100 | GKO_INT64 file_i = (blocks + block_id)->start_f; 101 | GKO_INT64 offset = 0; 102 | int fd; 103 | unsigned tmp_hash = 0; 104 | 105 | if (FAIL_CHECK(-1 == (fd = open((files + ((blocks + block_id)->start_f))->name, 106 | O_RDONLY | O_NOFOLLOW)))) 107 | { 108 | GKOLOG(WARNING, "file open() error!"); 109 | } 110 | memset(buf, 0, BLOCK_SIZE); 111 | offset = (blocks + block_id)->start_off; 112 | while (read_counter < (blocks + block_id)->size) 113 | { 114 | GKO_INT64 tmp = pread(fd, buf + read_counter, 115 | (blocks + block_id)->size - read_counter, offset); 116 | if (FAIL_CHECK(tmp < 0)) 117 | { 118 | GKOLOG(WARNING, "pread failed"); 119 | } 120 | if (LIKELY(tmp)) 121 | { 122 | ///printf("read: %ld\n", tmp); 123 | tmp_hash = xor_hash(buf + read_counter, (int) tmp, tmp_hash); 124 | read_counter += tmp; 125 | offset += tmp; 126 | } 127 | else 128 | { 129 | close(fd); 130 | ///if the next if a nonfile then next 131 | file_i = next_f(jo, file_i); 132 | if (FAIL_CHECK(-1 133 | == (fd = open( 134 | (files + ((blocks + block_id)->start_f) 135 | + file_i)->name, O_RDONLY | O_NOFOLLOW)))) 136 | { 137 | fprintf(stderr, "filename: %s\n", 138 | (files + ((blocks + block_id)->start_f) + file_i)->name); 139 | GKOLOG(WARNING, "filename: %s", 140 | (files + ((blocks + block_id)->start_f) + file_i)->name); 141 | } 142 | offset = 0; 143 | 144 | } 145 | } 146 | (blocks + block_id)->digest = tmp_hash; 147 | /// printf("buf: %d\n", sizeof(buf)); 148 | /// memset(buf, 0, sizeof(buf)); 149 | /// printf("buf: %d\n", sizeof(buf)); 150 | close(fd); 151 | return tmp_hash; 152 | } 153 | 154 | /** 155 | * @brief xor hash the file given 156 | * 157 | * @see 158 | * @note 159 | * @author auxten 160 | * @date 2011-8-1 161 | **/ 162 | unsigned xor_hash_file(unsigned value, FILE * fd, off_t * off, size_t * count, 163 | u_char * buf) 164 | { 165 | fseeko(fd, *off, SEEK_SET); 166 | if (FAIL_CHECK(*count != fread(buf, sizeof(char), *count, fd))) 167 | { 168 | GKOLOG(FATAL, "fread error"); 169 | } 170 | ///fprintf(stderr, "#######################buf: %s\n", buf); 171 | return xor_hash(buf, *count, value); 172 | } 173 | 174 | -------------------------------------------------------------------------------- /src/hash/xor_hash.h: -------------------------------------------------------------------------------- 1 | /** 2 | * xor_hash.h 3 | * 4 | * Created on: 2011-5-9 5 | * Author: auxten 6 | **/ 7 | #include "../gingko.h" 8 | 9 | #ifndef XOR_HASH_H_ 10 | #define XOR_HASH_H_ 11 | 12 | /// shift Macro 13 | //#define ROLL(h) (((h) << 7) ^ ((h) >> 25)) 14 | #define ROLL(h) (h * 16777619) 15 | 16 | /// xor hash a given length buf 17 | unsigned xor_hash(const void *key, int len, unsigned hval); 18 | /// check if the fnv check sum is OK 19 | char digest_ok(void * buf, s_block_t * b); 20 | /// xor hash all blocks for a job given 21 | int xor_hash_all(s_job_t * jo, hash_worker_thread_arg arg[]); 22 | #endif /** XOR_HASH_H_ **/ 23 | -------------------------------------------------------------------------------- /src/hermes_errno.h: -------------------------------------------------------------------------------- 1 | /* 2 | * gko_errno.h 3 | * 4 | * Created on: May 17, 2012 5 | * Author: auxten 6 | */ 7 | 8 | #ifndef GKO_ERRNO_H_ 9 | #define GKO_ERRNO_H_ 10 | 11 | enum error_no { 12 | //////////////////// DO NOT USE DIRECTLY below ////////////////////// 13 | /// succ or fail 14 | INVILID = -1, 15 | SUCC = 0, 16 | FAIL = 1, 17 | ING = 2, /// doing something. 18 | RESUME = 3, 19 | PAUSE = 4, 20 | DELETED = 5, 21 | STOP = 6, 22 | FINISHED = 7, 23 | 24 | /// 失败原因 25 | ERROR = 10, /// 其它各种失败 26 | RECV_TIMEOUT = 20, 27 | SEND_TIMEOUT = 30, 28 | RESETED = 40, 29 | RECV_ERROR = 50, 30 | SEND_ERROR = 60, 31 | AGENT_ERROR = 70, 32 | 33 | /// 任务阶段 34 | TODO = 100, 35 | DISPATCH = 200, 36 | EXECUTE = 300, 37 | GKO_MYSQL = 400, 38 | SERVER_INTERNAL = 500, 39 | DNS_RESOLVE = 600, 40 | //////////////////// DO NOT USE DIRECTLY above ////////////////////// 41 | ////以上不要直接使用 42 | 43 | /// 任务分发结果 44 | DISPATCH_ING = DISPATCH + ING, /// 正在分发任务 45 | DISPATCH_SUCC = DISPATCH + SUCC, 46 | DISPATCH_SEND_TIMEOUT = DISPATCH + SEND_TIMEOUT + FAIL, /// 网络发送超时 47 | DISPATCH_RECV_TIMEOUT = DISPATCH + RECV_TIMEOUT + FAIL, /// 发送任务获取agent收到确认超时 48 | DISPATCH_SEND_ERROR = DISPATCH + SEND_ERROR + FAIL, /// 网络连接错误,eg. reset 49 | DISPATCH_RECV_ERROR = DISPATCH + RECV_ERROR + FAIL, /// 网络连接错误,eg. reset 50 | DISPATCH_AGENT_FAIL = DISPATCH + AGENT_ERROR + FAIL, /// agent接受任务后立刻返回失败 51 | 52 | /// 任务执行结果 53 | EXECUTE_SUCC = EXECUTE + SUCC, /// 54 | EXECUTE_FAIL = EXECUTE + FAIL, /// agent执行失败 55 | 56 | /// 服务器内部状态 57 | SERVER_INTERNAL_ERROR = SERVER_INTERNAL + ERROR + FAIL, 58 | 59 | /// DNS解析错误 60 | DNS_RESOLVE_FAIL = DNS_RESOLVE + ERROR + FAIL, 61 | 62 | }; 63 | #endif /* GKO_ERRNO_H_ */ 64 | -------------------------------------------------------------------------------- /src/libgko.pc.in: -------------------------------------------------------------------------------- 1 | prefix=@prefix@ 2 | exec_prefix=@exec_prefix@ 3 | libdir=@libdir@ 4 | includedir=@includedir@ 5 | venderlibsinc=@venderlibsinc@ 6 | venderincs=@venderincs@ 7 | 8 | Name: gko_pool 9 | Description: Async server and client framework 10 | URL: http://optool.net 11 | Version: 0.8 12 | Requires: 13 | Libs: -L${libdir} -Wl,-lgko @venderlibsinc@ -lpthread 14 | Cflags: -I${includedir} @venderincs@ 15 | -------------------------------------------------------------------------------- /src/limit.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * limit.cpp 3 | * 4 | * Created on: Mar 9, 2012 5 | * Author: auxten 6 | **/ 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "gingko.h" 17 | 18 | 19 | ///mutex for up bandwidth limit 20 | static pthread_mutex_t g_bw_up_mutex = PTHREAD_MUTEX_INITIALIZER; 21 | ///mutex for down bandwidth limit 22 | static pthread_mutex_t g_bw_down_mutex = PTHREAD_MUTEX_INITIALIZER; 23 | ///mutex for up bandwidth limit 24 | static pthread_mutex_t g_disk_r_mutex = PTHREAD_MUTEX_INITIALIZER; 25 | ///mutex for down bandwidth limit 26 | static pthread_mutex_t g_disk_w_mutex = PTHREAD_MUTEX_INITIALIZER; 27 | ///mutex for make seed limit 28 | static pthread_mutex_t g_mk_seed_mutex = PTHREAD_MUTEX_INITIALIZER; 29 | 30 | /** 31 | * @brief limit down rate 32 | * 33 | * @see 34 | * @note bwlimit modified from scp 35 | * @author auxten 36 | * @date 2011-8-1 37 | **/ 38 | void bw_down_limit(int amount, int limit_rate) 39 | { 40 | static struct timeval bw_down_start; 41 | static struct timeval bw_down_end; 42 | static int down_lamt; 43 | static int down_thresh = 16384; 44 | GKO_UINT64 waitlen; 45 | struct timespec ts; 46 | struct timespec rm; 47 | 48 | if (amount <= 0 || limit_rate <= 0) 49 | { 50 | return; 51 | } 52 | pthread_mutex_lock(&g_bw_down_mutex); 53 | 54 | if (UNLIKELY(!timerisset(&bw_down_start))) 55 | { 56 | gettimeofday(&bw_down_start, NULL); 57 | goto DISK_W_UNLOCK_RET; 58 | } 59 | 60 | down_lamt += amount; 61 | if (down_lamt < down_thresh) 62 | { 63 | goto DISK_W_UNLOCK_RET; 64 | } 65 | 66 | gettimeofday(&bw_down_end, NULL); 67 | timersub(&bw_down_end, &bw_down_start, &bw_down_end); 68 | if (!timerisset(&bw_down_end)) 69 | { 70 | goto DISK_W_UNLOCK_RET; 71 | } 72 | 73 | waitlen = (GKO_UINT64) 1000000L * down_lamt / limit_rate; 74 | 75 | bw_down_start.tv_sec = waitlen / 1000000L; 76 | bw_down_start.tv_usec = waitlen % 1000000L; 77 | 78 | if (timercmp(&bw_down_start, &bw_down_end, >)) 79 | { 80 | timersub(&bw_down_start, &bw_down_end, &bw_down_end); 81 | 82 | /** Adjust the wait time **/ 83 | if (bw_down_end.tv_sec) 84 | { 85 | down_thresh /= 2; 86 | if (down_thresh < 2048) 87 | { 88 | down_thresh = 2048; 89 | } 90 | } 91 | else if (bw_down_end.tv_usec < 100) 92 | { 93 | down_thresh *= 2; 94 | if (down_thresh > 32768) 95 | { 96 | down_thresh = 32768; 97 | } 98 | } 99 | 100 | TIMEVAL_TO_TIMESPEC(&bw_down_end, &ts); 101 | ///GKOLOG(WARNING, "sleep for: %d usec", (&bw_down_end)->tv_usec); 102 | while (nanosleep(&ts, &rm) == -1) 103 | { 104 | if (errno != EINTR) 105 | { 106 | break; 107 | } 108 | ts = rm; 109 | } 110 | } 111 | 112 | down_lamt = 0; 113 | gettimeofday(&bw_down_start, NULL); 114 | 115 | DISK_W_UNLOCK_RET: 116 | pthread_mutex_unlock(&g_bw_down_mutex); 117 | return; 118 | } 119 | 120 | /** 121 | * @brief limit up rate 122 | * 123 | * @see 124 | * @note bwlimit modified from scp 125 | * @author auxten 126 | * @date 2011-8-1 127 | **/ 128 | void bw_up_limit(int amount, int limit_rate) 129 | { 130 | static struct timeval bw_up_start; 131 | static struct timeval bw_up_end; 132 | static int up_lamt; 133 | static int up_thresh = 16384; 134 | GKO_UINT64 waitlen; 135 | struct timespec ts; 136 | struct timespec rm; 137 | 138 | if (amount <= 0 || limit_rate <= 0) 139 | { 140 | return; 141 | } 142 | pthread_mutex_lock(&g_bw_up_mutex); 143 | 144 | if (UNLIKELY(!timerisset(&bw_up_start))) 145 | { 146 | gettimeofday(&bw_up_start, NULL); 147 | goto UP_UNLOCK_RET; 148 | } 149 | 150 | up_lamt += amount; 151 | if (up_lamt < up_thresh) 152 | { 153 | goto UP_UNLOCK_RET; 154 | } 155 | 156 | gettimeofday(&bw_up_end, NULL); 157 | timersub(&bw_up_end, &bw_up_start, &bw_up_end); 158 | if (!timerisset(&bw_up_end)) 159 | { 160 | goto UP_UNLOCK_RET; 161 | } 162 | 163 | waitlen = (GKO_UINT64) 1000000L * up_lamt / limit_rate; 164 | 165 | bw_up_start.tv_sec = waitlen / 1000000L; 166 | bw_up_start.tv_usec = waitlen % 1000000L; 167 | 168 | if (timercmp(&bw_up_start, &bw_up_end, >)) 169 | { 170 | timersub(&bw_up_start, &bw_up_end, &bw_up_end); 171 | 172 | /** Adjust the wait time **/ 173 | if (bw_up_end.tv_sec) 174 | { 175 | up_thresh /= 2; 176 | if (up_thresh < 2048) 177 | { 178 | up_thresh = 2048; 179 | } 180 | } 181 | else if (bw_up_end.tv_usec < 100) 182 | { 183 | up_thresh *= 2; 184 | if (up_thresh > 32768) 185 | { 186 | up_thresh = 32768; 187 | } 188 | } 189 | 190 | TIMEVAL_TO_TIMESPEC(&bw_up_end, &ts); 191 | ///GKOLOG(WARNING, "sleep for: %d usec", (&bw_up_end)->tv_usec); 192 | while (nanosleep(&ts, &rm) == -1) 193 | { 194 | if (errno != EINTR) 195 | { 196 | break; 197 | } 198 | ts = rm; 199 | } 200 | } 201 | 202 | up_lamt = 0; 203 | gettimeofday(&bw_up_start, NULL); 204 | 205 | UP_UNLOCK_RET: 206 | pthread_mutex_unlock(&g_bw_up_mutex); 207 | return; 208 | } 209 | 210 | /** 211 | * @brief limit disk write rate 212 | * 213 | * @see 214 | * @note bwlimit modified from scp 215 | * @author auxten 216 | * @date 2011-8-1 217 | **/ 218 | void disk_w_limit(int amount, int limit_rate) 219 | { 220 | static struct timeval disk_w_start; 221 | static struct timeval disk_w_end; 222 | static int write_lamt; 223 | static int write_thresh = 16384; 224 | GKO_UINT64 waitlen; 225 | struct timespec ts; 226 | struct timespec rm; 227 | 228 | if (amount <= 0 || limit_rate <= 0) 229 | { 230 | return; 231 | } 232 | pthread_mutex_lock(&g_disk_w_mutex); 233 | 234 | if (UNLIKELY(!timerisset(&disk_w_start))) 235 | { 236 | gettimeofday(&disk_w_start, NULL); 237 | goto DISK_W_UNLOCK_RET; 238 | } 239 | 240 | write_lamt += amount; 241 | if (write_lamt < write_thresh) 242 | { 243 | goto DISK_W_UNLOCK_RET; 244 | } 245 | 246 | gettimeofday(&disk_w_end, NULL); 247 | timersub(&disk_w_end, &disk_w_start, &disk_w_end); 248 | if (!timerisset(&disk_w_end)) 249 | { 250 | goto DISK_W_UNLOCK_RET; 251 | } 252 | 253 | waitlen = (GKO_UINT64) 1000000L * write_lamt / limit_rate; 254 | 255 | disk_w_start.tv_sec = waitlen / 1000000L; 256 | disk_w_start.tv_usec = waitlen % 1000000L; 257 | 258 | if (timercmp(&disk_w_start, &disk_w_end, >)) 259 | { 260 | timersub(&disk_w_start, &disk_w_end, &disk_w_end); 261 | 262 | /** Adjust the wait time **/ 263 | if (disk_w_end.tv_sec) 264 | { 265 | write_thresh /= 2; 266 | if (write_thresh < 2048) 267 | { 268 | write_thresh = 2048; 269 | } 270 | } 271 | else if (disk_w_end.tv_usec < 100) 272 | { 273 | write_thresh *= 2; 274 | if (write_thresh > 32768) 275 | { 276 | write_thresh = 32768; 277 | } 278 | } 279 | 280 | TIMEVAL_TO_TIMESPEC(&disk_w_end, &ts); 281 | ///GKOLOG(WARNING, "sleep for: %d usec", (&disk_w_end)->tv_usec); 282 | while (nanosleep(&ts, &rm) == -1) 283 | { 284 | if (errno != EINTR) 285 | { 286 | break; 287 | } 288 | ts = rm; 289 | } 290 | } 291 | 292 | write_lamt = 0; 293 | gettimeofday(&disk_w_start, NULL); 294 | 295 | DISK_W_UNLOCK_RET: 296 | pthread_mutex_unlock(&g_disk_w_mutex); 297 | return; 298 | } 299 | 300 | /** 301 | * @brief limit disk read rate 302 | * 303 | * @see 304 | * @note bwlimit modified from scp 305 | * @author auxten 306 | * @date 2011-8-1 307 | **/ 308 | void disk_r_limit(int amount, int limit_rate) 309 | { 310 | static struct timeval disk_r_start; 311 | static struct timeval disk_r_end; 312 | static int read_lamt; 313 | static int read_thresh = 16384; 314 | GKO_UINT64 waitlen; 315 | struct timespec ts; 316 | struct timespec rm; 317 | 318 | if (amount <= 0 || limit_rate <= 0) 319 | { 320 | return; 321 | } 322 | pthread_mutex_lock(&g_disk_r_mutex); 323 | 324 | if (UNLIKELY(!timerisset(&disk_r_start))) 325 | { 326 | gettimeofday(&disk_r_start, NULL); 327 | goto DISK_R_UNLOCK_RET; 328 | } 329 | 330 | read_lamt += amount; 331 | if (read_lamt < read_thresh) 332 | { 333 | goto DISK_R_UNLOCK_RET; 334 | } 335 | 336 | gettimeofday(&disk_r_end, NULL); 337 | timersub(&disk_r_end, &disk_r_start, &disk_r_end); 338 | if (!timerisset(&disk_r_end)) 339 | { 340 | goto DISK_R_UNLOCK_RET; 341 | } 342 | 343 | waitlen = (GKO_UINT64) 1000000L * read_lamt / limit_rate; 344 | 345 | disk_r_start.tv_sec = waitlen / 1000000L; 346 | disk_r_start.tv_usec = waitlen % 1000000L; 347 | 348 | if (timercmp(&disk_r_start, &disk_r_end, >)) 349 | { 350 | timersub(&disk_r_start, &disk_r_end, &disk_r_end); 351 | 352 | /** Adjust the wait time **/ 353 | if (disk_r_end.tv_sec) 354 | { 355 | read_thresh /= 2; 356 | if (read_thresh < 2048) 357 | { 358 | read_thresh = 2048; 359 | } 360 | } 361 | else if (disk_r_end.tv_usec < 100) 362 | { 363 | read_thresh *= 2; 364 | if (read_thresh > 32768) 365 | { 366 | read_thresh = 32768; 367 | } 368 | } 369 | 370 | TIMEVAL_TO_TIMESPEC(&disk_r_end, &ts); 371 | ///GKOLOG(WARNING, "sleep for: %d usec", (&disk_r_end)->tv_usec); 372 | while (nanosleep(&ts, &rm) == -1) 373 | { 374 | if (errno != EINTR) 375 | { 376 | break; 377 | } 378 | ts = rm; 379 | } 380 | } 381 | 382 | read_lamt = 0; 383 | gettimeofday(&disk_r_start, NULL); 384 | 385 | DISK_R_UNLOCK_RET: 386 | pthread_mutex_unlock(&g_disk_r_mutex); 387 | return; 388 | } 389 | 390 | 391 | /** 392 | * @brief limit make seed rate 393 | * 394 | * @see 395 | * @note bwlimit modified from scp 396 | * @author auxten 397 | * @date 2011-8-1 398 | **/ 399 | void mk_seed_limit(int amount, int limit_rate) 400 | { 401 | static struct timeval mk_seed_start; 402 | static struct timeval mk_seed_end; 403 | static int mk_seed_lamt; 404 | static int mk_seed_thresh = 16384; 405 | GKO_UINT64 waitlen; 406 | struct timespec ts; 407 | struct timespec rm; 408 | if (amount <= 0 || limit_rate <= 0) 409 | { 410 | return; 411 | } 412 | pthread_mutex_lock(&g_mk_seed_mutex); 413 | 414 | if (UNLIKELY(!timerisset(&mk_seed_start))) 415 | { 416 | gettimeofday(&mk_seed_start, NULL); 417 | goto MK_SEED_UNLOCK_RET; 418 | } 419 | 420 | mk_seed_lamt += amount; 421 | if (mk_seed_lamt < mk_seed_thresh) 422 | { 423 | goto MK_SEED_UNLOCK_RET; 424 | } 425 | 426 | gettimeofday(&mk_seed_end, NULL); 427 | timersub(&mk_seed_end, &mk_seed_start, &mk_seed_end); 428 | if (!timerisset(&mk_seed_end)) 429 | { 430 | goto MK_SEED_UNLOCK_RET; 431 | } 432 | 433 | waitlen = (GKO_UINT64) 1000000L * mk_seed_lamt / limit_rate; 434 | 435 | mk_seed_start.tv_sec = waitlen / 1000000L; 436 | mk_seed_start.tv_usec = waitlen % 1000000L; 437 | 438 | if (timercmp(&mk_seed_start, &mk_seed_end, >)) 439 | { 440 | timersub(&mk_seed_start, &mk_seed_end, &mk_seed_end); 441 | 442 | /** Adjust the wait time **/ 443 | if (mk_seed_end.tv_sec) 444 | { 445 | mk_seed_thresh /= 2; 446 | if (mk_seed_thresh < 2048) 447 | { 448 | mk_seed_thresh = 2048; 449 | } 450 | } 451 | else if (mk_seed_end.tv_usec < 100) 452 | { 453 | mk_seed_thresh *= 2; 454 | if (mk_seed_thresh > 32768) 455 | { 456 | mk_seed_thresh = 32768; 457 | } 458 | } 459 | 460 | TIMEVAL_TO_TIMESPEC(&mk_seed_end, &ts); 461 | ///GKOLOG(WARNING, "sleep for: %d usec", (&mk_seed_end)->tv_usec); 462 | while (nanosleep(&ts, &rm) == -1) 463 | { 464 | if (errno != EINTR) 465 | { 466 | break; 467 | } 468 | ts = rm; 469 | } 470 | } 471 | 472 | mk_seed_lamt = 0; 473 | gettimeofday(&mk_seed_start, NULL); 474 | 475 | MK_SEED_UNLOCK_RET: 476 | pthread_mutex_unlock(&g_mk_seed_mutex); 477 | return; 478 | } 479 | 480 | -------------------------------------------------------------------------------- /src/limit.h: -------------------------------------------------------------------------------- 1 | /** 2 | * limit.h 3 | * 4 | * Created on: Mar 9, 2012 5 | * Author: auxten 6 | **/ 7 | 8 | #ifndef LIMIT_H_ 9 | #define LIMIT_H_ 10 | 11 | /// limit download rate 12 | void bw_down_limit(int amount, int limit_rate); 13 | /// limit upload rate 14 | void bw_up_limit(int amount, int limit_rate); 15 | /// limit disk write rate 16 | void disk_w_limit(int amount, int limit_rate); 17 | /// limit disk read rate 18 | void disk_r_limit(int amount, int limit_rate); 19 | /// limit make seed rate 20 | void mk_seed_limit(int amount, int limit_rate); 21 | 22 | #endif /** LIMIT_H_ **/ 23 | -------------------------------------------------------------------------------- /src/log.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * log.cpp 3 | * 4 | * Created on: 2011-7-13 5 | * Author: auxten 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "gingko.h" 17 | #include "log.h" 18 | 19 | extern s_gingko_global_t gko; 20 | pthread_mutex_t g_logcut_lock = PTHREAD_MUTEX_INITIALIZER; 21 | /// TLS, for print proformance time in log 22 | static pthread_key_t g_proformace_timer_key; 23 | static pthread_once_t key_once = PTHREAD_ONCE_INIT; 24 | 25 | /** 26 | * @brief loglevel 27 | * 28 | * @see 29 | * @note 30 | * @author auxten 31 | * @date 2011-8-1 32 | **/ 33 | static const char * LOG_DIC[] = 34 | { "FATAL", "WARN ", "NOTIC", "TRACE", "DEBUG", }; 35 | 36 | /** 37 | * @brief generate the time string according to the TIME_FORMAT 38 | * 39 | * @see 40 | * @note 41 | * @author auxten 42 | * @date 2011-8-1 43 | **/ 44 | size_t gettimestr(char * time, const char * format, struct timeval * tv) 45 | { 46 | struct tm ltime; 47 | time_t curtime; 48 | gettimeofday(tv, NULL); 49 | curtime = tv->tv_sec; 50 | ///Format time 51 | return strftime(time, 25, format, localtime_r(&curtime, <ime)); 52 | } 53 | 54 | static void make_key() 55 | { 56 | pthread_key_create(&g_proformace_timer_key, NULL); 57 | } 58 | 59 | static struct timeval * get_timer(void) 60 | { 61 | void *p = NULL; 62 | 63 | pthread_once(&key_once, make_key); 64 | p = pthread_getspecific(g_proformace_timer_key); 65 | if (p == NULL) 66 | { 67 | p = calloc(1, sizeof(struct timeval)); 68 | pthread_setspecific(g_proformace_timer_key, p); 69 | } 70 | 71 | return (struct timeval *) p; 72 | } 73 | 74 | /** 75 | * @brief log handler 76 | * 77 | * @see 78 | * @note 79 | * @author auxten 80 | * @date 2011-8-1 81 | **/ 82 | void gko_log_flf(const u_int8_t log_level, const char *file, const int line, const char *func, const char *fmt, ...) 83 | { 84 | if (gko.opt.to_debug || log_level < DEBUG ) 85 | { 86 | int errnum = errno; 87 | va_list args; 88 | va_start(args, fmt); 89 | char logstr[MAX_LOG_BYTE]; 90 | char oldlogpath[MAX_PATH_LEN]; 91 | char rmOldLogs[MAX_PATH_LEN]; 92 | static FILE * lastfp = NULL; 93 | static GKO_INT64 counter = 1; 94 | struct timeval last_timeval; 95 | struct timeval time_diff; 96 | struct timeval * time_p = get_timer(); 97 | long usec_diff; 98 | int len = 0; 99 | int rm_cmd_len; 100 | 101 | memcpy(&last_timeval, time_p, sizeof(last_timeval)); 102 | 103 | len += snprintf(logstr, sizeof(logstr), "%s: [%u]", LOG_DIC[log_level], gko_gettid()); 104 | len += gettimestr(logstr + len, TIME_FORMAT, time_p); 105 | timersub(time_p, &last_timeval, &time_diff); 106 | usec_diff = time_diff.tv_sec * 1000000 + time_diff.tv_usec; 107 | if (usec_diff < 0) 108 | usec_diff = 0; 109 | 110 | len += snprintf(logstr + len, sizeof(logstr) - len, "[%s:%d @%s][%ldus]\t", file, line, func, usec_diff); 111 | len += vsnprintf(logstr + len, sizeof(logstr) - len, fmt, args); 112 | 113 | logstr[sizeof(logstr) - 3] = '\0'; 114 | len = MIN(sizeof(logstr) - 3, len); 115 | 116 | if (log_level < NOTICE) 117 | { 118 | len += snprintf(logstr + len, sizeof(logstr) - len, "; "); 119 | strerror_r(errnum, logstr + len, 120 | sizeof(logstr) - len); 121 | } 122 | 123 | pthread_mutex_lock(&g_logcut_lock); 124 | if (gko.opt.logpath[0] == '\0') 125 | { 126 | gko.log_fp = stdout; 127 | } 128 | else 129 | { 130 | counter ++; 131 | if (counter % MAX_LOG_REOPEN_LINE == 0) 132 | { 133 | if (lastfp) 134 | { 135 | fclose(lastfp); 136 | } 137 | lastfp = gko.log_fp; 138 | if (counter % MAX_LOG_LINE == 0) 139 | { 140 | strncpy(oldlogpath, gko.opt.logpath, MAX_PATH_LEN - 1); 141 | gettimestr(oldlogpath + strlen(oldlogpath), OLD_LOG_TIME, time_p); 142 | rename(gko.opt.logpath, oldlogpath); 143 | rm_cmd_len = snprintf(rmOldLogs, MAX_PATH_LEN - 1, 144 | "/bin/ls -t %s.201* | /usr/bin/tail -n +%u | /usr/bin/xargs /bin/rm -f", 145 | gko.opt.logpath, MAX_LOG_KEEPED); 146 | if (strlen(gko.opt.logpath) > 0 && 147 | rm_cmd_len < MAX_PATH_LEN && 148 | rm_cmd_len > 0) 149 | { 150 | rmOldLogs[rm_cmd_len] = '\0'; 151 | system(rmOldLogs); 152 | } 153 | 154 | } 155 | gko.log_fp = fopen(gko.opt.logpath, "a+"); 156 | } 157 | if(UNLIKELY(! gko.log_fp)) 158 | { 159 | gko.log_fp = fopen(gko.opt.logpath, "a+"); 160 | if(! gko.log_fp) 161 | { 162 | perror("Cann't open log file"); 163 | _exit(1); 164 | } 165 | } 166 | } 167 | fprintf(gko.log_fp, "%s\n", logstr); 168 | if (usec_diff >= 1000000 || 169 | (counter % MAX_LOG_FLUSH_LINE == 0)) 170 | { 171 | fflush(gko.log_fp); 172 | } 173 | pthread_mutex_unlock(&g_logcut_lock); 174 | 175 | va_end(args); 176 | } 177 | return; 178 | 179 | } 180 | 181 | int lock_log(void) 182 | { 183 | return pthread_mutex_lock(&g_logcut_lock); 184 | } 185 | 186 | int unlock_log(void) 187 | { 188 | return pthread_mutex_unlock(&g_logcut_lock); 189 | } 190 | 191 | int reinit_log_lock(void) 192 | { 193 | return pthread_mutex_init(&g_logcut_lock, NULL); 194 | } 195 | -------------------------------------------------------------------------------- /src/log.h: -------------------------------------------------------------------------------- 1 | /** 2 | * log.h 3 | * 4 | * Created on: Mar 9, 2012 5 | * Author: auxten 6 | * 7 | **/ 8 | 9 | #ifndef LOG_H_ 10 | #define LOG_H_ 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | ///fatal 22 | static const u_int8_t FATAL = 0; 23 | ///warning 24 | static const u_int8_t WARNING = 1; 25 | ///notice 26 | static const u_int8_t NOTICE = 2; 27 | ///trace 28 | static const u_int8_t TRACE = 3; 29 | ///debug 30 | static const u_int8_t DEBUG = 4; 31 | 32 | /// to print the file line func 33 | /// MUST NOT use it in "const char *fmt, ..." type func 34 | //#define FLF(a) "{%s:%d %s} %s",__FILE__,__LINE__,__func__,#a 35 | #define FLF(a) a 36 | 37 | #define GKOLOG(level, args...) gko_log_flf(level, __FILE__, __LINE__, __func__, args) 38 | 39 | /// log handler 40 | void gko_log_flf(const u_int8_t log_level, const char *file, const int line, const char *func, const char *fmt, ...); 41 | int lock_log(void); 42 | int unlock_log(void); 43 | int reinit_log_lock(void); 44 | 45 | #endif /** LOG_H_ **/ 46 | -------------------------------------------------------------------------------- /src/memory.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * memory.cpp 3 | * 4 | * Created on: Jun 8, 2012 5 | * Author: auxten 6 | * 7 | * only support little endian : x86 8 | */ 9 | 10 | //#define MEM_TEST 11 | #define _XOPEN_SOURCE 600 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "memory.h" 19 | 20 | #include "log.h" 21 | 22 | 23 | gkoAlloc::gkoAlloc(void) 24 | { 25 | pthread_mutex_init(&alloc_lock, NULL); 26 | memset((void *) m_map, 0, M_MAP_SIZE * sizeof(u_int8_t)); 27 | memset((void *) bucket_s, 0, BUCKET_COUNT * sizeof(void *)); 28 | memset((void *) bucket_used, 0, BUCKET_COUNT * sizeof(int16_t)); 29 | latest_bucket = 0; 30 | } 31 | 32 | void * gkoAlloc::id2addr(int block_id) 33 | { 34 | if (block_id < 0) 35 | return NULL; 36 | 37 | int bucket_no = block_id / BUCKET_CAPACITY; 38 | int bucket_offset = SLOT_SIZE * (block_id % BUCKET_CAPACITY); 39 | 40 | return ((char *)bucket_s[bucket_no]) + bucket_offset; 41 | } 42 | 43 | int gkoAlloc::get_bit(u_int8_t * b) 44 | { 45 | /** 46 | * idx 01234567 47 | * byte 11111010 48 | * 49 | * return 5 and 50 | * byte 11111110 51 | */ 52 | 53 | int i; 54 | for (i = 0; i < 8; i++) 55 | { 56 | if ((u_int8_t)((*b >> 7 - i) << 7) == (u_int8_t)0u) 57 | break; 58 | } 59 | 60 | *b |= (u_int8_t)( 1u << 7 - i); 61 | 62 | return i; 63 | } 64 | 65 | int gkoAlloc::free_bit(u_int8_t * b, int index) 66 | { 67 | /** 68 | * idx 01234567 69 | * byte 11111110 70 | * 71 | * return 5 and 72 | * byte 11111010 73 | */ 74 | 75 | *b ^= (u_int8_t)( 1u << 7 - index); 76 | 77 | return index; 78 | } 79 | 80 | int gkoAlloc::get_block(void) 81 | { 82 | int i; 83 | int the_bucket; 84 | int idx = INVILID_BLOCK; 85 | u_int8_t * p_idx; 86 | u_int64_t * bucket_start_idx; 87 | u_int64_t * bucket_end_idx; 88 | u_int64_t * bucket_idx; 89 | 90 | pthread_mutex_lock(&alloc_lock); 91 | for (i = 0; i < BUCKET_COUNT; i++) 92 | { 93 | the_bucket = (latest_bucket + i) % BUCKET_COUNT; 94 | if (bucket_used[the_bucket] < BUCKET_CAPACITY) 95 | { 96 | // latest_bucket = the_bucket; 97 | break; 98 | } 99 | } 100 | 101 | if (i == BUCKET_COUNT) 102 | { 103 | fprintf(stderr, "out of memory in pool\n"); 104 | GKOLOG(FATAL, "out of memory in pool"); 105 | idx = INVILID_BLOCK; 106 | goto GET_BLOCK_RET; 107 | } 108 | 109 | if (!bucket_s[the_bucket]) 110 | { 111 | void * ptr; 112 | if (!posix_memalign(&ptr, SLOT_SIZE, BUCKET_SIZE)) 113 | { 114 | bucket_s[the_bucket] = ptr; 115 | bucket_used[the_bucket] = 0; 116 | } 117 | else 118 | { 119 | fprintf(stderr, "posix_memalign fail\n"); 120 | // GKOLOG(FATAL, "posix_memalign fail"); 121 | idx = INVILID_BLOCK; 122 | goto GET_BLOCK_RET; 123 | } 124 | } 125 | 126 | bucket_start_idx = (u_int64_t *) &(this->m_map[the_bucket * BUCKET_CAPACITY / 8]); 127 | bucket_end_idx = (u_int64_t *) &(this->m_map[(the_bucket + 1) * BUCKET_CAPACITY / 8]); 128 | for (bucket_idx = bucket_start_idx; 129 | bucket_idx < bucket_end_idx; 130 | bucket_idx++) 131 | { 132 | if (*(u_int64_t *) bucket_idx != ~0uLL) 133 | { 134 | if (*(u_int32_t *) bucket_idx != ~0u) 135 | { 136 | if (*((u_int16_t *) bucket_idx) != (u_int16_t) ~0u) 137 | { 138 | if (*(u_int8_t *) bucket_idx != (u_int8_t) ~0u) 139 | { 140 | p_idx = (u_int8_t *) bucket_idx + 0; 141 | } 142 | else 143 | { 144 | p_idx = (u_int8_t *) bucket_idx + 1; 145 | } 146 | } 147 | else 148 | { 149 | if (*((u_int8_t *) bucket_idx + 2) != (u_int8_t) ~0u) 150 | { 151 | p_idx = (u_int8_t *) bucket_idx + 2; 152 | } 153 | else 154 | { 155 | p_idx = (u_int8_t *) bucket_idx + 3; 156 | } 157 | 158 | } 159 | } 160 | else 161 | { 162 | if (*((u_int16_t *) bucket_idx + 2) != (u_int16_t) ~0u) 163 | { 164 | if (*((u_int8_t *) bucket_idx + 4) != (u_int8_t) ~0u) 165 | { 166 | p_idx = (u_int8_t *) bucket_idx + 4; 167 | } 168 | else 169 | { 170 | p_idx = (u_int8_t *) bucket_idx + 5; 171 | } 172 | } 173 | else 174 | { 175 | if (*((u_int8_t *) bucket_idx + 6) != (u_int8_t) ~0u) 176 | { 177 | p_idx = (u_int8_t *) bucket_idx + 6; 178 | } 179 | else 180 | { 181 | p_idx = (u_int8_t *) bucket_idx + 7; 182 | } 183 | 184 | } 185 | } 186 | idx = get_bit(p_idx) + 187 | 8 * (p_idx - (u_int8_t *) bucket_start_idx) + 188 | the_bucket * BUCKET_CAPACITY; 189 | bucket_used[the_bucket] ++; 190 | break; 191 | } 192 | else 193 | { 194 | continue; 195 | } 196 | } 197 | 198 | GET_BLOCK_RET: 199 | pthread_mutex_unlock(&alloc_lock); 200 | return idx; 201 | } 202 | 203 | void gkoAlloc::free_block(int block_id) 204 | { 205 | if (block_id < 0) 206 | return; 207 | 208 | int bucket_no = block_id / BUCKET_CAPACITY; 209 | 210 | pthread_mutex_lock(&alloc_lock); 211 | free_bit(&m_map[block_id / 8], block_id % 8); 212 | 213 | if(--bucket_used[bucket_no] == 0 && bucket_no) 214 | { 215 | free(bucket_s[bucket_no]); 216 | bucket_s[bucket_no] = NULL; 217 | } 218 | else 219 | { 220 | latest_bucket = bucket_no; 221 | } 222 | pthread_mutex_unlock(&alloc_lock); 223 | 224 | } 225 | 226 | #ifdef MEM_TEST 227 | int main() 228 | { 229 | gkoAlloc mem; 230 | for (int i = 0; i < BUCKET_CAPACITY - 1; i++) 231 | { 232 | int k = mem.get_block(); 233 | printf("%d, %d\n", i, k); 234 | if (i != k) 235 | { 236 | break; 237 | } 238 | } 239 | int blk1 = mem.get_block(); 240 | int blk2 = mem.get_block(); 241 | int blk3 = mem.get_block(); 242 | printf("%p\n", mem.id2addr(blk1)); 243 | printf("%p\n", mem.id2addr(blk2)); 244 | printf("%p\n", mem.id2addr(blk3)); 245 | mem.free_block(blk1); 246 | mem.free_block(blk2); 247 | mem.free_block(blk3); 248 | return 0; 249 | } 250 | #endif 251 | -------------------------------------------------------------------------------- /src/memory.h: -------------------------------------------------------------------------------- 1 | /* 2 | * memory.h 3 | * 4 | * Created on: Jun 8, 2012 5 | * Author: auxten 6 | */ 7 | 8 | #ifndef GKO_MEMORY_H_ 9 | #define GKO_MEMORY_H_ 10 | 11 | #include 12 | #include 13 | 14 | static const u_int32_t SLOT_SIZE = 4 * 1024; /// one page is good 15 | static const u_int32_t SLOT_COUNT = 1 * 1024 * 1024; 16 | static const u_int32_t M_MAP_SIZE = SLOT_COUNT / sizeof(u_int8_t); /// bitmap 17 | static const u_int32_t BUCKET_SIZE = 16 * 1024 * 1024; 18 | static const int32_t BUCKET_CAPACITY = BUCKET_SIZE / SLOT_SIZE; /// capacity, 4096 19 | static const int32_t BUCKET_COUNT = SLOT_COUNT / BUCKET_CAPACITY; /// 256 20 | static const int INVILID_BLOCK = -1; 21 | 22 | 23 | class gkoAlloc 24 | { 25 | private: 26 | pthread_mutex_t alloc_lock; 27 | u_int8_t m_map[M_MAP_SIZE]; /// 1MB can fit L2 cache 28 | void * bucket_s[BUCKET_COUNT]; 29 | int16_t bucket_used[BUCKET_COUNT]; 30 | int latest_bucket; 31 | int get_bit(u_int8_t * b); 32 | int free_bit(u_int8_t * b, int index); 33 | 34 | public: 35 | gkoAlloc(void); 36 | int get_block(void); 37 | int get_clear_block(void); 38 | int get2x_block(int block_id); 39 | void free_block(int block_id); 40 | int clear_block(void *block, int c, size_t size); 41 | void * id2addr(int block_id); 42 | }; 43 | 44 | #endif /* GKO_MEMORY_H_ */ 45 | -------------------------------------------------------------------------------- /src/path.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * path.cpp 3 | * 4 | * Created on: 2011-5-11 5 | * Author: auxten 6 | **/ 7 | #include 8 | #include 9 | #include "gingko.h" 10 | #include "hash/xor_hash.h" 11 | #include "log.h" 12 | #include "path.h" 13 | 14 | /** 15 | * @brief ../path//// TO ../path 16 | * @brief ../path/ TO ../path 17 | * @brief ../path TO ../path 18 | * 19 | * @see 20 | * @note 21 | * @author auxten 22 | * @date 2011-8-1 23 | **/ 24 | int inplace_strip_tailing_slash(char * path) 25 | { 26 | if (path == NULL) 27 | { 28 | GKOLOG(WARNING, "passed NULL p"); 29 | return -1; 30 | } 31 | char * p = path; 32 | for (int i = strlen(path) - 1; i > 0; i--) 33 | { 34 | if (*(p + i) == '/') 35 | { 36 | *(p + i) = '\0'; 37 | } 38 | else 39 | { 40 | break; 41 | } 42 | } 43 | return 0; 44 | } 45 | 46 | /** 47 | * @brief ../path/// TO ../path/ 48 | * @brief ../path/ TO ../path/ 49 | * @brief ../path TO ../path/ 50 | * 51 | * @see 52 | * @note 53 | * @author auxten 54 | * @date 2011-8-1 55 | **/ 56 | int inplace_add_tailing_slash(char * path) 57 | { 58 | if (path == NULL) 59 | { 60 | GKOLOG(WARNING, "passed NULL p"); 61 | return -1; 62 | } 63 | char * p = path; 64 | for (int i = strlen(path) - 1; i > 0; i--) 65 | { 66 | if (*(p + i) == '/') 67 | { 68 | *(p + i) = '\0'; 69 | } 70 | else 71 | { 72 | break; 73 | } 74 | } 75 | strncat(path, "/", MAX_PATH_LEN - strlen(path)); 76 | return 0; 77 | } 78 | 79 | /** 80 | * @brief get the base name of a string, 81 | * @brief if out not NULL, cp the base path to out 82 | * @brief return the base path len 83 | * 84 | * @see 85 | * @note 86 | * for example: 87 | * /home/work/opdir -> 11 88 | * ^ 89 | * ./dir -> 2 90 | * ^ 91 | * dir -> 0 92 | * ^ 93 | * ../file -> 3 94 | * ^ 95 | * @author auxten 96 | * @date 2011-8-1 97 | **/ 98 | int get_base_name_index(char * out, const char * in) 99 | { 100 | if (in == NULL) 101 | { 102 | GKOLOG(WARNING, FLF("passed NULL p at get_base_name_index")); 103 | return -1; 104 | } 105 | char * p = (char *) in; 106 | int i; 107 | int len = strlen(in); 108 | if (!len) 109 | { 110 | GKOLOG(WARNING, FLF("input string len == 0")); 111 | return -1; 112 | } 113 | for (i = len; i > 0; i--) 114 | { 115 | if (*(p + i) == '/') 116 | { 117 | i++; 118 | break; 119 | } 120 | } 121 | if (out) 122 | { 123 | strncpy(out, p + i, MAX_PATH_LEN); 124 | } 125 | return i; 126 | } 127 | 128 | /** 129 | * @brief in : const char * dir_name, const char * base_name 130 | * @brief out : dir_name/base_name 131 | * 132 | * @see 133 | * @note 134 | * @author auxten 135 | * @date 2011-8-1 136 | **/ 137 | int merge_path(char * out, const char * dir_name, const char * base_name) 138 | { 139 | if (!out || !dir_name || !base_name) 140 | { 141 | GKOLOG(WARNING, "passed NULL p"); 142 | return -1; 143 | } 144 | strncpy(out, dir_name, MAX_PATH_LEN); 145 | inplace_strip_tailing_slash(out); 146 | strncat(out, "/", MAX_PATH_LEN - strlen(out)); 147 | strncat(out, base_name, MAX_PATH_LEN - strlen(out)); 148 | if (strlen(out) == MAX_PATH_LEN) 149 | { 150 | GKOLOG(WARNING, "path too long"); 151 | return -1; 152 | } 153 | return 0; 154 | } 155 | 156 | /** 157 | * @brief change remote path to local path, store it in path 158 | * 159 | * @see 160 | * @note 161 | * path: ../test/.DS_Store 162 | * req_path: ../test 163 | * local_path: ../output2/ 164 | * if (dst_path_exist) 165 | * path output: ../output2/test/.DS_Store 166 | * else 167 | * path output: ../output2/.DS_Store 168 | * @author auxten 169 | * @date 2011-8-1 170 | **/ 171 | int change_to_local_path(char * path, const char * req_path, 172 | const char * local_path, char dst_path_exist) 173 | { 174 | if (!path || !req_path || !local_path) 175 | { 176 | GKOLOG(FATAL, "%s %s passed NULL p", __FILE__, __func__); 177 | return -1; 178 | } 179 | char base_name[MAX_PATH_LEN] = "\0"; 180 | char tmp_req_path[MAX_PATH_LEN] = "\0"; 181 | strncpy(tmp_req_path, req_path, MAX_PATH_LEN); /// ../test 182 | ///printf("tmp2: %s", tmp2); 183 | inplace_strip_tailing_slash(tmp_req_path); /// ../test 184 | ///printf("tmp2: %s", tmp2); 185 | int d; 186 | if (dst_path_exist) 187 | { 188 | d = get_base_name_index(NULL, tmp_req_path); 189 | } 190 | else 191 | { 192 | d = strlen(tmp_req_path) + 1; 193 | } 194 | 195 | if (d < 0) 196 | { 197 | GKOLOG(FATAL, FLF("change_to_local_path failed")); 198 | return -1; 199 | } 200 | ///printf("d: %d,path+d: %s", d, tmp2+d); 201 | strncpy(base_name, path + d, MAX_PATH_LEN); /// test/.DS_Store 202 | merge_path(path, local_path, base_name); /// ../output2/ + test/.DS_Store 203 | inplace_strip_tailing_slash(path); /// ../test 204 | return 0; 205 | } 206 | 207 | /** 208 | * @brief change current working dir path to absolute path 209 | * 210 | * @see 211 | * @note 212 | * result is stored in abs_path 213 | * return abs_path on succeed else NULL 214 | * @author auxten 215 | * @date 2011-8-1 216 | **/ 217 | GKO_STATIC_FUNC char * cwd_path_to_abs_path(char * abs_path, const char * oldpath) 218 | { 219 | if (!abs_path || !oldpath) 220 | { 221 | GKOLOG(FATAL, "%s %s passed NULL p", __FILE__, __func__); 222 | return NULL; 223 | } 224 | if ((strlen(oldpath) < 1)) 225 | { 226 | GKOLOG(WARNING, "invalid oldpath"); 227 | return NULL; 228 | } 229 | else 230 | { 231 | if (*oldpath == '/') ///already a abs path 232 | { 233 | strncpy(abs_path, oldpath, MAX_PATH_LEN); 234 | } 235 | else 236 | { 237 | if (!(getcwd(abs_path, MAX_PATH_LEN))) 238 | { 239 | GKOLOG(WARNING, "getcwd error"); 240 | return NULL; 241 | } 242 | inplace_add_tailing_slash(abs_path); 243 | strncat(abs_path, oldpath, MAX_PATH_LEN - strlen(abs_path)); 244 | } 245 | return abs_path; 246 | } 247 | } 248 | 249 | /** 250 | * @brief get the symlink dest's absolute path, store it in abs_path 251 | * 252 | * @see 253 | * @note 254 | * @author auxten 255 | * @date 2011-8-1 256 | **/ 257 | char * symlink_dest_to_abs_path(char * abs_path, const char * symlink) 258 | { 259 | if (!abs_path || !symlink) 260 | { 261 | GKOLOG(FATAL, "%s %s passed NULL p", __FILE__, __func__); 262 | return NULL; 263 | } 264 | 265 | memset(abs_path, 0, MAX_PATH_LEN); 266 | /** read the symlink first **/ 267 | if (readlink(symlink, abs_path, MAX_PATH_LEN) < 0) 268 | { 269 | GKOLOG(WARNING, "read synlink '%s' dest failed", symlink); 270 | return NULL; 271 | } 272 | 273 | /** if starts with a '/', that's it!! **/ 274 | if (*abs_path == '/') 275 | {/** symlink to a abs path **/ 276 | GKOLOG(NOTICE, "symlink to a abs path:)"); 277 | return abs_path; 278 | } 279 | else 280 | {/** symlink to a relative path, need convert **/ 281 | char tmp_sympath[MAX_PATH_LEN]; 282 | strncpy(tmp_sympath, abs_path, MAX_PATH_LEN); 283 | if (!cwd_path_to_abs_path(abs_path, symlink)) 284 | { 285 | GKOLOG(WARNING, "cwd_path_to_abs_path failed '%s' '%s'", abs_path, 286 | symlink); 287 | return NULL; 288 | } 289 | int idx = get_base_name_index(NULL, abs_path); 290 | if (idx < 0) 291 | { 292 | GKOLOG(FATAL, "get_base_name_index failed: '%s'", abs_path); 293 | return NULL; 294 | } 295 | *(abs_path + idx) = '\0'; 296 | strncat(abs_path, tmp_sympath, MAX_PATH_LEN); 297 | return abs_path; 298 | } 299 | 300 | } 301 | 302 | /** 303 | * @brief generate snap file path with requested localpath and requested remote uri 304 | * 305 | * @see 306 | * @note 307 | * return the hash result from xor_hash(uri, strlen(uri), 0) 308 | * the snap file path is stored in snap_fpath 309 | * @return 0 for error 310 | * uri_hash for succ 311 | * @author auxten 312 | * @date 2011-8-1 313 | **/ 314 | unsigned gen_snap_fpath(char *snap_fpath, const char * localpath, 315 | const char * uri) 316 | { 317 | if (!snap_fpath || !localpath || !uri) 318 | { 319 | GKOLOG(FATAL, "%s %s passed NULL p", __FILE__, __func__); 320 | return 0; 321 | } 322 | 323 | char out_path[MAX_PATH_LEN] = {'\0'}; 324 | strncpy(out_path, localpath, MAX_PATH_LEN); 325 | inplace_strip_tailing_slash(out_path); 326 | 327 | if (!cwd_path_to_abs_path(snap_fpath, out_path)) 328 | { 329 | GKOLOG(WARNING, "cwd_path_to_abs_path failed '%s' '%s'", snap_fpath, 330 | out_path); 331 | return 0; 332 | } 333 | /** cut the tailing file in path **/ 334 | if (!(path_type(snap_fpath) & GKO_DIR)) 335 | { 336 | int idx; 337 | if ((idx = get_base_name_index(NULL, snap_fpath)) < 0) 338 | { 339 | GKOLOG(WARNING, "gen_snap_fpath failed '%s' '%s' '%s'", snap_fpath, 340 | out_path, uri); 341 | return 0; 342 | } 343 | *(snap_fpath + idx) = '\0'; 344 | } 345 | 346 | int snap_fpath_len = strlen(snap_fpath); 347 | unsigned uri_hash = xor_hash(uri, strlen(uri), 0); 348 | snprintf(snap_fpath + snap_fpath_len, MAX_PATH_LEN - snap_fpath_len, 349 | "/%s%u", GKO_SNAP_FILE, uri_hash); 350 | 351 | GKOLOG(NOTICE, "gko.snap_fpath: '%s'", snap_fpath); 352 | return uri_hash; 353 | } 354 | 355 | /** 356 | * @brief get the dest type, return: 357 | * 358 | * @see 359 | * @note 360 | * GKO_FILE = 0001; ///file and for file test.eg: flag & GKO_FILE 361 | * GKO_DIR = 0002; ///dir and for dir test 362 | * GKO_NONE = 0004; ///nonexisted and for nonexisted test 363 | * GKO_OTHR = 0000; ///other 364 | * GKO_LFILE = 0011; ///symlink to file 365 | * GKO_LDIR = 0012; ///symlink to dir 366 | * GKO_LNONE = 0014; ///symlink to nonexist 367 | * GKO_LOTHR = 0010; ///symlink to other 368 | * GKO_LINK = 0010; ///for symlink test 369 | * GKO_ERR = 0100; ///error 370 | * @author auxten 371 | * @date 2011-8-1 372 | **/ 373 | int path_type(const char * p) 374 | { 375 | if (!p) 376 | { 377 | GKOLOG(FATAL, FLF("null passed to path_type")); 378 | return GKO_ERR; 379 | } 380 | struct stat dest_stat; 381 | char path[MAX_PATH_LEN]; 382 | strncpy(path, p, MAX_PATH_LEN); 383 | inplace_strip_tailing_slash(path); 384 | if (lstat(path, &dest_stat) < 0) 385 | { 386 | if (errno == ENOENT) 387 | { 388 | GKOLOG(NOTICE, "non existed path '%s'", path); 389 | return GKO_NONE; 390 | } 391 | GKOLOG(WARNING, "stat dest_stat failed %d", errno); 392 | return GKO_ERR; 393 | } 394 | else 395 | { 396 | switch (dest_stat.st_mode & S_IFMT) 397 | { 398 | case S_IFREG: 399 | return GKO_FILE; 400 | 401 | case S_IFDIR: 402 | return GKO_DIR; 403 | 404 | case S_IFLNK: 405 | { 406 | struct stat symstat; 407 | if (stat(path, &symstat) < 0) 408 | { 409 | if (errno == ENOENT) 410 | { 411 | GKOLOG(WARNING, "non existed sympath '%s'", path); 412 | return GKO_LNONE; 413 | } 414 | GKOLOG(WARNING, "stat symstat failed %d", errno); 415 | return GKO_ERR; 416 | } 417 | else 418 | { 419 | switch (symstat.st_mode & S_IFMT) 420 | { 421 | case S_IFREG: 422 | return GKO_LFILE; 423 | 424 | case S_IFDIR: 425 | return GKO_LDIR; 426 | 427 | default: 428 | GKOLOG(WARNING, 429 | "symbol path '%s' not a regular file or dir", 430 | path); 431 | return GKO_LOTHR; 432 | } 433 | } 434 | break; 435 | } 436 | 437 | default: 438 | GKOLOG(WARNING, "'%s' not a regular file or dir", path); 439 | return GKO_OTHR; 440 | } 441 | } 442 | } 443 | 444 | /** 445 | * @brief make dir and symlink, create the file and check the size 446 | * 447 | * @see 448 | * @note 449 | * if size not matched, continue flag is canceled. all file 450 | * will be truncate to the expected size. 451 | * if the symlink is already exist, unlink it and create a 452 | * new one 453 | * if the dir is already exist, leave it there 454 | * @author auxten 455 | * @date 2011-8-1 456 | **/ 457 | int mk_dir_symlink_file(s_job_t * jo, char * to_continue) 458 | { 459 | if (!jo || !to_continue) 460 | { 461 | GKOLOG(FATAL, FLF("null pointer")); 462 | return -1; 463 | } 464 | s_file_t * tmp; 465 | for (GKO_INT64 i = 0; i < jo->file_count; i++) 466 | { 467 | tmp = jo->files + i; 468 | int fd = -1; 469 | GKOLOG(NOTICE, "0%3o\t%lld\t%s\t%s", tmp->mode & 0777, tmp->size, 470 | (jo->files + i)->name, tmp->sympath); 471 | switch (tmp->size) 472 | { 473 | /** 474 | * to make sure client can write to every file transfered, 475 | * we create the dir and file with mode |= S_IWUSR 476 | * we will correct file, dir mode after transfer is over. 477 | **/ 478 | case -1: /// dir 479 | if (mkdir(tmp->name, tmp->mode|S_IWUSR) && errno != EEXIST) 480 | { 481 | GKOLOG(FATAL, "mkdir error"); 482 | return -1; 483 | } 484 | break; 485 | 486 | case -2: /// symbol link 487 | if (symlink(tmp->sympath, tmp->name)) 488 | { 489 | if (errno == EEXIST) 490 | { 491 | if (unlink(tmp->name)) 492 | { 493 | GKOLOG(FATAL, "unlink existed symlink '%s' error", 494 | tmp->name); 495 | } 496 | if (symlink(tmp->sympath, tmp->name)) 497 | { 498 | GKOLOG(FATAL, "re-create symlink '%s' to '%s' error", 499 | tmp->name, tmp->sympath); 500 | } 501 | } 502 | else 503 | { 504 | GKOLOG(FATAL, "symlink error"); 505 | return -1; 506 | } 507 | } 508 | break; 509 | 510 | default: ///regular file 511 | if (-1 == (fd = open(tmp->name, CREATE_OPEN_FLAG, tmp->mode|S_IWUSR))) 512 | { 513 | GKOLOG(FATAL, "make or open new file error"); 514 | return -1; 515 | } 516 | else 517 | { 518 | struct stat f_stat; 519 | /// if to_continue flag is on, but the file size existed 520 | /// doesn't match the one in seed. cancel the to_continue 521 | /// flag. 522 | if (*to_continue) 523 | { 524 | if (fstat(fd, &f_stat)) 525 | { 526 | GKOLOG( 527 | WARNING, 528 | "fstat file '%s' error, continue flag canceled", 529 | tmp->name); 530 | *to_continue = 0; 531 | } 532 | if (f_stat.st_size != tmp->size) 533 | { 534 | GKOLOG( 535 | WARNING, 536 | "file '%s' size not matched, continue flag canceled", 537 | tmp->name); 538 | *to_continue = 0; 539 | } 540 | } 541 | if (ftruncate(fd, tmp->size)) 542 | { 543 | GKOLOG(FATAL, "truncate file error"); 544 | close(fd); 545 | return -1; 546 | } 547 | close(fd); 548 | } 549 | break; 550 | } 551 | } 552 | return 0; 553 | } 554 | 555 | /** 556 | * @brief correct the file and dir mode, cause for write in we create them with mode|S_IWUSR 557 | * and take care of the setgit of dir 558 | * 559 | * @see http://www.gnu.org/software/coreutils/manual/html_node/Directory-Setuid-and-Setgid.html 560 | * @note 561 | * @author auxten 562 | * @date 2011-8-10 563 | **/ 564 | int correct_mode(s_job_t * jo) 565 | { 566 | s_file_t * f_p; 567 | for (GKO_INT64 i = 0; i < jo->file_count; i++) 568 | { 569 | f_p = jo->files + i; 570 | if ((f_p->size == -1) || (!(f_p->mode & S_IWUSR))) 571 | { 572 | if (chmod(f_p->name, f_p->mode)) 573 | { 574 | GKOLOG(FATAL, "chmod for '%s' to 0%3o failed!!!", f_p->name, 575 | f_p->mode & 0777); 576 | return -1; 577 | } 578 | GKOLOG(TRACE, "chmod for '%s' to 0%3o succeed", f_p->name, 579 | f_p->mode & 0777); 580 | } 581 | } 582 | 583 | return 0; 584 | } 585 | /** 586 | * @brief convert the remote path recv from JOIN to local path 587 | * 588 | * @see 589 | * @note 590 | * @author auxten 591 | * @date 2011-8-1 592 | **/ 593 | int process_path(s_job_t * jo) 594 | { 595 | char out_path[MAX_PATH_LEN] = {'\0'}; 596 | strncpy(out_path, jo->path, MAX_PATH_LEN); 597 | inplace_strip_tailing_slash(out_path); 598 | int out_type = path_type(out_path); 599 | if (out_type & GKO_NONE) 600 | {/** non-existed dest or dest is symlink to non-existed **/ 601 | if (out_type & GKO_LINK) 602 | {/** dest is symlink to non-existed **/ 603 | GKOLOG(FATAL, "destination: '%s' is a symlink to non-exist", 604 | out_path); 605 | return -1; 606 | } 607 | else if ((jo->file_count == 1 && jo->files->size >= 0) && 608 | (jo->path)[strlen(jo->path) - 1] == '/') 609 | {/** dest is a non existed dir path like './non/' but job is a singal file **/ 610 | GKOLOG(FATAL, "downloading a file to non existed path '%s'", 611 | jo->path); 612 | return -1; 613 | } 614 | else 615 | { /** non-existed dest **/ 616 | /** determine if the base dir is existed **/ 617 | char out_base_path[MAX_PATH_LEN]; 618 | //strncpy(out_base_path, out_path, MAX_PATH_LEN); 619 | cwd_path_to_abs_path(out_base_path, out_path); 620 | 621 | int base_idx = get_base_name_index(NULL, out_base_path); 622 | if (base_idx < 0) 623 | { 624 | GKOLOG(FATAL, "process_path failed"); 625 | return -1; 626 | } 627 | out_base_path[base_idx] = '\0'; 628 | int out_base_type = path_type(out_base_path); 629 | if (out_base_type & GKO_DIR) 630 | {/** out base path is a dir or symlink to dir **/ 631 | for (int i = 0; i < jo->file_count; i++) 632 | { 633 | if (FAIL_CHECK( 634 | change_to_local_path((jo->files + i)->name, jo->uri, out_path, 0))) 635 | { 636 | GKOLOG( 637 | FATAL, 638 | "change to local path error, name: '%s', uri: '%s', path: '%s'", 639 | (jo->files + i)->name, jo->uri, out_path); 640 | return -1; 641 | } 642 | //GKOLOG(NOTICE, "path: '%s'", (jo->files + i)->name); 643 | } 644 | return 0; 645 | } 646 | else 647 | {/** out base path is not dir, nor symlink to dir **/ 648 | GKOLOG(FATAL, "base path: '%s' is non-dir", out_base_path); 649 | return -1; 650 | } 651 | } 652 | } 653 | else 654 | {/** dest path existed **/ 655 | if (out_type & GKO_FILE) 656 | {/** dest path is file or symlink to file **/ 657 | if (jo->file_count == 1) 658 | { 659 | if ((jo->files)->size == -1) 660 | {/** dest path is dir **/ 661 | GKOLOG(FATAL, "can't overwrite dir: '%s' on file: '%s'", 662 | (jo->files)->name, out_path); 663 | return -1; 664 | } 665 | else if ((jo->files)->size == -2) 666 | {/** dest path is symlink **/ 667 | GKOLOG(FATAL, "can't overwrite symlink: '%s' on file: '%s'", 668 | (jo->files)->name, out_path); 669 | return -1; 670 | } 671 | else 672 | {/** dest path is file **/ 673 | strncpy((jo->files)->name, out_path, MAX_PATH_LEN); 674 | return 0; 675 | } 676 | } 677 | else 678 | { /** file count != 1, 679 | indicating that job is a dir, dest is a existed file **/ 680 | GKOLOG(FATAL, "can't overwrite dir: '%s' on file: '%s'", 681 | (jo->files)->name, out_path); 682 | return -1; 683 | } 684 | } 685 | else if (out_type & GKO_DIR) 686 | {/** dest is a existed dir **/ 687 | for (int i = 0; i < jo->file_count; i++) 688 | { 689 | if (FAIL_CHECK(change_to_local_path((jo->files + i)->name, jo->uri, out_path, 1))) 690 | { 691 | GKOLOG( 692 | FATAL, 693 | "change to local path error, name: '%s', uri: '%s', path: '%s'", 694 | (jo->files + i)->name, jo->uri, out_path); 695 | return -1; 696 | } 697 | } 698 | return 0; 699 | } 700 | else 701 | {/** dest is non-regular file or dir **/ 702 | GKOLOG(FATAL, "the dest: '%s' is non-regular file or dir", out_path); 703 | return -1; 704 | } 705 | } 706 | return 0; 707 | } 708 | 709 | -------------------------------------------------------------------------------- /src/path.h: -------------------------------------------------------------------------------- 1 | /** 2 | * path.h 3 | * 4 | * Created on: 2011-5-11 5 | * Author: auxten 6 | **/ 7 | 8 | #ifndef PATH_H_ 9 | #define PATH_H_ 10 | 11 | ///file and for file test.eg: flag & GKO_FILE 12 | static const int GKO_FILE = 0001; 13 | ///dir and for dir test 14 | static const int GKO_DIR = 0002; 15 | ///nonexisted and for nonexisted test 16 | static const int GKO_NONE = 0004; 17 | ///other 18 | static const int GKO_OTHR = 0000; 19 | ///symlink to file 20 | static const int GKO_LFILE = 0011; 21 | ///symlink to dir 22 | static const int GKO_LDIR = 0012; 23 | ///symlink to nonexist 24 | static const int GKO_LNONE = 0014; 25 | ///symlink to other 26 | static const int GKO_LOTHR = 0010; 27 | ///for symlink test 28 | static const int GKO_LINK = 0010; 29 | ///error 30 | static const int GKO_ERR = 0100; 31 | 32 | 33 | /// ../path//// TO ../path 34 | int inplace_strip_tailing_slash(char * path); 35 | /// ../path TO ../path/ 36 | int inplace_add_tailing_slash(char * path); 37 | /// get the base name of a string 38 | int get_base_name_index(char * in, const char * out); 39 | /// merge into dir_name/base_name 40 | int merge_path(char * out, const char * dir_name, const char * base_name); 41 | /// change remote path to local path, store it in path 42 | int change_to_local_path(char * path, const char * req_path, 43 | const char * local_path, char dst_path_exist); 44 | /// get the symlink dest's absolute path, store it in abs_path 45 | char * symlink_dest_to_abs_path(char * abs_path, const char * symlink); 46 | /// generate snap file path with requested localpath and requested remote uri 47 | unsigned gen_snap_fpath(char *snap_fpath, const char * localpath, 48 | const char * uri); 49 | /// generate snap file path with requested localpath and requested remote uri 50 | int path_type(const char * path); 51 | /// generate snap file path with requested localpath and requested remote uri 52 | int mk_dir_symlink_file(s_job_t * jo, char * to_continue); 53 | /// correct the file and dir mode, cause for write in we create them with mode|S_IWUSR 54 | int correct_mode(s_job_t * jo); 55 | /// convert the remote path recv from JOIN to local path 56 | int process_path(s_job_t * jo); 57 | 58 | #endif /** PATH_H_ **/ 59 | -------------------------------------------------------------------------------- /src/socket.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * socket.cpp 3 | * 4 | * Created on: Mar 9, 2012 5 | * Author: auxten 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "config.h" 14 | #include "gingko.h" 15 | #include "log.h" 16 | #include "socket.h" 17 | #ifdef __APPLE__ 18 | #include 19 | #else 20 | #include 21 | #endif /** __APPLE__ **/ 22 | 23 | char * addr_itoa(in_addr_t address, char * str) 24 | { 25 | u_int32_t addr = (u_int32_t) address; 26 | snprintf(str, 16, "%u.%u.%u.%u", (addr) & 255u, (addr >> 8) & 255u, (addr >> 16) & 255u, (addr >> 24) & 255u); 27 | return str; 28 | } 29 | 30 | /** 31 | * @brief Set non-blocking 32 | * 33 | * @see 34 | * @note 35 | * @author auxten 36 | * @date 2011-8-1 37 | **/ 38 | int setnonblock(int fd) 39 | { 40 | int flags; 41 | 42 | flags = fcntl(fd, F_GETFL); 43 | if (flags < 0) 44 | { 45 | return flags; 46 | } 47 | 48 | if (!(flags & O_NONBLOCK)) 49 | { 50 | flags |= O_NONBLOCK; 51 | if (fcntl(fd, F_SETFL, flags) < 0) 52 | { 53 | return -1; 54 | } 55 | } 56 | return 0; 57 | } 58 | 59 | /** 60 | * @brief Set blocking 61 | * 62 | * @see 63 | * @note 64 | * @author auxten 65 | * @date 2011-8-1 66 | **/ 67 | int setblock(int fd) 68 | { 69 | int flags; 70 | 71 | flags = fcntl(fd, F_GETFL); 72 | if (flags < 0) 73 | { 74 | return flags; 75 | } 76 | 77 | if (flags & O_NONBLOCK) 78 | { 79 | flags &= ~O_NONBLOCK; 80 | if (fcntl(fd, F_SETFL, flags) < 0) 81 | { 82 | return -1; 83 | } 84 | } 85 | return 0; 86 | } 87 | 88 | /** 89 | * @brief connect to a host 90 | * 91 | * @see 92 | * @note 93 | * h: pointer to s_host_t 94 | * recv_sec: receive timeout seconds, 0 for never timeout 95 | * return the socket when succ 96 | * return < 0 when error, specially HOST_DOWN_FAIL indicate host dead 97 | * @author auxten 98 | * @date 2011-8-1 99 | **/ 100 | int connect_host(const s_host_t * h, const int recv_sec, const int send_sec) 101 | { 102 | int sock = -1; 103 | int ret; 104 | int select_ret; 105 | int res; 106 | socklen_t res_size = sizeof res; 107 | struct sockaddr_in channel; 108 | in_addr_t host; 109 | int addr_len; 110 | struct timeval recv_timeout; 111 | struct timeval send_timeout; 112 | #if HAVE_POLL 113 | #else 114 | fd_set wset; 115 | #endif /* HAVE_POLL */ 116 | 117 | addr_len = getaddr_my(h->addr, &host); 118 | if (FAIL_CHECK(!addr_len)) 119 | { 120 | GKOLOG(WARNING, "gethostbyname %s error", h->addr); 121 | ret = -1; 122 | goto CONNECT_END; 123 | } 124 | sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); 125 | if (FAIL_CHECK(sock < 0)) 126 | { 127 | GKOLOG(WARNING, "get socket error"); 128 | ret = -1; 129 | goto CONNECT_END; 130 | } 131 | 132 | recv_timeout.tv_usec = 0; 133 | recv_timeout.tv_sec = recv_sec ? recv_sec : RCV_TIMEOUT; 134 | send_timeout.tv_usec = 0; 135 | send_timeout.tv_sec = send_sec ? send_sec : SND_TIMEOUT; 136 | 137 | memset(&channel, 0, sizeof(channel)); 138 | channel.sin_family = AF_INET; 139 | memcpy(&channel.sin_addr.s_addr, &host, addr_len); 140 | channel.sin_port = htons(h->port); 141 | 142 | /** set the connect non-blocking then blocking for add timeout on connect **/ 143 | if (FAIL_CHECK(setnonblock(sock) < 0)) 144 | { 145 | GKOLOG(WARNING, "set socket non-blocking error"); 146 | ret = -1; 147 | goto CONNECT_END; 148 | } 149 | 150 | /** connect and send the msg **/ 151 | if (FAIL_CHECK(connect(sock, (struct sockaddr *) &channel, sizeof(channel)) && 152 | errno != EINPROGRESS)) 153 | { 154 | GKOLOG(WARNING, "connect error"); 155 | ret = HOST_DOWN_FAIL; 156 | goto CONNECT_END; 157 | } 158 | 159 | /** Wait for write bit to be set **/ 160 | #if HAVE_POLL 161 | { 162 | struct pollfd pollfd; 163 | 164 | pollfd.fd = sock; 165 | pollfd.events = POLLOUT; 166 | 167 | /* send_sec is in seconds, timeout in ms */ 168 | select_ret = poll(&pollfd, 1, (int)(send_sec * 1000 + 1)); 169 | } 170 | #else 171 | { 172 | FD_ZERO(&wset); 173 | FD_SET(sock, &wset); 174 | select_ret = select(sock + 1, 0, &wset, 0, &send_timeout); 175 | } 176 | #endif /* HAVE_POLL */ 177 | if (select_ret < 0) 178 | { 179 | GKOLOG(WARNING, "select/poll error on connect"); 180 | ret = HOST_DOWN_FAIL; 181 | goto CONNECT_END; 182 | } 183 | if (!select_ret) 184 | { 185 | GKOLOG(WARNING, "connect timeout on connect"); 186 | ret = HOST_DOWN_FAIL; 187 | goto CONNECT_END; 188 | } 189 | 190 | /** 191 | * check if connection is RESETed, maybe this is the 192 | * best way to do that 193 | * SEE: http://cr.yp.to/docs/connect.html 194 | **/ 195 | (void) getsockopt(sock, SOL_SOCKET, SO_ERROR, &res, &res_size); 196 | if (CONNECT_DEST_DOWN(res)) 197 | { 198 | // GKOLOG(NOTICE, "connect dest is down errno: %d", res); 199 | ret = HOST_DOWN_FAIL; 200 | goto CONNECT_END; 201 | } 202 | 203 | ///GKOLOG(WARNING, "selected %d ret %d, time %d", sock, select_ret, send_timeout.tv_sec); 204 | /** set back blocking **/ 205 | if (FAIL_CHECK(setblock(sock) < 0)) 206 | { 207 | GKOLOG(WARNING, "set socket non-blocking error"); 208 | ret = -1; 209 | goto CONNECT_END; 210 | } 211 | 212 | /** set recv & send timeout **/ 213 | if (FAIL_CHECK(setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *) &recv_timeout, 214 | sizeof(struct timeval)))) 215 | { 216 | GKOLOG(WARNING, "setsockopt SO_RCVTIMEO error"); 217 | ret = -1; 218 | goto CONNECT_END; 219 | } 220 | if (FAIL_CHECK(setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *) &send_timeout, 221 | sizeof(struct timeval)))) 222 | { 223 | GKOLOG(WARNING, "setsockopt SO_SNDTIMEO error"); 224 | ret = -1; 225 | goto CONNECT_END; 226 | } 227 | 228 | ret = sock; 229 | 230 | CONNECT_END: 231 | /// 232 | if (ret < 0 && sock >= 0) 233 | { 234 | close_socket(sock); 235 | } 236 | return ret; 237 | } 238 | 239 | /** 240 | * @brief gracefully close a socket, for client side 241 | * 242 | * @see 243 | * @note 244 | * @author auxten 245 | * @date 2011-8-1 246 | **/ 247 | int close_socket(int sock) 248 | { 249 | /// if (shutdown(sock, 2)) { 250 | /// GKOLOG(WARNING, "shutdown sock error"); 251 | /// return -1; 252 | /// } 253 | // struct linger so_linger; 254 | // so_linger.l_onoff = 1; /// close right now, no time_wait at serv 255 | // so_linger.l_linger = 0; /// at most wait for 1s 256 | // if (FAIL_CHECK(setsockopt(sock, SOL_SOCKET, SO_LINGER, &so_linger, sizeof(so_linger)))) 257 | // { 258 | // GKOLOG(WARNING, "set so_linger failed"); 259 | // } 260 | if (sock < 0) 261 | return 0; 262 | if (FAIL_CHECK(close(sock))) 263 | { 264 | GKOLOG(WARNING, "close sock error"); 265 | return -1; 266 | } 267 | return 0; 268 | } 269 | 270 | -------------------------------------------------------------------------------- /src/socket.h: -------------------------------------------------------------------------------- 1 | /** 2 | * socket.h 3 | * 4 | * Created on: Mar 9, 2012 5 | * Author: auxten 6 | **/ 7 | 8 | #ifndef SOCKET_H_ 9 | #define SOCKET_H_ 10 | 11 | /// in_addr_t to ascii 12 | char * addr_itoa(in_addr_t address, char * str); 13 | /// Set non-blocking 14 | int setnonblock(int fd); 15 | /// Set blocking 16 | int setblock(int fd); 17 | /// gracefully close a socket, for client side 18 | int close_socket(int sock); 19 | /// connect to a host 20 | int connect_host(const s_host_t * h, const int recv_sec, const int send_sec); 21 | 22 | #endif /** SOCKET_H_ **/ 23 | --------------------------------------------------------------------------------