├── AUTHORS ├── COPYING ├── ChangeLog ├── INSTALL ├── Makefile.am ├── NEWS ├── NOTICE ├── README.md ├── bootstrap ├── configure.in ├── doc ├── Makefile ├── doc.en.md ├── doc.ja.md ├── kumo-gateway.1 ├── kumo-gateway.1.txt ├── kumo-manager.1 ├── kumo-manager.1.txt ├── kumo-server.1 ├── kumo-server.1.txt ├── kumoctl.1 ├── kumoctl.1.txt ├── kumohash.1 ├── kumohash.1.txt ├── kumolog.1 ├── kumolog.1.txt ├── kumomergedb.1 ├── kumomergedb.1.txt ├── kumostat.1 ├── kumostat.1.txt ├── kumotop.1 ├── kumotop.1.txt └── user.css ├── preprocess ├── rt ├── 01_mgr1_kill_1_srv.rt ├── 02_mgr2_kill_1_mgr.rt ├── 03_mgr1_kill_2_srvs.rt ├── 04_mgr2_kill_2_srvs.rt ├── 05_mgr2_kill_1_srv_1_mgr.rt ├── 06_mgr2_kill_1_mgr_1_srv.rt ├── 07_mgr2_kill_2_srvs_while_replacing.rt ├── 08_mgr2_kill_2_srvs_while_replacing2.rt ├── 09_cas.rt ├── README ├── chukan.rb └── common.rb ├── src ├── Makefile.am ├── command │ ├── Makefile.am │ ├── kumoctl │ ├── kumohash │ ├── kumolog │ ├── kumostat │ ├── kumotop │ └── mergedb.cc ├── gate │ ├── Makefile.am │ ├── cloudy.cc │ ├── cloudy.h │ ├── interface.h │ ├── memcache_binary.cc │ ├── memcache_binary.h │ ├── memcache_text.cc │ ├── memcache_text.h │ └── memproto │ │ ├── memproto.c │ │ ├── memproto.h │ │ ├── memtext.c │ │ ├── memtext.h │ │ └── memtext.rl ├── kazuhiki │ ├── Makefile.am │ ├── kazuhiki.cc │ └── kazuhiki.h ├── log │ ├── Makefile.am │ ├── logpack.c │ ├── logpack.h │ ├── logpack.hpp.erb │ ├── logpacker.cc │ ├── logpacker.h │ ├── mlogger.cc │ ├── mlogger.h.erb │ ├── mlogger_null.cc │ ├── mlogger_null.h │ ├── mlogger_ostream.cc │ ├── mlogger_ostream.h │ ├── mlogger_syslog.cc │ ├── mlogger_syslog.h │ ├── mlogger_tty.cc │ └── mlogger_tty.h ├── logic │ ├── Makefile.am │ ├── boot.cc │ ├── boot.h │ ├── client_logic.h │ ├── clock.h │ ├── clock_logic.h │ ├── cluster_logic.h │ ├── gateway.proto.h │ ├── gateway │ │ ├── Makefile.am │ │ ├── framework.cc │ │ ├── framework.h │ │ ├── gate.cc │ │ ├── init.h │ │ ├── main.cc │ │ ├── mod_cache.cc │ │ ├── mod_cache.h │ │ ├── mod_network.cc │ │ ├── mod_store.cc │ │ └── mod_store.h │ ├── global.h │ ├── hash.cc │ ├── hash.h │ ├── manager.proto.h │ ├── manager │ │ ├── Makefile.am │ │ ├── framework.cc │ │ ├── framework.h │ │ ├── init.h │ │ ├── main.cc │ │ ├── mod_control.cc │ │ ├── mod_network.cc │ │ └── mod_replace.cc │ ├── msgtype.h │ ├── protogen │ ├── role.h │ ├── rpc_server.h │ ├── server.proto.h │ ├── server │ │ ├── Makefile.am │ │ ├── framework.cc │ │ ├── framework.h │ │ ├── init.h │ │ ├── main.cc │ │ ├── mod_control.cc │ │ ├── mod_network.cc │ │ ├── mod_replace.cc │ │ ├── mod_replace_stream.cc │ │ ├── mod_store.cc │ │ ├── zconnection.h │ │ ├── zmmap_stream.cc │ │ └── zmmap_stream.h │ ├── wavy_server.cc │ └── wavy_server.h ├── mp │ ├── Makefile.am │ ├── exception.h │ ├── functional.h │ ├── memory.h │ ├── object_callback.pre.h │ ├── pp.h │ ├── pthread.h │ ├── pthread_impl.h │ ├── shared_buffer.h │ ├── shared_buffer_impl.h │ ├── source.h │ ├── source_impl.h │ ├── stream_buffer.h │ ├── stream_buffer_impl.h │ ├── utility.h │ ├── wavy.h │ └── wavy │ │ ├── core.pre.h │ │ ├── output.h │ │ └── singleton.pre.h ├── mpsrc │ ├── Makefile.am │ ├── wavy_connect.cc │ ├── wavy_core.cc │ ├── wavy_core.h │ ├── wavy_edge.h │ ├── wavy_edge_epoll.h │ ├── wavy_edge_eventport.h │ ├── wavy_edge_kqueue.h │ ├── wavy_listen.cc │ ├── wavy_output.cc │ └── wavy_timer.cc ├── rpc │ ├── Makefile.am │ ├── address.cc │ ├── address.h │ ├── client.h │ ├── client_tmpl.h │ ├── cluster.cc │ ├── cluster.h │ ├── connection.h │ ├── exception.h │ ├── message.h │ ├── protocol.h │ ├── request.h │ ├── responder.h │ ├── responder_impl.h │ ├── retry.h │ ├── rpc.h │ ├── server.cc │ ├── server.h │ ├── session.cc │ ├── session.h │ ├── session_impl.h │ ├── transport.h │ ├── transport_impl.h │ ├── types.h │ ├── vrefbuffer.h │ ├── wavy.h │ ├── weak_responder.h │ └── weak_responder_impl.h └── storage │ ├── Makefile.am │ ├── buffer_queue.h │ ├── interface.h │ ├── luxio.cc │ ├── storage.cc │ ├── storage.h │ ├── tcadb.cc │ ├── tcbdb.cc │ └── tchdb.cc └── test ├── Makefile ├── check_set_get_delete.c ├── memstrike.c ├── test_delete.c ├── test_get.c ├── test_get_noset.c ├── test_multi.c ├── test_set.c └── test_set_get_delete.c /AUTHORS: -------------------------------------------------------------------------------- 1 | FURUHASHI Sadayuki 2 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | Copyright (C) 2009 FURUHASHI Sadayuki 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | 2 | 2010-12-14 version 0.4.12: 3 | 4 | * gateway: added --key-prefix option that adds prefix to keys automatically 5 | * server: added --replace-memory-limit option (a.mishima++) 6 | 7 | 2010-08-06 version 0.4.12: 8 | 9 | * kumoctl: purge used callback Proc to save memory (hirose31++) 10 | 11 | 2010-07-29 version 0.4.11: 12 | 13 | * gateway: text protocol gateway accepts libmemcached's peculiar cas request 14 | 15 | 2010-07-14 version 0.4.10: 16 | 17 | * gateway: binary protocol gateway doesn't send "not found" response on getq command when the key is expired 18 | 19 | 2010-07-06 version 0.4.9: 20 | 21 | * server: makes -gS work 22 | * server: makes attach/detach-noreplace work 23 | 24 | 2010-05-25 version 0.4.8: 25 | 26 | * kumostat: added stats subcommand 27 | * server: sends keepalive packet during replacing (breaks backward compatibility) 28 | 29 | 2010-05-29 version 0.4.7: 30 | 31 | * make --keepalive-internval (-k) option work. 32 | 33 | 2010-05-27 version 0.4.6: 34 | 35 | * kumotop: shows 'status' column: inactive, Copying, WAIT, Deleting, ready or ? 36 | 37 | 2010-05-22 version 0.4.5: 38 | 39 | * gateway: fixes memory leak problem on binary protocol. 40 | 41 | 2010-05-21 version 0.4.4: 42 | 43 | * gateway: returns 'out of memory' instead of 'invalid argument' on server error 44 | on binary protocol. 45 | * configure: added --with-jemalloc option that links jemalloc. 46 | 47 | 2010-05-18 version 0.4.3: 48 | 49 | * gateway: Fixed memory leak problem on memcached binary protocol 50 | * gateway: improve performance on binary protocol when -E or -F is enabled. 51 | 52 | 2010-05-15 version 0.4.2: 53 | 54 | * Fixed the problem that get_multi always returns a CAS of 0 55 | * Supported 'replace' command with CAS value on memcached binary protocol 56 | 57 | 2010-05-14 version 0.4.1: 58 | 59 | * Fixed replication behavior on CAS 60 | * Fixed error response message of CAS on memcached text protocol 61 | * Support flags, expiration time and CAS on memcached binary protocol 62 | 63 | 2010-05-14 version 0.4.0: 64 | 65 | * CAS (Compare-and-Swap) support 66 | 67 | 2010-05-13 version 0.3.7: 68 | 69 | * create backup on background thread 70 | 71 | 2010-05-10 version 0.3.6: 72 | 73 | * fixed SEGV on memcached gateway. 74 | 75 | 2010-04-30 version 0.3.5: 76 | 77 | * Memcached gateway saves expiration time if -E option is specified. 78 | 79 | 2010-04-22 version 0.3.4: 80 | 81 | * Memcached gateway supports memcached client for java. 82 | 83 | 2010-04-20 version 0.3.3: 84 | 85 | * Memcached gateway saves flags if -F option is specified. 86 | 87 | 2010-03-18 version 0.3.2: 88 | 89 | * Memcached gateway supports 'version' command 90 | * ./configure checks compiler's __sync_* atomic operations support 91 | 92 | 2010-01-20 version 0.3.1: 93 | 94 | * Make it compilable on FreeBSD. 95 | * Link msgpackc library. (--as-needed proglem) 96 | * Update documents. 97 | 98 | 2010-01-18 version 0.3.0: 99 | 100 | * First public release. 101 | 102 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | REVISION = $(shell git show --pretty=format:'%T %ad' | head -n 1) 3 | export REVISION 4 | 5 | SUBDIRS = src 6 | 7 | DOC_FILES = \ 8 | README.md \ 9 | NOTICE \ 10 | doc/kumo-gateway.1.txt \ 11 | doc/kumo-manager.1.txt \ 12 | doc/kumo-server.1.txt \ 13 | doc/kumoctl.1.txt \ 14 | doc/kumohash.1.txt \ 15 | doc/kumolog.1.txt \ 16 | doc/kumomergedb.1.txt \ 17 | doc/kumostat.1.txt \ 18 | doc/kumotop.1.txt \ 19 | doc/kumo-gateway.1 \ 20 | doc/kumo-manager.1 \ 21 | doc/kumo-server.1 \ 22 | doc/kumoctl.1 \ 23 | doc/kumohash.1 \ 24 | doc/kumolog.1 \ 25 | doc/kumomergedb.1 \ 26 | doc/kumostat.1 \ 27 | doc/kumotop.1 \ 28 | doc/doc.en.md \ 29 | doc/doc.ja.md 30 | 31 | 32 | EXTRA_DIST = $(DOC_FILES) 33 | 34 | man_MANS = \ 35 | doc/kumo-gateway.1 \ 36 | doc/kumo-manager.1 \ 37 | doc/kumo-server.1 \ 38 | doc/kumoctl.1 \ 39 | doc/kumohash.1 \ 40 | doc/kumolog.1 \ 41 | doc/kumomergedb.1 \ 42 | doc/kumostat.1 \ 43 | doc/kumotop.1 44 | 45 | -------------------------------------------------------------------------------- /NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/etolabo/kumofs/b0f7ce8263194dbfa92d10ae75849e676e77c067/NEWS -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | MessagePack is developed by FURUHASHI Sadayuki, licensed under Apache License, 2 | Version 2.0. The original software and related information is available at 3 | http://msgpack.sourceforge.jp/. 4 | 5 | This software includes MessagePack fast log library which is developed by 6 | FURUHASHI Sadayuki, licensed under Apache License, Version 2.0. 7 | 8 | The memcached protocol parser is provided by memproto library copyright by 9 | FURUHASHI Sadayuki, which is open source source software licensed under 10 | Apache License, Version 2.0. 11 | 12 | This software includes mpio library developed by FURUHASHI Sadayuki, licensed 13 | under Apache License, Version 2.0. 14 | 15 | This software includes Kazuhiki 3 library developed by FURUHASHI Sadayuki, 16 | licensed under Apache License, Version 2.0. 17 | 18 | This product includes software developed by the OpenSSL Project for use in the 19 | OpenSSL Toolkit.(http://www.openssl.org/) 20 | This product includes cryptographic software written by Eric Young 21 | (eay@cryptsoft.com) 22 | This product includes software written by Tim Hudson (tjh@cryptsoft.com) 23 | 24 | I have had the support and encouragement of Etolabo Corp. for developing this 25 | software. I would like to thank for the aid that made it possible to achieve 26 | this work. 27 | 28 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | 2 | DOCS = \ 3 | kumo-gateway.1 \ 4 | kumo-manager.1 \ 5 | kumo-server.1 \ 6 | kumoctl.1 \ 7 | kumohash.1 \ 8 | kumolog.1 \ 9 | kumomergedb.1 \ 10 | kumostat.1 \ 11 | kumotop.1 12 | 13 | all: $(DOCS) 14 | 15 | clean: kumo-gateway.1.clean kumo-manager.1.clean kumo-server.1.clean kumoctl.1.clean kumohash.1.clean kumolog.1.clean kumomergedb.1.clean kumostat.1.clean kumotop.1.clean 16 | 17 | %.sdoc : %.txt 18 | wfdoc -i $< -o $@ 19 | 20 | %.html : %.txt 21 | wfdoc -f html -i $< -o $@ 22 | 23 | %.tex : %.sdoc 24 | sdoc -toc -format:latex2e -latex2e.driver=dvipdfm -latex2e.ref=HyperRefLaTeX2eRefHandler $< 25 | 26 | %.etex : %.tex 27 | wftex $< $@ 28 | 29 | %.toc : %.etex 30 | platex $< 31 | 32 | %.dvi : %.etex %.toc 33 | platex $< 34 | 35 | %.pdf : %.dvi 36 | dvipdfmx $< 37 | rm -f $*.aux $*.log 38 | 39 | %.clean : %.txt 40 | rm -f $*.sdoc $*.tex $*.etex $*.dvi $*.toc $*.aux $*.out $*.log $*.pdf 41 | 42 | %.distclean : %.txt 43 | rm -f $*.sdoc $*.tex $*.etex $*.dvi $*.toc $*.aux $*.out $*.log $*.pdf $*.mk 44 | 45 | %.1 : %.1.txt 46 | wfdoc -f man $< -o $@ 47 | 48 | -------------------------------------------------------------------------------- /doc/kumo-gateway.1: -------------------------------------------------------------------------------- 1 | .TH KUMO-GATEWAY "1" "July 2009" "kumo-gateway" 2 | .SH NAME 3 | kumo-gateway 4 | .SH SYNOPSIS 5 | kumo-gateway -m -p [-t port] [-b port] [-c port] 6 | .SH DESCRIPTION 7 | 8 | .B kumo-gateway 9 | receives requests from applications and relays it to the kumo-servers. 10 | Run this daemon on the hosts that runs applications that uses kumofs. 11 | .SH OPTIONS 12 | .TP 13 | .B -m --manager1 14 | address of manager 1 15 | .TP 16 | .B -p --manager2 17 | address of manager 2 18 | .TP 19 | .B -lc --local-cache 20 | local cache (Tokyo Cabinet abstract database) 21 | .TP 22 | .B -t <[addr:]port=11411> --memproto-text 23 | memcached text protocol listen port 24 | .TP 25 | .B -b <[addr:]port=11511> --memproto-binary 26 | memcached binary protocol listen port 27 | .TP 28 | .B -c <[addr:]port=11611> --cloudy 29 | asynchronous memcached binary protocol listen port 30 | .TP 31 | .B -F --memproto-save-flag 32 | save flags on memcached text protocol 33 | .TP 34 | .B -As --async-replicate-set 35 | send response without waiting replication on set 36 | .TP 37 | .B -Ad --async-replicate-delete 38 | send response without waiting replication on delete 39 | .TP 40 | .B -G --get-retry 41 | get retry limit 42 | .TP 43 | .B -S --set-retry 44 | set retry limit 45 | .TP 46 | .B -D --delete-retry 47 | delete retry limit 48 | .TP 49 | .B -rn --renew-threshold 50 | hash space renew threshold 51 | .TP 52 | .B -k --keepalive-interval 53 | keepalive interval in seconds 54 | .TP 55 | .B -Ys --connect-timeout 56 | connect timeout time in seconds 57 | .TP 58 | .B -Yn --connect-retry-limit 59 | connect retry limit 60 | .TP 61 | .B -Ci --clock-interval 62 | clock interval in seconds 63 | .TP 64 | .B -TW --write-threads 65 | number of threads for asynchronous writing 66 | .TP 67 | .B -TR --read-threads 68 | number of threads for asynchronous reading 69 | .TP 70 | .B -o --log 71 | output logs to the file 72 | .TP 73 | .B -g --binary-log 74 | enable binary log 75 | .TP 76 | .B -v --verbose 77 | 78 | .TP 79 | .B -d --daemon 80 | 81 | .SH EXAMPLE 82 | [on cli1 ]$ kumo-gateway -v -m mgr1 -p mgr2 -t 11211 -As -Ad 83 | .PP 84 | [localhost]$ kumo-gateway -v -m localhost -t 11211 -d kumo-gateway.pid 85 | .SH SEE ALSO 86 | kumolog(1). 87 | -------------------------------------------------------------------------------- /doc/kumo-gateway.1.txt: -------------------------------------------------------------------------------- 1 | *? KUMO-GATEWAY "1" "July 2009" "kumo-gateway" 2 | *NAME 3 | kumo-gateway 4 | 5 | *SYNOPSIS 6 | kumo-gateway -m -p [-t port] [-b port] [-c port] 7 | 8 | *DESCRIPTION 9 | ''kumo-gateway'' receives requests from applications and relays it to the kumo-servers. 10 | Run this daemon on the hosts that runs applications that uses kumofs. 11 | 12 | *OPTIONS 13 | ::?-m --manager1 14 | ::=address of manager 1 15 | ::?-p --manager2 16 | ::=address of manager 2 17 | ::?-lc --local-cache 18 | ::=local cache (Tokyo Cabinet abstract database) 19 | ::?-t <[addr:]port=11411> --memproto-text 20 | ::=memcached text protocol listen port 21 | ::?-b <[addr:]port=11511> --memproto-binary 22 | ::=memcached binary protocol listen port 23 | ::?-c <[addr:]port=11611> --cloudy 24 | ::=asynchronous memcached binary protocol listen port 25 | ::?-F --memproto-save-flag 26 | ::=save flags on memcached text protocol 27 | ::?-As --async-replicate-set 28 | ::=send response without waiting replication on set 29 | ::?-Ad --async-replicate-delete 30 | ::=send response without waiting replication on delete 31 | ::?-G --get-retry 32 | ::=get retry limit 33 | ::?-S --set-retry 34 | ::=set retry limit 35 | ::?-D --delete-retry 36 | ::=delete retry limit 37 | ::?-rn --renew-threshold 38 | ::=hash space renew threshold 39 | ::?-k --keepalive-interval 40 | ::=keepalive interval in seconds 41 | ::?-Ys --connect-timeout 42 | ::=connect timeout time in seconds 43 | ::?-Yn --connect-retry-limit 44 | ::=connect retry limit 45 | ::?-Ci --clock-interval 46 | ::=clock interval in seconds 47 | ::?-TW --write-threads 48 | ::=number of threads for asynchronous writing 49 | ::?-TR --read-threads 50 | ::=number of threads for asynchronous reading 51 | ::?-o --log 52 | ::=output logs to the file 53 | ::?-g --binary-log 54 | ::=enable binary log 55 | ::?-v --verbose 56 | ::= 57 | ::?-d --daemon 58 | ::= 59 | 60 | *EXAMPLE 61 | [on cli1 ]$ kumo-gateway -v -m mgr1 -p mgr2 -t 11211 -As -Ad &br; 62 | [localhost]$ kumo-gateway -v -m localhost -t 11211 -d kumo-gateway.pid 63 | 64 | *SEE ALSO 65 | kumolog(1). 66 | 67 | -------------------------------------------------------------------------------- /doc/kumo-manager.1: -------------------------------------------------------------------------------- 1 | .TH KUMO-MANAGER "1" "July 2009" "kumo-manager" 2 | .SH NAME 3 | kumo-manager 4 | .SH SYNOPSIS 5 | kumo-manager -l -p 6 | .SH DESCRIPTION 7 | 8 | .B kumo-manager 9 | manages kumo-servers. Run this daemon on one or two hosts. 10 | You can get live status of the kumo-manager using 11 | .B kumoctl 12 | command. 13 | .SH OPTIONS 14 | .TP 15 | .B -p --partner 16 | master-slave replication partner 17 | .TP 18 | .B -a --auto-replace 19 | enable auto replacing 20 | .TP 21 | .B -Rs --replace-delay 22 | delay time of auto replacing in sec. 23 | .TP 24 | .B -k --keepalive-interval 25 | keepalive interval in seconds 26 | .TP 27 | .B -Ys --connect-timeout 28 | connect timeout time in seconds 29 | .TP 30 | .B -Yn --connect-retry-limit 31 | connect retry limit 32 | .TP 33 | .B -Ci --clock-interval 34 | clock interval in seconds 35 | .TP 36 | .B -TW --write-threads 37 | number of threads for asynchronous writing 38 | .TP 39 | .B -TR --read-threads 40 | number of threads for asynchronous reading 41 | .TP 42 | .B -o --log 43 | output logs to the file 44 | .TP 45 | .B -g --binary-log 46 | enable binary log 47 | .TP 48 | .B -v --verbose 49 | 50 | .TP 51 | .B -d --daemon 52 | 53 | .SH AUTO REPLACING 54 | See 55 | .B kumoctl(1) 56 | for details. 57 | .SH EXAMPLE 58 | [on mgr1 ]$ kumo-manager -v -l mgr1 -p mgr2 -o - 59 | .PP 60 | [on mgr2 ]$ kumo-manager -v -l mgr2 -p mgr1 -g errlog.mpac 61 | .PP 62 | [localhost]$ kumo-manager -v -l localhost -Ys 4 -Yn 2 63 | .SH SEE ALSO 64 | kumoctl(1), kumolog(1). 65 | -------------------------------------------------------------------------------- /doc/kumo-manager.1.txt: -------------------------------------------------------------------------------- 1 | *? KUMO-MANAGER "1" "July 2009" "kumo-manager" 2 | *NAME 3 | kumo-manager 4 | 5 | *SYNOPSIS 6 | kumo-manager -l -p 7 | 8 | *DESCRIPTION 9 | ''kumo-manager'' manages kumo-servers. Run this daemon on one or two hosts. 10 | You can get live status of the kumo-manager using ''kumoctl'' command. 11 | 12 | *OPTIONS 13 | ::?-p --partner 14 | ::=master-slave replication partner 15 | ::?-a --auto-replace 16 | ::=enable auto replacing 17 | ::?-Rs --replace-delay 18 | ::=delay time of auto replacing in sec. 19 | ::?-k --keepalive-interval 20 | ::=keepalive interval in seconds 21 | ::?-Ys --connect-timeout 22 | ::=connect timeout time in seconds 23 | ::?-Yn --connect-retry-limit 24 | ::=connect retry limit 25 | ::?-Ci --clock-interval 26 | ::=clock interval in seconds 27 | ::?-TW --write-threads 28 | ::=number of threads for asynchronous writing 29 | ::?-TR --read-threads 30 | ::=number of threads for asynchronous reading 31 | ::?-o --log 32 | ::=output logs to the file 33 | ::?-g --binary-log 34 | ::=enable binary log 35 | ::?-v --verbose 36 | ::= 37 | ::?-d --daemon 38 | ::= 39 | 40 | *AUTO REPLACING 41 | See ''kumoctl(1)'' for details. 42 | 43 | *EXAMPLE 44 | [on mgr1 ]$ kumo-manager -v -l mgr1 -p mgr2 -o - &br; 45 | [on mgr2 ]$ kumo-manager -v -l mgr2 -p mgr1 -g errlog.mpac &br; 46 | [localhost]$ kumo-manager -v -l localhost -Ys 4 -Yn 2 47 | 48 | *SEE ALSO 49 | kumoctl(1), kumolog(1). 50 | 51 | -------------------------------------------------------------------------------- /doc/kumo-server.1: -------------------------------------------------------------------------------- 1 | .TH KUMO-SERVER "1" "July 2009" "kumo-server" 2 | .SH NAME 3 | kumo-server 4 | .SH SYNOPSIS 5 | kumo-server -m -p -l -s 6 | .SH DESCRIPTION 7 | 8 | .B kumo-server 9 | stores actual data. You can add kumo-servers after buiding the cluster. 10 | You can get live status of the kumo-manager using 11 | .B kumostat 12 | and 13 | .B kumotop 14 | command. 15 | .SH OPTIONS 16 | .TP 17 | .B -l --listen 18 | listen address 19 | .TP 20 | .B -L --stream-listen 21 | listen port for replacing stream 22 | .TP 23 | .B -f --offer-tmp 24 | path to temporary directory for replacing 25 | .TP 26 | .B -s --store 27 | path to database 28 | .TP 29 | .B -m --manager1 30 | address of manager 1 31 | .TP 32 | .B -p --manager2 33 | address of manager 2 34 | .TP 35 | .B -S --replicate-set-retry 36 | replicate set retry limit 37 | .TP 38 | .B -D --replicate-delete-retry 39 | replicate delete retry limit 40 | .TP 41 | .B -gN --garbage-min-time 42 | minimum time to maintenance deleted key 43 | .TP 44 | .B -gX --garbage-max-time 45 | maximum time to maintenance deleted key 46 | .TP 47 | .B -gS --garbage-mem-limit 48 | maximum memory usage to memory deleted key 49 | .TP 50 | .B -k --keepalive-interval 51 | keepalive interval in seconds 52 | .TP 53 | .B -Ys --connect-timeout 54 | connect timeout time in seconds 55 | .TP 56 | .B -Yn --connect-retry-limit 57 | connect retry limit 58 | .TP 59 | .B -Ci --clock-interval 60 | clock interval in seconds 61 | .TP 62 | .B -TW --write-threads 63 | number of threads for asynchronous writing 64 | .TP 65 | .B -TR --read-threads 66 | number of threads for asynchronous reading 67 | .TP 68 | .B -o --log 69 | output logs to the file 70 | .TP 71 | .B -g --binary-log 72 | enable binary log 73 | .TP 74 | .B -v --verbose 75 | 76 | .TP 77 | .B -d --daemon 78 | 79 | .SH EXAMPLE 80 | [on svr1 ]$ kumo-server -v -l svr1 -m mgr1 -p mgr2 -s /var/kumodb.tch -gN 5 -gS 20480 81 | .PP 82 | [on svr2 ]$ kumo-server -v -l svr2 -m mgr1 -p mgr2 -s /var/kumodb.tch -TR 6 -TW 4 83 | .PP 84 | [on svr3 ]$ kumo-server -v -l svr3 -m mgr1 -p mgr2 -s /var/kumodb.tch -Ci 16 85 | .PP 86 | [localhost]$ kumo-server -v -m localhost -l localhost:19801 -L 19901 -s ./database1.tch 87 | .PP 88 | [localhost]$ kumo-server -v -m localhost -l localhost:19802 -L 19902 -s ./database2.tch 89 | .PP 90 | [localhost]$ kumo-server -v -m localhost -l localhost:19803 -L 19902 -s ./database3.tch 91 | .PP 92 | .SH SEE ALSO 93 | kumostat(1), kumotop(1), kumolog(1), kumomergedb(1), kumohash(1). 94 | -------------------------------------------------------------------------------- /doc/kumo-server.1.txt: -------------------------------------------------------------------------------- 1 | *? KUMO-SERVER "1" "July 2009" "kumo-server" 2 | *NAME 3 | kumo-server 4 | 5 | *SYNOPSIS 6 | kumo-server -m -p -l -s 7 | 8 | *DESCRIPTION 9 | ''kumo-server'' stores actual data. You can add kumo-servers after buiding the cluster. 10 | You can get live status of the kumo-manager using ''kumostat'' and ''kumotop'' command. 11 | 12 | *OPTIONS 13 | ::?-l --listen 14 | ::=listen address 15 | ::?-L --stream-listen 16 | ::=listen port for replacing stream 17 | ::?-f --offer-tmp 18 | ::=path to temporary directory for replacing 19 | ::?-s --store 20 | ::=path to database 21 | ::?-m --manager1 22 | ::=address of manager 1 23 | ::?-p --manager2 24 | ::=address of manager 2 25 | ::?-S --replicate-set-retry 26 | ::=replicate set retry limit 27 | ::?-D --replicate-delete-retry 28 | ::=replicate delete retry limit 29 | ::?-gN --garbage-min-time 30 | ::=minimum time to maintenance deleted key 31 | ::?-gX --garbage-max-time 32 | ::=maximum time to maintenance deleted key 33 | ::?-gS --garbage-mem-limit 34 | ::=maximum memory usage to memory deleted key 35 | ::?-k --keepalive-interval 36 | ::=keepalive interval in seconds 37 | ::?-Ys --connect-timeout 38 | ::=connect timeout time in seconds 39 | ::?-Yn --connect-retry-limit 40 | ::=connect retry limit 41 | ::?-Ci --clock-interval 42 | ::=clock interval in seconds 43 | ::?-TW --write-threads 44 | ::=number of threads for asynchronous writing 45 | ::?-TR --read-threads 46 | ::=number of threads for asynchronous reading 47 | ::?-o --log 48 | ::=output logs to the file 49 | ::?-g --binary-log 50 | ::=enable binary log 51 | ::?-v --verbose 52 | ::= 53 | ::?-d --daemon 54 | ::= 55 | 56 | *EXAMPLE 57 | [on svr1 ]$ kumo-server -v -l svr1 -m mgr1 -p mgr2 -s /var/kumodb.tch -gN 5 -gS 20480 &br; 58 | [on svr2 ]$ kumo-server -v -l svr2 -m mgr1 -p mgr2 -s /var/kumodb.tch -TR 6 -TW 4 &br; 59 | [on svr3 ]$ kumo-server -v -l svr3 -m mgr1 -p mgr2 -s /var/kumodb.tch -Ci 16 &br; 60 | [localhost]$ kumo-server -v -m localhost -l localhost:19801 -L 19901 -s ./database1.tch &br; 61 | [localhost]$ kumo-server -v -m localhost -l localhost:19802 -L 19902 -s ./database2.tch &br; 62 | [localhost]$ kumo-server -v -m localhost -l localhost:19803 -L 19902 -s ./database3.tch &br; 63 | 64 | *SEE ALSO 65 | kumostat(1), kumotop(1), kumolog(1), kumomergedb(1), kumohash(1). 66 | 67 | -------------------------------------------------------------------------------- /doc/kumoctl.1: -------------------------------------------------------------------------------- 1 | .TH KUMOCTL "1" "July 2009" "kumoctl" 2 | .SH NAME 3 | kumoctl 4 | .SH SYNOPSIS 5 | kumoctl address[:port=19750] command [options] 6 | .SH DESCRIPTION 7 | Show status of kumo-manager and send control commands to kumo-manager. 8 | .SH COMMANDS 9 | .TP 10 | .B status 11 | show status of the cluster 12 | .TP 13 | .B attach 14 | attach all new servers and start replace 15 | .TP 16 | .B attach-noreplace 17 | attach all new servers 18 | .TP 19 | .B detach 20 | detach all fault servers and start replace 21 | .TP 22 | .B detach-noreplace 23 | detach all fault servers 24 | .TP 25 | .B replace 26 | start replace without attach/detach 27 | .TP 28 | .B full-replace 29 | start full-replace (repair consistency) 30 | .TP 31 | .B backup [suffix=20090304] 32 | create backup with specified suffix 33 | .TP 34 | .B enable-auto-replace 35 | enable auto replace 36 | .TP 37 | .B disable-auto-replace 38 | disable auto replace 39 | .SH STATUS 40 | .TP 41 | .B hash space timestamp 42 | The time that the list of attached kumo-servers is updated. It is updated when new kumo-server is added or existing kumo-server is down. 43 | .TP 44 | .B attached node 45 | The list of attached kumo-servers. (active) is normal node and (fault) is fault node or recoverd but not re-attached node. 46 | .TP 47 | .B not attached node 48 | The list of recognized but not-attached nodes. 49 | .SH AUTO REPLACING 50 | By default, you have to attach new kumo-servers manually using 51 | .B kumoctl 52 | command. If the auto replacing is enabled, new kumo-servers are attached automatically. 53 | This is experimental feature. 54 | .SH EXAMPLE 55 | $ kumoctl mgr1 status 56 | .PP 57 | $ kumoctl mgr1 attach 58 | .SH SEE ALSO 59 | kumo-manager(1). 60 | -------------------------------------------------------------------------------- /doc/kumoctl.1.txt: -------------------------------------------------------------------------------- 1 | *? KUMOCTL "1" "July 2009" "kumoctl" 2 | *NAME 3 | kumoctl 4 | 5 | *SYNOPSIS 6 | kumoctl address[:port=19750] command [options] 7 | 8 | *DESCRIPTION 9 | Show status of kumo-manager and send control commands to kumo-manager. 10 | 11 | *COMMANDS 12 | :status :show status of the cluster 13 | :attach :attach all new servers and start replace 14 | :attach-noreplace :attach all new servers 15 | :detach :detach all fault servers and start replace 16 | :detach-noreplace :detach all fault servers 17 | :replace :start replace without attach/detach 18 | :full-replace :start full-replace (repair consistency) 19 | :backup [suffix=20090304] :create backup with specified suffix 20 | :enable-auto-replace :enable auto replace 21 | :disable-auto-replace :disable auto replace 22 | 23 | *STATUS 24 | :hash space timestamp :The time that the list of attached kumo-servers is updated. It is updated when new kumo-server is added or existing kumo-server is down. 25 | :attached node :The list of attached kumo-servers. ''(active)'' is normal node and ''(fault)'' is fault node or recoverd but not re-attached node. 26 | :not attached node :The list of recognized but not-attached nodes. 27 | 28 | *AUTO REPLACING 29 | By default, you have to attach new kumo-servers manually using ''kumoctl'' command. If the auto replacing is enabled, new kumo-servers are attached automatically. 30 | This is experimental feature. 31 | 32 | *EXAMPLE 33 | $ kumoctl mgr1 status &br; 34 | $ kumoctl mgr1 attach 35 | 36 | *SEE ALSO 37 | kumo-manager(1). 38 | 39 | -------------------------------------------------------------------------------- /doc/kumohash.1: -------------------------------------------------------------------------------- 1 | .TH KUMOHASH "1" "July 2009" "kumohash" 2 | .SH NAME 3 | kumohash 4 | .SH SYNOPSIS 5 | kumohash server-address[:port=19800] ... -- command [options] 6 | 7 | .PP 8 | kumohash -m manager-address[:port=19700] command [options] 9 | .SH DESCRIPTION 10 | kumohash is a Consistent Hashing simulator. 11 | .SH COMMANDS 12 | .TP 13 | .B hash key... 14 | calculate hash of keys 15 | .TP 16 | .B assign keys... 17 | calculate assign node 18 | .TP 19 | .B dump 20 | dump hash space 21 | .SH EXAMPLE 22 | $ kumohash svr1 svr2 svr3 svr4 -- dump 23 | .PP 24 | $ kumohash -m mgr1 assign "key1" "key2" "key3" 25 | -------------------------------------------------------------------------------- /doc/kumohash.1.txt: -------------------------------------------------------------------------------- 1 | *? KUMOHASH "1" "July 2009" "kumohash" 2 | *NAME 3 | kumohash 4 | 5 | *SYNOPSIS 6 | kumohash server-address[:port=19800] ... -- command [options] 7 | &br; 8 | kumohash -m manager-address[:port=19700] command [options] 9 | 10 | *DESCRIPTION 11 | kumohash is a Consistent Hashing simulator. 12 | 13 | *COMMANDS 14 | :hash key... :calculate hash of keys 15 | :assign keys... :calculate assign node 16 | :dump :dump hash space 17 | 18 | *EXAMPLE 19 | $ kumohash svr1 svr2 svr3 svr4 -- dump &br; 20 | $ kumohash -m mgr1 assign "key1" "key2" "key3" 21 | 22 | -------------------------------------------------------------------------------- /doc/kumolog.1: -------------------------------------------------------------------------------- 1 | .TH KUMOLOG "1" "July 2009" "kumolog" 2 | .SH NAME 3 | kumolog 4 | .SH SYNOPSIS 5 | kumolog [options] 6 | .SH DESCRIPTION 7 | Convert binary logs into human-readable format. 8 | .SH OPTIONS 9 | .TP 10 | .B -f, --follow 11 | same as `tail -f' 12 | .TP 13 | .B -t, --tail 14 | show only last N logs (default: N=10) 15 | .TP 16 | .B -h, --head 17 | show only first N logs (default: N=10) 18 | .TP 19 | .B -n, --lines=[-]N 20 | specify N 21 | .SH EXAMPLE 22 | $ kumolog -f svr1_log.mpac 23 | .PP 24 | $ kumolog -h -n 50 svr1_log.mpac 25 | -------------------------------------------------------------------------------- /doc/kumolog.1.txt: -------------------------------------------------------------------------------- 1 | *? KUMOLOG "1" "July 2009" "kumolog" 2 | *NAME 3 | kumolog 4 | 5 | *SYNOPSIS 6 | kumolog [options] 7 | 8 | *DESCRIPTION 9 | Convert binary logs into human-readable format. 10 | 11 | *OPTIONS 12 | :-f, --follow : same as `tail -f' 13 | :-t, --tail : show only last N logs (default: N=10) 14 | :-h, --head : show only first N logs (default: N=10) 15 | :-n, --lines=[-]N : specify N 16 | 17 | *EXAMPLE 18 | $ kumolog -f svr1_log.mpac &br; 19 | $ kumolog -h -n 50 svr1_log.mpac 20 | 21 | -------------------------------------------------------------------------------- /doc/kumomergedb.1: -------------------------------------------------------------------------------- 1 | .TH KUMOMERGEDB "1" "July 2009" "kumomergedb" 2 | .SH NAME 3 | kumomergedb -- merge database files 4 | .SH SYNOPSIS 5 | kumomergedb ... 6 | .SH DESCRIPTION 7 | Merge multiple database files into one database file. This command is 8 | useful to collect database files created by `kumoctl backup' command. 9 | .SH EXAMPLE 10 | $ kumomergedb backup.tch-20090101 svr1.tch-20090101 svr2.tch-20090101 11 | .SH SEE ALSO 12 | kumoctl(1). 13 | -------------------------------------------------------------------------------- /doc/kumomergedb.1.txt: -------------------------------------------------------------------------------- 1 | *? KUMOMERGEDB "1" "July 2009" "kumomergedb" 2 | *NAME 3 | kumomergedb -- merge database files 4 | 5 | *SYNOPSIS 6 | kumomergedb ... 7 | 8 | *DESCRIPTION 9 | Merge multiple database files into one database file. This command is 10 | useful to collect database files created by `kumoctl backup' command. 11 | 12 | *EXAMPLE 13 | $ kumomergedb backup.tch-20090101 svr1.tch-20090101 svr2.tch-20090101 14 | 15 | *SEE ALSO 16 | kumoctl(1). 17 | 18 | -------------------------------------------------------------------------------- /doc/kumostat.1: -------------------------------------------------------------------------------- 1 | .TH KUMOSTAT "1" "July 2009" "kumostat" 2 | .SH NAME 3 | kumostat 4 | .SH SYNOPSIS 5 | kumostat server-address[:port=19800] command 6 | .PP 7 | kumostat -m manager-address[:port=19700] command 8 | .SH DESCRIPTION 9 | Get status from Server node. 10 | .SH COMMANDS 11 | .TP 12 | .B pid 13 | get pid of server process 14 | .TP 15 | .B uptime 16 | get uptime 17 | .TP 18 | .B time 19 | get UNIX time 20 | .TP 21 | .B version 22 | get version 23 | .TP 24 | .B cmd_get 25 | get total number of processed get requests 26 | .TP 27 | .B cmd_set 28 | get total number of processed set requests 29 | .TP 30 | .B cmd_delete 31 | get total number of processed delete requests 32 | .TP 33 | .B items 34 | get number of stored items 35 | .TP 36 | .B rhs 37 | get rhs (routing table for Get) 38 | .TP 39 | .B whs 40 | get whs (routing table for Set/Delete) 41 | .TP 42 | .B hscheck 43 | check if rhs == whs 44 | .TP 45 | .B set_delay 46 | maximize throughput at the expense of latency 47 | .TP 48 | .B unset_delay 49 | maximize throughput at the expense of latency 50 | .SH DELAY MODE 51 | If the delay mode is enabled, req/sec throughput is maximized at the expense of latency. This mode actually disables TCP_NODELAY option of all sockets. 52 | The practice of this feature is as following: 53 | .PP 54 | - run some scripts that watches load of the kumo-server. (see source code of kumotop) 55 | .PP 56 | - if the load of kumo-server becomes high, enable delay mode by 57 | .B kumostat set_delay 58 | command. 59 | .PP 60 | - if the load of kumo-server becomes low, disable delay mode by 61 | .B kumostat unset_delay 62 | command. 63 | .PP 64 | .SH EXAMPLE 65 | $ kumostat svr1 items 66 | .PP 67 | $ kumotop -m mgr1 hscheck 68 | .SH SEE ALSO 69 | kumo-server(1). 70 | -------------------------------------------------------------------------------- /doc/kumostat.1.txt: -------------------------------------------------------------------------------- 1 | *? KUMOSTAT "1" "July 2009" "kumostat" 2 | *NAME 3 | kumostat 4 | 5 | *SYNOPSIS 6 | kumostat server-address[:port=19800] command &br; 7 | kumostat -m manager-address[:port=19700] command 8 | 9 | *DESCRIPTION 10 | Get status from Server node. 11 | 12 | *COMMANDS 13 | :pid :get pid of server process 14 | :uptime :get uptime 15 | :time :get UNIX time 16 | :version :get version 17 | :cmd_get :get total number of processed get requests 18 | :cmd_set :get total number of processed set requests 19 | :cmd_delete :get total number of processed delete requests 20 | :items :get number of stored items 21 | :rhs :get rhs (routing table for Get) 22 | :whs :get whs (routing table for Set/Delete) 23 | :hscheck :check if rhs == whs 24 | :set_delay :maximize throughput at the expense of latency 25 | :unset_delay :maximize throughput at the expense of latency 26 | 27 | *DELAY MODE 28 | If the delay mode is enabled, req/sec throughput is maximized at the expense of latency. This mode actually disables TCP_NODELAY option of all sockets. 29 | The practice of this feature is as following: &br; 30 | - run some scripts that watches load of the kumo-server. (see source code of kumotop) &br; 31 | - if the load of kumo-server becomes high, enable delay mode by ''kumostat set_delay'' command. &br; 32 | - if the load of kumo-server becomes low, disable delay mode by ''kumostat unset_delay'' command. &br; 33 | 34 | *EXAMPLE 35 | $ kumostat svr1 items &br; 36 | $ kumotop -m mgr1 hscheck 37 | 38 | *SEE ALSO 39 | kumo-server(1). 40 | 41 | -------------------------------------------------------------------------------- /doc/kumotop.1: -------------------------------------------------------------------------------- 1 | .TH KUMOTOP "1" "July 2009" "kumotop" 2 | .SH NAME 3 | kumotop 4 | .SH SYNOPSIS 5 | kumotop server-address[:port=19800] ... 6 | .PP 7 | kumotop -m manager-address[:port=19700] 8 | .SH DESCRIPTION 9 | Show status of kumo-server and renew it continuously like 10 | .B top 11 | command. 12 | .SH EXAMPLE 13 | $ kumotop svr1 svr2 svr3 svr4 14 | .PP 15 | $ kumotop -m mgr1 16 | .SH SEE ALSO 17 | kumo-server(1). 18 | -------------------------------------------------------------------------------- /doc/kumotop.1.txt: -------------------------------------------------------------------------------- 1 | *? KUMOTOP "1" "July 2009" "kumotop" 2 | *NAME 3 | kumotop 4 | 5 | *SYNOPSIS 6 | kumotop server-address[:port=19800] ... &br; 7 | kumotop -m manager-address[:port=19700] 8 | 9 | *DESCRIPTION 10 | Show status of kumo-server and renew it continuously like ''top'' command. 11 | 12 | *EXAMPLE 13 | $ kumotop svr1 svr2 svr3 svr4 &br; 14 | $ kumotop -m mgr1 15 | 16 | *SEE ALSO 17 | kumo-server(1). 18 | 19 | -------------------------------------------------------------------------------- /doc/user.css: -------------------------------------------------------------------------------- 1 | 2 | body { 3 | margin: 0 auto; 4 | font-size: 94%; 5 | line-height: 180%; 6 | max-width: 50em; 7 | } 8 | 9 | h1 { 10 | margin: 2em 0.2em 0 0.3em; 11 | padding: 0.1em 0.1em 0.1em 0.4em; 12 | border-left: 5px solid red; 13 | } 14 | 15 | h2 { 16 | border-bottom: 1px solid red; 17 | padding-left: 0.3em; 18 | margin-top: 2em; 19 | } 20 | 21 | h3 { 22 | border-bottom: 1px solid #aaaaff; 23 | padding-left: 0.5em; 24 | margin-top: 2em; 25 | } 26 | 27 | h4 { 28 | padding-left: 0.6em; 29 | margin-bottom: 0; 30 | } 31 | 32 | p { 33 | margin: 1em; 34 | } 35 | 36 | pre { 37 | color: white; 38 | background-color: #333333; 39 | border: 1px solid black; 40 | padding: 0.5em; 41 | margin: 1em; 42 | } 43 | 44 | dl { 45 | margin: 1em; 46 | } 47 | 48 | dt { 49 | font-weight: bold; 50 | margin-top: 0.1em; 51 | } 52 | 53 | /*body > ul { 54 | border: 1px dashed gray; 55 | padding: 0.5em 0.5em 0.5em 2.5em; 56 | margin: 1em 1em 2em 1em; 57 | }*/ 58 | 59 | -------------------------------------------------------------------------------- /rt/01_mgr1_kill_1_srv.rt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | $LOAD_PATH << File.dirname(__FILE__) 3 | require 'common' 4 | include Chukan::Test 5 | 6 | LOOP_RESTART = (ARGV[0] || ENV["LOOP_RESTART"] || (ENV["HEAVY"] ? 20 : 3)).to_i 7 | SLEEP = (ARGV[1] || 1).to_i 8 | NUM_STORE = (ARGV[2] || 100).to_i 9 | NUM_THREAD = (ARGV[3] || 1).to_i 10 | 11 | mgr, gw, srv1, srv2, srv3 = init_cluster(false, 3) 12 | 13 | mgrs = [ref(mgr)] 14 | srvs = [ref(srv1), ref(srv2), ref(srv3)] 15 | 16 | test "run normally" do 17 | tester = RandomTester.start_threads(gw, NUM_THREAD, NUM_STORE) 18 | 19 | LOOP_RESTART.times { 20 | sleep SLEEP 21 | k = srvs.choice 22 | k.set restart_srv(k.get, mgr) 23 | } 24 | 25 | tester.each {|ra| ra.stop } 26 | tester.each {|ra| ra.join } 27 | 28 | true 29 | end 30 | term_daemons *((mgrs + srvs).map {|r| r.get } + [gw]) 31 | 32 | -------------------------------------------------------------------------------- /rt/02_mgr2_kill_1_mgr.rt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | $LOAD_PATH << File.dirname(__FILE__) 3 | require 'common' 4 | include Chukan::Test 5 | 6 | LOOP_RESTART = (ARGV[0] || ENV["LOOP_RESTART"] || (ENV["HEAVY"] ? 20 : 3)).to_i 7 | SLEEP = (ARGV[1] || 2).to_i 8 | NUM_STORE = (ARGV[2] || 100).to_i 9 | NUM_THREAD = (ARGV[3] || 1).to_i 10 | 11 | mgr1, mgr2, gw, srv1, srv2, srv3 = init_cluster(true, 3) 12 | 13 | mgrs = [ref(mgr1), ref(mgr2)] 14 | srvs = [ref(srv1), ref(srv2), ref(srv3)] 15 | 16 | test "run normally" do 17 | tester = RandomTester.start_threads(gw, NUM_THREAD, NUM_STORE) 18 | 19 | LOOP_RESTART.times { 20 | sleep SLEEP 21 | case rand(2) 22 | when 0 23 | mgr1 = restart_mgr(mgr1, mgr2) 24 | when 1 25 | mgr2 = restart_mgr(mgr2, mgr1) 26 | end 27 | } 28 | 29 | tester.each {|ra| ra.stop } 30 | tester.each {|ra| ra.join } 31 | 32 | true 33 | end 34 | term_daemons(mgr1, mgr2, gw, srv1, srv2, srv3) 35 | 36 | -------------------------------------------------------------------------------- /rt/03_mgr1_kill_2_srvs.rt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | $LOAD_PATH << File.dirname(__FILE__) 3 | require 'common' 4 | include Chukan::Test 5 | 6 | LOOP_RESTART = (ARGV[0] || ENV["LOOP_RESTART"] || (ENV["HEAVY"] ? 20 : 3)).to_i 7 | SLEEP = (ARGV[1] || 2).to_i 8 | NUM_STORE = (ARGV[2] || 100).to_i 9 | NUM_THREAD = (ARGV[3] || 1).to_i 10 | 11 | mgr, gw, srv1, srv2, srv3 = init_cluster(false, 3) 12 | 13 | mgrs = [ref(mgr)] 14 | srvs = [ref(srv1), ref(srv2), ref(srv3)] 15 | 16 | test "run normally" do 17 | tester = RandomTester.start_threads(gw, NUM_THREAD, NUM_STORE) 18 | 19 | LOOP_RESTART.times { 20 | sleep SLEEP 21 | k1, k2 = srvs.shuffle[0,2] 22 | 23 | k1.get.kill 24 | k2.get.kill 25 | k1.get.join 26 | k2.get.join 27 | 28 | k1.set start_srv(k1.get, mgr) 29 | k2.set start_srv(k2.get, mgr) 30 | } 31 | 32 | tester.each {|ra| ra.stop } 33 | tester.each {|ra| ra.join } 34 | 35 | true 36 | end 37 | term_daemons *((mgrs + srvs).map {|r| r.get } + [gw]) 38 | 39 | -------------------------------------------------------------------------------- /rt/04_mgr2_kill_2_srvs.rt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | $LOAD_PATH << File.dirname(__FILE__) 3 | require 'common' 4 | include Chukan::Test 5 | 6 | LOOP_RESTART = (ARGV[0] || ENV["LOOP_RESTART"] || (ENV["HEAVY"] ? 20 : 3)).to_i 7 | SLEEP = (ARGV[1] || 2).to_i 8 | NUM_STORE = (ARGV[2] || 100).to_i 9 | NUM_THREAD = (ARGV[3] || 1).to_i 10 | 11 | mgr1, mgr2, gw, srv1, srv2, srv3 = init_cluster(true, 3) 12 | 13 | mgrs = [ref(mgr1), ref(mgr2)] 14 | srvs = [ref(srv1), ref(srv2), ref(srv3)] 15 | 16 | test "run normally" do 17 | tester = RandomTester.start_threads(gw, NUM_THREAD, NUM_STORE) 18 | 19 | LOOP_RESTART.times { 20 | sleep SLEEP 21 | k1, k2 = srvs.shuffle[0,2] 22 | 23 | k1.get.kill 24 | k2.get.kill 25 | k1.get.join 26 | k2.get.join 27 | 28 | k1.set start_srv(k1.get, mgr1, mgr2) 29 | k2.set start_srv(k2.get, mgr1, mgr2) 30 | } 31 | 32 | tester.each {|ra| ra.stop } 33 | tester.each {|ra| ra.join } 34 | 35 | true 36 | end 37 | term_daemons *((mgrs + srvs).map {|r| r.get } + [gw]) 38 | 39 | -------------------------------------------------------------------------------- /rt/05_mgr2_kill_1_srv_1_mgr.rt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | $LOAD_PATH << File.dirname(__FILE__) 3 | require 'common' 4 | include Chukan::Test 5 | 6 | LOOP_RESTART = (ARGV[0] || ENV["LOOP_RESTART"] || (ENV["HEAVY"] ? 20 : 3)).to_i 7 | SLEEP = (ARGV[1] || 2).to_i 8 | NUM_STORE = (ARGV[2] || 100).to_i 9 | NUM_THREAD = (ARGV[3] || 1).to_i 10 | 11 | mgr1, mgr2, gw, srv1, srv2, srv3 = init_cluster(true, 3) 12 | 13 | mgrs = [ref(mgr1), ref(mgr2)] 14 | srvs = [ref(srv1), ref(srv2), ref(srv3)] 15 | 16 | test "run normally" do 17 | tester = RandomTester.start_threads(gw, NUM_THREAD, NUM_STORE) 18 | 19 | LOOP_RESTART.times { 20 | sleep SLEEP 21 | 22 | m, pt = *mgrs.shuffle 23 | k = srvs.choice 24 | 25 | m.get.kill 26 | k.get.kill 27 | m.get.join 28 | k.get.join 29 | 30 | m.set start_mgr(m.get, pt.get) 31 | k.set start_srv(k.get, pt.get, m.get) 32 | } 33 | 34 | tester.each {|ra| ra.stop } 35 | tester.each {|ra| ra.join } 36 | 37 | true 38 | end 39 | term_daemons *((mgrs + srvs).map {|r| r.get } + [gw]) 40 | 41 | -------------------------------------------------------------------------------- /rt/06_mgr2_kill_1_mgr_1_srv.rt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | $LOAD_PATH << File.dirname(__FILE__) 3 | require 'common' 4 | include Chukan::Test 5 | 6 | LOOP_RESTART = (ARGV[0] || ENV["LOOP_RESTART"] || (ENV["HEAVY"] ? 20 : 3)).to_i 7 | SLEEP = (ARGV[1] || 2).to_i 8 | NUM_STORE = (ARGV[2] || 100).to_i 9 | NUM_THREAD = (ARGV[3] || 1).to_i 10 | 11 | mgr1, mgr2, gw, srv1, srv2, srv3 = init_cluster(true, 3) 12 | 13 | mgrs = [ref(mgr1), ref(mgr2)] 14 | srvs = [ref(srv1), ref(srv2), ref(srv3)] 15 | 16 | test "run normally" do 17 | tester = RandomTester.start_threads(gw, NUM_THREAD, NUM_STORE) 18 | 19 | LOOP_RESTART.times { 20 | sleep SLEEP 21 | 22 | m, pt = *mgrs.shuffle 23 | k = srvs.choice 24 | 25 | k.get.kill 26 | m.get.kill 27 | k.get.join 28 | m.get.join 29 | 30 | k.set start_srv(k.get, pt.get, m.get) 31 | m.set start_mgr(m.get, pt.get) 32 | } 33 | 34 | tester.each {|ra| ra.stop } 35 | tester.each {|ra| ra.join } 36 | 37 | true 38 | end 39 | term_daemons *((mgrs + srvs).map {|r| r.get } + [gw]) 40 | 41 | -------------------------------------------------------------------------------- /rt/07_mgr2_kill_2_srvs_while_replacing.rt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | $LOAD_PATH << File.dirname(__FILE__) 3 | require 'common' 4 | include Chukan::Test 5 | 6 | LOOP_RESTART = (ARGV[0] || ENV["LOOP_RESTART"] || (ENV["HEAVY"] ? 30 : 3)).to_i 7 | SLEEP = (ARGV[1] || 5).to_i 8 | NUM_STORE = (ARGV[2] || 100).to_i 9 | NUM_THREAD = (ARGV[3] || 1).to_i 10 | 11 | mgr1, mgr2, gw, srv1, srv2, srv3 = init_cluster(true, 3) 12 | 13 | mgrs = [ref(mgr1), ref(mgr2)] 14 | srvs = [ref(srv1), ref(srv2), ref(srv3)] 15 | 16 | test "run normally" do 17 | 18 | tester = RandomTester.start_threads(gw, NUM_THREAD, NUM_STORE) 19 | 20 | LOOP_RESTART.times { 21 | sleep SLEEP 22 | 23 | k1, k2 = srvs.shuffle[0, 2] 24 | mgr = mgrs.choice.get 25 | 26 | mgr.stdout_join("lost node") do 27 | k1.get.kill.join 28 | end 29 | mgr.stdout_join("server connected") do 30 | k1.set Server.new(k1.get.index, mgr1, mgr2) 31 | end 32 | sleep 2 33 | 34 | ctl = nil 35 | mgr.stdout_join("start replace copy") do 36 | ctl = mgr.attach 37 | end 38 | 39 | # don't join 40 | 41 | mgr.stdout_join("lost node") do 42 | k2.get.kill.join 43 | end 44 | mgr.stdout_join("server connected") do 45 | k2.set Server.new(k2.get.index, mgr1, mgr2) 46 | end 47 | sleep 2 48 | 49 | ctl.join 50 | mgr.stdout_join("replace finished") do 51 | mgr.attach.join 52 | end 53 | } 54 | 55 | tester.each {|ra| ra.stop } 56 | tester.each {|ra| ra.join } 57 | 58 | true 59 | end 60 | term_daemons *((mgrs + srvs).map {|r| r.get } + [gw]) 61 | 62 | -------------------------------------------------------------------------------- /rt/08_mgr2_kill_2_srvs_while_replacing2.rt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | $LOAD_PATH << File.dirname(__FILE__) 3 | require 'common' 4 | include Chukan::Test 5 | 6 | LOOP_RESTART = (ARGV[0] || ENV["LOOP_RESTART"] || (ENV["HEAVY"] ? 30 : 3)).to_i 7 | SLEEP = (ARGV[1] || 5).to_i 8 | NUM_STORE = (ARGV[2] || 100).to_i 9 | NUM_THREAD = (ARGV[3] || 1).to_i 10 | 11 | mgr1, mgr2, gw, srv1, srv2, srv3, srv4, srv5, srv6 = init_cluster(true, 6) 12 | 13 | mgrs = [ref(mgr1), ref(mgr2)] 14 | srvs = [ref(srv1), ref(srv2), ref(srv3), ref(srv4), ref(srv5), ref(srv6)] 15 | 16 | test "run normally" do 17 | 18 | tester = RandomTester.start_threads(gw, NUM_THREAD, NUM_STORE) 19 | 20 | LOOP_RESTART.times { 21 | sleep SLEEP 22 | 23 | k1, k2 = srvs.shuffle[0, 2] 24 | mgr = mgrs.choice.get 25 | 26 | mgr.stdout_join("lost node") do 27 | k1.get.kill.join 28 | end 29 | mgr.stdout_join("server connected") do 30 | k1.set Server.new(k1.get.index, mgr1, mgr2) 31 | end 32 | sleep 2 33 | 34 | ctl = nil 35 | mgr.stdout_join("start replace copy") do 36 | ctl = mgr.attach 37 | end 38 | 39 | # don't join 40 | 41 | mgr.stdout_join("lost node") do 42 | k2.get.kill.join 43 | end 44 | mgr.stdout_join("server connected") do 45 | k2.set Server.new(k2.get.index, mgr1, mgr2) 46 | end 47 | sleep 2 48 | 49 | ctl.join 50 | mgr.stdout_join("replace finished") do 51 | mgr.attach.join 52 | end 53 | } 54 | 55 | tester.each {|ra| ra.stop } 56 | tester.each {|ra| ra.join } 57 | 58 | true 59 | end 60 | term_daemons *((mgrs + srvs).map {|r| r.get } + [gw]) 61 | 62 | -------------------------------------------------------------------------------- /rt/09_cas.rt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | $LOAD_PATH << File.dirname(__FILE__) 3 | require 'common' 4 | include Chukan::Test 5 | 6 | LOOP_RESTART = (ARGV[0] || ENV["LOOP_RESTART"] || (ENV["HEAVY"] ? 20 : 3)).to_i 7 | NUM_STORE = (ARGV[2] || 500).to_i 8 | NUM_THREAD = (ARGV[3] || 1).to_i 9 | 10 | mgr, gw, srv1, srv2, srv3 = init_cluster(false, 3) 11 | 12 | mgrs = [ref(mgr)] 13 | srvs = [ref(srv1), ref(srv2), ref(srv3)] 14 | 15 | pid = Process.pid 16 | keyf = "#{pid}-key%d" 17 | valf = "val%d" 18 | 19 | test "run normally" do 20 | NUM_STORE.times do |i| 21 | test "set initial" do 22 | c = gw.client 23 | key = keyf % i 24 | val = valf % i 25 | begin 26 | c.set(key, val) 27 | true 28 | rescue 29 | raise "set failed #{key.inspect} => #{val.inspect}: #{$!.inspect}" 30 | end 31 | end 32 | end 33 | 34 | LOOP_RESTART.times { 35 | NUM_STORE.times do |i| 36 | test "cas fail" do 37 | c = gw.client 38 | key = keyf % i 39 | val = valf % i 40 | 41 | begin 42 | 43 | result = c.cas(key) do |val1| 44 | 45 | test "cas success" do 46 | begin 47 | c.cas(key) do |val2| 48 | val 49 | end 50 | rescue 51 | raise "cas failed #{key.inspect} => #{val.inspect}: #{$!.inspect}" 52 | end 53 | end 54 | 55 | val 56 | end 57 | 58 | if result =~ /EXISTS/ 59 | true 60 | else 61 | $stderr.puts "cas unexpectedly succeeded #{key.inspect} => #{val.inspect}: #{result.inspect}" 62 | false 63 | end 64 | 65 | rescue 66 | raise "cas failed #{key.inspect} => #{val.inspect}: #{$!.inspect}" 67 | end 68 | 69 | end 70 | end 71 | 72 | k = srvs.choice 73 | k.set restart_srv(k.get, mgr) 74 | } 75 | 76 | true 77 | end 78 | 79 | term_daemons *((mgrs + srvs).map {|r| r.get } + [gw]) 80 | 81 | -------------------------------------------------------------------------------- /rt/README: -------------------------------------------------------------------------------- 1 | 2 | == How to test == 3 | 4 | 1. ./bootstrap && ./configure && make 5 | 6 | 2. gem install Ruby-MemCache 7 | 8 | 3. prove rt/*.rt 9 | 10 | 11 | = Tests = 12 | 13 | == 01_mgr1_kill_1_srv == 14 | - kumo-manager 1台 15 | - kummo-server 3台 16 | loop { 17 | 1. kumo-server をランダムに１台選んでkillする 18 | 2. killした kumo-server を再起動する 19 | } 20 | 21 | 22 | == 02_mgr2_kill_1_mgr == 23 | - kumo-manager 2台 24 | - kummo-server 3台 25 | loop { 26 | 1. kumo-manager をランダムに１台選んでkillする 27 | 2. killした kumo-manager を再起動する 28 | } 29 | 30 | 31 | == 03_mgr1_kill_2_srvs == 32 | - kumo-manager 1台 33 | - kummo-server 3台 34 | loop { 35 | 1. kumo-server をランダムに１台選んでkillする 36 | 2. kumo-server をランダムにもう１台選んでkillする 37 | 3. killした kumo-server をすべて再起動する 38 | } 39 | 40 | 41 | == 04_mgr2_kill_2_srvs == 42 | - kumo-manager 2台 43 | - kummo-server 3台 44 | loop { 45 | 1. kumo-server をランダムに１台選んでkillする 46 | 2. kumo-server をランダムにもう１台選んでkillする 47 | 3. killした kumo-server をすべて再起動する 48 | } 49 | 50 | 51 | == 05_mgr2_kill_1_srv_1_mgr == 52 | - kumo-manager 2台 53 | - kummo-server 3台 54 | loop { 55 | 1. kumo-manager をランダムに１台選んでkillする 56 | 2. kumo-server をランダムに１台選んでkillする 57 | 3. killした kumo-manager を再起動する 58 | 4. killした kumo-server を再起動する 59 | } 60 | 61 | 62 | == 06_mgr2_kill_1_mgr_1_srv == 63 | - kumo-manager 2台 64 | - kummo-server 3台 65 | loop { 66 | 1. kumo-server をランダムに１台選んでkillする 67 | 2. kumo-manager をランダムに１台選んでkillする 68 | 3. killした kumo-server を再起動する 69 | 4. killした kumo-manager を再起動する 70 | } 71 | 72 | 73 | == 07_mgr2_kill_2_srvs_while_replacing == 74 | - kumo-manager 2台 75 | - kummo-server 3台 76 | loop { 77 | 1. kumo-server をランダムに１台選んでkillする 78 | 2. killした kumo-server を再起動する 79 | 3. 再配置している最中に別の kumo-server を１台選んでkillする 80 | 4. killした kumo-server を再起動する 81 | } 82 | 83 | 84 | == 08_mgr2_kill_2_srvs_while_replacing2 == 85 | - kumo-manager 2台 86 | - kummo-server 6台 87 | loop { 88 | 1. kumo-server をランダムに１台選んでkillする 89 | 2. killした kumo-server を再起動する 90 | 3. 再配置している最中に別の kumo-server を１台選んでkillする 91 | 4. killした kumo-server を再起動する 92 | } 93 | 94 | -------------------------------------------------------------------------------- /src/Makefile.am: -------------------------------------------------------------------------------- 1 | export REVISION 2 | 3 | SUBDIRS = mp mpsrc log kazuhiki rpc storage gate logic command 4 | 5 | -------------------------------------------------------------------------------- /src/command/Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | AM_CPPFLAGS = -I.. -I../logic 3 | AM_C_CPPFLAGS = -I.. -I../logic 4 | 5 | bin_PROGRAMS = kumomergedb 6 | bin_SCRIPTS = kumoctl kumostat kumotop kumolog kumohash 7 | 8 | EXTRA_DIST = $(bin_SCRIPTS) 9 | 10 | kumomergedb_SOURCES = \ 11 | mergedb.cc 12 | 13 | kumomergedb_LDADD = \ 14 | ../storage/libkumo_storage.a \ 15 | ../log/libkumo_log.a 16 | 17 | -------------------------------------------------------------------------------- /src/command/mergedb.cc: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "log/mlogger.h" 19 | #include "log/mlogger_ostream.h" 20 | #include "storage/storage.h" 21 | #include 22 | 23 | template 24 | struct auto_array { 25 | auto_array() : m(NULL) { } 26 | auto_array(T* p) : m(p) { } 27 | ~auto_array() { delete[] m; } 28 | T& operator[] (size_t i) { return m[i]; } 29 | private: 30 | T* m; 31 | auto_array(const auto_array&); 32 | }; 33 | 34 | 35 | using namespace kumo; 36 | 37 | struct for_each_update { 38 | for_each_update(Storage* dstdb, uint64_t* total, uint64_t* merged) : 39 | m_total(total), m_merged(merged), m_dstdb(dstdb) { } 40 | 41 | void operator() (Storage::iterator& kv) 42 | { 43 | ++*m_total; 44 | 45 | if(kv.keylen() < Storage::KEY_META_SIZE) { return; } 46 | if(kv.vallen() < Storage::VALUE_META_SIZE) { return; } 47 | 48 | if( m_dstdb->update(kv.key(), kv.keylen(), kv.val(), kv.vallen()) ) { 49 | ++*m_merged; 50 | } 51 | } 52 | 53 | private: 54 | uint64_t *m_total; 55 | uint64_t *m_merged; 56 | Storage* m_dstdb; 57 | }; 58 | 59 | 60 | int main(int argc, char* argv[]) 61 | { 62 | if(argc <= 3) { 63 | std::cerr << "usage: "< ..." << std::endl; 64 | return 1; 65 | } 66 | 67 | const char* dst = argv[1]; 68 | unsigned int nsrcs = argc - 2; 69 | char* const* psrcs = argv + 2; 70 | 71 | mlogger::reset(new mlogger_ostream(mlogger::TRACE, std::cout)); 72 | 73 | { 74 | // init src databases 75 | auto_array< std::auto_ptr > srcdbs(new std::auto_ptr[nsrcs]); 76 | for(unsigned int i=0; i < nsrcs; ++i) { 77 | srcdbs[i].reset(new Storage(psrcs[i], 0, 0, 0)); 78 | } 79 | 80 | // init dst database 81 | std::auto_ptr dstdb(new Storage(dst, 0, 0, 0)); 82 | 83 | uint64_t total = 0; 84 | uint64_t merged = 0; 85 | for(unsigned int i=0; i < nsrcs; ++i) { 86 | std::cout << "merging "<for_each( 89 | for_each_update(dstdb.get(), &total, &merged), 90 | ClockTime(0) ); 91 | 92 | //std::cout << srcdbs[i]->error() << std::endl; // FIXME 93 | std::cout << " merged " << merged << " records of " << total << " records" << std::endl; 94 | } 95 | 96 | std::cout << "closing "< 24 | #include 25 | #include 26 | 27 | namespace kumo { 28 | namespace gate { 29 | 30 | 31 | typedef rpc::wavy wavy; 32 | using rpc::shared_zone; 33 | using rpc::auto_zone; 34 | 35 | 36 | class gate { 37 | public: 38 | gate() { } 39 | virtual ~gate() { } 40 | virtual void run() = 0; 41 | }; 42 | 43 | 44 | uint64_t stdhash(const char* key, size_t keylen); 45 | void fatal_stop(); 46 | 47 | 48 | struct res_get { 49 | int error; 50 | const char* key; 51 | uint32_t keylen; 52 | 53 | uint64_t hash; 54 | 55 | char* val; 56 | uint32_t vallen; 57 | uint64_t clocktime; 58 | }; 59 | 60 | typedef void (*callback_get)(void* user, res_get& res, auto_zone z); 61 | 62 | struct req_get { 63 | req_get() : has_user_hash(false) { } 64 | 65 | const char* key; 66 | uint32_t keylen; 67 | 68 | bool has_user_hash; 69 | uint64_t user_hash; 70 | 71 | shared_zone life; 72 | callback_get callback; 73 | void* user; 74 | 75 | void submit(); 76 | }; 77 | 78 | 79 | enum set_op_t { 80 | OP_SET = 0, 81 | OP_SET_ASYNC = 1, 82 | OP_CAS = 2, 83 | OP_APPEND = 3, 84 | OP_PREPEND = 4, 85 | }; 86 | 87 | 88 | struct res_set { 89 | int error; 90 | 91 | const char* key; 92 | uint32_t keylen; 93 | 94 | uint64_t hash; 95 | 96 | const char* val; 97 | uint32_t vallen; 98 | uint64_t clocktime; 99 | 100 | bool cas_success; 101 | }; 102 | 103 | typedef void (*callback_set)(void* user, res_set& res, auto_zone z); 104 | 105 | struct req_set { 106 | req_set() : has_user_hash(false), operation(OP_SET) { } 107 | 108 | const char* key; 109 | uint32_t keylen; 110 | 111 | bool has_user_hash; 112 | uint64_t user_hash; 113 | 114 | const char* val; 115 | uint32_t vallen; 116 | 117 | set_op_t operation; 118 | uint64_t clocktime; 119 | 120 | shared_zone life; 121 | callback_set callback; 122 | void* user; 123 | 124 | void submit(); 125 | }; 126 | 127 | 128 | struct res_delete { 129 | int error; 130 | 131 | const char* key; 132 | uint32_t keylen; 133 | 134 | uint64_t hash; 135 | 136 | bool deleted; 137 | }; 138 | 139 | typedef void (*callback_delete)(void* user, res_delete& res, auto_zone z); 140 | 141 | struct req_delete { 142 | req_delete() : has_user_hash(false), async(false) { } 143 | 144 | const char* key; 145 | uint32_t keylen; 146 | 147 | bool has_user_hash; 148 | uint64_t user_hash; 149 | 150 | bool async; 151 | 152 | shared_zone life; 153 | callback_delete callback; 154 | void* user; 155 | 156 | void submit(); 157 | }; 158 | 159 | 160 | } // namespace gate 161 | } // namespace kumo 162 | 163 | #endif /* gate/interface.h */ 164 | 165 | -------------------------------------------------------------------------------- /src/gate/memcache_binary.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef KUMO_GATE_MEMCACHE_BINARY_H__ 19 | #define KUMO_GATE_MEMCACHE_BINARY_H__ 20 | 21 | #include "gate/interface.h" 22 | 23 | namespace kumo { 24 | 25 | 26 | class MemcacheBinary : public gate::gate { 27 | public: 28 | MemcacheBinary(int lsock, bool save_flag, bool save_exptime); 29 | ~MemcacheBinary(); 30 | 31 | void run(); 32 | 33 | private: 34 | int m_lsock; 35 | 36 | private: 37 | MemcacheBinary(); 38 | MemcacheBinary(const MemcacheBinary&); 39 | }; 40 | 41 | 42 | } // namespace kumo 43 | 44 | #endif /* gate/memcache_binary.h */ 45 | 46 | -------------------------------------------------------------------------------- /src/gate/memcache_text.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef KUMO_GATE_MEMCACHE_TEXT_H__ 19 | #define KUMO_GATE_MEMCACHE_TEXT_H__ 20 | 21 | #include "gate/interface.h" 22 | 23 | namespace kumo { 24 | 25 | 26 | class MemcacheText : public gate::gate { 27 | public: 28 | MemcacheText(int lsock, bool save_flag, bool save_exptime); 29 | ~MemcacheText(); 30 | 31 | void run(); 32 | 33 | private: 34 | int m_lsock; 35 | 36 | private: 37 | MemcacheText(); 38 | MemcacheText(const MemcacheText&); 39 | }; 40 | 41 | 42 | } // namespace kumo 43 | 44 | #endif /* gate/memcache_text.h */ 45 | 46 | -------------------------------------------------------------------------------- /src/kazuhiki/Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | noinst_LIBRARIES = libkazuhiki.a 3 | 4 | libkazuhiki_a_SOURCES = \ 5 | kazuhiki.cc 6 | 7 | noinst_HEADERS = \ 8 | kazuhiki.h 9 | 10 | -------------------------------------------------------------------------------- /src/log/Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | noinst_LIBRARIES = libkumo_log.a 3 | 4 | libkumo_log_a_SOURCES = \ 5 | mlogger.cc \ 6 | mlogger_null.cc \ 7 | mlogger_ostream.cc \ 8 | mlogger_syslog.cc \ 9 | mlogger_tty.cc \ 10 | logpack.c \ 11 | logpacker.cc 12 | 13 | noinst_HEADERS = \ 14 | mlogger.h \ 15 | mlogger_null.h \ 16 | mlogger_ostream.h \ 17 | mlogger_syslog.h \ 18 | mlogger_tty.h \ 19 | logpack.h \ 20 | logpack.hpp \ 21 | logpacker.h 22 | 23 | EXTRA_DIST = \ 24 | mlogger.h.erb \ 25 | logpack.hpp.erb 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/log/logpack.h: -------------------------------------------------------------------------------- 1 | /* 2 | * MessagePack fast log format 3 | * 4 | * Copyright (C) 2008-2009 FURUHASHI Sadayuki 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | #ifndef LOGPACK_H__ 19 | #define LOGPACK_H__ 20 | 21 | #include 22 | #include 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | 29 | typedef struct logpack_t logpack_t; 30 | 31 | logpack_t* logpack_new(const char* fname); 32 | logpack_t* logpack_new_pshared(const char* fname); 33 | int logpack_reopen(logpack_t* lpk); 34 | void logpack_free(logpack_t* lpk); 35 | 36 | int logpack_write_raw(logpack_t* lpk, const char* buf, size_t size); 37 | 38 | #if 0 39 | typedef struct { 40 | msgpack_pack_t packer; 41 | msgpack_sbuffer buffer; 42 | } logpack_log_t; 43 | 44 | int logpack_log_init(logpack_log_t* pac, const char* name, uint16_t version); 45 | void logpack_log_destroy(logpack_log_t* pac); 46 | int logpack_write(logpack_t* lpk, const logpack_log_t* pac); 47 | #endif 48 | 49 | 50 | #ifdef __cplusplus 51 | } 52 | #endif 53 | 54 | #endif /* logpack.h */ 55 | 56 | -------------------------------------------------------------------------------- /src/log/logpack.hpp.erb: -------------------------------------------------------------------------------- 1 | // 2 | // MessagePack fast log format 3 | // 4 | // Copyright (C) 2008-2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef LOGPACK_HPP__ 19 | #define LOGPACK_HPP__ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include "logpack.h" 26 | 27 | 28 | class logpack { 29 | public: 30 | struct PSHARED_TAG { }; 31 | logpack(const char* fname); 32 | logpack(const char* fname, PSHARED_TAG); 33 | ~logpack(); 34 | 35 | int reopen(); 36 | 37 | public: 38 | #define MSGPACK_LOG_BEGIN(NUM) \ 39 | msgpack::sbuffer buf; \ 40 | { \ 41 | uint64_t zero = 0; \ 42 | buf.write((const char*)&zero, 4); \ 43 | } \ 44 | msgpack::packer m(buf); \ 45 | m.pack_array(3); \ 46 | m.pack_uint32(fixstr(name)); \ 47 | m.pack_uint16(version); \ 48 | m.pack_map(NUM); 49 | #define MSGPACK_LOG_ADD(k, v) \ 50 | m.pack_uint32(fixstr(k)); \ 51 | m << v; 52 | #define MSGPACK_LOG_END \ 53 | char* p = (char*)buf.data(); \ 54 | size_t sz = buf.size(); \ 55 | *(uint32_t*)p = htonl(sz - 4); \ 56 | return write_raw(p, sz); 57 | 58 | <%1.upto(16) {|i|%> 59 | template , size_t L<%=j%>, typename T<%=j%><%}%>> 60 | int write(const char (&name)[LN], uint16_t version<%1.upto(i) {|j|%>, const char (&k<%=j%>)[L<%=j%>], T<%=j%> v<%=j%><%}%>) 61 | { 62 | MSGPACK_LOG_BEGIN(<%=i%>); 63 | <%1.upto(i) {|j|%> 64 | MSGPACK_LOG_ADD(k<%=j%>, v<%=j%>); 65 | <%}%> 66 | MSGPACK_LOG_END; 67 | } 68 | <%}%> 69 | 70 | private: 71 | <%1.upto(5) {|i|%> 72 | static uint32_t fixstr(const char (&str)[<%=i%>]) 73 | { return 0<%0.upto(i-2) {|j|%> | (((uint32_t)str[<%=j%>])<<(<%=i-2-j%>*8))<%}%>; } 74 | <%}%> 75 | 76 | int write_raw(char* p, size_t sz); 77 | 78 | private: 79 | logpack_t* m_logpack; 80 | 81 | private: 82 | logpack(); 83 | logpack(const logpack&); 84 | }; 85 | 86 | 87 | inline logpack::logpack(const char* fname) 88 | { 89 | m_logpack = logpack_new(fname); 90 | if(!m_logpack) { 91 | throw std::runtime_error("failed to initialize msgpack::logpack"); 92 | } 93 | } 94 | 95 | inline logpack::logpack(const char* fname, PSHARED_TAG) 96 | { 97 | m_logpack = logpack_new_pshared(fname); 98 | if(!m_logpack) { 99 | throw std::runtime_error("failed to initialize msgpack::logpack"); 100 | } 101 | } 102 | 103 | inline int logpack::reopen() 104 | { 105 | return logpack_reopen(m_logpack); 106 | } 107 | 108 | inline logpack::~logpack() 109 | { 110 | logpack_free(m_logpack); 111 | } 112 | 113 | inline int logpack::write_raw(char* buf, size_t size) 114 | { 115 | return logpack_write_raw(m_logpack, buf, size); 116 | } 117 | 118 | 119 | #endif /* logpack.hpp */ 120 | 121 | -------------------------------------------------------------------------------- /src/log/logpacker.cc: -------------------------------------------------------------------------------- 1 | // 2 | // MessagePack fast log format 3 | // 4 | // Copyright (C) 2008-2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "logpacker.h" 19 | 20 | void logpacker::initialize(const char* fname) 21 | { 22 | s_instance.reset(new logpack(fname)); 23 | } 24 | 25 | void logpacker::destroy() 26 | { 27 | s_instance.reset(); 28 | } 29 | 30 | std::auto_ptr logpacker::s_instance; 31 | 32 | -------------------------------------------------------------------------------- /src/log/logpacker.h: -------------------------------------------------------------------------------- 1 | // 2 | // MessagePack fast log format 3 | // 4 | // Copyright (C) 2008-2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef LOGPACKER_H__ 19 | #define LOGPACKER_H__ 20 | 21 | #include "logpack.hpp" 22 | #include 23 | 24 | class logpacker { 25 | public: 26 | static void initialize(const char* fname); 27 | static void destroy(); 28 | static void reopen() { s_instance->reopen(); } 29 | static bool is_active() { return !!s_instance.get(); } 30 | static logpack& instance() { return *s_instance; } 31 | private: 32 | static std::auto_ptr s_instance; 33 | }; 34 | 35 | #define LOGPACK(name, version, ...) \ 36 | do { \ 37 | if(logpacker::is_active()) { \ 38 | logpacker::instance().write(name, version, __VA_ARGS__); \ 39 | } \ 40 | } while(0) 41 | 42 | #define TLOGPACK(name, version, ...) \ 43 | LOGPACK(name, version, "time", time(NULL), __VA_ARGS__) 44 | 45 | #endif /* logpacker.h */ 46 | 47 | -------------------------------------------------------------------------------- /src/log/mlogger.cc: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "mlogger.h" 19 | 20 | mlogger* mlogger::s_logger; 21 | 22 | void mlogger::reset(mlogger* lg) 23 | { 24 | if(s_logger) { delete s_logger; } 25 | s_logger = lg; 26 | } 27 | 28 | void mlogger::destroy() 29 | { 30 | delete s_logger; 31 | s_logger = NULL; 32 | } 33 | 34 | 35 | mlogger::mlogger(level runtime_level) : 36 | m_runtime_level(runtime_level) {} 37 | 38 | mlogger::~mlogger() {} 39 | -------------------------------------------------------------------------------- /src/log/mlogger_null.cc: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "mlogger.h" 19 | #include 20 | 21 | mlogger_null::mlogger_null() : 22 | mlogger((level)((int)FATAL+1)) 23 | {} 24 | 25 | mlogger_null::~mlogger_null() 26 | {} 27 | 28 | void mlogger_null::log_impl(level lv, std::string& str) 29 | { } 30 | 31 | -------------------------------------------------------------------------------- /src/log/mlogger_null.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef MLOGGER_NULL_H__ 19 | #define MLOGGER_NULL_H__ 20 | 21 | #include "mlogger.h" 22 | 23 | class mlogger_null : public mlogger { 24 | public: 25 | mlogger_null(); 26 | ~mlogger_null(); 27 | 28 | void log_impl(level lv, std::string& str); 29 | }; 30 | 31 | #endif /* mlogger_null.h */ 32 | 33 | -------------------------------------------------------------------------------- /src/log/mlogger_ostream.cc: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "mlogger_ostream.h" 19 | #include 20 | 21 | mlogger_ostream::mlogger_ostream(level runtime_level, std::ostream& stream) : 22 | mlogger(runtime_level), 23 | m_stream(stream) 24 | {} 25 | 26 | mlogger_ostream::~mlogger_ostream() 27 | {} 28 | 29 | void mlogger_ostream::log_impl(level lv, std::string& str) 30 | { 31 | m_stream << str << std::endl; 32 | } 33 | 34 | -------------------------------------------------------------------------------- /src/log/mlogger_ostream.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef MLOGGER_OSTREAM_H__ 19 | #define MLOGGER_OSTREAM_H__ 20 | 21 | #include "mlogger.h" 22 | 23 | class mlogger_ostream : public mlogger { 24 | public: 25 | mlogger_ostream(level runtime_level, std::ostream& stream); 26 | ~mlogger_ostream(); 27 | 28 | void log_impl(level lv, std::string& str); 29 | 30 | private: 31 | std::ostream& m_stream; 32 | }; 33 | 34 | #endif /* mlogger_ostream.h */ 35 | 36 | -------------------------------------------------------------------------------- /src/log/mlogger_syslog.cc: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "mlogger_syslog.h" 19 | #include 20 | 21 | mlogger_syslog::mlogger_syslog(level runtime_level, const char* ident, int facility, int option) : 22 | mlogger(runtime_level) 23 | { 24 | ::openlog(ident, option, facility); 25 | } 26 | 27 | mlogger_syslog::~mlogger_syslog() 28 | { 29 | ::closelog(); 30 | } 31 | 32 | void mlogger_syslog::log_impl(level lv, std::string& str) 33 | { 34 | int priority = LOG_DEBUG; 35 | switch(lv) { 36 | case TRACE: 37 | case DEBUG: 38 | priority = LOG_DEBUG; 39 | break; 40 | case INFO: 41 | priority = LOG_INFO; 42 | break; 43 | case WARN: 44 | priority = LOG_NOTICE; 45 | break; 46 | case ERROR: 47 | priority = LOG_ERR; 48 | break; 49 | case FATAL: 50 | priority = LOG_CRIT; 51 | break; 52 | } 53 | 54 | ::syslog(priority, "%s", str.c_str()); 55 | } 56 | 57 | -------------------------------------------------------------------------------- /src/log/mlogger_syslog.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef MLOGGER_SYSLOG_H__ 19 | #define MLOGGER_SYSLOG_H__ 20 | 21 | #include "mlogger.h" 22 | #include 23 | 24 | class mlogger_syslog : public mlogger { 25 | public: 26 | mlogger_syslog(level runtime_level, const char* ident, int facility = LOG_USER, int option = 0); 27 | ~mlogger_syslog(); 28 | 29 | void log_impl(level lv, std::string& str); 30 | }; 31 | 32 | #endif /* mlogger_syslog.h */ 33 | 34 | -------------------------------------------------------------------------------- /src/log/mlogger_tty.cc: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "mlogger_tty.h" 19 | #include 20 | #include 21 | 22 | #define TTY_COLOR_RESET "\033]R" 23 | #define TTY_COLOR_CRE "\033[K" 24 | #define TTY_COLOR_CLEAR "\033c" 25 | #define TTY_COLOR_NORMAL "\033[0;39m" 26 | #define TTY_COLOR_RED "\033[1;31m" 27 | #define TTY_COLOR_GREEN "\033[1;32m" 28 | #define TTY_COLOR_YELLOW "\033[1;33m" 29 | #define TTY_COLOR_BLUE "\033[1;34m" 30 | #define TTY_COLOR_MAGENTA "\033[1;35m" 31 | #define TTY_COLOR_CYAN "\033[1;36m" 32 | #define TTY_COLOR_WHITE "\033[1;37m" 33 | 34 | static const char* const color_table[] = { 35 | TTY_COLOR_NORMAL, 36 | TTY_COLOR_WHITE, 37 | TTY_COLOR_GREEN, 38 | TTY_COLOR_YELLOW, 39 | TTY_COLOR_MAGENTA, 40 | TTY_COLOR_RED, 41 | }; 42 | 43 | mlogger_tty::mlogger_tty(level runtime_level, std::ostream& stream) : 44 | mlogger(runtime_level), 45 | m_stream(stream) 46 | {} 47 | 48 | mlogger_tty::~mlogger_tty() 49 | {} 50 | 51 | void mlogger_tty::log_impl(level lv, std::string& str) 52 | { 53 | // output atomically 54 | 55 | size_t sz = 56 | strlen(color_table[lv]) 57 | + str.size() 58 | + strlen(TTY_COLOR_NORMAL "\n"); 59 | 60 | char* buf = (char*)::malloc(sz); 61 | if(!buf) { throw std::bad_alloc(); } 62 | 63 | char* p = buf; 64 | memcpy(p, color_table[lv], strlen(color_table[lv])); 65 | p += strlen(color_table[lv]); 66 | memcpy(p, str.data(), str.size()); 67 | p += str.size(); 68 | memcpy(p, TTY_COLOR_NORMAL "\n", strlen(TTY_COLOR_NORMAL "\n")); 69 | 70 | try { 71 | m_stream.write(buf, sz) << std::flush; 72 | } catch (...) { 73 | free(buf); 74 | throw; 75 | } 76 | free(buf); 77 | } 78 | 79 | 80 | -------------------------------------------------------------------------------- /src/log/mlogger_tty.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef MLOGGER_TTY_H__ 19 | #define MLOGGER_TTY_H__ 20 | 21 | #include "mlogger.h" 22 | 23 | class mlogger_tty : public mlogger { 24 | public: 25 | mlogger_tty(level runtime_level, std::ostream& stream); 26 | ~mlogger_tty(); 27 | 28 | void log_impl(level lv, std::string& str); 29 | 30 | private: 31 | std::ostream& m_stream; 32 | }; 33 | 34 | #endif /* mlogger_tty.h */ 35 | 36 | -------------------------------------------------------------------------------- /src/logic/Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | SUBDIRS = manager server gateway 3 | 4 | noinst_LIBRARIES = libkumo_logic.a 5 | bin_PROGRAMS = kumo-manager kumo-server kumo-gateway 6 | 7 | AM_CPPFLAGS = -I.. -DREVISION='"$(REVISION)"' 8 | AM_C_CPPFLAGS = -I.. -DREVISION='"$(REVISION)"' 9 | 10 | 11 | libkumo_logic_a_SOURCES = \ 12 | boot.cc \ 13 | hash.cc \ 14 | wavy_server.cc 15 | 16 | 17 | kumo_manager_SOURCES = \ 18 | manager/framework.cc \ 19 | manager/main.cc \ 20 | manager/mod_control.cc \ 21 | manager/mod_network.cc \ 22 | manager/mod_replace.cc 23 | 24 | kumo_manager_LDADD = \ 25 | libkumo_logic.a \ 26 | ../kazuhiki/libkazuhiki.a \ 27 | ../rpc/libkumo_cluster.a \ 28 | ../log/libkumo_log.a \ 29 | ../mpsrc/libmpio.a 30 | 31 | 32 | kumo_server_SOURCES = \ 33 | server/framework.cc \ 34 | server/main.cc \ 35 | server/zmmap_stream.cc \ 36 | server/mod_control.cc \ 37 | server/mod_network.cc \ 38 | server/mod_replace.cc \ 39 | server/mod_replace_stream.cc \ 40 | server/mod_store.cc 41 | 42 | kumo_server_LDADD = \ 43 | libkumo_logic.a \ 44 | ../storage/libkumo_storage.a \ 45 | ../kazuhiki/libkazuhiki.a \ 46 | ../rpc/libkumo_cluster.a \ 47 | ../log/libkumo_log.a \ 48 | ../mpsrc/libmpio.a 49 | 50 | 51 | kumo_gateway_SOURCES = \ 52 | gateway/framework.cc \ 53 | gateway/gate.cc \ 54 | gateway/main.cc \ 55 | gateway/mod_network.cc \ 56 | gateway/mod_cache.cc \ 57 | gateway/mod_store.cc 58 | 59 | kumo_gateway_LDADD = \ 60 | libkumo_logic.a \ 61 | ../gate/libkumo_gate.a \ 62 | ../kazuhiki/libkazuhiki.a \ 63 | ../log/libkumo_log.a \ 64 | ../rpc/libkumo_rpc.a \ 65 | ../mpsrc/libmpio.a 66 | 67 | 68 | noinst_HEADERS = \ 69 | server/proto.h \ 70 | gateway/proto.h \ 71 | manager/proto.h \ 72 | boot.h \ 73 | client_logic.h \ 74 | clock.h \ 75 | clock_logic.h \ 76 | cluster_logic.h \ 77 | global.h \ 78 | hash.h \ 79 | msgtype.h \ 80 | role.h \ 81 | rpc_server.h \ 82 | wavy_server.h \ 83 | manager/framework.h \ 84 | manager/init.h \ 85 | server/framework.h \ 86 | server/init.h \ 87 | server/zmmap_stream.h \ 88 | server/zconnection.h \ 89 | gateway/framework.h \ 90 | gateway/init.h \ 91 | gateway/mod_cache.h \ 92 | gateway/mod_store.h 93 | 94 | EXTRA_DIST = \ 95 | protogen \ 96 | gateway.proto.h \ 97 | manager.proto.h \ 98 | server.proto.h 99 | 100 | 101 | # work around for duplicated file name 102 | kumo_manager_CFLAGS = $(AM_CFLAGS) 103 | kumo_manager_CXXFLAGS = $(AM_CXXFLAGS) 104 | kumo_server_CFLAGS = $(AM_CFLAGS) 105 | kumo_server_CXXFLAGS = $(AM_CXXFLAGS) 106 | kumo_gateway_CFLAGS = $(AM_CFLAGS) 107 | kumo_gateway_CXXFLAGS = $(AM_CXXFLAGS) 108 | 109 | 110 | -------------------------------------------------------------------------------- /src/logic/boot.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef LOGIC_BOOT_H__ 19 | #define LOGIC_BOOT_H__ 20 | 21 | #include "kazuhiki/kazuhiki.h" 22 | #include "rpc/address.h" 23 | #include "log/mlogger_tty.h" 24 | #include "log/mlogger_ostream.h" 25 | #include "log/logpacker.h" 26 | #include "logic/global.h" 27 | 28 | namespace kumo { 29 | 30 | 31 | class scoped_listen_tcp { 32 | public: 33 | scoped_listen_tcp(struct sockaddr_in addr); 34 | ~scoped_listen_tcp(); 35 | 36 | public: 37 | static int listen(const rpc::address& addr); 38 | 39 | public: 40 | int sock() const 41 | { 42 | return m_sock; 43 | } 44 | 45 | rpc::address addr() const 46 | { 47 | return rpc::address(m_addr); 48 | } 49 | 50 | private: 51 | rpc::address m_addr; 52 | int m_sock; 53 | 54 | private: 55 | scoped_listen_tcp(); 56 | scoped_listen_tcp(const scoped_listen_tcp&); 57 | }; 58 | 59 | 60 | void do_daemonize(bool close_stdio, const char* pidfile); 61 | 62 | void init_mlogger(const std::string& logfile, bool use_tty, mlogger::level level); 63 | 64 | struct rpc_args { 65 | rpc_args(); 66 | ~rpc_args(); 67 | 68 | bool verbose; 69 | 70 | bool logfile_set; 71 | std::string logfile; 72 | 73 | bool logpack_path_set; 74 | std::string logpack_path; 75 | 76 | bool pidfile_set; 77 | std::string pidfile; 78 | 79 | const char* prog; 80 | 81 | double keepalive_interval; // sec 82 | unsigned long keepalive_interval_usec; // convert 83 | 84 | double clock_interval; // sec 85 | unsigned long clock_interval_usec; // convert 86 | 87 | double connect_timeout_sec; 88 | unsigned int connect_timeout_msec; // convert 89 | 90 | unsigned short connect_retry_limit; 91 | 92 | unsigned short wthreads; 93 | unsigned short rthreads; 94 | 95 | public: 96 | virtual void set_basic_args(); 97 | virtual void show_usage(); 98 | 99 | void parse(int argc, char** argv); 100 | 101 | protected: 102 | virtual void convert(); 103 | }; 104 | 105 | 106 | struct cluster_args : rpc_args { 107 | cluster_args(); 108 | ~cluster_args(); 109 | 110 | virtual void set_basic_args(); 111 | virtual void show_usage(); 112 | 113 | struct sockaddr_in cluster_addr_in; 114 | rpc::address cluster_addr; // convert 115 | int cluster_lsock; // convert 116 | 117 | protected: 118 | virtual void convert(); 119 | }; 120 | 121 | 122 | } // namespace kumo 123 | 124 | #endif /* logic/boot.h */ 125 | 126 | -------------------------------------------------------------------------------- /src/logic/client_logic.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef LOGIC_CLIENT_LOGIC__ 19 | #define LOGIC_CLIENT_LOGIC__ 20 | 21 | #include "rpc/client.h" 22 | #include "logic/rpc_server.h" 23 | #include "logic/hash.h" 24 | #include "logic/clock.h" 25 | #include "logic/global.h" 26 | 27 | namespace kumo { 28 | 29 | 30 | template 31 | class client_logic : public rpc_server, public rpc::client { 32 | public: 33 | client_logic( 34 | unsigned int connect_timeout_msec, 35 | unsigned short connect_retry_limit) : 36 | rpc::client(connect_timeout_msec, connect_retry_limit) { } 37 | }; 38 | 39 | 40 | typedef mp::shared_ptr shared_session; 41 | typedef mp::weak_ptr weak_session; 42 | 43 | 44 | } // namespace kumo 45 | 46 | #endif /* logic/client_logic.h */ 47 | 48 | -------------------------------------------------------------------------------- /src/logic/clock_logic.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef LOGIC_CLOCK_LOGIC__ 19 | #define LOGIC_CLOCK_LOGIC__ 20 | 21 | #include "logic/clock.h" 22 | 23 | namespace kumo { 24 | 25 | 26 | class clock_logic { 27 | public: 28 | clock_logic() 29 | { 30 | clock_update_time(); 31 | } 32 | 33 | ~clock_logic() { } 34 | 35 | // interrupt rpc_server::timer_handler and call me regularly. 36 | void clock_update_time() 37 | { 38 | m_time = time(NULL); 39 | } 40 | 41 | public: 42 | void clock_update(Clock c) 43 | { 44 | m_clock.update(c.get()); 45 | } 46 | 47 | ClockTime clocktime_now() const 48 | { 49 | return ClockTime(m_clock.get(), m_time); 50 | } 51 | 52 | Clock clock_incr() 53 | { 54 | return Clock( m_clock.get_incr() ); 55 | } 56 | 57 | ClockTime clock_incr_clocktime() 58 | { 59 | return ClockTime(m_clock.get_incr(), m_time); 60 | } 61 | 62 | uint32_t clock_get_time() const 63 | { 64 | return m_time; 65 | } 66 | 67 | private: 68 | Clock m_clock; 69 | uint32_t m_time; 70 | }; 71 | 72 | 73 | } // namespace kumo 74 | 75 | #endif /* logic/clock_logic.h */ 76 | 77 | -------------------------------------------------------------------------------- /src/logic/cluster_logic.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef LOGIC_CLUSTER_LOGIC__ 19 | #define LOGIC_CLUSTER_LOGIC__ 20 | 21 | #include "rpc/cluster.h" 22 | #include "logic/rpc_server.h" 23 | #include "logic/hash.h" 24 | #include "logic/clock.h" 25 | #include "logic/global.h" 26 | #include "logic/role.h" 27 | 28 | namespace kumo { 29 | 30 | 31 | using rpc::role_type; 32 | using rpc::weak_node; 33 | using rpc::shared_node; 34 | using rpc::shared_peer; 35 | 36 | 37 | template 38 | class cluster_logic : public rpc_server, public rpc::cluster { 39 | public: 40 | cluster_logic( 41 | role_type self_id, 42 | const address& self_addr, 43 | unsigned int connect_timeout_msec, 44 | unsigned short connect_retry_limit) : 45 | rpc::cluster( 46 | self_id, 47 | self_addr, 48 | connect_timeout_msec, 49 | connect_retry_limit) { } 50 | 51 | protected: 52 | void listen_cluster(int fd) 53 | { 54 | using namespace mp::placeholders; 55 | wavy::listen(fd, mp::bind( 56 | &Framework::cluster_accepted, this, 57 | _1, _2)); 58 | } 59 | 60 | private: 61 | void cluster_accepted(int fd, int err) 62 | { 63 | if(fd < 0) { 64 | LOG_FATAL("accept failed: ",strerror(err)); 65 | static_cast(this)->signal_end(); 66 | return; 67 | } 68 | LOG_DEBUG("accept cluster fd=",fd); 69 | static_cast(this)->rpc::cluster::accepted(fd); 70 | } 71 | }; 72 | 73 | 74 | #define REQUIRE_HSLK const pthread_scoped_lock& hslk 75 | #define REQUIRE_RELK const pthread_scoped_lock& relk 76 | #define REQUIRE_STLK const pthread_scoped_lock& stlk 77 | 78 | #define REQUIRE_HSLK_RDLOCK const pthread_scoped_rdlock& hslk 79 | #define REQUIRE_HSLK_WRLOCK const pthread_scoped_wrlock& hslk 80 | 81 | 82 | } // namespace kumo 83 | 84 | #endif /* logic/cluster_logic.h */ 85 | 86 | -------------------------------------------------------------------------------- /src/logic/gateway.proto.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "gateway/proto.h" 19 | #include "logic/msgtype.h" 20 | #include "logic/client_logic.h" 21 | #include 22 | #include 23 | #include 24 | 25 | namespace kumo { 26 | namespace gateway { 27 | 28 | 29 | @message mod_network_t::HashSpacePush = 3 30 | 31 | 32 | @rpc mod_network_t 33 | message HashSpacePush { 34 | msgtype::HSSeed wseed; 35 | msgtype::HSSeed rseed; 36 | // acknowledge: true 37 | }; 38 | 39 | public: 40 | void keep_alive(); 41 | 42 | void renew_hash_space(); 43 | void renew_hash_space_for(const address& addr); 44 | RPC_REPLY_DECL(HashSpaceRequest, from, res, err, z); 45 | @end 46 | 47 | 48 | } // namespace gateway 49 | } // namespace kumo 50 | 51 | -------------------------------------------------------------------------------- /src/logic/gateway/Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | PROTOGEN_AUTOGEN_FILES = \ 3 | proto.h \ 4 | mod_network.h 5 | 6 | EXTRA_DIST = \ 7 | $(PROTOGEN_AUTOGEN_FILES) 8 | 9 | -------------------------------------------------------------------------------- /src/logic/gateway/framework.cc: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "gateway/framework.h" 19 | 20 | namespace kumo { 21 | namespace gateway { 22 | 23 | 24 | std::auto_ptr net; 25 | std::auto_ptr share; 26 | 27 | 28 | void framework::dispatch( 29 | shared_session from, weak_responder response, 30 | rpc::method_id method, rpc::msgobj param, auto_zone z) 31 | try { 32 | switch(method.get()) { 33 | RPC_DISPATCH(mod_network, HashSpacePush); 34 | default: 35 | throw unknown_method_error(); 36 | } 37 | } 38 | DISPATCH_CATCH(method, response) 39 | 40 | 41 | void framework::session_lost(const address& addr, shared_session& s) 42 | { 43 | LOG_WARN("lost session ",addr); 44 | if(addr == share->manager1() || addr == share->manager2()) { 45 | mod_network.renew_hash_space_for(addr); 46 | } 47 | } 48 | 49 | 50 | } // namespace gateway 51 | } // namespace kumo 52 | 53 | -------------------------------------------------------------------------------- /src/logic/gateway/gate.cc: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "gateway/framework.h" 19 | 20 | namespace kumo { 21 | namespace gate { 22 | 23 | 24 | uint64_t stdhash(const char* key, size_t keylen) 25 | { 26 | return HashSpace::hash(key, keylen); 27 | } 28 | 29 | void fatal_stop() 30 | { 31 | gateway::net->signal_end(); 32 | } 33 | 34 | 35 | void req_get::submit() 36 | { 37 | gateway::net->mod_store.Get(*this); 38 | } 39 | 40 | void req_set::submit() 41 | { 42 | gateway::net->mod_store.Set(*this); 43 | } 44 | 45 | void req_delete::submit() 46 | { 47 | gateway::net->mod_store.Delete(*this); 48 | } 49 | 50 | 51 | } // namespace gate 52 | } // namespace kumo 53 | 54 | -------------------------------------------------------------------------------- /src/logic/gateway/init.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef GATEWAY_INIT_H__ 19 | #define GATEWAY_INIT_H__ 20 | 21 | #include "gateway/framework.h" 22 | 23 | namespace kumo { 24 | namespace gateway { 25 | 26 | 27 | template 28 | framework::framework(const Config& cfg) : 29 | client_logic( 30 | cfg.connect_timeout_msec, 31 | cfg.connect_retry_limit) 32 | { 33 | if(!cfg.local_cache.empty()) { 34 | mod_cache.init(cfg.local_cache.c_str()); 35 | } 36 | } 37 | 38 | template 39 | void framework::run(const Config& cfg) 40 | { 41 | init_wavy(cfg.rthreads, cfg.wthreads); // wavy_server 42 | start_timeout_step(cfg.clock_interval_usec); // rpc_server 43 | start_keepalive(cfg.keepalive_interval_usec); // rpc_server 44 | mod_network.renew_hash_space(); 45 | TLOGPACK("SW",2, 46 | "mgr1", share->manager1(), 47 | "mgr2", share->manager2()); 48 | } 49 | 50 | template 51 | resource::resource(const Config& cfg) : 52 | m_manager1(cfg.manager1), 53 | m_manager2(cfg.manager2), 54 | m_cfg_async_replicate_set(cfg.async_replicate_set), 55 | m_cfg_async_replicate_delete(cfg.async_replicate_delete), 56 | m_cfg_get_retry_num(cfg.get_retry_num), 57 | m_cfg_set_retry_num(cfg.set_retry_num), 58 | m_cfg_delete_retry_num(cfg.delete_retry_num), 59 | m_cfg_renew_threshold(cfg.renew_threshold), 60 | m_cfg_key_prefix(cfg.key_prefix), 61 | m_error_count(0) 62 | { } 63 | 64 | template 65 | static void init(const Config& cfg) 66 | { 67 | share.reset(new resource(cfg)); 68 | net.reset(new framework(cfg)); 69 | } 70 | 71 | 72 | } // namespace kumo 73 | } // namespace gateway 74 | 75 | #endif /* gateway/init.h */ 76 | 77 | -------------------------------------------------------------------------------- /src/logic/gateway/mod_cache.cc: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "gateway/framework.h" 19 | #include "storage/storage.h" 20 | 21 | namespace kumo { 22 | namespace gateway { 23 | 24 | 25 | mod_cache_t::mod_cache_t() : m_db(NULL) { } 26 | 27 | mod_cache_t::~mod_cache_t() 28 | { 29 | if(m_db) { 30 | tcadbclose(m_db); 31 | tcadbdel(m_db); 32 | } 33 | } 34 | 35 | void mod_cache_t::init(const char* name) 36 | { 37 | m_db = tcadbnew(); 38 | if(!m_db) { 39 | throw std::bad_alloc(); 40 | } 41 | 42 | if(!tcadbopen(m_db, name)) { 43 | tcadbdel(m_db); 44 | m_db = NULL; 45 | throw std::runtime_error("failed to open local cache db"); 46 | } 47 | } 48 | 49 | bool mod_cache_t::get_real(const msgtype::DBKey& key, msgtype::DBValue* result_val, 50 | msgpack::zone* z) 51 | { 52 | int len; 53 | char* val = (char*)tcadbget(m_db, key.data(), key.size(), &len); // FIXME key.raw_data() is invalid 54 | if(!val) { 55 | return false; 56 | } 57 | if(len < static_cast(Storage::VALUE_META_SIZE)) { 58 | ::free(val); 59 | return false; 60 | } 61 | 62 | z->push_finalizer(&::free, (void*)val); 63 | *result_val = msgtype::DBValue(val, len); 64 | 65 | return true; 66 | } 67 | 68 | namespace { 69 | static void* cache_update_proc(const void* vbuf, int vsiz, int *sp, void* op) 70 | { 71 | const msgtype::DBValue* val = 72 | reinterpret_cast(op); 73 | 74 | if(static_cast(vsiz) < Storage::VALUE_CLOCKTIME_SIZE || 75 | Storage::clocktime_of((const char*)vbuf) < val->clocktime()) { 76 | 77 | void* mem = ::malloc(val->raw_size()); 78 | if(!mem) { 79 | return NULL; // FIXME 80 | } 81 | 82 | *sp = val->raw_size(); 83 | memcpy(mem, val->raw_data(), val->raw_size()); 84 | return mem; 85 | 86 | } else { 87 | return NULL; 88 | } 89 | } 90 | } // noname namespace 91 | 92 | void mod_cache_t::update_real(const msgtype::DBKey& key, const msgtype::DBValue& val) 93 | { 94 | tcadbputproc(m_db, 95 | key.data(), key.size(), // FIXME key.raw_data()? 96 | val.raw_data(), val.raw_size(), 97 | &cache_update_proc, 98 | const_cast(reinterpret_cast(&val))); 99 | } 100 | 101 | 102 | } // namespace gateway 103 | } // namespace kumo 104 | 105 | -------------------------------------------------------------------------------- /src/logic/gateway/mod_cache.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef GATEWAY_MOD_CACHE_H__ 19 | #define GATEWAY_MOD_CACHE_H__ 20 | 21 | #include 22 | #include 23 | #include "logic/msgtype.h" 24 | 25 | namespace kumo { 26 | namespace gateway { 27 | 28 | 29 | class mod_cache_t { 30 | public: 31 | mod_cache_t(); 32 | ~mod_cache_t(); 33 | 34 | public: 35 | void init(const char* name); 36 | 37 | public: 38 | bool get(const msgtype::DBKey& key, msgtype::DBValue* result_val, 39 | msgpack::zone* z) 40 | { 41 | if(!m_db) { return false; } 42 | return get_real(key, result_val, z); 43 | } 44 | 45 | void update(const msgtype::DBKey& key, const msgtype::DBValue& val) 46 | { 47 | if(!m_db) { return; } 48 | return update_real(key, val); 49 | } 50 | 51 | private: 52 | bool get_real(const msgtype::DBKey& key, msgtype::DBValue* result_val, 53 | msgpack::zone* z); 54 | 55 | void update_real(const msgtype::DBKey& key, const msgtype::DBValue& val); 56 | 57 | public: 58 | TCADB* m_db; 59 | }; 60 | 61 | 62 | } // namespace gateway 63 | } // namespace kumo 64 | 65 | #endif /* gateway/mod_cache.h */ 66 | 67 | -------------------------------------------------------------------------------- /src/logic/gateway/mod_network.cc: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "gateway/framework.h" 19 | #include "gateway/mod_network.h" 20 | #include "manager/mod_network.h" 21 | 22 | namespace kumo { 23 | namespace gateway { 24 | 25 | 26 | RPC_IMPL(mod_network_t, HashSpacePush, req, z, response) 27 | { 28 | LOG_DEBUG("HashSpacePush"); 29 | 30 | { 31 | pthread_scoped_wrlock hslk(share->hs_rwlock()); 32 | share->update_whs(req.param().wseed, hslk); 33 | share->update_rhs(req.param().rseed, hslk); 34 | } 35 | 36 | response.result(true); 37 | } 38 | 39 | 40 | void mod_network_t::renew_hash_space() 41 | { 42 | shared_zone nullz; 43 | manager::mod_network_t::HashSpaceRequest param; 44 | 45 | rpc::callback_t callback( BIND_RESPONSE(mod_network_t, HashSpaceRequest) ); 46 | 47 | net->get_session(share->manager1())->call( 48 | param, nullz, callback, 10); 49 | 50 | if(!share->manager2().connectable()) { return; } 51 | 52 | net->get_session(share->manager2())->call( 53 | param, nullz, callback, 10); 54 | } 55 | 56 | void mod_network_t::renew_hash_space_for(const address& addr) 57 | { 58 | shared_session ns(net->get_session(addr)); 59 | shared_zone nullz; 60 | manager::mod_network_t::HashSpaceRequest param; 61 | ns->call(param, nullz, 62 | BIND_RESPONSE(mod_network_t, HashSpaceRequest), 10); 63 | } 64 | 65 | void mod_network_t::keep_alive() 66 | { 67 | shared_zone nullz; 68 | manager::mod_network_t::HashSpaceRequest param; 69 | shared_session ns; 70 | 71 | ns = net->get_session(share->manager1()); 72 | if(!ns->is_bound()) { // FIXME 73 | ns->call(param, nullz, 74 | BIND_RESPONSE(mod_network_t, HashSpaceRequest), 10); 75 | } 76 | 77 | if(!share->manager2().connectable()) { return; } 78 | 79 | ns = net->get_session(share->manager2()); 80 | if(!ns->is_bound()) { // FIXME 81 | ns->call(param, nullz, 82 | BIND_RESPONSE(mod_network_t, HashSpaceRequest), 10); 83 | } 84 | } 85 | 86 | 87 | RPC_REPLY_IMPL(mod_network_t, HashSpaceRequest, from, res, err, z) 88 | { 89 | if(!err.is_nil()) { 90 | LOG_ERROR("HashSpaceRequest failed ",err); 91 | if(SESSION_IS_ACTIVE(from)) { 92 | shared_zone nullz; 93 | manager::mod_network_t::HashSpaceRequest param; 94 | 95 | from->call(param, nullz, 96 | BIND_RESPONSE(mod_network_t, HashSpaceRequest), 10); 97 | } // retry on Gateway::session_lost() if the node is lost 98 | } else { 99 | gateway::mod_network_t::HashSpacePush st(res.convert()); 100 | { 101 | pthread_scoped_wrlock hslk(share->hs_rwlock()); 102 | share->update_whs(st.wseed, hslk); 103 | share->update_rhs(st.rseed, hslk); 104 | } 105 | } 106 | } 107 | 108 | 109 | } // namespace gateway 110 | } // namespace kumo 111 | 112 | -------------------------------------------------------------------------------- /src/logic/gateway/mod_store.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef GATEWAY_MOD_STORE_H__ 19 | #define GATEWAY_MOD_STORE_H__ 20 | 21 | #include "gate/interface.h" 22 | #include "server/mod_store.h" 23 | 24 | namespace kumo { 25 | namespace gateway { 26 | 27 | 28 | class mod_store_t { 29 | public: 30 | mod_store_t(); 31 | ~mod_store_t(); 32 | 33 | public: 34 | void Get(gate::req_get& req); 35 | 36 | void Set(gate::req_set& req); 37 | 38 | void Delete(gate::req_delete& req); 39 | 40 | private: 41 | RPC_REPLY_DECL(Get, from, res, err, z, 42 | rpc::retry* retry, 43 | gate::callback_get callback, void* user); 44 | 45 | RPC_REPLY_DECL(GetIfModified, from, res, err, z, 46 | rpc::retry* retry, 47 | gate::callback_get callback, void* user, 48 | msgtype::DBValue* cached_val); 49 | 50 | RPC_REPLY_DECL(Set, from, res, err, z, 51 | rpc::retry* retry, 52 | gate::callback_set callback, void* user); 53 | 54 | RPC_REPLY_DECL(Delete, from, res, err, z, 55 | rpc::retry* retry, 56 | gate::callback_delete callback, void* user); 57 | }; 58 | 59 | 60 | } // namespace gateway 61 | } // namespace kumo 62 | 63 | #endif /* gateway/mod_store.h */ 64 | 65 | -------------------------------------------------------------------------------- /src/logic/global.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef LOGIC_GLOBAL_H__ 19 | #define LOGIC_GLOBAL_H__ 20 | 21 | #include "log/mlogger.h" 22 | #include "config.h" 23 | 24 | #define NUM_REPLICATION 2 25 | 26 | #ifndef MANAGER_DEFAULT_PORT 27 | #define MANAGER_DEFAULT_PORT 19700 28 | #endif 29 | 30 | #ifndef SERVER_DEFAULT_PORT 31 | #define SERVER_DEFAULT_PORT 19800 32 | #endif 33 | 34 | #ifndef SERVER_STREAM_DEFAULT_PORT 35 | #define SERVER_STREAM_DEFAULT_PORT 19900 36 | #endif 37 | 38 | #endif /* logic/global.h */ 39 | 40 | -------------------------------------------------------------------------------- /src/logic/manager/Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | PROTOGEN_AUTOGEN_FILES = \ 3 | proto.h \ 4 | mod_control.h \ 5 | mod_network.h \ 6 | mod_replace.h 7 | 8 | EXTRA_DIST = \ 9 | $(PROTOGEN_AUTOGEN_FILES) 10 | 11 | -------------------------------------------------------------------------------- /src/logic/manager/framework.cc: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "manager/framework.h" 19 | 20 | namespace kumo { 21 | namespace manager { 22 | 23 | 24 | std::auto_ptr net; 25 | std::auto_ptr share; 26 | 27 | 28 | void framework::cluster_dispatch( 29 | shared_node from, weak_responder response, 30 | rpc::method_id method, rpc::msgobj param, auto_zone z) 31 | try { 32 | // FIXME try & catch 33 | switch(method.get()) { 34 | RPC_DISPATCH(mod_network, KeepAlive); 35 | RPC_DISPATCH(mod_network, WHashSpaceRequest); 36 | RPC_DISPATCH(mod_network, RHashSpaceRequest); 37 | RPC_DISPATCH(mod_network, HashSpaceSync); 38 | RPC_DISPATCH(mod_replace, ReplaceCopyEnd); 39 | RPC_DISPATCH(mod_replace, ReplaceDeleteEnd); 40 | RPC_DISPATCH(mod_replace, ReplaceElection); 41 | default: 42 | throw unknown_method_error(); 43 | } 44 | } 45 | DISPATCH_CATCH(method, response) 46 | 47 | void framework::subsystem_dispatch( 48 | shared_peer from, weak_responder response, 49 | rpc::method_id method, rpc::msgobj param, auto_zone z) 50 | try { 51 | // FIXME try & catch 52 | switch(method.get()) { 53 | RPC_DISPATCH(mod_network, HashSpaceRequest); 54 | RPC_DISPATCH(mod_control, GetNodesInfo); 55 | RPC_DISPATCH(mod_control, AttachNewServers); 56 | RPC_DISPATCH(mod_control, DetachFaultServers); 57 | RPC_DISPATCH(mod_control, CreateBackup); 58 | RPC_DISPATCH(mod_control, SetAutoReplace); 59 | RPC_DISPATCH(mod_control, StartReplace); 60 | default: 61 | throw unknown_method_error(); 62 | } 63 | } 64 | DISPATCH_CATCH(method, response) 65 | 66 | 67 | void framework::new_node(address addr, role_type id, shared_node n) 68 | { 69 | LOG_WARN("new node ",(uint16_t)id," ",addr); 70 | if(id == ROLE_MANAGER) { 71 | if(addr != share->partner()) { 72 | TLOGPACK("eP",3, 73 | "addr",addr); 74 | LOG_ERROR("unknown partner node"); 75 | // FIXME 76 | return; 77 | } 78 | LOG_INFO("partner connected ",addr); 79 | { 80 | pthread_scoped_lock hslk(share->hs_mutex()); 81 | mod_network.sync_hash_space_partner(hslk); 82 | } 83 | return; 84 | 85 | } else if(id == ROLE_SERVER) { 86 | // FIXME delayed change 87 | mod_replace.add_server(addr, n); 88 | return; 89 | 90 | } else { 91 | LOG_ERROR("unkown node id ",(uint16_t)id); 92 | } 93 | } 94 | 95 | void framework::lost_node(address addr, role_type id) 96 | { 97 | LOG_WARN("lost node ",id," ",addr); 98 | if(id == ROLE_MANAGER) { 99 | return; 100 | 101 | } else if(id == ROLE_SERVER) { 102 | // FIXME delayed change 103 | mod_replace.remove_server(addr); 104 | return; 105 | } 106 | } 107 | 108 | 109 | } // namespace manager 110 | } // namespace kumo 111 | 112 | -------------------------------------------------------------------------------- /src/logic/manager/init.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef MANAGER_INIT_H__ 19 | #define MANAGER_INIT_H__ 20 | 21 | #include "manager/framework.h" 22 | 23 | namespace kumo { 24 | namespace manager { 25 | 26 | 27 | template 28 | framework::framework(const Config& cfg) : 29 | cluster_logic( 30 | ROLE_MANAGER, 31 | cfg.cluster_addr, 32 | cfg.connect_timeout_msec, 33 | cfg.connect_retry_limit) 34 | { } 35 | 36 | template 37 | void framework::run(const Config& cfg) 38 | { 39 | init_wavy(cfg.rthreads, cfg.wthreads); // wavy_server 40 | listen_cluster(cfg.cluster_lsock); // cluster_logic 41 | start_timeout_step(cfg.clock_interval_usec); // rpc_server 42 | start_keepalive(cfg.keepalive_interval_usec); // rpc_server 43 | LOG_INFO("start manager ",addr()); 44 | TLOGPACK("SM",2, 45 | "addr", cfg.cluster_addr, 46 | "Padd", share->partner()); 47 | } 48 | 49 | template 50 | resource::resource(const Config& cfg) : 51 | m_partner(cfg.partner), 52 | m_cfg_auto_replace(cfg.auto_replace), 53 | m_cfg_replace_delay_seconds(cfg.replace_delay_seconds) 54 | { } 55 | 56 | template 57 | static void init(const Config& cfg) 58 | { 59 | share.reset(new resource(cfg)); 60 | net.reset(new framework(cfg)); 61 | } 62 | 63 | 64 | } // namespace manager 65 | } // namespace kumo 66 | 67 | #endif /* manager/init.h */ 68 | 69 | -------------------------------------------------------------------------------- /src/logic/manager/main.cc: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "logic/boot.h" 19 | #include "manager/framework.h" 20 | #include "manager/init.h" 21 | 22 | using namespace kumo; 23 | 24 | struct arg_t : cluster_args { 25 | unsigned short replace_delay_seconds; 26 | 27 | bool auto_replace; 28 | 29 | bool partner_set; 30 | struct sockaddr_in partner_in; 31 | rpc::address partner; // convert 32 | 33 | virtual void convert() 34 | { 35 | cluster_args::convert(); 36 | partner = rpc::address(partner_in); 37 | } 38 | 39 | arg_t(int argc, char** argv) : 40 | replace_delay_seconds(4) 41 | { 42 | using namespace kazuhiki; 43 | set_basic_args(); 44 | on("-l", "--listen", 45 | type::connectable(&cluster_addr_in, MANAGER_DEFAULT_PORT)); 46 | on("-p", "--partner", &partner_set, 47 | type::connectable(&partner_in, MANAGER_DEFAULT_PORT)); 48 | on("-a", "--auto-replace", 49 | type::boolean(&auto_replace)); 50 | on("-Rs", "--replace-delay", 51 | type::numeric(&replace_delay_seconds, replace_delay_seconds)); 52 | parse(argc, argv); 53 | } 54 | 55 | void show_usage() 56 | { 57 | std::cout << 58 | "usage: "<" 60 | " -p \n" 61 | "\n" 62 | " -p " 63 | "--partner master-slave replication partner\n" 64 | " -a " 65 | "--auto-replace enable auto replacing\n" 66 | " -Rs " 67 | "--replace-delay delay time of auto replacing in sec.\n" 68 | ; 69 | cluster_args::show_usage(); 70 | } 71 | }; 72 | 73 | int main(int argc, char* argv[]) 74 | { 75 | arg_t arg(argc, argv); 76 | 77 | // initialize logger first 78 | mlogger::level loglevel = (arg.verbose ? mlogger::TRACE : mlogger::WARN); 79 | init_mlogger(arg.logfile, arg.pidfile.empty(), loglevel); 80 | 81 | // daemonize 82 | if(!arg.pidfile.empty()) { 83 | do_daemonize(!arg.logfile.empty(), arg.pidfile.c_str()); 84 | } 85 | 86 | // initialize binary logger 87 | if(arg.logpack_path_set) { 88 | logpacker::initialize(arg.logpack_path.c_str()); 89 | } 90 | 91 | // run server 92 | manager::init(arg); 93 | manager::net->run(arg); 94 | manager::net->join(); 95 | 96 | return 0; 97 | } 98 | 99 | -------------------------------------------------------------------------------- /src/logic/role.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef LOGIC_ROLE_H__ 19 | #define LOGIC_ROLE_H__ 20 | 21 | #include "rpc/types.h" 22 | 23 | namespace kumo { 24 | 25 | static const rpc::role_type ROLE_MANAGER = 0; 26 | static const rpc::role_type ROLE_SERVER = 1; 27 | 28 | } // namespace kumo 29 | 30 | #endif /* logic/role.h */ 31 | 32 | -------------------------------------------------------------------------------- /src/logic/server/Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | PROTOGEN_AUTOGEN_FILES = \ 3 | proto.h \ 4 | mod_control.h \ 5 | mod_network.h \ 6 | mod_replace.h \ 7 | mod_replace_stream.h \ 8 | mod_store.h 9 | 10 | EXTRA_DIST = \ 11 | $(PROTOGEN_AUTOGEN_FILES) 12 | 13 | -------------------------------------------------------------------------------- /src/logic/server/framework.cc: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "server/framework.h" 19 | 20 | namespace kumo { 21 | namespace server { 22 | 23 | 24 | std::auto_ptr net; 25 | std::auto_ptr share; 26 | 27 | 28 | void framework::cluster_dispatch( 29 | shared_node from, weak_responder response, 30 | rpc::method_id method, rpc::msgobj param, auto_zone z) 31 | try { 32 | switch(method.get()) { 33 | RPC_DISPATCH(mod_network, KeepAlive); 34 | RPC_DISPATCH(mod_network, HashSpaceSync); 35 | RPC_DISPATCH(mod_store, ReplicateSet); 36 | RPC_DISPATCH(mod_store, ReplicateDelete); 37 | RPC_DISPATCH(mod_replace, ReplaceCopyStart); 38 | RPC_DISPATCH(mod_replace, ReplaceDeleteStart); 39 | RPC_DISPATCH(mod_replace_stream, ReplaceOffer); 40 | RPC_DISPATCH(mod_control, CreateBackup); 41 | default: 42 | throw unknown_method_error(); 43 | } 44 | } 45 | DISPATCH_CATCH(method, response) 46 | 47 | void framework::subsystem_dispatch( 48 | shared_peer from, weak_responder response, 49 | rpc::method_id method, rpc::msgobj param, auto_zone z) 50 | try { 51 | switch(method.get()) { 52 | RPC_DISPATCH(mod_store, Get); 53 | RPC_DISPATCH(mod_store, Set); 54 | RPC_DISPATCH(mod_store, Delete); 55 | RPC_DISPATCH(mod_store, GetIfModified); 56 | RPC_DISPATCH(mod_control, GetStatus); 57 | RPC_DISPATCH(mod_control, SetConfig); 58 | default: 59 | throw unknown_method_error(); 60 | } 61 | } 62 | DISPATCH_CATCH(method, response) 63 | 64 | 65 | void framework::end_preprocess() 66 | { 67 | mod_replace_stream.stop_stream(); 68 | } 69 | 70 | 71 | void framework::new_node(address addr, role_type id, shared_node n) 72 | { 73 | // XXX 74 | LOG_WARN("new node ",(uint16_t)id," ",addr); 75 | if(addr == share->manager1()) { 76 | mod_network.renew_r_hash_space(); 77 | mod_network.renew_w_hash_space(); 78 | } else if(share->manager2().connectable() && addr == share->manager2()) { 79 | mod_network.renew_r_hash_space(); 80 | mod_network.renew_w_hash_space(); 81 | } 82 | } 83 | 84 | void framework::lost_node(address addr, role_type id) 85 | { 86 | // XXX 87 | LOG_WARN("lost node ",(uint16_t)id," ",addr); 88 | } 89 | 90 | 91 | } // namespace server 92 | } // namespace kumo 93 | 94 | -------------------------------------------------------------------------------- /src/logic/server/init.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef SERVER_INIT_H__ 19 | #define SERVER_INIT_H__ 20 | 21 | #include "server/framework.h" 22 | 23 | namespace kumo { 24 | namespace server { 25 | 26 | 27 | template 28 | framework::framework(const Config& cfg) : 29 | cluster_logic( 30 | ROLE_SERVER, 31 | cfg.cluster_addr, 32 | cfg.connect_timeout_msec, 33 | cfg.connect_retry_limit), 34 | mod_replace_stream(cfg.stream_addr) 35 | { } 36 | 37 | template 38 | void framework::run(const Config& cfg) 39 | { 40 | init_wavy(cfg.rthreads, cfg.wthreads); // wavy_server 41 | listen_cluster(cfg.cluster_lsock); // cluster_logic 42 | start_timeout_step(cfg.clock_interval_usec); // rpc_server 43 | start_keepalive(cfg.keepalive_interval_usec); // rpc_server 44 | mod_replace_stream.init_stream(cfg.stream_lsock); 45 | LOG_INFO("start server ",addr()); 46 | TLOGPACK("SS",2, 47 | "addr", cfg.cluster_addr, 48 | "db", cfg.dbpath, 49 | "mgr1", share->manager1(), 50 | "mgr2", share->manager2(), 51 | "sadd", cfg.stream_addr, 52 | "tmpd", share->cfg_offer_tmpdir(), 53 | "bkup", share->cfg_db_backup_basename()); 54 | } 55 | 56 | template 57 | resource::resource(const Config& cfg) : 58 | m_db(*cfg.db), 59 | m_manager1(cfg.manager1), 60 | m_manager2(cfg.manager2), 61 | 62 | m_cfg_offer_tmpdir(cfg.offer_tmpdir), 63 | m_cfg_db_backup_basename(cfg.db_backup_basename), 64 | m_cfg_replicate_set_retry_num(cfg.replicate_set_retry_num), 65 | m_cfg_replicate_delete_retry_num(cfg.replicate_delete_retry_num), 66 | m_cfg_replace_set_limit_mem(cfg.replace_set_limit_mem), 67 | 68 | m_stat_start_time(time(NULL)), 69 | m_stat_num_get(0), 70 | m_stat_num_set(0), 71 | m_stat_num_delete(0) 72 | { } 73 | 74 | template 75 | static void init(const Config& cfg) 76 | { 77 | share.reset(new resource(cfg)); 78 | net.reset(new framework(cfg)); 79 | } 80 | 81 | 82 | 83 | } // namespace server 84 | } // namespace kumo 85 | 86 | #endif /* server/init.h */ 87 | 88 | -------------------------------------------------------------------------------- /src/logic/server/zmmap_stream.cc: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "server/zmmap_stream.h" 19 | #include 20 | 21 | namespace kumo { 22 | namespace server { 23 | 24 | 25 | zmmap_stream::zmmap_stream(int fd) : 26 | m_fd(fd) 27 | { 28 | m_z.zalloc = Z_NULL; 29 | m_z.zfree = Z_NULL; 30 | m_z.opaque = Z_NULL; 31 | if(deflateInit(&m_z, Z_DEFAULT_COMPRESSION) != Z_OK) { 32 | throw std::runtime_error(m_z.msg); 33 | } 34 | 35 | if(::ftruncate(m_fd, ZMMAP_STREAM_INITIAL_SIZE) < 0) { 36 | deflateEnd(&m_z); 37 | throw mp::system_error(errno, "failed to truncate offer storage"); 38 | } 39 | 40 | m_map = (char*)::mmap(NULL, ZMMAP_STREAM_INITIAL_SIZE, 41 | PROT_WRITE, MAP_SHARED, m_fd, 0); 42 | if(m_map == MAP_FAILED) { 43 | deflateEnd(&m_z); 44 | throw mp::system_error(errno, "failed to mmap offer storage"); 45 | } 46 | 47 | m_z.avail_out = ZMMAP_STREAM_INITIAL_SIZE; 48 | m_z.next_out = (Bytef*)m_map; 49 | } 50 | 51 | zmmap_stream::~zmmap_stream() 52 | { 53 | size_t used = (char*)m_z.next_out - m_map; 54 | size_t csize = used + m_z.avail_out; 55 | ::munmap(m_map, csize); 56 | //::ftruncate(m_fd, used); 57 | deflateEnd(&m_z); 58 | } 59 | 60 | void zmmap_stream::flush() 61 | { 62 | while(true) { 63 | switch(deflate(&m_z, Z_FINISH)) { 64 | 65 | case Z_STREAM_END: 66 | return; 67 | 68 | case Z_OK: 69 | break; 70 | 71 | default: 72 | throw std::runtime_error("deflate flush failed"); 73 | } 74 | 75 | expand_map(m_z.avail_in); 76 | } 77 | } 78 | 79 | void zmmap_stream::expand_map(size_t req) 80 | { 81 | size_t used = (char*)m_z.next_out - m_map; 82 | size_t csize = used + m_z.avail_out; 83 | size_t nsize = csize * 2; 84 | while(nsize < req) { nsize *= 2; } 85 | 86 | if(::ftruncate(m_fd, nsize) < 0 ) { 87 | throw mp::system_error(errno, "failed to resize offer storage"); 88 | } 89 | 90 | #ifdef __linux__ 91 | void* tmp = ::mremap(m_map, csize, nsize, MREMAP_MAYMOVE); 92 | if(tmp == MAP_FAILED) { 93 | throw mp::system_error(errno, "failed to mremap offer storage"); 94 | } 95 | m_map = (char*)tmp; 96 | 97 | #else 98 | if(::munmap(m_map, csize) < 0) { 99 | throw mp::system_error(errno, "failed to munmap offer storage"); 100 | } 101 | m_map = NULL; 102 | m_z.next_out = NULL; 103 | m_z.avail_out = 0; 104 | 105 | m_map = (char*)::mmap(NULL, nsize, 106 | PROT_WRITE, MAP_SHARED, m_fd, 0); 107 | if(m_map == MAP_FAILED) { 108 | throw mp::system_error(errno, "failed to mmap"); 109 | } 110 | 111 | #endif 112 | m_z.next_out = (Bytef*)(m_map + used); 113 | m_z.avail_out = nsize - used; 114 | } 115 | 116 | 117 | } // namespace server 118 | } // namespace kumo 119 | 120 | -------------------------------------------------------------------------------- /src/logic/server/zmmap_stream.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef SERVER_ZMMAP_STREAM_H__ 19 | #define SERVER_ZMMAP_STREAM_H__ 20 | 21 | #include 22 | #include 23 | 24 | #ifndef ZMMAP_STREAM_INITIAL_SIZE 25 | #define ZMMAP_STREAM_INITIAL_SIZE (1024*1024) 26 | #endif 27 | 28 | #ifndef ZMMAP_STREAM_RESERVE_SIZE 29 | #define ZMMAP_STREAM_RESERVE_SIZE (8*1024) 30 | #endif 31 | 32 | namespace kumo { 33 | namespace server { 34 | 35 | 36 | class zmmap_stream { 37 | public: 38 | zmmap_stream(int fd); 39 | ~zmmap_stream(); 40 | size_t size() const; 41 | 42 | void write(const void* buf, size_t len); 43 | void flush(); 44 | 45 | private: 46 | z_stream m_z; 47 | 48 | char* m_map; 49 | int m_fd; 50 | void expand_map(size_t req); 51 | 52 | private: 53 | zmmap_stream(); 54 | zmmap_stream(const zmmap_stream&); 55 | }; 56 | 57 | 58 | inline size_t zmmap_stream::size() const 59 | { 60 | return (char*)m_z.next_out - m_map; 61 | } 62 | 63 | 64 | inline void zmmap_stream::write(const void* buf, size_t len) 65 | { 66 | m_z.next_in = (Bytef*)buf; 67 | m_z.avail_in = len; 68 | 69 | do { 70 | if(m_z.avail_out < ZMMAP_STREAM_RESERVE_SIZE) { 71 | expand_map(ZMMAP_STREAM_INITIAL_SIZE); 72 | } 73 | 74 | if(deflate(&m_z, Z_NO_FLUSH) != Z_OK) { 75 | throw std::runtime_error("deflate failed"); 76 | } 77 | 78 | } while(m_z.avail_in > 0); 79 | } 80 | 81 | 82 | } // namespace server 83 | } // namespace kumo 84 | 85 | #endif /* server/zmmap_stream.h */ 86 | 87 | -------------------------------------------------------------------------------- /src/logic/wavy_server.cc: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "logic/wavy_server.h" 19 | #include 20 | 21 | namespace kumo { 22 | 23 | 24 | wavy_server::wavy_server() 25 | { 26 | wavy::initialize(0, 0); 27 | } 28 | 29 | wavy_server::~wavy_server() { } 30 | 31 | 32 | void wavy_server::do_after(unsigned int steps, mp::function func) 33 | { 34 | mp::pthread_scoped_lock dalk(m_do_after_mutex); 35 | m_do_after.push_back( do_after_entry(steps, func) ); 36 | } 37 | 38 | void wavy_server::step_do_after() 39 | { 40 | do_after_t fire; 41 | 42 | { 43 | mp::pthread_scoped_lock dalk(m_do_after_mutex); 44 | for(do_after_t::iterator it(m_do_after.begin()); it != m_do_after.end(); ) { 45 | if(it->remain_steps == 0) { 46 | fire.splice(fire.end(), m_do_after, it++); 47 | } else { 48 | --it->remain_steps; 49 | ++it; 50 | } 51 | } 52 | } 53 | 54 | for(do_after_t::iterator it(fire.begin()); it != fire.end(); ++it) { 55 | try { 56 | // FIXME wavy::submit? 57 | it->func(); 58 | } catch (...) { } // FIXME log 59 | } 60 | } 61 | 62 | 63 | namespace { 64 | // avoid compile error 65 | typedef void (*sigend_callback)(void*, int); 66 | static sigend_callback get_signal_handler() 67 | { 68 | sigend_callback f = &mp::object_callback:: 69 | mem_fun; 70 | return f; 71 | } 72 | } // noname namespace 73 | 74 | 75 | void wavy_server::init_wavy(unsigned short rthreads, unsigned short wthreads) 76 | { 77 | // ignore SIGPIPE 78 | if( signal(SIGPIPE, SIG_IGN) == SIG_ERR ) { 79 | perror("signal"); 80 | throw mp::system_error(errno, "signal"); 81 | } 82 | 83 | // initialize signal handler before starting threads 84 | sigset_t ss; 85 | sigemptyset(&ss); 86 | sigaddset(&ss, SIGHUP); 87 | sigaddset(&ss, SIGINT); 88 | sigaddset(&ss, SIGTERM); 89 | 90 | s_pth.reset( new mp::pthread_signal(ss, 91 | get_signal_handler(), 92 | reinterpret_cast(this)) ); 93 | 94 | wavy::add_core_thread(rthreads); 95 | wavy::add_output_thread(wthreads); 96 | } 97 | 98 | void wavy_server::join() 99 | { 100 | wavy::join(); 101 | } 102 | 103 | 104 | void wavy_server::signal_handler(int signo) 105 | { 106 | if(signo == SIGINT || signo == SIGTERM) { 107 | signal_end(); 108 | } else { 109 | signal_hup(); 110 | } 111 | } 112 | 113 | 114 | // dummy function 115 | static void finished() { } 116 | 117 | void wavy_server::signal_end() 118 | { 119 | wavy::end(); 120 | wavy::submit(finished); // submit dummy function 121 | end_preprocess(); 122 | LOG_INFO("end"); 123 | } 124 | 125 | void wavy_server::signal_hup() 126 | { 127 | LOG_INFO("SIGHUP"); 128 | if(logpacker::is_active()) { 129 | logpacker::reopen(); 130 | } 131 | } 132 | 133 | 134 | } // namespace kumo 135 | 136 | -------------------------------------------------------------------------------- /src/logic/wavy_server.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef LOGIC_WAVY_SERVER_H__ 19 | #define LOGIC_WAVY_SERVER_H__ 20 | 21 | #include "rpc/wavy.h" 22 | #include "log/mlogger.h" 23 | #include "log/logpacker.h" 24 | #include 25 | #include 26 | #include 27 | 28 | namespace kumo { 29 | 30 | 31 | using rpc::wavy; 32 | 33 | 34 | class wavy_server { 35 | public: 36 | wavy_server(); 37 | ~wavy_server(); 38 | 39 | void do_after(unsigned int steps, mp::function func); 40 | 41 | protected: 42 | // call this function before starting any threads 43 | void init_wavy(unsigned short rthreads, unsigned short wthreads); 44 | 45 | virtual void end_preprocess() { } 46 | 47 | void step_do_after(); 48 | 49 | public: 50 | virtual void join(); 51 | 52 | void signal_handler(int signo); 53 | void signal_end(); 54 | void signal_hup(); 55 | 56 | private: 57 | unsigned short m_core_threads; 58 | unsigned short m_output_threads; 59 | std::auto_ptr s_pth; 60 | 61 | struct do_after_entry { 62 | do_after_entry(unsigned int steps, mp::function f) : 63 | remain_steps(steps), func(f) { } 64 | unsigned int remain_steps; 65 | mp::function func; 66 | }; 67 | 68 | mp::pthread_mutex m_do_after_mutex; 69 | typedef std::list do_after_t; 70 | do_after_t m_do_after; 71 | }; 72 | 73 | 74 | } // namespace kumo 75 | 76 | #endif /* logic/wavy_server.h */ 77 | 78 | -------------------------------------------------------------------------------- /src/mp/Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | EXTRA_DIST = \ 3 | object_callback.pre.h \ 4 | wavy/core.pre.h \ 5 | wavy/singleton.pre.h 6 | 7 | noinst_HEADERS = \ 8 | exception.h \ 9 | functional.h \ 10 | memory.h \ 11 | object_callback.h \ 12 | pp.h \ 13 | pthread.h \ 14 | pthread_impl.h \ 15 | shared_buffer.h \ 16 | shared_buffer_impl.h \ 17 | source.h \ 18 | source_impl.h \ 19 | stream_buffer.h \ 20 | stream_buffer_impl.h \ 21 | utility.h \ 22 | wavy/core.h \ 23 | wavy/output.h \ 24 | wavy/singleton.h \ 25 | wavy.h 26 | 27 | -------------------------------------------------------------------------------- /src/mp/exception.h: -------------------------------------------------------------------------------- 1 | // 2 | // mpio exception 3 | // 4 | // Copyright (C) 2008 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #ifndef MP_EXCEPTION_H__ 20 | #define MP_EXCEPTION_H__ 21 | 22 | #include 23 | #include 24 | #include 25 | 26 | namespace mp { 27 | 28 | 29 | struct system_error : std::runtime_error { 30 | system_error(int errno_, const std::string& msg) : 31 | std::runtime_error(msg + ": " + strerror(errno_)) {} 32 | }; 33 | 34 | 35 | struct event_error : system_error { 36 | event_error(int errno_, const std::string& msg) : 37 | system_error(errno, msg) {} 38 | }; 39 | 40 | 41 | } // namespace mp 42 | 43 | 44 | #endif /* mp/exception.h */ 45 | 46 | -------------------------------------------------------------------------------- /src/mp/functional.h: -------------------------------------------------------------------------------- 1 | // 2 | // mp::functional 3 | // 4 | // Copyright (C) 2008 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #ifndef MP_FUNCTIONAL_H__ 20 | #define MP_FUNCTIONAL_H__ 21 | 22 | #ifdef MP_FUNCTIONAL_BOOST 23 | #include 24 | namespace mp { 25 | using std::tr1::function; 26 | using std::tr1::bind; 27 | namespace placeholders { 28 | using namespace std::tr1::placeholders; 29 | } 30 | } 31 | #else 32 | #ifdef MP_FUNCTIONAL_BOOST_ORG 33 | #include 34 | #include 35 | namespace mp { 36 | using boost::function; 37 | using boost::bind; 38 | namespace placeholders { } 39 | } 40 | #else 41 | #ifndef MP_FUNCTIONAL_STANDARD 42 | #include 43 | namespace mp { 44 | using std::tr1::function; 45 | using std::tr1::bind; 46 | namespace placeholders { 47 | using namespace std::tr1::placeholders; 48 | } 49 | } 50 | #else 51 | #include 52 | namespace mp { 53 | using std::function; 54 | using std::bind; 55 | namespace placeholders { 56 | using namespace std::placeholders; 57 | } 58 | } 59 | #endif 60 | #endif 61 | #endif 62 | 63 | #endif /* mp/functional.h */ 64 | 65 | -------------------------------------------------------------------------------- /src/mp/memory.h: -------------------------------------------------------------------------------- 1 | // 2 | // mp::memory 3 | // 4 | // Copyright (C) 2008 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #ifndef MP_MEMORY_H__ 20 | #define MP_MEMORY_H__ 21 | 22 | #ifdef MP_MEMORY_BOOST 23 | #include 24 | namespace mp { 25 | using std::tr1::shared_ptr; 26 | using std::tr1::wak_ptr; 27 | //using std::tr2::scoped_ptr; 28 | using std::tr1::static_pointer_cast; 29 | using std::tr1::dynamic_pointer_cast; 30 | } 31 | #else 32 | #ifdef MP_MEMORY_BOOST_ORG 33 | #include 34 | #include 35 | //#include 36 | namespace mp { 37 | using boost::shared_ptr; 38 | using boost::weak_ptr; 39 | //using boost::scoped_ptr; 40 | using boost::static_pointer_cast; 41 | using boost::dynamic_pointer_cast; 42 | } 43 | #else 44 | #ifndef MP_MEMORY_STANDARD 45 | #include 46 | namespace mp { 47 | using std::tr1::shared_ptr; 48 | using std::tr1::weak_ptr; 49 | //using std::tr2::scoped_ptr; 50 | using std::tr1::static_pointer_cast; 51 | using std::tr1::dynamic_pointer_cast; 52 | } 53 | #else 54 | #include 55 | namespace mp { 56 | using std::shared_ptr; 57 | using std::weak_ptr; 58 | //using std::scoped_ptr; 59 | using std::static_pointer_cast; 60 | using std::dynamic_pointer_cast; 61 | } 62 | #endif 63 | #endif 64 | #endif 65 | 66 | #endif /* mp/memory.h */ 67 | 68 | -------------------------------------------------------------------------------- /src/mp/object_callback.pre.h: -------------------------------------------------------------------------------- 1 | // 2 | // mp::object_callback 3 | // 4 | // Copyright (C) 2008 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #ifndef MP_OBJECT_CALLBACK_H__ 20 | #define MP_OBJECT_CALLBACK_H__ 21 | 22 | #include "mp/memory.h" 23 | 24 | namespace mp { 25 | 26 | 27 | template 28 | static void object_destructor(void* obj) 29 | { 30 | reinterpret_cast(obj)->~T(); 31 | } 32 | 33 | 34 | template 35 | static void object_delete(void* obj) 36 | { 37 | delete reinterpret_cast(obj); 38 | } 39 | 40 | 41 | template 42 | struct object_callback; 43 | 44 | template 45 | struct object_callback 46 | { 47 | template 48 | static R mem_fun(void* obj) 49 | { 50 | return (reinterpret_cast(obj)->*MemFun)(); 51 | } 52 | 53 | template 54 | static R const_mem_fun(const void* obj) 55 | { 56 | return (reinterpret_cast(obj)->*MemFun)(); 57 | } 58 | 59 | template 60 | static R shared_fun(shared_ptr obj) 61 | { 62 | return (obj.get()->*MemFun)(); 63 | } 64 | }; 65 | 66 | MP_ARGS_BEGIN 67 | template 68 | struct object_callback 69 | { 70 | template 71 | static R mem_fun(void* obj, MP_ARGS_PARAMS) 72 | { 73 | return (reinterpret_cast(obj)->*MemFun)(MP_ARGS_FUNC); 74 | } 75 | 76 | template 77 | static R const_mem_fun(const void* obj, MP_ARGS_PARAMS) 78 | { 79 | return (reinterpret_cast(obj)->*MemFun)(MP_ARGS_FUNC); 80 | } 81 | 82 | template 83 | static R shared_fun(shared_ptr obj, MP_ARGS_PARAMS) 84 | { 85 | return (obj.get()->*MemFun)(MP_ARGS_FUNC); 86 | } 87 | }; 88 | 89 | MP_ARGS_END 90 | 91 | } // namespace mp 92 | 93 | #endif /* mp/object_callback.h */ 94 | 95 | -------------------------------------------------------------------------------- /src/mp/pp.h: -------------------------------------------------------------------------------- 1 | #ifndef MP_PP_H__ 2 | #define MP_PP_H__ 3 | 4 | #define MP_PP_STR(s) #s 5 | #define MP_PP_XSTR(s) MP_PP_STR(s) 6 | #define MP_PP_CONCAT(a,b) a##b 7 | #define MP_PP_XCONCAT(a,b) MP_PP_CONCAT(a,b) 8 | #define MP_PP_HEADER(dir, prefix, file, suffix) \ 9 | MP_PP_XSTR( \ 10 | MP_PP_XCONCAT(dir/prefix ## file, suffix).h \ 11 | ) 12 | 13 | #endif /* mp/pp.h */ 14 | 15 | -------------------------------------------------------------------------------- /src/mp/shared_buffer.h: -------------------------------------------------------------------------------- 1 | // 2 | // mp::shared_buffer 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #ifndef MP_SHARED_BUFFER_H__ 20 | #define MP_SHARED_BUFFER_H__ 21 | 22 | #include 23 | #include 24 | 25 | #ifndef MP_SHARED_BUFFER_INITIAL_BUFFER_SIZE 26 | #define MP_SHARED_BUFFER_INITIAL_BUFFER_SIZE 8*1024 27 | #endif 28 | 29 | namespace mp { 30 | 31 | 32 | class shared_buffer { 33 | public: 34 | shared_buffer(size_t init_size = MP_SHARED_BUFFER_INITIAL_BUFFER_SIZE); 35 | ~shared_buffer(); 36 | 37 | public: 38 | void reserve(size_t len, size_t init_size = MP_SHARED_BUFFER_INITIAL_BUFFER_SIZE); 39 | 40 | void* buffer(); 41 | size_t buffer_capacity() const; 42 | 43 | class reference { 44 | public: 45 | reference(); 46 | reference(void* p); 47 | reference(const reference& o); 48 | ~reference(); 49 | void clear(); 50 | void reset(void* p); 51 | void swap(reference& x); 52 | private: 53 | void* m; 54 | }; 55 | 56 | void* allocate(size_t size, reference* result_ref = NULL, 57 | size_t init_size = MP_SHARED_BUFFER_INITIAL_BUFFER_SIZE); 58 | 59 | private: 60 | char* m_buffer; 61 | size_t m_used; 62 | size_t m_free; 63 | 64 | private: 65 | void expand_buffer(size_t len, size_t init_size); 66 | 67 | typedef volatile unsigned int count_t; 68 | static void init_count(void* d); 69 | static void decr_count(void* d); 70 | static void incr_count(void* d); 71 | static count_t get_count(void* d); 72 | 73 | private: 74 | shared_buffer(const shared_buffer&); 75 | }; 76 | 77 | 78 | } // namespace mp 79 | 80 | #include "mp/shared_buffer_impl.h" 81 | 82 | #endif /* mp/shared_buffer.h */ 83 | 84 | -------------------------------------------------------------------------------- /src/mp/source.h: -------------------------------------------------------------------------------- 1 | // 2 | // mp::source 3 | // 4 | // Copyright (C) 2008 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #ifndef MP_SOURCE_H__ 20 | #define MP_SOURCE_H__ 21 | 22 | #include 23 | 24 | #ifndef MP_SOURCE_DEFAULT_ALLOCATION_SIZE 25 | #define MP_SOURCE_DEFAULT_ALLOCATION_SIZE 32*1024 26 | #endif 27 | 28 | #ifndef MP_SOURCE_DEFAULT_LOTS_IN_CHUNK 29 | #define MP_SOURCE_DEFAULT_LOTS_IN_CHUNK 4 30 | #endif 31 | 32 | namespace mp { 33 | 34 | static const size_t SOURCE_DEFAULT_ALLOCATION_SIZE = MP_SOURCE_DEFAULT_ALLOCATION_SIZE; 35 | static const size_t SOURCE_DEFAULT_LOTS_IN_CHUNK = MP_SOURCE_DEFAULT_LOTS_IN_CHUNK; 36 | 37 | template < size_t EstimatedAllocationSize = SOURCE_DEFAULT_ALLOCATION_SIZE, 38 | size_t OptimalLotsInChunk = SOURCE_DEFAULT_LOTS_IN_CHUNK > 39 | class source { 40 | public: 41 | source(); 42 | ~source(); 43 | 44 | public: 45 | //! Allocate memory from the pool. 46 | /* The allocated memory have to be freed using free() function. */ 47 | void* malloc(size_t size); 48 | 49 | //! Free the allocated memory. 50 | void free(void* x); 51 | 52 | private: 53 | struct chunk_t { 54 | chunk_t* next; 55 | chunk_t* prev; 56 | size_t free; 57 | size_t lots; 58 | size_t size; 59 | }; 60 | struct data_t { 61 | chunk_t* chunk; 62 | }; 63 | 64 | private: 65 | chunk_t* m_free; 66 | chunk_t* m_used; 67 | 68 | private: 69 | void* expand_free(size_t req); 70 | void splice_to_used(chunk_t* chunk); 71 | void splice_to_free(chunk_t* chunk); 72 | 73 | private: 74 | source(const source&); 75 | }; 76 | 77 | 78 | } // namespace mp 79 | 80 | #include "mp/source_impl.h" 81 | 82 | #endif /* mp/source.h */ 83 | 84 | -------------------------------------------------------------------------------- /src/mp/stream_buffer.h: -------------------------------------------------------------------------------- 1 | // 2 | // mp::stream_buffer 3 | // 4 | // Copyright (C) 2008 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #ifndef MP_STREAM_BUFFER_H__ 20 | #define MP_STREAM_BUFFER_H__ 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #ifndef MP_STREAM_BUFFER_INITIAL_BUFFER_SIZE 28 | #define MP_STREAM_BUFFER_INITIAL_BUFFER_SIZE 8*1024 29 | #endif 30 | 31 | namespace mp { 32 | 33 | 34 | class stream_buffer { 35 | public: 36 | stream_buffer(size_t initial_buffer_size = MP_STREAM_BUFFER_INITIAL_BUFFER_SIZE); 37 | ~stream_buffer(); 38 | 39 | public: 40 | void reserve_buffer(size_t len, size_t initial_buffer_size = MP_STREAM_BUFFER_INITIAL_BUFFER_SIZE); 41 | 42 | void* buffer(); 43 | size_t buffer_capacity() const; 44 | void buffer_consumed(size_t len); 45 | 46 | void* data(); 47 | size_t data_size() const; 48 | void data_used(size_t len); 49 | 50 | class reference { 51 | public: 52 | reference(); 53 | reference(const reference& o); 54 | ~reference(); 55 | void clear(); 56 | void push(void* d); 57 | void swap(reference& x); 58 | private: 59 | std::vector m_array; 60 | struct each_incr; 61 | struct each_decr; 62 | private: 63 | void move(void* d); 64 | friend class stream_buffer; 65 | }; 66 | 67 | reference* release(); 68 | void release_to(reference* to); 69 | 70 | private: 71 | char* m_buffer; 72 | size_t m_used; 73 | size_t m_free; 74 | size_t m_off; 75 | reference m_ref; 76 | 77 | private: 78 | void expand_buffer(size_t len, size_t initial_buffer_size); 79 | 80 | typedef volatile unsigned int count_t; 81 | static void init_count(void* d); 82 | static void decr_count(void* d); 83 | static void incr_count(void* d); 84 | static count_t get_count(void* d); 85 | 86 | private: 87 | stream_buffer(const stream_buffer&); 88 | }; 89 | 90 | 91 | } // namespace mp 92 | 93 | #include "mp/stream_buffer_impl.h" 94 | 95 | #endif /* mp/stream_buffer.h */ 96 | 97 | -------------------------------------------------------------------------------- /src/mp/utility.h: -------------------------------------------------------------------------------- 1 | // 2 | // mpio utility 3 | // 4 | // Copyright (C) 2008 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #ifndef MP_UTILITY_H__ 20 | #define MP_UTILITY_H__ 21 | 22 | #include "mp/exception.h" 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | namespace mp { 29 | 30 | 31 | template 32 | struct thread_tag { 33 | static const unsigned int ID = ThreadID; 34 | }; 35 | 36 | typedef thread_tag<0> main_thread_tag; 37 | 38 | 39 | inline void set_nonblock(int fd) 40 | { 41 | if( ::fcntl(fd, F_SETFL, O_NONBLOCK) < 0 ) { 42 | throw system_error(errno, "failed to set nonblock flag"); 43 | } 44 | } 45 | 46 | 47 | } // namespace mp 48 | 49 | #endif /* mp/utility.h */ 50 | 51 | -------------------------------------------------------------------------------- /src/mp/wavy.h: -------------------------------------------------------------------------------- 1 | // 2 | // mp::wavy 3 | // 4 | // Copyright (C) 2008 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #ifndef MP_WAVY_H__ 20 | #define MP_WAVY_H__ 21 | 22 | #include "mp/wavy/core.h" 23 | #include "mp/wavy/output.h" 24 | 25 | #endif /* mp/wavy.h */ 26 | 27 | -------------------------------------------------------------------------------- /src/mp/wavy/output.h: -------------------------------------------------------------------------------- 1 | // 2 | // mp::wavy::output 3 | // 4 | // Copyright (C) 2008 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #ifndef MP_WAVY_OUTPUT_H__ 20 | #define MP_WAVY_OUTPUT_H__ 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | namespace mp { 28 | namespace wavy { 29 | 30 | 31 | class output { 32 | public: 33 | output(); 34 | ~output(); 35 | 36 | void add_thread(size_t num); 37 | 38 | void end(); 39 | 40 | void join(); 41 | void detach(); 42 | 43 | public: 44 | typedef void (*finalize_t)(void* user); 45 | struct request { 46 | request() : finalize(NULL), user(NULL) { } 47 | request(finalize_t f, void* u) : finalize(f), user(u) { } 48 | finalize_t finalize; 49 | void* user; 50 | }; 51 | 52 | void write(int fd, const char* buf, size_t buflen); 53 | void writev(int fd, const iovec* vec, size_t veclen); 54 | 55 | void write(int fd, const char* buf, size_t buflen, request req); 56 | void write(int fd, const char* buf, size_t buflen, finalize_t finalize, void* user); 57 | void writev(int fd, const iovec* vec, size_t veclen, request req); 58 | void writev(int fd, const iovec* vec, size_t veclen, finalize_t finalize, void* user); 59 | 60 | void writev(int fd, const iovec* bufvec, const request* reqvec, size_t veclen); 61 | 62 | private: 63 | class impl; 64 | const std::auto_ptr m_impl; 65 | 66 | output(const output&); 67 | }; 68 | 69 | 70 | } // namespace wavy 71 | } // namespace mp 72 | 73 | #endif /* mp/wavy/output.h */ 74 | 75 | -------------------------------------------------------------------------------- /src/mpsrc/Makefile.am: -------------------------------------------------------------------------------- 1 | noinst_LIBRARIES = libmpio.a 2 | 3 | libmpio_a_CPPFLAGS = -I.. 4 | 5 | libmpio_a_SOURCES = \ 6 | wavy_core.cc \ 7 | wavy_connect.cc \ 8 | wavy_listen.cc \ 9 | wavy_output.cc \ 10 | wavy_timer.cc 11 | 12 | noinst_HEADERS = \ 13 | wavy_core.h \ 14 | wavy_edge.h \ 15 | wavy_edge_epoll.h \ 16 | wavy_edge_kqueue.h 17 | 18 | -------------------------------------------------------------------------------- /src/mpsrc/wavy_core.h: -------------------------------------------------------------------------------- 1 | // 2 | // mp::wavy::core 3 | // 4 | // Copyright (C) 2008-2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #ifndef WAVY_CORE_H__ 20 | #define WAVY_CORE_H__ 21 | 22 | #include "mp/wavy/core.h" 23 | #include "mp/pthread.h" 24 | #include "wavy_edge.h" 25 | 26 | namespace mp { 27 | namespace wavy { 28 | 29 | 30 | class core::impl { 31 | public: 32 | impl(); 33 | ~impl(); 34 | 35 | public: 36 | void add_thread(size_t num); 37 | 38 | void end(); 39 | bool is_end() const; 40 | 41 | void join(); 42 | void detach(); 43 | 44 | class connect_thread; 45 | 46 | class listen_handler; 47 | void listen(int lsock, listen_callback_t callback); 48 | 49 | class timer_thread; 50 | 51 | public: 52 | inline void add_impl(int fd, handler* newh); 53 | inline void submit_impl(task_t& f); 54 | 55 | public: 56 | void operator() (); 57 | 58 | private: 59 | volatile size_t m_off; 60 | volatile size_t m_num; 61 | volatile bool m_pollable; 62 | 63 | edge::backlog m_backlog; 64 | 65 | typedef shared_ptr shared_handler; 66 | shared_handler* m_state; 67 | 68 | edge m_edge; 69 | 70 | pthread_mutex m_mutex; 71 | pthread_cond m_cond; 72 | 73 | volatile bool m_end_flag; 74 | 75 | typedef std::queue task_queue_t; 76 | task_queue_t m_task_queue; 77 | 78 | private: 79 | typedef std::vector workers_t; 80 | workers_t m_workers; 81 | 82 | private: 83 | impl(const impl&); 84 | }; 85 | 86 | 87 | } // namespace wavy 88 | } // namespace mp 89 | 90 | #endif /* wavy_core.h */ 91 | 92 | -------------------------------------------------------------------------------- /src/mpsrc/wavy_edge.h: -------------------------------------------------------------------------------- 1 | #ifndef MP_WAVY_EDGE_H__ 2 | #define MP_WAVY_EDGE_H__ 3 | 4 | #include "mp/pp.h" 5 | 6 | #ifndef MP_WAVY_EDGE 7 | # if defined(HAVE_SYS_EPOLL_H) 8 | # define MP_WAVY_EDGE epoll 9 | # elif defined(HAVE_SYS_EVENT_H) 10 | # define MP_WAVY_EDGE kqueue 11 | # elif defined(HAVE_PORT_H) 12 | # define MP_WAVY_EDGE eventport 13 | # else 14 | # if defined(__linux__) 15 | # define MP_WAVY_EDGE epoll 16 | # elif defined(__APPLE__) && defined(__MACH__) 17 | # define MP_WAVY_EDGE kqueue 18 | # elif defined(__FreeBSD__) || defined(__NetBSD__) 19 | # define MP_WAVY_EDGE kqueue 20 | # elif defined(__sun__) 21 | # define MP_WAVY_EDGE eventport 22 | # else 23 | # error this system is not supported. 24 | # endif 25 | # endif 26 | #endif 27 | 28 | #define MP_WAVY_EDGE_HEADER(sys) \ 29 | MP_PP_HEADER(mpsrc, wavy_edge_, sys, ) 30 | 31 | #ifndef MP_WAVY_EDGE_BACKLOG_SIZE 32 | #define MP_WAVY_EDGE_BACKLOG_SIZE 256 33 | #endif 34 | 35 | #include MP_WAVY_EDGE_HEADER(MP_WAVY_EDGE) 36 | 37 | #endif /* wavy_edge.h */ 38 | 39 | -------------------------------------------------------------------------------- /src/mpsrc/wavy_edge_epoll.h: -------------------------------------------------------------------------------- 1 | // 2 | // mp::wavy::edge 3 | // 4 | // Copyright (C) 2008 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #ifndef MP_WAVY_EDGE_EPOLL_H__ 20 | #define MP_WAVY_EDGE_EPOLL_H__ 21 | 22 | #include "mp/exception.h" 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | namespace mp { 29 | namespace wavy { 30 | 31 | 32 | static const short EVEDGE_READ = EPOLLIN; 33 | static const short EVEDGE_WRITE = EPOLLOUT; 34 | 35 | 36 | class edge { 37 | public: 38 | edge() : m_ep(epoll_create(MP_WAVY_EDGE_BACKLOG_SIZE)) 39 | { 40 | if(m_ep < 0) { 41 | throw system_error(errno, "failed to initialize epoll"); 42 | } 43 | } 44 | 45 | ~edge() 46 | { 47 | ::close(m_ep); 48 | } 49 | 50 | int add_notify(int fd, short event) 51 | { 52 | struct epoll_event ev; 53 | ::memset(&ev, 0, sizeof(ev)); // FIXME 54 | ev.events = event | EPOLLONESHOT; 55 | ev.data.fd = fd; 56 | return epoll_ctl(m_ep, EPOLL_CTL_ADD, fd, &ev); 57 | } 58 | 59 | int shot_reactivate(int fd, short event) 60 | { 61 | struct epoll_event ev; 62 | ::memset(&ev, 0, sizeof(ev)); // FIXME 63 | ev.events = event | EPOLLONESHOT; 64 | ev.data.fd = fd; 65 | return epoll_ctl(m_ep, EPOLL_CTL_MOD, fd, &ev); 66 | } 67 | 68 | int shot_remove(int fd, short event) 69 | { 70 | return epoll_ctl(m_ep, EPOLL_CTL_DEL, fd, NULL); 71 | } 72 | 73 | int remove(int fd, short event) 74 | { 75 | return epoll_ctl(m_ep, EPOLL_CTL_DEL, fd, NULL); 76 | } 77 | 78 | struct backlog { 79 | backlog() 80 | { 81 | buf = (struct epoll_event*)::calloc( 82 | sizeof(struct epoll_event), 83 | MP_WAVY_EDGE_BACKLOG_SIZE); 84 | if(!buf) { throw std::bad_alloc(); } 85 | } 86 | 87 | ~backlog() 88 | { 89 | ::free(buf); 90 | } 91 | 92 | int operator[] (int n) 93 | { 94 | return buf[n].data.fd; 95 | } 96 | 97 | private: 98 | struct epoll_event* buf; 99 | friend class edge; 100 | backlog(const backlog&); 101 | }; 102 | 103 | int wait(backlog* result) 104 | { 105 | return wait(result, -1); 106 | } 107 | 108 | int wait(backlog* result, int timeout_msec) 109 | { 110 | return epoll_wait(m_ep, result->buf, 111 | MP_WAVY_EDGE_BACKLOG_SIZE, timeout_msec); 112 | } 113 | 114 | private: 115 | int m_ep; 116 | 117 | private: 118 | edge(const edge&); 119 | }; 120 | 121 | 122 | } // namespace wavy 123 | } // namespace mp 124 | 125 | #endif /* wavy_edge_kqueue.h */ 126 | 127 | -------------------------------------------------------------------------------- /src/mpsrc/wavy_edge_kqueue.h: -------------------------------------------------------------------------------- 1 | // 2 | // mp::wavy::edge 3 | // 4 | // Copyright (C) 2008 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #ifndef MP_WAVY_EDGE_KQUEUE_H__ 20 | #define MP_WAVY_EDGE_KQUEUE_H__ 21 | 22 | #include "mp/exception.h" 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | namespace mp { 29 | namespace wavy { 30 | 31 | 32 | static const short EVEDGE_READ = EVFILT_READ; 33 | static const short EVEDGE_WRITE = EVFILT_WRITE; 34 | 35 | 36 | class edge { 37 | public: 38 | edge() : m_kq(kqueue()) 39 | { 40 | if(m_kq < 0) { 41 | throw system_error(errno, "failed to initialize kqueue"); 42 | } 43 | } 44 | 45 | ~edge() 46 | { 47 | ::close(m_kq); 48 | } 49 | 50 | int add_notify(int fd, short event) 51 | { 52 | struct kevent kev; 53 | EV_SET(&kev, fd, event, EV_ADD|EV_ONESHOT, 0, 0, NULL); 54 | return kevent(m_kq, &kev, 1, NULL, 0, NULL); 55 | } 56 | 57 | int shot_reactivate(int fd, short event) 58 | { 59 | return add_notify(fd, event); 60 | } 61 | 62 | int shot_remove(int fd, short event) 63 | { return 0; } 64 | 65 | int remove(int fd, short event) 66 | { 67 | struct kevent kev; 68 | EV_SET(&kev, fd, event, EV_DELETE, 0, 0, NULL); 69 | return kevent(m_kq, &kev, 1, NULL, 0, NULL); 70 | } 71 | 72 | struct backlog { 73 | backlog() 74 | { 75 | buf = (struct kevent*)::calloc( 76 | sizeof(struct kevent), 77 | MP_WAVY_EDGE_BACKLOG_SIZE); 78 | if(!buf) { throw std::bad_alloc(); } 79 | } 80 | 81 | ~backlog() 82 | { 83 | ::free(buf); 84 | } 85 | 86 | int operator[] (int n) const 87 | { 88 | return buf[n].ident; 89 | } 90 | 91 | private: 92 | struct kevent* buf; 93 | friend class edge; 94 | backlog(const backlog&); 95 | }; 96 | 97 | int wait(backlog* result) 98 | { 99 | return kevent(m_kq, NULL, 0, result->buf, 100 | MP_WAVY_EDGE_BACKLOG_SIZE, NULL); 101 | } 102 | 103 | int wait(backlog* result, int timeout_msec) 104 | { 105 | struct timespec ts; 106 | ts.tv_sec = timeout_msec / 1000; 107 | ts.tv_nsec = (timeout_msec % 1000) * 1000000; 108 | return kevent(m_kq, NULL, 0, result->buf, 109 | MP_WAVY_EDGE_BACKLOG_SIZE, &ts); 110 | } 111 | 112 | private: 113 | int m_kq; 114 | 115 | private: 116 | edge(const edge&); 117 | }; 118 | 119 | 120 | } // namespace wavy 121 | } // namespace mp 122 | 123 | #endif /* wavy_edge_kqueue.h */ 124 | 125 | -------------------------------------------------------------------------------- /src/mpsrc/wavy_listen.cc: -------------------------------------------------------------------------------- 1 | // 2 | // mp::wavy::core::listen 3 | // 4 | // Copyright (C) 2008 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #include "wavy_core.h" 20 | #include "mp/exception.h" 21 | 22 | namespace mp { 23 | namespace wavy { 24 | 25 | 26 | class core::impl::listen_handler : public handler { 27 | public: 28 | listen_handler(int fd, core* c, listen_callback_t callback) : 29 | handler(fd), m_core(c), m_callback(callback) { } 30 | 31 | ~listen_handler() { } 32 | 33 | void read_event() 34 | { 35 | while(true) { 36 | int err = 0; 37 | int sock = ::accept(fd(), NULL, NULL); 38 | if(sock < 0) { 39 | if(errno == EAGAIN || errno == EINTR) { 40 | return; 41 | } else { 42 | err = errno; 43 | } 44 | } else if(sock == 0) { 45 | err = errno; 46 | } 47 | 48 | try { 49 | m_core->submit(m_callback, sock, err); 50 | } catch(...) { } 51 | 52 | if(err) { 53 | throw system_error(errno, "mp::wvy::accept: accept failed"); 54 | } 55 | } 56 | } 57 | 58 | private: 59 | core* m_core; 60 | listen_callback_t m_callback; 61 | }; 62 | 63 | 64 | void core::listen(int lsock, listen_callback_t callback) 65 | { 66 | add(lsock, this, callback); 67 | } 68 | 69 | 70 | } // namespace wavy 71 | } // namespace mp 72 | 73 | -------------------------------------------------------------------------------- /src/mpsrc/wavy_timer.cc: -------------------------------------------------------------------------------- 1 | // 2 | // mp::wavy::core::timer 3 | // 4 | // Copyright (C) 2008 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #include "wavy_core.h" 20 | #include 21 | 22 | namespace mp { 23 | namespace wavy { 24 | 25 | 26 | class core::impl::timer_thread { 27 | public: 28 | timer_thread(core* c, 29 | const timespec* interval, 30 | timer_callback_t callback) : 31 | m_interval(*interval), 32 | m_core(c), m_callback(callback) { } 33 | 34 | void operator() () 35 | { 36 | while(!m_core->is_end()) { 37 | nanosleep(&m_interval, NULL); 38 | m_core->submit(m_callback); 39 | } 40 | } 41 | 42 | private: 43 | const timespec m_interval; 44 | core* m_core; 45 | timer_callback_t m_callback; 46 | timer_thread(); 47 | }; 48 | 49 | void core::timer(const timespec* interval, timer_callback_t callback) 50 | { 51 | add_thread(1); 52 | impl::timer_thread t(this, interval, callback); 53 | submit(t); 54 | } 55 | 56 | 57 | } // namespace wavy 58 | } // namespace mp 59 | 60 | -------------------------------------------------------------------------------- /src/rpc/Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | noinst_LIBRARIES = libkumo_rpc.a libkumo_cluster.a 3 | 4 | libkumo_rpc_a_CPPFLAGS = -I.. 5 | 6 | libkumo_cluster_a_CPPFLAGS = -I.. 7 | 8 | libkumo_rpc_a_SOURCES = \ 9 | address.cc \ 10 | session.cc 11 | 12 | libkumo_cluster_a_SOURCES = \ 13 | $(libkumo_rpc_a_SOURCES) \ 14 | server.cc \ 15 | cluster.cc 16 | 17 | noinst_HEADERS = \ 18 | address.h \ 19 | client.h \ 20 | client_tmpl.h \ 21 | cluster.h \ 22 | connection.h \ 23 | exception.h \ 24 | message.h \ 25 | protocol.h \ 26 | request.h \ 27 | responder.h \ 28 | responder_impl.h \ 29 | retry.h \ 30 | rpc.h \ 31 | server.h \ 32 | session.h \ 33 | session_impl.h \ 34 | transport.h \ 35 | transport_impl.h \ 36 | types.h \ 37 | vrefbuffer.h \ 38 | wavy.h \ 39 | weak_responder.h \ 40 | weak_responder_impl.h 41 | 42 | -------------------------------------------------------------------------------- /src/rpc/exception.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef RPC_EXCEPTION_H__ 19 | #define RPC_EXCEPTION_H__ 20 | 21 | #include 22 | 23 | namespace rpc { 24 | 25 | 26 | struct connection_error : public std::exception { 27 | connection_error() { } 28 | }; 29 | 30 | struct connection_closed_error : public connection_error { 31 | connection_closed_error() { } 32 | const char* what() const throw() { return "connection closed"; } 33 | }; 34 | 35 | struct connection_broken_error : public connection_error { 36 | connection_broken_error() { } 37 | const char* what() const throw() { return "connection broken"; } 38 | }; 39 | 40 | 41 | } // namespace rpc 42 | 43 | #endif /* rpc/exception.h */ 44 | 45 | -------------------------------------------------------------------------------- /src/rpc/message.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef RPC_MESSAGE_H__ 19 | #define RPC_MESSAGE_H__ 20 | 21 | #include 22 | 23 | namespace rpc { 24 | 25 | 26 | template 27 | struct message { 28 | typedef Session session_type; 29 | typedef rpc::method method; 30 | }; 31 | 32 | 33 | } // namespace rpc 34 | 35 | #endif /* rpc/message.h */ 36 | 37 | -------------------------------------------------------------------------------- /src/rpc/request.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef RPC_REQUEST_H__ 19 | #define RPC_REQUEST_H__ 20 | 21 | namespace rpc { 22 | 23 | 24 | template 25 | class request; 26 | 27 | 28 | template 29 | class request { 30 | public: 31 | request(basic_shared_session from, msgobj param) : 32 | m_from(from) 33 | { 34 | param.convert(&m_param); 35 | } 36 | 37 | public: 38 | Parameter& param() 39 | { 40 | return m_param; 41 | } 42 | 43 | const Parameter& param() const 44 | { 45 | return m_param; 46 | } 47 | 48 | basic_shared_session& session() 49 | { 50 | return m_from; 51 | } 52 | 53 | private: 54 | Parameter m_param; 55 | basic_shared_session m_from; 56 | 57 | private: 58 | request(); 59 | }; 60 | 61 | 62 | 63 | } // namespace rpc 64 | 65 | #endif /* rpc/request.h */ 66 | 67 | -------------------------------------------------------------------------------- /src/rpc/responder.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef RPC_RESPONDER_H__ 19 | #define RPC_RESPONDER_H__ 20 | 21 | #include "rpc/types.h" 22 | 23 | namespace rpc { 24 | 25 | 26 | class responder { 27 | public: 28 | responder(int fd, msgid_t msgid); 29 | 30 | ~responder(); 31 | 32 | template 33 | void result(Result res); 34 | 35 | template 36 | void result(Result res, auto_zone z); 37 | 38 | template 39 | void error(Error err); 40 | 41 | template 42 | void error(Error err, auto_zone z); 43 | 44 | void null(); 45 | 46 | void send_response(const char* buf, size_t buflen, auto_zone z); 47 | 48 | void send_responsev(const struct iovec* vb, size_t count, auto_zone z); 49 | 50 | private: 51 | template 52 | void call(Result& res, Error& err); 53 | 54 | template 55 | void call(Result& res, Error& err, auto_zone z); 56 | 57 | private: 58 | int m_fd; 59 | msgid_t m_msgid; 60 | 61 | private: 62 | responder(); 63 | }; 64 | 65 | 66 | } // namespace rpc 67 | 68 | #endif /* rpc/responder.h */ 69 | 70 | -------------------------------------------------------------------------------- /src/rpc/responder_impl.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef RPC_RESPONDER_IMPL_H__ 19 | #define RPC_RESPONDER_IMPL_H__ 20 | 21 | #include 22 | #include "rpc/vrefbuffer.h" 23 | #include "rpc/wavy.h" 24 | 25 | namespace rpc { 26 | 27 | 28 | inline responder::responder(int fd, msgid_t msgid) : 29 | m_fd(fd), m_msgid(msgid) { } 30 | 31 | inline responder::~responder() { } 32 | 33 | template 34 | void responder::result(Result res) 35 | { 36 | msgpack::type::nil err; 37 | call(res, err); 38 | } 39 | 40 | template 41 | void responder::result(Result res, auto_zone z) 42 | { 43 | msgpack::type::nil err; 44 | call(res, err, z); 45 | } 46 | 47 | template 48 | void responder::error(Error err) 49 | { 50 | msgpack::type::nil res; 51 | call(res, err); 52 | } 53 | 54 | template 55 | void responder::error(Error err, auto_zone z) 56 | { 57 | msgpack::type::nil res; 58 | call(res, err, z); 59 | } 60 | 61 | inline void responder::null() 62 | { 63 | msgpack::type::nil res; 64 | msgpack::type::nil err; 65 | call(res, err); 66 | } 67 | 68 | template 69 | inline void responder::call(Result& res, Error& err) 70 | { 71 | msgpack::sbuffer buf; // FIXME use vrefbuffer? 72 | rpc_response msgres(res, err, m_msgid); 73 | msgpack::pack(buf, msgres); 74 | 75 | wavy::request req(&::free, buf.data()); 76 | wavy::write(m_fd, buf.data(), buf.size(), req); 77 | buf.release(); 78 | } 79 | 80 | template 81 | inline void responder::call(Result& res, Error& err, auto_zone z) 82 | { 83 | vrefbuffer* buf = z->allocate(); 84 | rpc_response msgres(res, err, m_msgid); 85 | msgpack::pack(*buf, msgres); 86 | 87 | wavy::request req(&mp::object_delete, z.get()); 88 | wavy::writev(m_fd, buf->vector(), buf->vector_size(), req); 89 | z.release(); 90 | } 91 | 92 | inline void responder::send_response(const char* buf, size_t buflen, auto_zone z) 93 | { 94 | wavy::request req(&mp::object_delete, z.get()); 95 | wavy::write(m_fd, buf, buflen, req); 96 | z.release(); 97 | } 98 | 99 | inline void responder::send_responsev(const struct iovec* vb, size_t count, auto_zone z) 100 | { 101 | wavy::request req(&mp::object_delete, z.get()); 102 | wavy::writev(m_fd, vb, count, req); 103 | z.release(); 104 | } 105 | 106 | 107 | } // namespace rpc 108 | 109 | #endif /* rpc/responder_impl.h */ 110 | 111 | -------------------------------------------------------------------------------- /src/rpc/retry.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef RPC_RETRY_H__ 19 | #define RPC_RETRY_H__ 20 | 21 | namespace rpc { 22 | 23 | 24 | template 25 | class retry { 26 | public: 27 | retry(Parameter param) : 28 | m_limit(0), m_param(param) { } 29 | 30 | void set_callback(rpc::callback_t callback) 31 | { 32 | m_callbck = callback; 33 | } 34 | 35 | public: 36 | bool retry_incr(unsigned short limit) 37 | { 38 | return ++m_limit <= limit; 39 | } 40 | 41 | template 42 | void call(Session s, rpc::shared_zone& life, unsigned short timeout_steps = 10) 43 | { 44 | s->call(m_param, life, m_callbck, timeout_steps); 45 | } 46 | 47 | unsigned short num_retried() const 48 | { 49 | return m_limit; 50 | } 51 | 52 | Parameter& param() 53 | { 54 | return m_param; 55 | } 56 | 57 | const Parameter& param() const 58 | { 59 | return m_param; 60 | } 61 | 62 | private: 63 | unsigned short m_limit; 64 | Parameter m_param; 65 | rpc::callback_t m_callbck; 66 | 67 | private: 68 | retry(); 69 | retry(const retry&); 70 | }; 71 | 72 | 73 | } // namespace rpc 74 | 75 | #endif /* rpc/retry.h */ 76 | 77 | -------------------------------------------------------------------------------- /src/rpc/rpc.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef RPC_RPC_H__ 19 | #define RPC_RPC_H__ 20 | 21 | #include "rpc/types.h" 22 | 23 | #include "rpc/address.h" 24 | #include "rpc/responder.h" 25 | #include "rpc/connection.h" 26 | 27 | #include "rpc/weak_responder.h" 28 | #include "rpc/session.h" 29 | #include "rpc/transport.h" 30 | 31 | #include "rpc/responder_impl.h" 32 | #include "rpc/session_impl.h" 33 | #include "rpc/transport_impl.h" 34 | #include "rpc/weak_responder_impl.h" 35 | 36 | #include "rpc/retry.h" 37 | #include "rpc/request.h" 38 | #include "rpc/message.h" 39 | 40 | #endif /* rpc/rpc.h */ 41 | 42 | -------------------------------------------------------------------------------- /src/rpc/server.cc: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #include "rpc/server.h" 19 | #include "rpc/protocol.h" 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | namespace rpc { 26 | 27 | 28 | server::server() { } 29 | 30 | server::~server() { } 31 | 32 | 33 | shared_peer server::accepted(int fd) 34 | { 35 | #ifndef NO_TCP_NODELAY 36 | int on = 1; 37 | ::setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on)); // ignore error 38 | #endif 39 | #ifndef NO_SO_LINGER 40 | struct linger opt = {0, 0}; 41 | ::setsockopt(fd, SOL_SOCKET, SO_LINGER, (void *)&opt, sizeof(opt)); // ignore error 42 | #endif 43 | basic_shared_session s(new peer(this)); 44 | wavy::add(fd, s, this); 45 | void* k = (void*)s.get(); 46 | 47 | pthread_scoped_lock lk(m_peers_mutex); 48 | m_peers.insert( peers_t::value_type(k, basic_weak_session(s)) ); 49 | return mp::static_pointer_cast(s); 50 | } 51 | 52 | 53 | void server::dispatch_request( 54 | basic_shared_session& s, weak_responder response, 55 | method_id method, msgobj param, auto_zone z) 56 | { 57 | dispatch(mp::static_pointer_cast(s), 58 | response, method, param, z); 59 | } 60 | 61 | 62 | void server::step_timeout() 63 | { 64 | pthread_scoped_lock lk(m_peers_mutex); 65 | for(peers_t::iterator it(m_peers.begin()), it_end(m_peers.end()); 66 | it != it_end; ) { 67 | basic_shared_session p(it->second.lock()); 68 | if(p && !p->is_lost()) { 69 | wavy::submit(&basic_session::step_timeout, p.get(), p); 70 | ++it; 71 | } else { 72 | m_peers.erase(it++); 73 | } 74 | } 75 | } 76 | 77 | void server::transport_lost_notify(basic_shared_session& s) 78 | { 79 | msgpack::object res; 80 | res.type = msgpack::type::NIL; 81 | msgpack::object err; 82 | err.type = msgpack::type::POSITIVE_INTEGER; 83 | err.via.u64 = protocol::NODE_LOST_ERROR; 84 | 85 | void* k = (void*)s.get(); 86 | { 87 | pthread_scoped_lock lk(m_peers_mutex); 88 | m_peers.erase(k); 89 | } 90 | 91 | s->force_lost(s, res, err); 92 | } 93 | 94 | 95 | } // namespace rpc 96 | 97 | -------------------------------------------------------------------------------- /src/rpc/server.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef RPC_SERVER_H__ 19 | #define RPC_SERVER_H__ 20 | 21 | #include "rpc/rpc.h" 22 | #include "log/mlogger.h" // FIXME 23 | #include 24 | #include 25 | #include 26 | 27 | namespace rpc { 28 | 29 | 30 | class server; 31 | 32 | class peer : public basic_session { 33 | public: 34 | peer(session_manager* mgr = NULL); 35 | }; 36 | 37 | typedef mp::shared_ptr shared_peer; 38 | typedef mp::weak_ptr weak_peer; 39 | 40 | 41 | class server : public session_manager, public transport_manager { 42 | public: 43 | typedef shared_peer shared_session; 44 | typedef weak_peer weak_session; 45 | 46 | server(); 47 | 48 | virtual ~server(); 49 | 50 | virtual void dispatch( 51 | shared_peer from, weak_responder response, 52 | method_id method, msgobj param, auto_zone z) = 0; 53 | 54 | public: 55 | // step timeout count. 56 | void step_timeout(); 57 | 58 | // add accepted connection 59 | shared_peer accepted(int fd); 60 | 61 | // apply function to all connected sessions. 62 | // F is required to implement 63 | // void operator() (shared_peer); 64 | template 65 | void for_each_peer(F f); 66 | 67 | protected: 68 | mp::pthread_mutex m_peers_mutex; 69 | typedef std::map peers_t; 70 | peers_t m_peers; 71 | 72 | public: 73 | virtual void dispatch_request( 74 | basic_shared_session& s, weak_responder response, 75 | method_id method, msgobj param, auto_zone z); 76 | 77 | virtual void transport_lost_notify(basic_shared_session& s); 78 | 79 | private: 80 | server(const server&); 81 | }; 82 | 83 | 84 | inline peer::peer(session_manager* mgr) : 85 | basic_session(mgr) { } 86 | 87 | 88 | namespace detail { 89 | template 90 | struct server_each_peer { 91 | server_each_peer(F f) : m(f) { } 92 | inline void operator() (std::pair& x) 93 | { 94 | basic_shared_session s(x.second.lock()); 95 | if(s && !s->is_lost()) { 96 | m(mp::static_pointer_cast(s)); 97 | } 98 | } 99 | private: 100 | F m; 101 | server_each_peer(); 102 | }; 103 | } // namespace detail 104 | 105 | template 106 | void server::for_each_peer(F f) 107 | { 108 | pthread_scoped_lock lk(m_peers_mutex); 109 | detail::server_each_peer e(f); 110 | std::for_each(m_peers.begin(), m_peers.end(), e); 111 | } 112 | 113 | 114 | } // namespace rpc 115 | 116 | #endif /* rpc/client.h */ 117 | 118 | -------------------------------------------------------------------------------- /src/rpc/session_impl.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef RPC_SESSION_IMPL_H__ 19 | #define RPC_SESSION_IMPL_H__ 20 | 21 | namespace rpc { 22 | 23 | 24 | inline bool basic_session::is_bound() const 25 | { 26 | return !m_binds.empty(); 27 | } 28 | 29 | inline bool basic_session::is_lost() const 30 | { 31 | return m_lost; 32 | } 33 | 34 | inline void basic_session::revive() 35 | { 36 | m_lost = false; 37 | } 38 | 39 | inline unsigned short basic_session::increment_connect_retried_count() 40 | { 41 | return ++m_connect_retried_count; // FIXME atomic? 42 | } 43 | 44 | inline unsigned short basic_session::connect_retried_count() 45 | { 46 | return m_connect_retried_count; 47 | } 48 | 49 | 50 | inline session::session(session_manager* mgr) : 51 | basic_session(mgr) { } 52 | 53 | inline session::~session() { cancel_pendings(); } 54 | 55 | 56 | inline session_manager* basic_session::get_manager() 57 | { 58 | return m_manager; 59 | } 60 | 61 | 62 | inline void basic_session::process_request( 63 | basic_shared_session& s, 64 | method_id method, msgobj param, 65 | msgid_t msgid, auto_zone z) 66 | { 67 | weak_responder response(s, msgid); 68 | get_manager()->dispatch_request(s, response, method, param, z); 69 | } 70 | 71 | 72 | template 73 | inline msgid_t basic_session::pack(vrefbuffer& buffer, Message& param) 74 | { 75 | msgid_t msgid = __sync_add_and_fetch(&m_msgid_rr, 1); 76 | rpc_request msgreq(typename Message::method(), param, msgid); 77 | msgpack::pack(buffer, msgreq); 78 | return msgid; 79 | } 80 | 81 | 82 | template 83 | inline void basic_session::call( 84 | Message& param, 85 | shared_zone life, callback_t callback, 86 | unsigned short timeout_steps) 87 | { 88 | LOG_DEBUG("send request method=",Message::method::id); 89 | 90 | std::auto_ptr buffer(new vrefbuffer()); 91 | msgid_t msgid = pack(*buffer, param); 92 | 93 | call_real(msgid, buffer, life, callback, timeout_steps); 94 | } 95 | 96 | 97 | template 98 | inline void session::call( 99 | Message& param, 100 | shared_zone life, callback_t callback, 101 | unsigned short timeout_steps) 102 | { 103 | LOG_DEBUG("send request method=",Message::method::id); 104 | 105 | std::auto_ptr buffer(new vrefbuffer()); 106 | msgid_t msgid = pack(*buffer, param); 107 | 108 | call_real(msgid, buffer, life, callback, timeout_steps); 109 | } 110 | 111 | 112 | } // namespace rpc 113 | 114 | #endif /* rpc/session_impl.h */ 115 | 116 | -------------------------------------------------------------------------------- /src/rpc/transport.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef RPC_TRANSPORT_H__ 19 | #define RPC_TRANSPORT_H__ 20 | 21 | #include "rpc/types.h" 22 | #include "rpc/connection.h" 23 | 24 | namespace rpc { 25 | 26 | 27 | struct transport_manager { 28 | virtual ~transport_manager() { } 29 | }; 30 | 31 | 32 | class basic_transport { 33 | public: 34 | basic_transport(int fd, basic_shared_session s, 35 | transport_manager* mgr = NULL); 36 | ~basic_transport(); 37 | 38 | public: 39 | // get transport manager 40 | transport_manager* get_manager(); 41 | 42 | // called from basic_session::shutdown() 43 | basic_shared_session shutdown(); 44 | 45 | public: 46 | void process_request(method_id method, msgobj param, 47 | msgid_t msgid, auto_zone& z); 48 | 49 | void process_response(msgobj res, msgobj err, 50 | msgid_t msgid, auto_zone& z); 51 | 52 | public: 53 | void send_data(const char* buf, size_t buflen, 54 | void (*finalize)(void*), void* data); 55 | 56 | void send_datav(vrefbuffer* buf, 57 | void (*finalize)(void*), void* data); 58 | 59 | protected: 60 | int m_fd; 61 | basic_shared_session m_session; 62 | 63 | private: 64 | transport_manager* m_manager; 65 | 66 | private: 67 | basic_transport(); 68 | basic_transport(const basic_transport&); 69 | }; 70 | 71 | 72 | class transport : public basic_transport, public connection { 73 | public: 74 | transport(int fd, basic_shared_session& s, 75 | transport_manager* mgr = NULL); 76 | 77 | virtual ~transport(); 78 | 79 | void process_request(method_id method, msgobj param, 80 | msgid_t msgid, auto_zone& z); 81 | 82 | void process_response(msgobj res, msgobj err, 83 | msgid_t msgid, auto_zone& z); 84 | 85 | private: 86 | transport(); 87 | transport(const transport&); 88 | }; 89 | 90 | 91 | } // namespace rpc 92 | 93 | #endif /* rpc/transport.h */ 94 | 95 | -------------------------------------------------------------------------------- /src/rpc/transport_impl.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef RPC_TRANSPORT_IMPL_H__ 19 | #define RPC_TRANSPORT_IMPL_H__ 20 | 21 | namespace rpc { 22 | 23 | 24 | inline basic_transport::basic_transport(int fd, 25 | basic_shared_session s, transport_manager* mgr) : 26 | m_fd(fd), 27 | m_session(s), 28 | m_manager(mgr) { } 29 | 30 | inline basic_transport::~basic_transport() { } 31 | 32 | 33 | inline transport_manager* basic_transport::get_manager() 34 | { 35 | return m_manager; 36 | } 37 | 38 | inline basic_shared_session basic_transport::shutdown() 39 | { 40 | ::shutdown(m_fd, SHUT_RD); // FIXME 41 | return m_session; 42 | } 43 | 44 | inline transport::transport(int fd, basic_shared_session& s, 45 | transport_manager* mgr) : 46 | basic_transport(fd, s, mgr), 47 | connection(fd) 48 | { 49 | m_session->bind_transport(this); 50 | } 51 | 52 | inline transport::~transport() 53 | { 54 | if(m_session) { 55 | m_session->unbind_transport(this, m_session); 56 | } 57 | } 58 | 59 | 60 | inline void basic_transport::process_request(method_id method, msgobj param, 61 | msgid_t msgid, auto_zone& z) 62 | { 63 | if(!m_session) { 64 | throw std::runtime_error("session unbound"); 65 | } 66 | m_session->process_request(m_session, method, param, msgid, z); 67 | } 68 | 69 | inline void transport::process_request(method_id method, msgobj param, 70 | msgid_t msgid, auto_zone& z) 71 | { 72 | basic_transport::process_request(method, param, msgid, z); 73 | } 74 | 75 | 76 | inline void basic_transport::process_response(msgobj result, msgobj error, 77 | msgid_t msgid, auto_zone& z) 78 | { 79 | m_session->process_response(m_session, result, error, msgid, z); 80 | } 81 | 82 | inline void transport::process_response(msgobj res, msgobj err, 83 | msgid_t msgid, auto_zone& z) 84 | { 85 | basic_transport::process_response(res, err, msgid, z); 86 | } 87 | 88 | 89 | inline void basic_transport::send_data( 90 | const char* buf, size_t buflen, 91 | void (*finalize)(void*), void* data) 92 | { 93 | wavy::request req(finalize, data); 94 | wavy::write(m_fd, buf, buflen, req); 95 | } 96 | 97 | inline void basic_transport::send_datav( 98 | vrefbuffer* buf, 99 | void (*finalize)(void*), void* data) 100 | { 101 | wavy::request req(finalize, data); 102 | wavy::writev(m_fd, buf->vector(), buf->vector_size(), req); 103 | } 104 | 105 | 106 | } // namespace rpc 107 | 108 | #endif /* rpc/transport_impl.h */ 109 | 110 | -------------------------------------------------------------------------------- /src/rpc/types.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef RPC_TYPES_H__ 19 | #define RPC_TYPES_H__ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | namespace rpc { 27 | 28 | 29 | typedef msgpack::object msgobj; 30 | typedef std::auto_ptr auto_zone; 31 | typedef mp::shared_ptr shared_zone; 32 | 33 | 34 | typedef uint32_t msgid_t; 35 | typedef uint8_t role_type; 36 | 37 | 38 | class transport; 39 | class session; 40 | class basic_transport; 41 | class basic_session; 42 | 43 | 44 | typedef mp::shared_ptr basic_shared_session; 45 | typedef mp::weak_ptr basic_weak_session; 46 | 47 | typedef mp::function callback_t; 48 | 49 | 50 | using mp::pthread_scoped_lock; 51 | using mp::pthread_scoped_rdlock; 52 | using mp::pthread_scoped_wrlock; 53 | 54 | 55 | } // namespace rpc 56 | 57 | #endif /* rpc/types.h */ 58 | -------------------------------------------------------------------------------- /src/rpc/vrefbuffer.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef RPC_VREFBUFFER_H__ 19 | #define RPC_VREFBUFFER_H__ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #ifndef VREFBUFFER_REF_SIZE 28 | #define VREFBUFFER_REF_SIZE 32 // FIXME 29 | #endif 30 | 31 | namespace rpc { 32 | 33 | 34 | class vrefbuffer { 35 | public: 36 | vrefbuffer(); 37 | ~vrefbuffer(); 38 | 39 | public: 40 | void append_ref(const char* buf, size_t len); 41 | void append_copy(const char* buf, size_t len); 42 | 43 | void write(const char* buf, size_t len); 44 | 45 | size_t vector_size() const; 46 | const struct iovec* vector() const; 47 | 48 | public: 49 | typedef std::vector vec_t; 50 | vec_t m_vec; 51 | 52 | msgpack::zone m_zone; 53 | 54 | private: 55 | vrefbuffer(const vrefbuffer&); 56 | }; 57 | 58 | 59 | inline vrefbuffer::vrefbuffer() 60 | { 61 | m_vec.reserve(4); // FIXME sizeof(struct iovec) * 4 < 72 62 | } 63 | 64 | inline vrefbuffer::~vrefbuffer() { } 65 | 66 | 67 | inline void vrefbuffer::append_ref(const char* buf, size_t len) 68 | { 69 | struct iovec v = {(void*)buf, len}; 70 | m_vec.push_back(v); 71 | } 72 | 73 | inline void vrefbuffer::append_copy(const char* buf, size_t len) 74 | { 75 | char* m = (char*)m_zone.malloc(len); 76 | memcpy(m, buf, len); 77 | if(!m_vec.empty() && ((const char*)m_vec.back().iov_base) + 78 | m_vec.back().iov_len == m) { 79 | m_vec.back().iov_len += len; 80 | } else { 81 | append_ref(m, len); 82 | } 83 | } 84 | 85 | 86 | inline void vrefbuffer::write(const char* buf, size_t len) 87 | { 88 | if(len > VREFBUFFER_REF_SIZE) { 89 | append_ref(buf, len); 90 | } else { 91 | append_copy(buf, len); 92 | } 93 | } 94 | 95 | 96 | inline size_t vrefbuffer::vector_size() const 97 | { 98 | return m_vec.size(); 99 | } 100 | 101 | inline const struct iovec* vrefbuffer::vector() const 102 | { 103 | return &m_vec[0]; 104 | } 105 | 106 | 107 | } // namespace rpc 108 | 109 | #endif /* rpc/vrefbuffer.h */ 110 | 111 | -------------------------------------------------------------------------------- /src/rpc/wavy.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef RPC_WAVY_H__ 19 | #define RPC_WAVY_H__ 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | namespace rpc { 26 | 27 | 28 | struct rpc_wavy { }; 29 | 30 | typedef mp::wavy::singleton wavy; 31 | 32 | 33 | } // namespace rpc 34 | 35 | #endif /* rpc/wavy.h */ 36 | 37 | -------------------------------------------------------------------------------- /src/rpc/weak_responder.h: -------------------------------------------------------------------------------- 1 | // 2 | // kumofs 3 | // 4 | // Copyright (C) 2009 FURUHASHI Sadayuki 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | #ifndef RPC_WEAK_RESPONDER_FWD_H__ 19 | #define RPC_WEAK_RESPONDER_FWD_H__ 20 | 21 | #include "rpc/types.h" 22 | 23 | namespace rpc { 24 | 25 | 26 | class weak_responder { 27 | public: 28 | weak_responder(basic_weak_session s, msgid_t msgid); 29 | 30 | ~weak_responder(); 31 | 32 | template 33 | void result(Result res); 34 | 35 | template 36 | void result(Result res, auto_zone& z); 37 | 38 | template 39 | void result(Result res, shared_zone& life); 40 | 41 | template 42 | void error(Error err); 43 | 44 | template 45 | void error(Error err, auto_zone& z); 46 | 47 | template 48 | void error(Error err, shared_zone& life); 49 | 50 | void null(); 51 | 52 | private: 53 | template 54 | void call(Result& res, Error& err); 55 | 56 | template 57 | void call(Result& res, Error& err, auto_zone& z); 58 | 59 | template 60 | void call(Result& res, Error& err, shared_zone& life); 61 | 62 | template 63 | void call_impl(Result& res, Error& err, ZoneType& life); 64 | 65 | private: 66 | basic_weak_session m_session; 67 | const msgid_t m_msgid; 68 | 69 | private: 70 | weak_responder(); 71 | }; 72 | 73 | 74 | } // namespace rpc 75 | 76 | #endif /* rpc/weak_responder_fwd.h */ 77 | 78 | -------------------------------------------------------------------------------- /src/storage/Makefile.am: -------------------------------------------------------------------------------- 1 | AM_CPPFLAGS = -I.. 2 | AM_C_CPPFLAGS = -I.. 3 | 4 | noinst_LIBRARIES = libkumo_storage.a 5 | 6 | if STORAGE_TCHDB 7 | libkumo_storage_a_SOURCES = storage.cc tchdb.cc 8 | endif 9 | 10 | if STORAGE_TCADB 11 | libkumo_storage_a_SOURCES = storage.cc tcadb.cc 12 | endif 13 | 14 | if STORAGE_TCBDB 15 | libkumo_storage_a_SOURCES = storage.cc tcbdb.cc 16 | endif 17 | 18 | if STORAGE_LUXIO 19 | libkumo_storage_a_SOURCES = storage.cc luxio.cc 20 | endif 21 | 22 | noinst_HEADERS = \ 23 | buffer_queue.h \ 24 | storage.h \ 25 | interface.h 26 | 27 | -------------------------------------------------------------------------------- /test/Makefile: -------------------------------------------------------------------------------- 1 | CXXFLAGS = -Wall -g -O0 2 | 3 | LDFLAGS = -lpthread -lm -Wl,-Bstatic -lmemcached -Wl,-Bdynamic 4 | 5 | TESTS = \ 6 | test_set \ 7 | test_get \ 8 | test_delete \ 9 | test_multi \ 10 | test_get_noset \ 11 | test_set_get_delete \ 12 | check_set_get_delete \ 13 | memstrike 14 | 15 | %: %.c 16 | $(CC) $< $(CXXFLAGS) $(LDFLAGS) -o $@ 17 | 18 | all: $(TESTS) 19 | 20 | .PHONY: clean 21 | clean: 22 | $(RM) $(TESTS) 23 | 24 | -------------------------------------------------------------------------------- /test/check_set_get_delete.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "libmemcached/memcached.h" 11 | 12 | #define KEY_PREFIX "key" 13 | #define VAL_PREFIX "val" 14 | 15 | void usage(void) 16 | { 17 | printf("usage: ./check_set_get_delete \n"); 18 | exit(1); 19 | } 20 | 21 | void pexit(const char* msg) 22 | { 23 | perror(msg); 24 | exit(1); 25 | } 26 | 27 | int main(int argc, char* argv[]) 28 | { 29 | if(argc < 4) { usage(); } 30 | 31 | char* host = argv[1]; 32 | 33 | unsigned short port = atoi(argv[2]); 34 | if(port == 0) { usage(); } 35 | 36 | uint32_t num = atoi(argv[3]); 37 | if(num == 0) { usage(); } 38 | 39 | memcached_st* mc = memcached_create(NULL); 40 | if(mc == NULL) { pexit("memcached_create"); } 41 | 42 | memcached_server_add(mc, host, port); 43 | 44 | // memcached_behavior_set(mc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1); 45 | 46 | char kbuf[strlen(KEY_PREFIX) + 11]; 47 | char vbuf[strlen(VAL_PREFIX) + 11]; 48 | 49 | while(1) { 50 | printf("s"); fflush(stdout); 51 | uint32_t i; 52 | for(i=0; i < num; ++i) { 53 | int klen = sprintf(kbuf, KEY_PREFIX "%d", i); 54 | int vlen = sprintf(vbuf, VAL_PREFIX "%d", i); 55 | memcached_return ret = memcached_set(mc, kbuf, klen, vbuf, vlen, 0, 0); 56 | if(ret != MEMCACHED_SUCCESS) { 57 | fprintf(stderr, "set failed %d '%s'\n", ret, kbuf); 58 | } 59 | } 60 | 61 | printf("g"); fflush(stdout); 62 | for(i=0; i < num; ++i) { 63 | int klen = sprintf(kbuf, KEY_PREFIX "%d", i); 64 | int vlen = sprintf(vbuf, VAL_PREFIX "%d", i); 65 | size_t vallen; 66 | uint32_t flags; 67 | memcached_return rc; 68 | char* val = memcached_get(mc, kbuf, klen, &vallen, &flags, &rc); 69 | if(!val || vallen != vlen || memcmp(val, vbuf, vlen) != 0) { 70 | fprintf(stderr, "false NEGATIVE get '%s'\n", kbuf); 71 | } 72 | } 73 | 74 | printf("d"); fflush(stdout); 75 | for(i=0; i < num; ++i) { 76 | int klen = sprintf(kbuf, KEY_PREFIX "%d", i); 77 | memcached_return ret = memcached_delete(mc, kbuf, klen, 0); 78 | if(ret != MEMCACHED_SUCCESS) { 79 | fprintf(stderr, "delete not performed %d '%s'\n", ret, kbuf); 80 | } 81 | } 82 | 83 | printf("g"); fflush(stdout); 84 | for(i=0; i < num; ++i) { 85 | int klen = sprintf(kbuf, KEY_PREFIX "%d", i); 86 | //int vlen = sprintf(vbuf, VAL_PREFIX "%d", i); 87 | size_t vallen; 88 | uint32_t flags; 89 | memcached_return rc; 90 | char* val = memcached_get(mc, kbuf, klen, &vallen, &flags, &rc); 91 | if(val) { 92 | fprintf(stderr, "false POSITIVE get '%s'\n", kbuf); 93 | } 94 | } 95 | 96 | } 97 | 98 | return 0; 99 | } 100 | 101 | -------------------------------------------------------------------------------- /test/test_delete.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "libmemcached/memcached.h" 11 | 12 | #define KEY_PREFIX "key" 13 | #define VAL_PREFIX "val" 14 | 15 | void usage(void) 16 | { 17 | printf("usage: ./test_delete \n"); 18 | exit(1); 19 | } 20 | 21 | void pexit(const char* msg) 22 | { 23 | perror(msg); 24 | exit(1); 25 | } 26 | 27 | int main(int argc, char* argv[]) 28 | { 29 | if(argc < 4) { usage(); } 30 | 31 | char* host = argv[1]; 32 | 33 | unsigned short port = atoi(argv[2]); 34 | if(port == 0) { usage(); } 35 | 36 | uint32_t num = atoi(argv[3]); 37 | if(num == 0) { usage(); } 38 | 39 | memcached_st* mc = memcached_create(NULL); 40 | if(mc == NULL) { pexit("memcached_create"); } 41 | 42 | memcached_server_add(mc, host, port); 43 | 44 | // memcached_behavior_set(mc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1); 45 | 46 | char kbuf[strlen(KEY_PREFIX) + 11]; 47 | char vbuf[strlen(VAL_PREFIX) + 11]; 48 | 49 | while(1) { 50 | uint32_t i; 51 | for(i=0; i < num; ++i) { 52 | int klen = sprintf(kbuf, KEY_PREFIX "%d", i); 53 | int vlen = sprintf(vbuf, VAL_PREFIX "%d", i); 54 | printf("set '%s' = '%s'\n", kbuf, vbuf); 55 | memcached_set(mc, kbuf, klen, vbuf, vlen, 0, 0); 56 | } 57 | 58 | for(i=0; i < num; ++i) { 59 | int klen = sprintf(kbuf, KEY_PREFIX "%d", i); 60 | printf("delete '%s'\n", kbuf); 61 | memcached_delete(mc, kbuf, klen, 0); 62 | } 63 | } 64 | 65 | return 0; 66 | } 67 | 68 | -------------------------------------------------------------------------------- /test/test_get.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "libmemcached/memcached.h" 11 | 12 | #define KEY_PREFIX "key" 13 | #define VAL_PREFIX "val" 14 | 15 | void usage(void) 16 | { 17 | printf("usage: ./test_get \n"); 18 | exit(1); 19 | } 20 | 21 | void pexit(const char* msg) 22 | { 23 | perror(msg); 24 | exit(1); 25 | } 26 | 27 | int main(int argc, char* argv[]) 28 | { 29 | if(argc < 4) { usage(); } 30 | 31 | char* host = argv[1]; 32 | 33 | unsigned short port = atoi(argv[2]); 34 | if(port == 0) { usage(); } 35 | 36 | uint32_t num = atoi(argv[3]); 37 | if(num == 0) { usage(); } 38 | 39 | memcached_st* mc = memcached_create(NULL); 40 | if(mc == NULL) { pexit("memcached_create"); } 41 | 42 | memcached_server_add(mc, host, port); 43 | 44 | // memcached_behavior_set(mc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1); 45 | 46 | char kbuf[strlen(KEY_PREFIX) + 11]; 47 | char vbuf[strlen(VAL_PREFIX) + 11]; 48 | 49 | uint32_t i; 50 | for(i=0; i < num; ++i) { 51 | int klen = sprintf(kbuf, KEY_PREFIX "%d", i); 52 | int vlen = sprintf(vbuf, VAL_PREFIX "%d", i); 53 | printf("set '%s' = '%s'\n", kbuf, vbuf); 54 | memcached_set(mc, kbuf, klen, vbuf, vlen, 0, 0); 55 | } 56 | 57 | while(1) { 58 | for(i=0; i < num; ++i) { 59 | int klen = sprintf(kbuf, KEY_PREFIX "%d", i); 60 | int vlen = sprintf(vbuf, VAL_PREFIX "%d", i); 61 | size_t vallen; 62 | uint32_t flags; 63 | memcached_return rc; 64 | char* val = memcached_get(mc, kbuf, klen, &vallen, &flags, &rc); 65 | if(!val) { 66 | fprintf(stderr, "** key '%s' not found **\n", kbuf); 67 | } else if(vallen != vlen || memcmp(val, vbuf, vlen) != 0) { 68 | fprintf(stderr, "** key '%s' not match ** '", kbuf); 69 | fwrite(val, vallen, 1, stderr); 70 | fprintf(stderr, "'\n"); 71 | } else { 72 | printf("get '%s' = '", kbuf); 73 | fwrite(val, vallen, 1, stdout); 74 | printf("'\n"); 75 | } 76 | } 77 | } 78 | 79 | return 0; 80 | } 81 | 82 | -------------------------------------------------------------------------------- /test/test_get_noset.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "libmemcached/memcached.h" 11 | 12 | #define KEY_PREFIX "key" 13 | #define VAL_PREFIX "val" 14 | 15 | void usage(void) 16 | { 17 | printf("usage: ./test_get_noset \n"); 18 | exit(1); 19 | } 20 | 21 | void pexit(const char* msg) 22 | { 23 | perror(msg); 24 | exit(1); 25 | } 26 | 27 | int main(int argc, char* argv[]) 28 | { 29 | if(argc < 4) { usage(); } 30 | 31 | char* host = argv[1]; 32 | 33 | unsigned short port = atoi(argv[2]); 34 | if(port == 0) { usage(); } 35 | 36 | uint32_t num = atoi(argv[3]); 37 | if(num == 0) { usage(); } 38 | 39 | memcached_st* mc = memcached_create(NULL); 40 | if(mc == NULL) { pexit("memcached_create"); } 41 | 42 | memcached_server_add(mc, host, port); 43 | 44 | // memcached_behavior_set(mc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1); 45 | 46 | char kbuf[strlen(KEY_PREFIX) + 11]; 47 | char vbuf[strlen(VAL_PREFIX) + 11]; 48 | 49 | uint32_t i; 50 | 51 | while(1) { 52 | for(i=0; i < num; ++i) { 53 | int klen = sprintf(kbuf, KEY_PREFIX "%d", i); 54 | int vlen = sprintf(vbuf, VAL_PREFIX "%d", i); 55 | size_t vallen; 56 | uint32_t flags; 57 | memcached_return rc; 58 | char* val = memcached_get(mc, kbuf, klen, &vallen, &flags, &rc); 59 | if(!val) { 60 | fprintf(stderr, "** key '%s' not found **\n", kbuf); 61 | } else if(vallen != vlen || memcmp(val, vbuf, vlen) != 0) { 62 | fprintf(stderr, "** key '%s' not match ** '", kbuf); 63 | fwrite(val, vallen, 1, stderr); 64 | fprintf(stderr, "'\n"); 65 | } else { 66 | printf("get '%s' = '", kbuf); 67 | fwrite(val, vallen, 1, stdout); 68 | printf("'\n"); 69 | } 70 | } 71 | } 72 | 73 | return 0; 74 | } 75 | 76 | -------------------------------------------------------------------------------- /test/test_set.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "libmemcached/memcached.h" 11 | 12 | #define KEY_PREFIX "key" 13 | #define VAL_PREFIX "val" 14 | 15 | void usage(void) 16 | { 17 | printf("usage: ./test_set \n"); 18 | exit(1); 19 | } 20 | 21 | void pexit(const char* msg) 22 | { 23 | perror(msg); 24 | exit(1); 25 | } 26 | 27 | int main(int argc, char* argv[]) 28 | { 29 | if(argc < 4) { usage(); } 30 | 31 | char* host = argv[1]; 32 | 33 | unsigned short port = atoi(argv[2]); 34 | if(port == 0) { usage(); } 35 | 36 | uint32_t num = atoi(argv[3]); 37 | if(num == 0) { usage(); } 38 | 39 | memcached_st* mc = memcached_create(NULL); 40 | if(mc == NULL) { pexit("memcached_create"); } 41 | 42 | memcached_server_add(mc, host, port); 43 | 44 | // memcached_behavior_set(mc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1); 45 | 46 | char kbuf[strlen(KEY_PREFIX) + 11]; 47 | char vbuf[strlen(VAL_PREFIX) + 11]; 48 | 49 | while(1) { 50 | uint32_t i; 51 | for(i=0; i < num; ++i) { 52 | int klen = sprintf(kbuf, KEY_PREFIX "%d", i); 53 | int vlen = sprintf(vbuf, VAL_PREFIX "%d", i); 54 | printf("set '%s' = '%s'\n", kbuf, vbuf); 55 | memcached_set(mc, kbuf, klen, vbuf, vlen, 0, 0); 56 | } 57 | } 58 | 59 | return 0; 60 | } 61 | 62 | -------------------------------------------------------------------------------- /test/test_set_get_delete.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "libmemcached/memcached.h" 11 | 12 | #define KEY_PREFIX "key" 13 | #define VAL_PREFIX "val" 14 | 15 | void usage(void) 16 | { 17 | printf("usage: ./test_set_get_delete \n"); 18 | exit(1); 19 | } 20 | 21 | void pexit(const char* msg) 22 | { 23 | perror(msg); 24 | exit(1); 25 | } 26 | 27 | int main(int argc, char* argv[]) 28 | { 29 | if(argc < 4) { usage(); } 30 | 31 | char* host = argv[1]; 32 | 33 | unsigned short port = atoi(argv[2]); 34 | if(port == 0) { usage(); } 35 | 36 | uint32_t num = atoi(argv[3]); 37 | if(num == 0) { usage(); } 38 | 39 | memcached_st* mc = memcached_create(NULL); 40 | if(mc == NULL) { pexit("memcached_create"); } 41 | 42 | memcached_server_add(mc, host, port); 43 | 44 | // memcached_behavior_set(mc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1); 45 | 46 | char kbuf[strlen(KEY_PREFIX) + 11]; 47 | char vbuf[strlen(VAL_PREFIX) + 11]; 48 | 49 | while(1) { 50 | uint32_t i; 51 | for(i=0; i < num; ++i) { 52 | int klen = sprintf(kbuf, KEY_PREFIX "%d", i); 53 | int vlen = sprintf(vbuf, VAL_PREFIX "%d", i); 54 | printf("set '%s' = '%s'\n", kbuf, vbuf); 55 | memcached_set(mc, kbuf, klen, vbuf, vlen, 0, 0); 56 | } 57 | 58 | for(i=0; i < num; ++i) { 59 | int klen = sprintf(kbuf, KEY_PREFIX "%d", i); 60 | int vlen = sprintf(vbuf, VAL_PREFIX "%d", i); 61 | size_t vallen; 62 | uint32_t flags; 63 | memcached_return rc; 64 | char* val = memcached_get(mc, kbuf, klen, &vallen, &flags, &rc); 65 | if(!val) { 66 | fprintf(stderr, "** key '%s' not found **\n", kbuf); 67 | } else if(vallen != vlen || memcmp(val, vbuf, vlen) != 0) { 68 | fprintf(stderr, "** key '%s' not match ** '", kbuf); 69 | fwrite(val, vallen, 1, stderr); 70 | fprintf(stderr, "'\n"); 71 | } else { 72 | printf("get '%s' = '", kbuf); 73 | fwrite(val, vallen, 1, stdout); 74 | printf("'\n"); 75 | } 76 | } 77 | 78 | for(i=0; i < num; ++i) { 79 | int klen = sprintf(kbuf, KEY_PREFIX "%d", i); 80 | printf("delete '%s'\n", kbuf); 81 | memcached_delete(mc, kbuf, klen, 0); 82 | } 83 | } 84 | 85 | return 0; 86 | } 87 | 88 | --------------------------------------------------------------------------------