├── php7_zeppelin ├── EXPERIMENTAL ├── CREDITS ├── config.w32 ├── .gitignore ├── zeppelin.php ├── tests │ └── 001.phpt ├── php_zeppelin.h ├── config.m4 └── zeppelin.cc ├── phpzeppelin ├── EXPERIMENTAL ├── CREDITS ├── change.sh ├── config.w32 ├── generate.sh ├── .gitignore ├── example_mget.php ├── zeppelin.php ├── example.php ├── example_options.php ├── config.m4 └── php_zeppelin.h ├── gozeppelin ├── src │ └── zeppelin │ │ ├── lib │ │ ├── include │ │ └── libzp │ │ └── zeppelin.go ├── README.md └── example.go ├── manager ├── golang │ ├── src │ │ └── zp_manager │ │ │ ├── lib │ │ │ └── include │ │ │ └── libzp │ ├── README.md │ └── manager.go ├── hostlist.example ├── utils │ ├── distribution.h │ ├── linenoise.h │ └── json.h ├── zp_loopset.cc ├── rocksdb_to_zp.cc ├── README.md ├── zp_benchmark.cc ├── Makefile ├── zp_expansion.cc └── zp_parade.cc ├── libzp └── libzp │ ├── .gitignore │ ├── src │ ├── build_version.cc.in │ ├── zp_conn.h │ ├── c_test.c │ ├── zp_meta.proto │ ├── zp_client.cc │ ├── zp_conn.cc │ ├── client.proto │ └── zp_entity.cc │ ├── include │ ├── build_version.h │ ├── zp_option.h │ ├── zp_client_c.h │ ├── zp_entity.h │ ├── zp_client.h │ ├── zp_cluster_c.h │ └── zp_cluster.h │ ├── example │ ├── zp_get.cc │ ├── zp_timeout.cc │ ├── zp_simple.cc │ ├── zp_hashtag.cc │ ├── zp_create_table.cc │ ├── Makefile │ ├── zp_async.cc │ ├── zp_mget.cc │ ├── zp_table.cc │ └── zp_parallel.cc │ ├── make_libzp_withdeps.sh │ └── Makefile ├── pyzeppelin ├── readme ├── setup.py ├── test_zp.py └── pyzeppelin.cc ├── .gitmodules ├── javazeppelin ├── README.md ├── zeppelin_jar │ ├── src │ │ └── net │ │ │ └── qihoo │ │ │ └── zeppelin │ │ │ ├── ZeppelinException.java │ │ │ └── Zeppelin.java │ └── build.xml ├── Makefile ├── test │ ├── TestZeppelin.java │ └── TestZeppelinThread.java ├── include │ └── net_qihoo_zeppelin_Zeppelin.h └── lib │ └── zeppelin.cc ├── .travis.yml ├── README.md ├── .gitignore └── LICENSE /php7_zeppelin/EXPERIMENTAL: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /phpzeppelin/EXPERIMENTAL: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /php7_zeppelin/CREDITS: -------------------------------------------------------------------------------- 1 | zeppelin -------------------------------------------------------------------------------- /phpzeppelin/CREDITS: -------------------------------------------------------------------------------- 1 | zeppelin 2 | -------------------------------------------------------------------------------- /gozeppelin/src/zeppelin/lib: -------------------------------------------------------------------------------- 1 | ../../../libzp/libzp/lib -------------------------------------------------------------------------------- /gozeppelin/src/zeppelin/include/libzp: -------------------------------------------------------------------------------- 1 | ../../../../libzp/libzp -------------------------------------------------------------------------------- /manager/golang/src/zp_manager/lib: -------------------------------------------------------------------------------- 1 | ../../../../libzp/libzp/lib -------------------------------------------------------------------------------- /manager/golang/src/zp_manager/include/libzp: -------------------------------------------------------------------------------- 1 | ../../../../../libzp/libzp -------------------------------------------------------------------------------- /libzp/libzp/.gitignore: -------------------------------------------------------------------------------- 1 | src/build_version.cc 2 | tags 3 | *.out 4 | *.d 5 | -------------------------------------------------------------------------------- /phpzeppelin/change.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cp modules/zeppelin.so /usr/local/php/lib/php/20100525/ 4 | -------------------------------------------------------------------------------- /gozeppelin/README.md: -------------------------------------------------------------------------------- 1 | ### 编译方法 2 | 3 | ``` 4 | cd ../libzp/libzp && ./make_libzp_withdeps.sh 5 | go build example.go 6 | ``` 7 | -------------------------------------------------------------------------------- /manager/golang/README.md: -------------------------------------------------------------------------------- 1 | ### 编译方法 2 | 3 | ``` 4 | cd ../../libzp/libzp/ && ./make_libzp_withdeps.sh 5 | export LD_LIBRARY_PATH=$PWD/src/zp_manager/lib 6 | ``` 7 | -------------------------------------------------------------------------------- /pyzeppelin/readme: -------------------------------------------------------------------------------- 1 | This is the python extension for zeppelin 2 | Install tips: 3 | To compile, run "python setup.py build". 4 | To auto-install, run "python setup.py install". 5 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "third/pink"] 2 | path = third/pink 3 | url = https://github.com/Qihoo360/pink.git 4 | [submodule "third/slash"] 5 | path = third/slash 6 | url = https://github.com/baotiao/slash.git 7 | -------------------------------------------------------------------------------- /libzp/libzp/src/build_version.cc.in: -------------------------------------------------------------------------------- 1 | #include "libzp/include/build_version.h" 2 | const char* libzp_build_git_sha = "libzp_build_git_sha:@@GIT_SHA@@"; 3 | const char* libzp_build_git_date = "libzp_build_git_date:@@GIT_DATE_TIME@@"; 4 | const char* libzp_build_compile_date = __DATE__; 5 | -------------------------------------------------------------------------------- /javazeppelin/README.md: -------------------------------------------------------------------------------- 1 | 1. To make 2 | 3 | make command will generate lib zeppelin.so in current dir and zeppelin.jar in ./zeppelin_jar dir. 4 | 5 | 2. To use 6 | 7 | javac -classpath {JAR_PATH}/zeppelin.jar {java file} 8 | 9 | java -classpath {JAR_PATH}/zeppelin.jar:. {java class} -------------------------------------------------------------------------------- /phpzeppelin/config.w32: -------------------------------------------------------------------------------- 1 | // $Id$ 2 | // vim:ft=javascript 3 | 4 | // If your extension references something external, use ARG_WITH 5 | // ARG_WITH("zeppelin", "for zeppelin support", "no"); 6 | 7 | // Otherwise, use ARG_ENABLE 8 | // ARG_ENABLE("zeppelin", "enable zeppelin support", "no"); 9 | 10 | if (PHP_ZEPPELIN != "no") { 11 | EXTENSION("zeppelin", "zeppelin.c"); 12 | } 13 | 14 | -------------------------------------------------------------------------------- /phpzeppelin/generate.sh: -------------------------------------------------------------------------------- 1 | if [[ $# < 1 ]]; then 2 | echo "Need PHP Path, such as /usr/local/php/" 3 | exit 4 | fi 5 | 6 | PHP_PATH=$1 7 | PHP_IZE=$PHP_PATH/bin/phpize 8 | PHP_EXE=$PHP_PATH/bin/php 9 | PHP_CON=$PHP_PATH/bin/php-config 10 | 11 | make -C ../libzp/libzp 12 | $PHP_IZE 13 | CXXFLAGS="-std=c++11 -g -O2" EXTRA_LDFLAGS="-lpthread" ./configure --with-php-config=$PHP_CON 14 | make 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | language: cpp 4 | 5 | os: 6 | - linux 7 | 8 | addons: 9 | apt: 10 | packages: ['libprotobuf-dev', 'libgoogle-glog-dev', 'protobuf-compiler'] 11 | 12 | compiler: 13 | - gcc 14 | 15 | language: cpp 16 | 17 | script: 18 | - make -C third/slash/slash 19 | - make -C third/pink/pink SLASH_PATH=../../slash 20 | - make -C libzp/libzp 21 | - make -C manager 22 | -------------------------------------------------------------------------------- /php7_zeppelin/config.w32: -------------------------------------------------------------------------------- 1 | // $Id$ 2 | // vim:ft=javascript 3 | 4 | // If your extension references something external, use ARG_WITH 5 | // ARG_WITH("zeppelin", "for zeppelin support", "no"); 6 | 7 | // Otherwise, use ARG_ENABLE 8 | // ARG_ENABLE("zeppelin", "enable zeppelin support", "no"); 9 | 10 | if (PHP_ZEPPELIN != "no") { 11 | EXTENSION("zeppelin", "zeppelin.cc", PHP_EXTNAME_SHARED, "/DZEND_ENABLE_STATIC_TSRMLS_CACHE=1"); 12 | } 13 | 14 | -------------------------------------------------------------------------------- /pyzeppelin/setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup, Extension 2 | setup(name = 'pyzeppelin', version = '0.0.2', ext_modules = [Extension('pyzeppelin', sources=['pyzeppelin.cc'], 3 | include_dirs=['../libzp', '../third/pink/', '../third/slash'], 4 | extra_objects=['../libzp/libzp/lib/libzp.a', '../third/pink/pink/lib/libpink.a', '../third/slash/slash/lib/libslash.a'], 5 | libraries=['pthread', 'protobuf'], 6 | extra_compile_args=['-std=c++11'] 7 | )]) 8 | -------------------------------------------------------------------------------- /manager/hostlist.example: -------------------------------------------------------------------------------- 1 | Cab_1 2 | 192.168.1.1 3 | 1001 4 | 1002 5 | 1003 6 | 1004 7 | 1005 8 | 192.168.1.2 9 | 1001 10 | 1002 11 | 1003 12 | 1004 13 | 1005 14 | Cab_2 15 | 192.168.1.3 16 | 1001 17 | 1002 18 | 1003 19 | 1004 20 | 1005 21 | 192.168.1.4 22 | 1001 23 | 1002 24 | 1003 25 | 1004 26 | 1005 27 | Cab_3 28 | 192.168.1.5 29 | 1001 30 | 1002 31 | 1003 32 | 1004 33 | 1005 34 | 192.168.1.6 35 | 1001 36 | 1002 37 | 1003 38 | 1004 39 | 1005 40 | -------------------------------------------------------------------------------- /php7_zeppelin/.gitignore: -------------------------------------------------------------------------------- 1 | .deps 2 | *.lo 3 | *.la 4 | .libs 5 | acinclude.m4 6 | aclocal.m4 7 | autom4te.cache 8 | build 9 | config.guess 10 | config.h 11 | config.h.in 12 | config.log 13 | config.nice 14 | config.status 15 | config.sub 16 | configure 17 | configure.in 18 | include 19 | install-sh 20 | libtool 21 | ltmain.sh 22 | Makefile 23 | Makefile.fragments 24 | Makefile.global 25 | Makefile.objects 26 | missing 27 | mkinstalldirs 28 | modules 29 | run-tests.php 30 | tests/*/*.diff 31 | tests/*/*.out 32 | tests/*/*.php 33 | tests/*/*.exp 34 | tests/*/*.log 35 | tests/*/*.sh 36 | -------------------------------------------------------------------------------- /phpzeppelin/.gitignore: -------------------------------------------------------------------------------- 1 | .deps 2 | *.lo 3 | *.la 4 | .libs 5 | acinclude.m4 6 | aclocal.m4 7 | autom4te.cache 8 | build 9 | config.guess 10 | config.h 11 | config.h.in 12 | config.log 13 | config.nice 14 | config.status 15 | config.sub 16 | configure 17 | configure.in 18 | include 19 | install-sh 20 | libtool 21 | ltmain.sh 22 | Makefile 23 | Makefile.fragments 24 | Makefile.global 25 | Makefile.objects 26 | missing 27 | mkinstalldirs 28 | modules 29 | run-tests.php 30 | tests/*/*.diff 31 | tests/*/*.out 32 | tests/*/*.php 33 | tests/*/*.exp 34 | tests/*/*.log 35 | tests/*/*.sh 36 | -------------------------------------------------------------------------------- /phpzeppelin/example_mget.php: -------------------------------------------------------------------------------- 1 | Set("example_mget_key_".$times, "example_mget_value_".$times); 8 | if ($ret == false) { 9 | echo "Set Error". PHP_EOL; 10 | break; 11 | } 12 | } 13 | 14 | $times = 5; 15 | $keys = array(); 16 | while ($times--) { 17 | array_push($keys, "example_mget_key_".$times); 18 | } 19 | 20 | var_dump($keys); 21 | 22 | $ret = $zp_mget->Mget($keys); 23 | if ($ret == false) { 24 | echo "Mget Error". PHP_EOL; 25 | } 26 | var_dump($ret); 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Zeppelin-client for [zeppelin](https://github.com/Qihoo360/zeppelin) 2 | 3 | [![Build Status](https://travis-ci.org/Qihoo360/zeppelin-client.svg?branch=master)](https://travis-ci.org/Qihoo360/zeppelin-client) 4 | 5 | - libzp: C++ client library 6 | - manager : Manager and checkup command line tools 7 | - zp_benchmark 8 | - zp_checkup 9 | - zp_expansion 10 | - zp_info 11 | - zp_manager 12 | - zp_parade 13 | - rocksdb_to_zp 14 | - phpzeppelin : PHP client extension 15 | - php7zeppelin : PHP7 client extension 16 | - pyzeppelin : Python client library 17 | - gozeppelin : Golang client library 18 | -------------------------------------------------------------------------------- /phpzeppelin/zeppelin.php: -------------------------------------------------------------------------------- 1 | "; 3 | 4 | if(!extension_loaded('zeppelin')) { 5 | dl('zeppelin.' . PHP_SHLIB_SUFFIX); 6 | } 7 | $module = 'zeppelin'; 8 | $functions = get_extension_funcs($module); 9 | echo "Functions available in the test extension:$br\n"; 10 | foreach($functions as $func) { 11 | echo $func."$br\n"; 12 | } 13 | echo "$br\n"; 14 | $function = 'confirm_' . $module . '_compiled'; 15 | if (extension_loaded($module)) { 16 | $str = $function($module); 17 | } else { 18 | $str = "Module $module is not compiled into PHP"; 19 | } 20 | echo "$str\n"; 21 | ?> 22 | -------------------------------------------------------------------------------- /php7_zeppelin/zeppelin.php: -------------------------------------------------------------------------------- 1 | "; 3 | 4 | if(!extension_loaded('zeppelin')) { 5 | dl('zeppelin.' . PHP_SHLIB_SUFFIX); 6 | } 7 | $module = 'zeppelin'; 8 | $functions = get_extension_funcs($module); 9 | echo "Functions available in the test extension:$br\n"; 10 | foreach($functions as $func) { 11 | echo $func."$br\n"; 12 | } 13 | echo "$br\n"; 14 | $function = 'confirm_' . $module . '_compiled'; 15 | if (extension_loaded($module)) { 16 | $str = $function($module); 17 | } else { 18 | $str = "Module $module is not compiled into PHP"; 19 | } 20 | echo "$str\n"; 21 | ?> 22 | -------------------------------------------------------------------------------- /php7_zeppelin/tests/001.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Check for zeppelin presence 3 | --SKIPIF-- 4 | 5 | --FILE-- 6 | 23 | --EXPECT-- 24 | zeppelin extension is available 25 | -------------------------------------------------------------------------------- /phpzeppelin/example.php: -------------------------------------------------------------------------------- 1 | Set("example_test_key", $num); 8 | if ($ret == false) { 9 | echo "Set Error". PHP_EOL; 10 | break; 11 | } 12 | $val = $zp->Get("example_test_key"); 13 | if ($val == false || $val != $num) { 14 | echo "Error, num: ". $num. " val: ". $val. PHP_EOL; 15 | var_dump($val); 16 | break; 17 | } 18 | $ret = $zp->Delete("example_test_key"); 19 | if ($ret == false) { 20 | echo "Delete Error". PHP_EOL; 21 | break; 22 | } 23 | 24 | $val = $zp->Get("example_test_key"); 25 | if ($val != false) { 26 | echo "Error, expect false, but: ". " val: ". $val. PHP_EOL; 27 | var_dump($val); 28 | break; 29 | } 30 | } 31 | echo "done". PHP_EOL; 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | libzp/libzp/lib 2 | libzp/libzp/example/output 3 | libzp/libzp/example/zp_async 4 | libzp/libzp/example/zp_create_table 5 | libzp/libzp/example/zp_get 6 | libzp/libzp/example/zp_mget 7 | libzp/libzp/example/zp_parallel 8 | libzp/libzp/example/zp_simple 9 | libzp/libzp/example/zp_timeout 10 | libzp/libzp/example/zp_hashtag 11 | libzp/libzp/example/zp_table 12 | 13 | manager/zp_benchmark 14 | manager/zp_checkup 15 | manager/zp_expansion 16 | manager/zp_info 17 | manager/zp_parade 18 | manager/golang/manager 19 | manager/zp_manager 20 | manager/rocksdb_to_zp 21 | manager/checkup_json_result 22 | manager/info_json_result 23 | manager/zp_loopset 24 | 25 | tags 26 | 27 | # Intermediate file 28 | libzp/libzp/src/client.pb.cc 29 | libzp/libzp/src/client.pb.h 30 | libzp/libzp/src/zp_meta.pb.cc 31 | libzp/libzp/src/zp_meta.pb.h 32 | *.o 33 | *.d.* 34 | *.out 35 | *.swp 36 | -------------------------------------------------------------------------------- /libzp/libzp/include/build_version.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 7 | // This source code is licensed under the BSD-style license found in the 8 | // LICENSE file in the root directory of this source tree. An additional grant 9 | // of patent rights can be found in the PATENTS file in the same directory. 10 | // 11 | #pragma once 12 | 13 | // this variable tells us about the git revision 14 | extern const char* libzp_build_git_sha; 15 | 16 | // Date on which the code was compiled: 17 | extern const char* libzp_build_compile_date; 18 | -------------------------------------------------------------------------------- /javazeppelin/zeppelin_jar/src/net/qihoo/zeppelin/ZeppelinException.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Qihoo 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:// www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package net.qihoo.zeppelin; 15 | 16 | public class ZeppelinException extends Exception { 17 | public ZeppelinException() { 18 | } 19 | public ZeppelinException(String msg) { 20 | super(msg); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /libzp/libzp/example/zp_get.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright [2016] qihoo" 3 | */ 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | #include "slash/include/slash_status.h" 10 | 11 | #include "libzp/include/zp_client.h" 12 | 13 | 14 | void usage() { 15 | std::cout << "usage:\n" 16 | << " zp_get host port table key\n"; 17 | } 18 | 19 | int main(int argc, char* argv[]) { 20 | if (argc != 5) { 21 | usage(); 22 | return -1; 23 | } 24 | 25 | // client handle io operation 26 | std::cout << "create client" << std::endl; 27 | libzp::Client* client = new libzp::Client(argv[1], atoi(argv[2]), argv[3]); 28 | libzp::Status s; 29 | 30 | std::string value; 31 | s = client->Get(argv[4], &value); 32 | if (!s.ok()) { 33 | std::cout << s.ToString() << std::endl; 34 | } else { 35 | std::cout << "get ok, value: " << value << std::endl; 36 | } 37 | 38 | delete client; 39 | return 1; 40 | } 41 | -------------------------------------------------------------------------------- /manager/utils/distribution.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef DISTRIBUTION_H_ 7 | #define DISTRIBUTION_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | namespace distribution { 18 | 19 | struct Host { 20 | // ip:port 21 | std::string host; 22 | // host id 23 | int host_id; 24 | // cabinet id 25 | int cab_id; 26 | }; 27 | 28 | extern std::vector > result; 29 | 30 | bool Load(const std::string& file); 31 | void Distribution(); 32 | void Checkup(); 33 | void Cleanup(); 34 | 35 | } // namespace distribution 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /javazeppelin/zeppelin_jar/build.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /phpzeppelin/example_options.php: -------------------------------------------------------------------------------- 1 | Set("example_options_test_key", $num); 10 | if ($ret == false) { 11 | echo "Set Error". PHP_EOL; 12 | break; 13 | } 14 | $val = $zp_options->Get("example_options_test_key"); 15 | if ($val == false || $val != $num) { 16 | echo "Error, num: ". $num. " val: ". $val. PHP_EOL; 17 | var_dump($val); 18 | break; 19 | } 20 | 21 | $ret = $zp_options->Delete("example_options_test_key"); 22 | if ($ret == false) { 23 | echo "Delete Error". PHP_EOL; 24 | break; 25 | } 26 | 27 | $val = $zp_options->Get("example_options_test_key"); 28 | if ($val != false) { 29 | echo "Error, expect false, but: ". " val: ". $val. PHP_EOL; 30 | var_dump($val); 31 | break; 32 | } 33 | $ret = $zp_options->Set("example_options_test_key", $num); 34 | if ($ret == false) { 35 | echo "Set Error". PHP_EOL; 36 | break; 37 | } 38 | } 39 | 40 | echo "done". PHP_EOL; 41 | -------------------------------------------------------------------------------- /pyzeppelin/test_zp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # test_zp.py 5 | import pyzeppelin 6 | # ip port table_name 7 | b = pyzeppelin.create_client("127.0.0.1", 9221, "test") 8 | 9 | test_key = "key" 10 | test_value = "value" 11 | 12 | test_key1 = "key1" 13 | test_value1 = "value1" 14 | (s, msg) = pyzeppelin.set(b, test_key, test_value) 15 | if s != 0: 16 | print msg 17 | (s, msg) = pyzeppelin.remove_client(b) 18 | 19 | (s, msg) = pyzeppelin.set(b, test_key1, test_value1) 20 | if s != 0: 21 | print msg 22 | (s, msg) = pyzeppelin.remove_client(b) 23 | 24 | (s, msg) = pyzeppelin.get(b, test_key) 25 | if s == 0 or s== 1: # s==1 msg ==None; the key not found 26 | result = msg 27 | print "get value:", result 28 | else: 29 | print msg 30 | (s, msg) = pyzeppelin.remove_client(b) 31 | 32 | (s, msg) = pyzeppelin.mget(b, [test_key, test_key1]) 33 | if (s == 0): 34 | print msg 35 | else: 36 | print msg 37 | (s, msg) = pyzeppelin.remove_client(b) 38 | 39 | (s, msg) = pyzeppelin.delete(b, "key") 40 | if s != 0: 41 | print msg 42 | (s, msg) = pyzeppelin.remove_client(b) 43 | -------------------------------------------------------------------------------- /javazeppelin/Makefile: -------------------------------------------------------------------------------- 1 | #zeppelin java lib 2 | 3 | #PARAM 4 | CC = g++ 5 | JNI_DIR = $(JAVA_HOME)/include/ 6 | JNI_MD_DIR = $(JAVA_HOME)/include/linux/ 7 | ZEPPELIN_HEAD_DIR = $(CURDIR)/../libzp/libzp/include 8 | SLASH_PATH = $(CURDIR)/../third/slash 9 | PINK_PATH = $(CURDIR)/../third/pink 10 | ZEPPELIN_PATH = $(CURDIR)/../libzp/ 11 | ZEPPELIN_SRC = $(CURDIR)/../libzp/libzp/src 12 | ZP_LIBRARY = $(CURDIR)/../libzp/libzp/lib/libzp.a 13 | LDFLAGS = -L/usr/local/protobuf/lib -pthread -lrt -lprotobuf 14 | PINK_LIBRARY=$(PINK_PATH)/pink/lib/libpink.a 15 | SLASH_LIBRARY=$(SLASH_PATH)/slash/lib/libslash.a 16 | DEP_LIBS = $(ZP_LIBRARY) $(PINK_LIBRARY) $(SLASH_LIBRARY) 17 | 18 | FLAGS = -g -Wall -D__STDC_FORMAT_MACROS -I$(JNI_DIR) -I$(JNI_MD_DIR) \ 19 | -I$(ZEPPELIN_PATH) -I $(ZEPPELIN_HEAD_DIR) -I$(SLASH_PATH) \ 20 | -I$(ZEPPELIN_SRC) -I$(PINK_PATH) -I./include -pthread 21 | SO_OBJS = libzeppelin.so 22 | SOFLAGS = -shared -fPIC 23 | C_SRC = lib/zeppelin.cc 24 | 25 | #.PHONY 26 | .PHONY : clean 27 | 28 | all: $(SO_OBJS) 29 | ant -buildfile zeppelin_jar/build.xml 30 | 31 | $(SO_OBJS): $(C_SRC) 32 | $(CC) $(SOFLAGS) $(FLAGS) -o $@ $^ $(DEP_LIBS) $(LDFLAGS) 33 | 34 | clean: 35 | rm $(SO_OBJS) 36 | rm -rf zeppelin_jar/classes 37 | rm -rf zeppelin_jar/doc 38 | rm zeppelin_jar/zeppelin.jar 39 | -------------------------------------------------------------------------------- /gozeppelin/example.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "zeppelin" 4 | import "fmt" 5 | 6 | func main() { 7 | op_timeout := 1000 8 | m1 := zeppelin.ZpNode{IP : "127.0.0.1", Port : 9221} 9 | m2 := zeppelin.ZpNode{IP : "127.0.0.1", Port : 9222} 10 | m3 := zeppelin.ZpNode{IP : "127.0.0.1", Port : 9223} 11 | table_name := "test" 12 | client := zeppelin.NewZpClient(table_name, op_timeout, m1, m2, m3) 13 | 14 | key := []byte("testkey") 15 | value := []byte("testvalue") 16 | key1 := []byte("testkey1") 17 | value1 := []byte("testvalue1") 18 | key2 := []byte("testkey2") 19 | value2 := []byte("testvalue2") 20 | 21 | ok, reason := client.Set(&key, &value, -1) 22 | if (!ok) { 23 | fmt.Println(reason); 24 | } 25 | 26 | ok, reason = client.Set(&key1, &value1, -1) 27 | if (!ok) { 28 | fmt.Println(reason); 29 | } 30 | 31 | ok, reason = client.Set(&key2, &value2, -1) 32 | if (!ok) { 33 | fmt.Println(reason); 34 | } 35 | 36 | ok, reason, v := client.Get(&key) 37 | if (!ok) { 38 | fmt.Println(reason); 39 | } 40 | fmt.Println(string(v[:])); 41 | 42 | mvalues := make(map[string]string) 43 | keys := []string {string(key1[:]), string(key2[:])} 44 | ok, reason = client.Mget(keys, &mvalues); 45 | if (!ok) { 46 | fmt.Println(reason); 47 | } 48 | for mk, mv := range mvalues { 49 | fmt.Println("key:", mk, "value:", mv); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /libzp/libzp/example/zp_timeout.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | 6 | #include "slash/include/slash_status.h" 7 | #include "libzp/include/zp_client.h" 8 | 9 | 10 | void usage() { 11 | std::cout << "usage:\n" 12 | << " zp_timeout host port table timeout\n"; 13 | } 14 | 15 | int main(int argc, char* argv[]) { 16 | if (argc != 5) { 17 | usage(); 18 | return -1; 19 | } 20 | 21 | // client handle io operation 22 | std::cout << "create client" << std::endl; 23 | libzp::Options opt; 24 | opt.op_timeout = atoi(argv[4]); 25 | opt.meta_addr.push_back(libzp::Node(argv[1], atoi(argv[2]))); 26 | 27 | libzp::Client* client = new libzp::Client(opt, argv[3]); 28 | 29 | std::string key("test_timeout"); 30 | libzp::Status s = client->Set(key, "value_timeout"); 31 | if (!s.ok()) { 32 | std::cout << s.ToString() << std::endl; 33 | } else { 34 | std::cout << "set ok."<< std::endl; 35 | } 36 | 37 | std::string value; 38 | s = client->Get(key, &value); 39 | if (!s.ok()) { 40 | std::cout << s.ToString() << std::endl; 41 | } else { 42 | std::cout << "get ok, value: " << value << std::endl; 43 | } 44 | 45 | s = client->Delete(key); 46 | if (!s.ok()) { 47 | std::cout << s.ToString() << std::endl; 48 | } else { 49 | std::cout << "delete ok." << std::endl; 50 | } 51 | 52 | delete client; 53 | return 1; 54 | } 55 | -------------------------------------------------------------------------------- /libzp/libzp/example/zp_simple.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "libzp/include/zp_client.h" 4 | void usage() { 5 | std::cout << "usage:\n" 6 | << " zp_timeout host port table timeout\n"; 7 | } 8 | 9 | int main(int argc, char* argv[]) { 10 | if (argc != 5) { 11 | usage(); 12 | return -1; 13 | } 14 | 15 | // Create client with ip, port and table name, the table must be already exist, you may create table by zp_manager first 16 | libzp::Options opt; 17 | opt.op_timeout = atoi(argv[4]); 18 | opt.meta_addr.push_back(libzp::Node(argv[1], atoi(argv[2]))); 19 | 20 | libzp::Client* client = new libzp::Client(opt, argv[3]); 21 | assert(client); 22 | // Set key value 23 | libzp::Status s = client->Set("key", "value"); 24 | if (!s.ok()) { 25 | std::cout << s.ToString() << std::endl; 26 | } else { 27 | std::cout << "set ok" << std::endl; 28 | } 29 | 30 | // Get value by key 31 | std::string var; 32 | s = client->Get("key", &var); 33 | if (!s.ok()) { 34 | std::cout << s.ToString() << std::endl; 35 | } else { 36 | std::cout << "get " << var << std::endl; 37 | } 38 | 39 | // Delete key 40 | s = client->Delete("key"); 41 | if (!s.ok()) { 42 | std::cout << s.ToString() << std::endl; 43 | } else { 44 | std::cout << "delete key" << std::endl; 45 | } 46 | 47 | // Remember to delete the client 48 | delete client; 49 | return 0; 50 | } 51 | -------------------------------------------------------------------------------- /libzp/libzp/make_libzp_withdeps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | CURDIR=$PWD 3 | TMPDIR=$CURDIR/tmp$UID 4 | PROJECT_DIR=$CURDIR/../.. 5 | 6 | # Update submodules 7 | cd $PROJECT_DIR && git submodule update --init 8 | if [[ -z $? ]]; then 9 | echo "git submodule failed" 10 | exit 1 11 | fi 12 | cd $CURDIR 13 | 14 | PINK_PATH=$PROJECT_DIR/third/pink 15 | SLASH_PATH=$PROJECT_DIR/third/slash 16 | 17 | make -C $SLASH_PATH/slash 18 | if [[ -z $? ]]; then 19 | echo "make slash failed" 20 | exit 1 21 | fi 22 | 23 | make -C $PINK_PATH/pink SLASH_PATH=$SLASH_PATH 24 | if [[ -z $? ]]; then 25 | echo "make pink failed" 26 | exit 1 27 | fi 28 | 29 | make PINK_PATH=$PINK_PATH SLASH_PATH=$SLASH_PATH 30 | if [[ -z $? ]]; then 31 | echo "make libzp failed" 32 | exit 1 33 | fi 34 | 35 | LIBZP_A=$CURDIR/lib/libzp.a 36 | LIBZP_SO=$CURDIR/lib/libzp.so 37 | LIBPINK_A=$PINK_PATH/pink/lib/libpink.a 38 | LIBSLASH_A=$SLASH_PATH/slash/lib/libslash.a 39 | 40 | # make libzp.a with pink and slash 41 | mkdir -p $TMPDIR && cd $TMPDIR 42 | ar x $LIBZP_A 43 | ar x $LIBPINK_A 44 | ar x $LIBSLASH_A 45 | 46 | rm -f $LIBZP_A $LIBZP_SO 47 | ar rs $LIBZP_A $TMPDIR/*.o 48 | g++ -o $LIBZP_SO -shared -fPIC $TMPDIR/*.o 49 | rm -rf $TMPDIR 50 | 51 | # make dist 52 | rm -rf $CURDIR/output 53 | mkdir $CURDIR/output 54 | mkdir -p $CURDIR/output/include/pink 55 | mkdir -p $CURDIR/output/include/slash 56 | cp -r $CURDIR/lib $CURDIR/output 57 | cp -r $PINK_PATH/pink/include $CURDIR/output/include/pink 58 | cp -r $SLASH_PATH/slash/include $CURDIR/output/include/slash 59 | 60 | echo "Done." 61 | -------------------------------------------------------------------------------- /javazeppelin/test/TestZeppelin.java: -------------------------------------------------------------------------------- 1 | import java.util.TreeMap; 2 | import java.util.Map; 3 | import net.qihoo.zeppelin.Zeppelin; 4 | import net.qihoo.zeppelin.ZeppelinException; 5 | 6 | public class TestZeppelin { 7 | public static void main(String args[]) { 8 | Zeppelin test = new Zeppelin("127.0.0.1", "9221", "test1"); 9 | try { 10 | String key1 = "key1"; 11 | String key2 = "key2"; 12 | String key3 = "key3"; 13 | String value1 = "value1"; 14 | String value2 = "value2"; 15 | String value3 = "value3"; 16 | 17 | boolean res = test.set(key1, value1); 18 | System.out.println("set " + key1 + " " + value1 + " "+ res); 19 | res = test.set(key2, value2); 20 | System.out.println("set " + key2 + " " + value2 + " " + res); 21 | res = test.set(key3, value3); 22 | System.out.println("set " + key3 + " " + value3 + " " + res); 23 | 24 | String value = test.get(key1); 25 | System.out.println("get " + key1 + ": " + value); 26 | 27 | res = test.delete(key1); 28 | System.out.println("delete " + key1 + ": " + res); 29 | res = test.delete("key9"); 30 | System.out.println("delete key9" + ": " + res); 31 | value = test.get(key1); 32 | System.out.println("get " + key1 + ": " + value); 33 | 34 | String[] keys = {key2, "key6"}; 35 | TreeMap tm = test.mget(keys); 36 | for (Map.Entry m:tm.entrySet()) { 37 | System.out.println("mget" + " " + m.getKey() + " " + m.getValue()); 38 | } 39 | test.removeZeppelin(); 40 | } catch(ZeppelinException e) { 41 | e.printStackTrace(); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /libzp/libzp/src/zp_conn.h: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright [2016] qihoo" 3 | */ 4 | 5 | #ifndef CLIENT_INCLUDE_ZP_CONN_H_ 6 | #define CLIENT_INCLUDE_ZP_CONN_H_ 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "slash/include/slash_mutex.h" 14 | #include "libzp/include/zp_entity.h" 15 | 16 | namespace pink { 17 | class PinkCli; 18 | } 19 | 20 | namespace libzp { 21 | 22 | enum TimeoutOptType { 23 | CONNECT = 0, 24 | SEND, 25 | RECV, 26 | }; 27 | 28 | struct ZpCli { 29 | explicit ZpCli(const Node& node); 30 | ~ZpCli(); 31 | bool TryKeepalive(); 32 | Status SetTimeout(uint64_t deadline, TimeoutOptType type); 33 | 34 | Node node; 35 | pink::PinkCli* cli; 36 | slash::Mutex cli_mu; 37 | uint64_t lastchecktime; 38 | }; 39 | 40 | class ConnectionPool { 41 | public : 42 | explicit ConnectionPool(size_t capacity); 43 | ~ConnectionPool(); 44 | std::shared_ptr GetConnection( 45 | const Node& node, uint64_t deadline, Status* sptr); 46 | void RemoveConnection(std::shared_ptr conn); 47 | std::shared_ptr GetExistConnection(); 48 | 49 | private: 50 | struct ZpCliItem { 51 | explicit ZpCliItem(std::shared_ptr zp_cli) 52 | : cli(zp_cli), 53 | next(nullptr), 54 | prev(nullptr) { 55 | } 56 | std::shared_ptr cli; 57 | ZpCliItem* next; 58 | ZpCliItem* prev; 59 | }; 60 | 61 | slash::Mutex pool_mu_; 62 | std::map conn_pool_; 63 | size_t capacity_; 64 | ZpCliItem lru_head_; 65 | 66 | void MoveToLRUHead(ZpCliItem* item); 67 | void InsertLRU(ZpCliItem* item); 68 | void RemoveFromLRU(ZpCliItem* item); 69 | }; 70 | 71 | } // namespace libzp 72 | #endif // CLIENT_INCLUDE_ZP_CONN_H_ 73 | -------------------------------------------------------------------------------- /javazeppelin/include/net_qihoo_zeppelin_Zeppelin.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class net_qihoo_zeppelin_Zeppelin */ 4 | 5 | #ifndef _Included_net_qihoo_zeppelin_Zeppelin 6 | #define _Included_net_qihoo_zeppelin_Zeppelin 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | /* 11 | * Class: net_qihoo_zeppelin_Zeppelin 12 | * Method: createZeppelin 13 | * Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V 14 | */ 15 | JNIEXPORT void JNICALL Java_net_qihoo_zeppelin_Zeppelin_createZeppelin 16 | (JNIEnv *, jobject, jstring, jstring, jstring); 17 | 18 | /* 19 | * Class: net_qihoo_zeppelin_Zeppelin 20 | * Method: removeZeppelin 21 | * Signature: ()V 22 | */ 23 | JNIEXPORT void JNICALL Java_net_qihoo_zeppelin_Zeppelin_removeZeppelin 24 | (JNIEnv *, jobject); 25 | 26 | /* 27 | * Class: net_qihoo_zeppelin_Zeppelin 28 | * Method: set 29 | * Signature: (Ljava/lang/String;Ljava/lang/String;)Z 30 | */ 31 | JNIEXPORT jboolean JNICALL Java_net_qihoo_zeppelin_Zeppelin_set 32 | (JNIEnv *, jobject, jstring, jstring); 33 | 34 | /* 35 | * Class: net_qihoo_zeppelin_Zeppelin 36 | * Method: get 37 | * Signature: (Ljava/lang/String;)Ljava/lang/String; 38 | */ 39 | JNIEXPORT jstring JNICALL Java_net_qihoo_zeppelin_Zeppelin_get 40 | (JNIEnv *, jobject, jstring); 41 | 42 | /* 43 | * Class: net_qihoo_zeppelin_Zeppelin 44 | * Method: mget 45 | * Signature: ([Ljava/lang/String;)Ljava/util/TreeMap; 46 | */ 47 | JNIEXPORT jobject JNICALL Java_net_qihoo_zeppelin_Zeppelin_mget 48 | (JNIEnv *, jobject, jobjectArray); 49 | 50 | /* 51 | * Class: net_qihoo_zeppelin_Zeppelin 52 | * Method: delete 53 | * Signature: (Ljava/lang/String;)Z 54 | */ 55 | JNIEXPORT jboolean JNICALL Java_net_qihoo_zeppelin_Zeppelin_delete 56 | (JNIEnv *, jobject, jstring); 57 | 58 | #ifdef __cplusplus 59 | } 60 | #endif 61 | #endif 62 | -------------------------------------------------------------------------------- /libzp/libzp/example/zp_hashtag.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "libzp/include/zp_cluster.h" 6 | #include "libzp/include/zp_client.h" 7 | #include "slash/include/slash_status.h" 8 | 9 | int main() { 10 | libzp::Options options; 11 | options.meta_addr.push_back(libzp::Node("127.0.0.1", 9221)); 12 | options.op_timeout = 100000; 13 | libzp::Cluster* cluster = new libzp::Cluster(options); 14 | slash::Status s; 15 | 16 | std::string table_name("test"); 17 | 18 | std::string hash_tag(libzp::kLBrace + "hashkey" + libzp::kRBrace); 19 | libzp::Cluster::Batch batch(hash_tag); 20 | batch.Write("testkey1", "testvalue1"); 21 | batch.Write("testkey2", "testvalue2"); 22 | batch.Write("testkey3", "testvalue3"); 23 | 24 | printf("WriteBatch\n"); 25 | s = cluster->WriteBatch(table_name, batch); 26 | if (!s.ok()) { 27 | std::cout << "WriteBatch Error: " << s.ToString() << std::endl; 28 | return -1; 29 | } 30 | 31 | printf("Write by hash tag\n"); 32 | std::string key1("example1{hashkey}key1"); 33 | std::string key2("example{hashkey}key2"); 34 | std::string key3("exa{hashkey}keykk3"); 35 | cluster->Set(table_name, key1, key1 + "value"); 36 | cluster->Set(table_name, key2, key2 + "value"); 37 | cluster->Set(table_name, key3, key3 + "value"); 38 | 39 | printf("ListbyTag\n"); 40 | std::map results; 41 | s = cluster->ListbyTag(table_name, hash_tag, &results); 42 | if (!s.ok()) { 43 | std::cout << "ListbyTag Error: " << s.ToString() << std::endl; 44 | return -1; 45 | } 46 | for (auto& kv : results) { 47 | printf("key: %s, value: %s\n", kv.first.c_str(), kv.second.c_str()); 48 | } 49 | 50 | printf("DeletebyTag\n"); 51 | s = cluster->DeletebyTag(table_name, hash_tag); 52 | if (!s.ok()) { 53 | std::cout << "DeletebyTag Error: " << s.ToString() << std::endl; 54 | return -1; 55 | } 56 | 57 | delete cluster; 58 | return 0; 59 | } 60 | -------------------------------------------------------------------------------- /libzp/libzp/example/zp_create_table.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright [2016] qihoo" 3 | */ 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "libzp/include/zp_cluster.h" 11 | #include "libzp/include/zp_client.h" 12 | 13 | 14 | void usage() { 15 | std::cout << "usage:\n" 16 | << " zp_create_table host port table_num partition_num\n"; 17 | } 18 | 19 | int main(int argc, char* argv[]) { 20 | if (argc != 5) { 21 | usage(); 22 | return -1; 23 | } 24 | 25 | libzp::Cluster* cluster = new libzp::Cluster(argv[1], atoi(argv[2])); 26 | 27 | libzp::Status s; 28 | int cnt = atoi(argv[3]); 29 | int partition_num = atoi(argv[4]); 30 | if (!s.ok()) { 31 | std::cout << s.ToString() << std::endl; 32 | return -1; 33 | } 34 | 35 | for(int i = 0; i < cnt; i++) { 36 | // needs connect to cluster first 37 | std::string table = "table" + std::to_string(i); 38 | std::cout << "Start " << i << std::endl; 39 | s = cluster->CreateTable(table, partition_num); 40 | if (!s.ok()) { 41 | std::cout << "Create table " << table << " failed, " << s.ToString() << std::endl; 42 | continue; 43 | } 44 | 45 | libzp::Status s; 46 | int retry = 10; 47 | for (int j = 0; j < retry && !s.ok(); j++) { 48 | // client handle io operation 49 | //std::cout << "create client" << std::endl; 50 | libzp::Client* client = new libzp::Client(argv[1], atoi(argv[2]), table); 51 | if (!s.ok()) { 52 | std::cout << "connect table failed, " << s.ToString() << std::endl; 53 | sleep(1); 54 | continue; 55 | } 56 | 57 | std::string key = "key" + std::to_string(i); 58 | s = client->Set(key, "value"); 59 | if (!s.ok()) { 60 | std::cout << s.ToString() << std::endl; 61 | } else { 62 | std::cout << "Table:" << table << " set key ok" << std::endl; 63 | } 64 | delete client; 65 | } 66 | } 67 | 68 | delete cluster; 69 | 70 | return 0; 71 | } 72 | -------------------------------------------------------------------------------- /libzp/libzp/example/Makefile: -------------------------------------------------------------------------------- 1 | CLEAN_FILES = # deliberately empty, so we can append below. 2 | CXX = g++ 3 | LDFLAGS= -lpthread -lrt -lprotobuf 4 | CXXFLAGS= -g -std=c++11 -fno-builtin-memcmp -msse -msse4.2 5 | PROFILING_FLAGS = -pg 6 | ARFLAGS = rs 7 | OPT= 8 | 9 | .PHONY: clean all 10 | 11 | all: zp_simple zp_timeout zp_get zp_mget zp_async zp_table zp_hashtag 12 | 13 | ifndef ZP_PATH 14 | $(warning Warning: missing zp path, using default) 15 | ZP_PATH = $(CURDIR)/../.. 16 | endif 17 | ZP_INCLUDE_DIR=$(ZP_PATH) 18 | ZP_LIBRARY=$(ZP_PATH)/libzp/lib/libzp.a 19 | 20 | ifndef PINK_PATH 21 | $(warning Warning: missing pink path, using default) 22 | PINK_PATH=$(CURDIR)/../../../third/pink 23 | endif 24 | PINK_INCLUDE_DIR=$(PINK_PATH) 25 | PINK_LIBRARY=$(PINK_PATH)/pink/lib/libpink.a 26 | 27 | ifndef SLASH_PATH 28 | $(warning Warning: missing slash path, using default) 29 | SLASH_PATH=$(CURDIR)/../../../third/slash 30 | endif 31 | SLASH_INCLUDE_DIR=$(SLASH_PATH) 32 | SLASH_LIBRARY=$(SLASH_PATH)/slash/lib/libslash.a 33 | 34 | CXXFLAGS+= -I$(PINK_INCLUDE_DIR) -I$(SLASH_INCLUDE_DIR) -I$(ZP_INCLUDE_DIR) 35 | DEP_LIBS = $(ZP_LIBRARY) $(PINK_LIBRARY) $(SLASH_LIBRARY) 36 | 37 | zp_simple: zp_simple.cc $(DEP_LIBS) 38 | $(CXX) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) 39 | 40 | zp_timeout: zp_timeout.cc $(DEP_LIBS) 41 | $(CXX) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) 42 | 43 | zp_get: zp_get.cc $(DEP_LIBS) 44 | $(CXX) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) 45 | 46 | zp_mget: zp_mget.cc $(DEP_LIBS) 47 | $(CXX) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) 48 | 49 | # zp_create_table: zp_create_table.cc $(DEP_LIBS) 50 | # $(CXX) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) 51 | 52 | zp_async: zp_async.cc $(DEP_LIBS) 53 | $(CXX) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) 54 | 55 | zp_table: zp_table.cc $(DEP_LIBS) 56 | $(CXX) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) 57 | 58 | zp_hashtag: zp_hashtag.cc $(DEP_LIBS) 59 | $(CXX) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) 60 | 61 | # zp_parallel: zp_parallel.cc $(DEP_LIBS) 62 | # $(CXX) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) 63 | 64 | clean: 65 | find . -name "*.[oda]" -exec rm -f {} \; 66 | rm -rf ./zp_simple ./zp_timeout ./zp_get ./zp_mget ./zp_create_table ./zp_async ./zp_parallel ./zp_table ./zp_hashtag 67 | -------------------------------------------------------------------------------- /manager/golang/manager.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "zp_manager" 4 | import "fmt" 5 | 6 | func main() { 7 | op_timeout := 1000 8 | m1 := zp_manager.ZpNode{IP : "127.0.0.1", Port : 9221} 9 | m2 := zp_manager.ZpNode{IP : "127.0.0.1", Port : 9222} 10 | cluster := zp_manager.NewZpCluster(op_timeout, m1, m2) 11 | 12 | cluster.Pull("test") 13 | 14 | suc, _, res := cluster.ListTable() 15 | if suc { 16 | for _, v := range *res { 17 | fmt.Println(v) 18 | } 19 | } 20 | 21 | suc, _, master, slaves := cluster.ListMeta() 22 | if suc { 23 | fmt.Println(*master) 24 | for _, v := range *slaves { 25 | fmt.Println(v) 26 | } 27 | } 28 | 29 | suc, _, nodes, status := cluster.ListNode() 30 | if suc { 31 | for _, v := range *nodes { 32 | fmt.Println(v) 33 | } 34 | for _, v := range *status { 35 | fmt.Println(v) 36 | } 37 | } 38 | 39 | key := []byte("testkey1") 40 | value := []byte("testvalue1") 41 | cluster.Set("test", &key, &value, -1) 42 | 43 | _, _, v := cluster.Get("test", &key) 44 | str := string(v[:]) 45 | fmt.Println(str) 46 | 47 | keys := make([]string, 1) 48 | keys = append(keys, "testkey1") 49 | result := make(map[string]string, 1) 50 | cluster.Mget("test", keys, &result) 51 | for k, v := range result { 52 | fmt.Println(k) 53 | fmt.Println(v) 54 | } 55 | 56 | // cluster.CreateTable("test4", 3) 57 | cluster.Delete("test", "testkey1") 58 | 59 | _, reason, qps, total_query := cluster.InfoQps("test") 60 | fmt.Println(reason) 61 | fmt.Printf("qps: %d\n", qps) 62 | fmt.Printf("total_query: %d\n", total_query) 63 | 64 | views := make(map[int]zp_manager.ZpPartitionView) 65 | d1 := zp_manager.ZpNode{IP : "127.0.0.1", Port : 13222} 66 | suc, reason = cluster.InfoRepl(d1, "test", &views) 67 | if suc { 68 | for k, v := range views { 69 | fmt.Printf("%d ", k) 70 | fmt.Println(v) 71 | } 72 | } else { 73 | fmt.Println(reason) 74 | } 75 | 76 | space_info := make(map[zp_manager.ZpNode]zp_manager.ZpSpaceInfo) 77 | suc, reason = cluster.InfoSpace("test", &space_info) 78 | if suc { 79 | for k, v := range space_info { 80 | fmt.Println(k, v) 81 | } 82 | } else { 83 | fmt.Println(reason) 84 | } 85 | 86 | var state zp_manager.ZpServerState 87 | suc, reason = cluster.InfoServer(d1, &state) 88 | if suc { 89 | fmt.Println(state) 90 | } else { 91 | fmt.Println(reason) 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /manager/zp_loopset.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright [2016] qihoo" 3 | */ 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "libzp/include/zp_cluster.h" 13 | 14 | 15 | void usage() { 16 | std::cout << "usage:\n" 17 | << " ./zp_loopset host port table partition value_len requrestnum\n"; 18 | } 19 | 20 | int main(int argc, char* argv[]) { 21 | if (argc != 7) { 22 | usage(); 23 | return -1; 24 | } 25 | std::string host = argv[1]; 26 | int port = atoi(argv[2]); 27 | std::string table = argv[3]; 28 | int pid = atoi(argv[4]); 29 | int value_len = atoi(argv[5]); 30 | int rnum = atoi(argv[6]); 31 | 32 | 33 | libzp::Options option; 34 | libzp::Node node(host, port); 35 | option.meta_addr.push_back(node); 36 | 37 | libzp::Cluster *cluster = new libzp::Cluster(option); 38 | libzp::Status s = cluster->Connect(); 39 | if (!s.ok()) { 40 | std::cout << "client " << std::this_thread::get_id() 41 | << " connect server failed: " << s.ToString() << std::endl; 42 | delete cluster; 43 | return -1; 44 | } 45 | s = cluster->Pull(table); 46 | if (!s.ok()) { 47 | std::cout << "client " << std::this_thread::get_id() 48 | << " pull table " << table << " failed: " << s.ToString() << std::endl; 49 | delete cluster; 50 | return -1; 51 | } 52 | 53 | std::string key; 54 | for (int i = 0; i < 10000000; ++i) { 55 | key = std::to_string(i); 56 | if (cluster->LocateKey(table, key) == pid) { 57 | break; 58 | } 59 | } 60 | if (key.empty()) { 61 | std::cout << "failed to find a key on " << pid; 62 | return -1; 63 | } 64 | 65 | // Set 66 | std::string* value = new std::string(value_len, 'a'); 67 | for (int i = 0; i < rnum; i++) { 68 | s= cluster->Set(table, key, *value); 69 | if (!s.ok()) { 70 | std::cout << "client set key " 71 | << key << " failed: " << s.ToString() << std::endl; 72 | delete value; 73 | delete cluster; 74 | return -1; 75 | } 76 | 77 | s= cluster->Delete(table, key); 78 | if (!s.ok()) { 79 | std::cout << "client delete key " 80 | << key << " failed: " << s.ToString() << std::endl; 81 | delete value; 82 | delete cluster; 83 | return -1; 84 | } 85 | } 86 | 87 | delete value; 88 | delete cluster; 89 | std::cout << "Bye!!" << std::endl; 90 | return 0; 91 | } 92 | -------------------------------------------------------------------------------- /javazeppelin/test/TestZeppelinThread.java: -------------------------------------------------------------------------------- 1 | import java.util.TreeMap; 2 | import java.util.Map; 3 | import net.qihoo.zeppelin.Zeppelin; 4 | import net.qihoo.zeppelin.ZeppelinException; 5 | 6 | public class TestZeppelinThread extends Thread { 7 | int pauseTime; 8 | public TestZeppelinThread(int pTime) { 9 | this.pauseTime = pTime; 10 | } 11 | 12 | 13 | public void run() { 14 | Zeppelin test = new Zeppelin("127.0.0.1", "9221", "test1"); 15 | for (int i = 0; i < 10; ++i) { 16 | this.doIt(test); 17 | try { 18 | Thread.sleep(pauseTime * 100); 19 | } catch (Exception e) { 20 | e.printStackTrace(); 21 | } 22 | } 23 | try { 24 | test.removeZeppelin(); 25 | } catch(ZeppelinException e) { 26 | e.printStackTrace(); 27 | } 28 | } 29 | 30 | public void doIt(Zeppelin test) { 31 | try { 32 | String key1 = "key1"; 33 | String key2 = "key2"; 34 | String key3 = "key3"; 35 | String value1 = "value1"; 36 | String value2 = "value2"; 37 | String value3 = "value3"; 38 | 39 | boolean res = test.set(key1, value1); 40 | System.out.println("thread" + pauseTime + " set " + key1 + " " + value1 + " "+ res); 41 | res = test.set(key2, value2); 42 | System.out.println("thread" + pauseTime + " set " + key2 + " " + value2 + " " + res); 43 | res = test.set(key3, value3); 44 | System.out.println("thread" + pauseTime + " set " + key3 + " " + value3 + " " + res); 45 | 46 | String value = test.get(key1); 47 | System.out.println("thread" + pauseTime + " get " + key1 + ": " + value); 48 | 49 | res = test.delete(key1); 50 | System.out.println("thread" + pauseTime + " delete " + key1 + ": " + res); 51 | res = test.delete("key9"); 52 | System.out.println("thread" + pauseTime + " delete key9" + ": " + res); 53 | value = test.get(key1); 54 | System.out.println("thread" + pauseTime + " get " + key1 + ": " + value); 55 | 56 | String[] keys = {key2, "key6"}; 57 | TreeMap tm = test.mget(keys); 58 | for (Map.Entry m:tm.entrySet()) { 59 | System.out.println("thread" + pauseTime + " mget" + " " + m.getKey() + " " + m.getValue()); 60 | } 61 | } catch(ZeppelinException e) { 62 | e.printStackTrace(); 63 | } 64 | } 65 | 66 | public static void main(String[] args) { 67 | for (int i = 1; i <= 100; i++) { 68 | TestZeppelinThread tqt = new TestZeppelinThread(i); 69 | tqt.start(); 70 | } 71 | } 72 | 73 | } 74 | 75 | 76 | -------------------------------------------------------------------------------- /php7_zeppelin/php_zeppelin.h: -------------------------------------------------------------------------------- 1 | /* 2 | +----------------------------------------------------------------------+ 3 | | PHP Version 7 | 4 | +----------------------------------------------------------------------+ 5 | | Copyright (c) 1997-2017 The PHP Group | 6 | +----------------------------------------------------------------------+ 7 | | This source file is subject to version 3.01 of the PHP license, | 8 | | that is bundled with this package in the file LICENSE, and is | 9 | | available through the world-wide-web at the following url: | 10 | | http://www.php.net/license/3_01.txt | 11 | | If you did not receive a copy of the PHP license and are unable to | 12 | | obtain it through the world-wide-web, please send a note to | 13 | | license@php.net so we can mail you a copy immediately. | 14 | +----------------------------------------------------------------------+ 15 | | Author: | 16 | +----------------------------------------------------------------------+ 17 | */ 18 | 19 | /* $Id$ */ 20 | 21 | #ifndef PHP_ZEPPELIN_H 22 | #define PHP_ZEPPELIN_H 23 | 24 | extern zend_module_entry zeppelin_module_entry; 25 | #define phpext_zeppelin_ptr &zeppelin_module_entry 26 | 27 | #define PHP_ZEPPELIN_VERSION "0.2.8" /* Replace with version number for your extension */ 28 | 29 | #ifdef PHP_WIN32 30 | # define PHP_ZEPPELIN_API __declspec(dllexport) 31 | #elif defined(__GNUC__) && __GNUC__ >= 4 32 | # define PHP_ZEPPELIN_API __attribute__ ((visibility("default"))) 33 | #else 34 | # define PHP_ZEPPELIN_API 35 | #endif 36 | 37 | #ifdef ZTS 38 | #include "TSRM.h" 39 | #endif 40 | 41 | /* 42 | Declare any global variables you may need between the BEGIN 43 | and END macros here: 44 | */ 45 | 46 | ZEND_BEGIN_MODULE_GLOBALS(zeppelin) 47 | void* g_zp_cli; 48 | ZEND_END_MODULE_GLOBALS(zeppelin) 49 | 50 | /* Always refer to the globals in your function as ZEPPELIN_G(variable). 51 | You are encouraged to rename these macros something shorter, see 52 | examples in any other php module directory. 53 | */ 54 | #define ZEPPELIN_G(v) ZEND_MODULE_GLOBALS_ACCESSOR(zeppelin, v) 55 | 56 | #if defined(ZTS) && defined(COMPILE_DL_ZEPPELIN) 57 | ZEND_TSRMLS_CACHE_EXTERN() 58 | #endif 59 | 60 | #endif /* PHP_ZEPPELIN_H */ 61 | 62 | 63 | /* 64 | * Local variables: 65 | * tab-width: 4 66 | * c-basic-offset: 4 67 | * End: 68 | * vim600: noet sw=4 ts=4 fdm=marker 69 | * vim<600: noet sw=4 ts=4 70 | */ 71 | -------------------------------------------------------------------------------- /libzp/libzp/include/zp_option.h: -------------------------------------------------------------------------------- 1 | #ifndef CLIENT_INCLUDE_ZP_OPTION_H_ 2 | #define CLIENT_INCLUDE_ZP_OPTION_H_ 3 | #include 4 | #include "slash/include/slash_status.h" 5 | #include "slash/include/slash_string.h" 6 | 7 | namespace libzp { 8 | 9 | using slash::Status; 10 | 11 | const std::string kLBrace = "#ZPLBRACE%#"; 12 | const std::string kRBrace = "#ZPRBRACE%#"; 13 | 14 | struct Node { 15 | Node(const std::string& other_ip, int other_port) 16 | : ip(other_ip), 17 | port(other_port) { 18 | } 19 | Node(const char* other_ip, int other_port) 20 | : ip(other_ip), 21 | port(other_port) { 22 | } 23 | Node(const std::string& ip_port) { 24 | slash::ParseIpPortString(ip_port, ip, port); 25 | } 26 | Node() : port(0) {} 27 | 28 | std::string ip; 29 | int port; 30 | 31 | std::string ToString() const { 32 | char buf[100]; 33 | sprintf(buf, "%s:%d", ip.c_str(), port); 34 | return std::string(buf); 35 | } 36 | 37 | bool operator < (const Node& other) const { 38 | return (slash::IpPortString(ip, port) < 39 | slash::IpPortString(other.ip, other.port)); 40 | } 41 | 42 | bool operator == (const Node& other) const { 43 | return (ip == other.ip && port == other.port); 44 | } 45 | 46 | bool operator != (const Node& other) const { 47 | return (ip != other.ip || port != other.port); 48 | } 49 | 50 | friend std::ostream& operator<< (std::ostream& stream, const Node& node) { 51 | stream << node.ip << ":" << node.port; 52 | return stream; 53 | } 54 | }; 55 | 56 | struct Options { 57 | // How many milliseconds to wait for a response 58 | // before returning an error from a zeppelin operation. 0 means no limit 59 | // Default 1000 60 | size_t op_timeout; 61 | 62 | // Data node connnection pool size 63 | // Default 32 64 | size_t connection_pool_capacity; 65 | 66 | // meta address collection 67 | std::vector meta_addr; 68 | 69 | Options() : op_timeout(1000), connection_pool_capacity(32) {} 70 | }; 71 | 72 | struct Result { 73 | Status ret; 74 | const std::string key; 75 | const std::string* value; 76 | const std::map* kvs; 77 | 78 | Result(Status r, const std::string& k) 79 | : ret(r), key(k), value(NULL), kvs(NULL) {} 80 | 81 | Result(Status r, const std::string& k, const std::string* v) 82 | : ret(r), key(k), value(v), kvs(NULL) {} 83 | 84 | Result(Status r, const std::map* vs) 85 | : ret(r), value(NULL), kvs(vs) {} 86 | }; 87 | 88 | typedef void (*zp_completion_t)(const struct Result& stat, 89 | void* data); 90 | 91 | } 92 | 93 | #endif 94 | -------------------------------------------------------------------------------- /manager/rocksdb_to_zp.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "rocksdb/db.h" 3 | 4 | #include "slash/include/env.h" 5 | #include "libzp/include/zp_cluster.h" 6 | 7 | void usage() { 8 | std::cout << "usage:\n" 9 | << " ./rocksdb_to_zp db_path meta_ip meta_port table_name\n"; 10 | } 11 | 12 | int main(int argc, char* argv[]) { 13 | if (argc != 5) { 14 | usage(); 15 | return -1; 16 | } 17 | std::string db_path(argv[1]); 18 | std::string meta_ip(argv[2]); 19 | int meta_port = atoi(argv[3]); 20 | std::string table_name(argv[4]); 21 | 22 | if (!slash::FileExists(db_path)) { 23 | std::cout << "Db not exist: " << db_path << std::endl; 24 | usage(); 25 | return -1; 26 | } 27 | 28 | // Connect zp 29 | libzp::Options option; 30 | libzp::Node node(meta_ip, meta_port); 31 | option.meta_addr.push_back(node); 32 | libzp::Cluster *cluster = new libzp::Cluster(option); 33 | libzp::Status s = cluster->Connect(); 34 | if (!s.ok()) { 35 | std::cout << "connect meta server failed! meta_ip: " << meta_ip 36 | << ", meta_port: " << meta_port 37 | << ", Error: "<< s.ToString() << std::endl; 38 | delete cluster; 39 | return -1; 40 | } 41 | 42 | // Open rocksdb 43 | rocksdb::DB* db; 44 | rocksdb::Options options; 45 | rocksdb::Status status = rocksdb::DB::Open(options, db_path, &db); 46 | if (!status.ok()) { 47 | std::cout << "Failed to open db: " << db_path 48 | << ", error: " << status.ToString() << std::endl; 49 | delete cluster; 50 | return -1; 51 | } 52 | 53 | std::cout << "Import data from rocksdb, db:" << db_path 54 | << ", meta_ip: " << meta_ip << ", meta_port: " << meta_port 55 | << ", table_name: " << table_name << ". y or n?"<< std::endl; 56 | char c; 57 | std::cin >> c; 58 | if (c != 'y') { 59 | delete cluster; 60 | delete db; 61 | return -1; 62 | } 63 | 64 | // Iterator all key value and write into zp 65 | int succ_num = 0, fail_num = 0; 66 | rocksdb::Iterator* it = db->NewIterator(rocksdb::ReadOptions()); 67 | for (it->SeekToFirst(); it->Valid(); it->Next()) { 68 | s = cluster->Set(table_name, it->key().ToString(), it->value().ToString()); 69 | if (!s.ok()) { 70 | std::cout << "set key " << it->key().ToString() << " failed: " 71 | << s.ToString() << std::endl; 72 | fail_num++; 73 | continue; 74 | } 75 | succ_num++; 76 | std::cout << "set key " << it->key().ToString() << " success." << std::endl; 77 | } 78 | std::cout << "Finished. succ: " << succ_num 79 | << ", fail: " << fail_num << std::endl; 80 | 81 | delete it; 82 | delete db; 83 | delete cluster; 84 | return 0; 85 | } 86 | 87 | -------------------------------------------------------------------------------- /phpzeppelin/config.m4: -------------------------------------------------------------------------------- 1 | dnl $Id$ 2 | dnl config.m4 for extension zeppelin 3 | 4 | dnl Comments in this file start with the string 'dnl'. 5 | dnl Remove where necessary. This file will not work 6 | dnl without editing. 7 | 8 | dnl If your extension references something external, use with: 9 | 10 | dnl PHP_ARG_WITH(zeppelin, for zeppelin support, 11 | dnl Make sure that the comment is aligned: 12 | dnl [ --with-zeppelin Include zeppelin support]) 13 | 14 | dnl Otherwise use enable: 15 | 16 | PHP_ARG_ENABLE(zeppelin, whether to enable zeppelin support, 17 | dnl Make sure that the comment is aligned: 18 | [ --enable-zeppelin Enable zeppelin support]) 19 | 20 | if test "$PHP_ZEPPELIN" != "no"; then 21 | dnl Write more examples of tests here... 22 | 23 | dnl # --with-zeppelin -> check with-path 24 | dnl SEARCH_PATH="/usr/local /usr" # you might want to change this 25 | dnl SEARCH_FOR="/include/zeppelin.h" # you most likely want to change this 26 | dnl if test -r $PHP_ZEPPELIN/$SEARCH_FOR; then # path given as parameter 27 | dnl ZEPPELIN_DIR=$PHP_ZEPPELIN 28 | dnl else # search default path list 29 | dnl AC_MSG_CHECKING([for zeppelin files in default path]) 30 | dnl for i in $SEARCH_PATH ; do 31 | dnl if test -r $i/$SEARCH_FOR; then 32 | dnl ZEPPELIN_DIR=$i 33 | dnl AC_MSG_RESULT(found in $i) 34 | dnl fi 35 | dnl done 36 | dnl fi 37 | dnl 38 | dnl if test -z "$ZEPPELIN_DIR"; then 39 | dnl AC_MSG_RESULT([not found]) 40 | dnl AC_MSG_ERROR([Please reinstall the zeppelin distribution]) 41 | dnl fi 42 | 43 | dnl # --with-zeppelin -> add include path 44 | PHP_ADD_INCLUDE(../libzp) 45 | PHP_ADD_INCLUDE(../third/pink) 46 | PHP_ADD_INCLUDE(../third/slash) 47 | 48 | dnl # --with-zeppelin -> check for lib and symbol presence 49 | dnl LIBNAME=zeppelin # you may want to change this 50 | dnl LIBSYMBOL=zeppelin # you most likely want to change this 51 | 52 | dnl PHP_CHECK_LIBRARY($LIBNAME,$LIBSYMBOL, 53 | dnl [ 54 | dnl PHP_ADD_LIBRARY_WITH_PATH($LIBNAME, $ZEPPELIN_DIR/lib, ZEPPELIN_SHARED_LIBADD) 55 | dnl AC_DEFINE(HAVE_ZEPPELINLIB,1,[ ]) 56 | dnl ],[ 57 | dnl AC_MSG_ERROR([wrong zeppelin lib version or lib not found]) 58 | dnl ],[ 59 | dnl -L$ZEPPELIN_DIR/lib -lm 60 | dnl ]) 61 | dnl 62 | dnl PHP_SUBST(ZEPPELIN_SHARED_LIBADD) 63 | 64 | PHP_REQUIRE_CXX() 65 | PHP_ADD_LIBRARY(stdc++, 1, EXTRA_LDFLAGS) 66 | PHP_ADD_LIBRARY(protobuf, 1, EXTRA_LDFLAGS) 67 | PHP_ADD_LIBRARY_WITH_PATH(slash, ../third/slash/slash/lib, EXTRA_LDFLAGS) 68 | PHP_ADD_LIBRARY_WITH_PATH(pink, ../third/pink/pink/lib, EXTRA_LDFLAGS) 69 | PHP_ADD_LIBRARY_WITH_PATH(zp, ../libzp/libzp/lib, EXTRA_LDFLAGS) 70 | 71 | CPPFILE="zeppelin.cc" 72 | 73 | PHP_NEW_EXTENSION(zeppelin, $CPPFILE, $ext_shared) 74 | fi 75 | -------------------------------------------------------------------------------- /php7_zeppelin/config.m4: -------------------------------------------------------------------------------- 1 | dnl $Id$ 2 | dnl config.m4 for extension zeppelin 3 | 4 | dnl Comments in this file start with the string 'dnl'. 5 | dnl Remove where necessary. This file will not work 6 | dnl without editing. 7 | 8 | dnl If your extension references something external, use with: 9 | 10 | dnl PHP_ARG_WITH(zeppelin, for zeppelin support, 11 | dnl Make sure that the comment is aligned: 12 | dnl [ --with-zeppelin Include zeppelin support]) 13 | 14 | dnl Otherwise use enable: 15 | 16 | PHP_ARG_ENABLE(zeppelin, whether to enable zeppelin support, 17 | dnl Make sure that the comment is aligned: 18 | [ --enable-zeppelin Enable zeppelin support]) 19 | 20 | if test "$PHP_ZEPPELIN" != "no"; then 21 | dnl Write more examples of tests here... 22 | 23 | dnl # --with-zeppelin -> check with-path 24 | dnl SEARCH_PATH="/usr/local /usr" # you might want to change this 25 | dnl SEARCH_FOR="/include/zeppelin.h" # you most likely want to change this 26 | dnl if test -r $PHP_ZEPPELIN/$SEARCH_FOR; then # path given as parameter 27 | dnl ZEPPELIN_DIR=$PHP_ZEPPELIN 28 | dnl else # search default path list 29 | dnl AC_MSG_CHECKING([for zeppelin files in default path]) 30 | dnl for i in $SEARCH_PATH ; do 31 | dnl if test -r $i/$SEARCH_FOR; then 32 | dnl ZEPPELIN_DIR=$i 33 | dnl AC_MSG_RESULT(found in $i) 34 | dnl fi 35 | dnl done 36 | dnl fi 37 | dnl 38 | dnl if test -z "$ZEPPELIN_DIR"; then 39 | dnl AC_MSG_RESULT([not found]) 40 | dnl AC_MSG_ERROR([Please reinstall the zeppelin distribution]) 41 | dnl fi 42 | 43 | dnl # --with-zeppelin -> add include path 44 | PHP_ADD_INCLUDE(../libzp) 45 | PHP_ADD_INCLUDE(../third/pink) 46 | PHP_ADD_INCLUDE(../third/slash) 47 | 48 | dnl # --with-zeppelin -> check for lib and symbol presence 49 | dnl LIBNAME=zeppelin # you may want to change this 50 | dnl LIBSYMBOL=zeppelin # you most likely want to change this 51 | 52 | dnl PHP_CHECK_LIBRARY($LIBNAME,$LIBSYMBOL, 53 | dnl [ 54 | dnl PHP_ADD_LIBRARY_WITH_PATH($LIBNAME, $ZEPPELIN_DIR/$PHP_LIBDIR, ZEPPELIN_SHARED_LIBADD) 55 | dnl AC_DEFINE(HAVE_ZEPPELINLIB,1,[ ]) 56 | dnl ],[ 57 | dnl AC_MSG_ERROR([wrong zeppelin lib version or lib not found]) 58 | dnl ],[ 59 | dnl -L$ZEPPELIN_DIR/$PHP_LIBDIR -lm 60 | dnl ]) 61 | dnl 62 | dnl PHP_SUBST(ZEPPELIN_SHARED_LIBADD) 63 | 64 | PHP_REQUIRE_CXX() 65 | PHP_ADD_LIBRARY(stdc++, 1, EXTRA_LDFLAGS) 66 | PHP_ADD_LIBRARY(protobuf, 1, EXTRA_LDFLAGS) 67 | PHP_ADD_LIBRARY_WITH_PATH(slash, ../third/slash/slash/lib, EXTRA_LDFLAGS) 68 | PHP_ADD_LIBRARY_WITH_PATH(pink, ../third/pink/pink/lib, EXTRA_LDFLAGS) 69 | PHP_ADD_LIBRARY_WITH_PATH(zp, ../libzp/libzp/lib, EXTRA_LDFLAGS) 70 | 71 | PHP_NEW_EXTENSION(zeppelin, zeppelin.cc, $ext_shared,, -DZEND_ENABLE_STATIC_TSRMLS_CACHE=1) 72 | fi 73 | -------------------------------------------------------------------------------- /libzp/libzp/include/zp_client_c.h: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright [2016] qihoo" 3 | */ 4 | #ifndef CLIENT_INCLUDE_ZP_CLIENT_C_H_ 5 | #define CLIENT_INCLUDE_ZP_CLIENT_C_H_ 6 | 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | 11 | typedef struct zp_status_t zp_status_t; 12 | typedef struct zp_option_t zp_option_t; 13 | typedef struct zp_client_t zp_client_t; 14 | typedef struct zp_node_t zp_node_t; 15 | typedef struct zp_node_vec_t zp_node_vec_t; 16 | typedef struct zp_string_t zp_string_t; 17 | typedef struct zp_string_vec_t zp_string_vec_t; 18 | 19 | // struct's constructor and destructor 20 | extern int zp_status_ok(const zp_status_t* s); 21 | extern zp_string_t* zp_status_tostring(const zp_status_t* s); 22 | extern void zp_status_destroy(zp_status_t* s); 23 | 24 | extern zp_option_t* zp_option_create(zp_node_vec_t* metas, int op_timeout); 25 | extern void zp_option_destroy(zp_option_t* option); 26 | 27 | extern zp_client_t* zp_client_create(const zp_option_t* options, const char* tb_name); 28 | extern void zp_client_destroy(zp_client_t* client); 29 | 30 | extern zp_node_t* zp_node_create1(const char* ip, int port); 31 | extern zp_node_t* zp_node_create(); 32 | extern void zp_node_destroy(zp_node_t* node); 33 | extern char* zp_node_ip(zp_node_t* node); 34 | extern int zp_node_port(zp_node_t* node); 35 | 36 | extern zp_node_vec_t* zp_nodevec_create(); 37 | extern void zp_nodevec_destroy(zp_node_vec_t* vec); 38 | extern void zp_nodevec_pushback(zp_node_vec_t* nodevec, const zp_node_t* node); 39 | extern zp_node_t* zp_nodevec_popback(zp_node_vec_t* nodevec); 40 | 41 | extern zp_string_t* zp_string_create1(const char* data, int length); 42 | extern zp_string_t* zp_string_create(); 43 | extern void zp_string_destroy(zp_string_t* str); 44 | extern char* zp_string_data(zp_string_t* str); 45 | extern int zp_string_length(zp_string_t* str); 46 | 47 | extern zp_string_vec_t* zp_strvec_create(); 48 | extern void zp_strvec_destroy(zp_string_vec_t* vec); 49 | extern void zp_strvec_pushback(zp_string_vec_t* nodevec, zp_string_t* str); 50 | extern zp_string_t* zp_strvec_popback(zp_string_vec_t* strvec); 51 | 52 | // Zeppelin client interface 53 | extern zp_status_t* zp_set( 54 | const zp_client_t* client, 55 | const zp_string_t* key, 56 | const zp_string_t* value, 57 | int ttl); 58 | 59 | extern zp_status_t* zp_get( 60 | const zp_client_t* client, 61 | const zp_string_t* key, 62 | zp_string_t* value); 63 | 64 | extern zp_status_t* zp_mget( 65 | const zp_client_t* client, 66 | zp_string_vec_t* keys, 67 | zp_string_vec_t* res_keys, 68 | zp_string_vec_t* res_values); 69 | 70 | extern zp_status_t* zp_delete( 71 | const zp_client_t* client, 72 | const zp_string_t* key); 73 | 74 | #ifdef __cplusplus 75 | } // extern "C" 76 | #endif 77 | 78 | #endif // CLIENT_INCLUDE_ZP_CLIENT_H_ 79 | -------------------------------------------------------------------------------- /libzp/libzp/src/c_test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "libzp/include/zp_cluster_c.h" 5 | 6 | int main() { 7 | zp_node_t* n = zp_node_create("127.0.0.1", 9221); 8 | zp_node_vec_t* metas = zp_nodevec_create(); 9 | zp_nodevec_pushback(metas, n); 10 | zp_option_t* option = zp_option_create(metas, 1000); 11 | 12 | zp_cluster_t* cluster = zp_cluster_create(option); 13 | 14 | char buf[1024]; 15 | 16 | // Pull 17 | zp_status_t* s = zp_pull(cluster, "test"); 18 | if (!zp_status_ok(s)) { 19 | zp_status_tostring(s, buf, sizeof(buf)); 20 | printf("error: %s\n", buf); 21 | } 22 | zp_status_destroy(s); 23 | 24 | // ListTable 25 | zp_string_vec_t* tables = zp_strvec_create(); 26 | s = zp_list_table(cluster, tables); 27 | if (!zp_status_ok(s)) { 28 | zp_status_tostring(s, buf, sizeof(buf)); 29 | printf("error: %s\n", buf); 30 | } 31 | zp_status_destroy(s); 32 | printf("ListTable:\n"); 33 | while (zp_strvec_popback(tables, buf, sizeof(buf)) > 0) { 34 | printf(" --- %s\n", buf); 35 | } 36 | zp_strvec_destroy(tables); 37 | 38 | // ListMeta 39 | zp_node_t* master = zp_node_create("", 0); 40 | zp_node_vec_t* slaves = zp_nodevec_create(); 41 | s = zp_list_meta(cluster, master, slaves); 42 | if (!zp_status_ok(s)) { 43 | zp_status_tostring(s, buf, sizeof(buf)); 44 | printf("error: %s\n", buf); 45 | } 46 | zp_status_destroy(s); 47 | printf("Meta master: %s:%d\n", master->ip, master->port); 48 | zp_node_t* slave; 49 | while ((slave = zp_nodevec_popback(slaves)) != NULL) { 50 | printf("Meta slave: %s:%d\n", slave->ip, slave->port); 51 | zp_node_destroy(slave); 52 | } 53 | zp_node_destroy(master); 54 | zp_nodevec_destroy(slaves); 55 | 56 | // ListNode 57 | zp_node_vec_t* nodes = zp_nodevec_create(); 58 | zp_string_vec_t* status = zp_strvec_create(); 59 | s = zp_list_node(cluster, nodes, status); 60 | if (!zp_status_ok(s)) { 61 | zp_status_tostring(s, buf, sizeof(buf)); 62 | printf("error: %s\n", buf); 63 | } 64 | zp_status_destroy(s); 65 | zp_node_t* node; 66 | while ((node = zp_nodevec_popback(nodes)) != NULL) { 67 | zp_strvec_popback(status, buf, sizeof(buf)); 68 | printf("node: %s:%d, %s\n", node->ip, node->port, buf); 69 | zp_node_destroy(node); 70 | } 71 | zp_nodevec_destroy(nodes); 72 | zp_strvec_destroy(status); 73 | 74 | // CreateTable 75 | const char* table_name = "test5"; 76 | int partition_num = 4; 77 | s = zp_create_table(cluster, table_name, partition_num); 78 | if (!zp_status_ok(s)) { 79 | zp_status_tostring(s, buf, sizeof(buf)); 80 | printf("error: %s\n", buf); 81 | } else { 82 | printf("create table %s success\n", table_name); 83 | } 84 | zp_status_destroy(s); 85 | 86 | zp_nodevec_destroy(metas); 87 | zp_node_destroy(n); 88 | zp_option_destroy(option); 89 | zp_cluster_destroy(cluster); 90 | } 91 | -------------------------------------------------------------------------------- /libzp/libzp/example/zp_async.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright [2016] qihoo" 3 | */ 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "libzp/include/zp_client.h" 9 | 10 | void usage() { 11 | std::cout << "usage:\n" 12 | << " zp_async host port count\n"; 13 | } 14 | 15 | static std::string user_data = "ASYNC"; 16 | 17 | void set_callback(const struct libzp::Result& stat, void* data) { 18 | std::cout << "set " << stat.key << "callback called. Result: "<< stat.ret.ToString() 19 | << ", usr data: "<< *(static_cast(data)) << std::endl; 20 | } 21 | 22 | void get_callback(const struct libzp::Result& stat, void* data) { 23 | std::cout << "get " << stat.key << " callback called. Result: "<< stat.ret.ToString() 24 | << ", value:" << *(stat.value) 25 | << ", usr data: "<< *(static_cast(data)) << std::endl; 26 | } 27 | 28 | void delete_callback(const struct libzp::Result& stat, void* data) { 29 | std::cout << "delete " << stat.key << " callback called. Result: "<< stat.ret.ToString() 30 | << ", usr data: "<< *(static_cast(data)) << std::endl; 31 | } 32 | 33 | void mget_callback(const struct libzp::Result& stat, void* data) { 34 | std::cout << "mget callback called. Result: "<< stat.ret.ToString() << std::endl; 35 | for (auto& get : *stat.kvs) { 36 | std::cout << get.first << " <-> " << get.second << std::endl; 37 | } 38 | std::cout << "usr data: "<< *(static_cast(data)) << std::endl; 39 | } 40 | 41 | int main(int argc, char* argv[]) { 42 | if (argc != 4) { 43 | usage(); 44 | return -1; 45 | } 46 | 47 | // client handle io operation 48 | std::cout << "create client" << std::endl; 49 | libzp::Client* client = new libzp::Client(argv[1], atoi(argv[2]), "parade"); 50 | std::cout << "connect cluster" << std::endl; 51 | int cnt = atoi(argv[3]); 52 | 53 | for(int i = 0; i < cnt; i++) { 54 | std::string key = "key" + std::to_string(i); 55 | std::string value(30, 'a'); 56 | client->Aset(key, value, set_callback, &user_data); 57 | } 58 | std::cout << std::endl; 59 | 60 | for(int i = 0; i < cnt; i++) { 61 | std::string key = "key" + std::to_string(i); 62 | client->Aget(key, get_callback, &user_data); 63 | } 64 | 65 | std::vector keys; 66 | for(int i = 0; i < cnt; i = i + 10) { 67 | keys.clear(); 68 | for (int j = 0; j < 10; ++j) { 69 | std::string key = "key" + std::to_string(i + j); 70 | keys.push_back(key); 71 | } 72 | client->Amget(keys, mget_callback, &user_data); 73 | } 74 | 75 | for(int i = 0; i < cnt; i++) { 76 | std::string key = "key" + std::to_string(i); 77 | client->Adelete(key, delete_callback, &user_data); 78 | } 79 | std::cout << std::endl; 80 | 81 | // wait async process finished 82 | sleep(10); 83 | 84 | 85 | delete client; 86 | } 87 | -------------------------------------------------------------------------------- /libzp/libzp/example/zp_mget.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright [2016] qihoo" 3 | */ 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "libzp/include/zp_client.h" 12 | 13 | 14 | void usage() { 15 | std::cout << "usage:\n" 16 | << " zp_cli host port cnt\n"; 17 | } 18 | 19 | int main(int argc, char* argv[]) { 20 | if (argc != 4) { 21 | usage(); 22 | return -1; 23 | } 24 | 25 | // client handle io operation 26 | std::cout << "create client" << std::endl; 27 | libzp::Client* client = new libzp::Client(argv[1], atoi(argv[2]), "parade"); 28 | std::cout << "connect cluster" << std::endl; 29 | int cnt = atoi(argv[3]); 30 | // needs connect to cluster first 31 | libzp::Status s; 32 | /* operation 33 | s = cluster->CreateTable("test", 24); 34 | */ 35 | if (!s.ok()) { 36 | std::cout << s.ToString() << std::endl; 37 | return -1; 38 | } 39 | std::vector keys; 40 | for(int i = 0; i < cnt; i++) { 41 | std::string key = "key" + std::to_string(i); 42 | keys.push_back(key); 43 | std::string value(1000, 'a'); 44 | s = client->Set(key, value); 45 | if (!s.ok()) { 46 | std::cout << s.ToString() << std::endl; 47 | } else { 48 | //std::cout << " - set ok, " << key << " -> " << value << std::endl; 49 | std::cout << " - set ok, " << std::endl; 50 | } 51 | } 52 | 53 | std::map kvs; 54 | std::cout << "Mget all keys exist begin ------------------------------------" << std::endl; 55 | s = client->Mget(keys, &kvs); 56 | std::cout << "Result :" << s.ToString() << std::endl; 57 | for (auto& kv : kvs) { 58 | std::cout << " - " << kv.first << " -> " << kv.second.substr(0, 100) << std::endl; 59 | } 60 | std::cout << "Mget all keys exist end ------------------------------------" << std::endl; 61 | sleep(60); 62 | 63 | std::cout << "Mget all keys exist begin ------------------------------------" << std::endl; 64 | s = client->Mget(keys, &kvs); 65 | std::cout << "Result :" << s.ToString() << std::endl; 66 | for (auto& kv : kvs) { 67 | std::cout << " - " << kv.first << " -> " << kv.second.substr(0, 100) << std::endl; 68 | } 69 | std::cout << "Mget all keys exist end ------------------------------------" << std::endl; 70 | sleep(60); 71 | 72 | std::cout << "Mget some keys not exist begin ------------------------------------" << std::endl; 73 | kvs.clear(); 74 | keys.push_back("not_exist"); 75 | keys.push_back("not_exist1"); 76 | s = client->Mget(keys, &kvs); 77 | std::cout << "Result :" << s.ToString() << std::endl; 78 | for (auto& kv : kvs) { 79 | if (kv.second.empty()) { 80 | std::cout << " - " << kv.first << " -> [emtpy] " << std::endl; 81 | } else { 82 | std::cout << " - " << kv.first << " -> " << kv.second.substr(0, 100) << std::endl; 83 | } 84 | } 85 | std::cout << "Mget some keys not exist end ------------------------------------" << std::endl; 86 | 87 | // Mget 88 | delete client; 89 | } 90 | -------------------------------------------------------------------------------- /manager/README.md: -------------------------------------------------------------------------------- 1 | ### zp_manager 2 | 3 | zeppelin的管理端命令行工具zp_manager，提供对集群的数据访问，建表，迁移，切主，状态查询等功能。 4 | 复杂功能如扩容，缩容，迁移，由基本命令setmaster, addslave, removeslave 组合完成 5 | 6 | #### 使用 7 | ``` 8 | ./zp_manager meta_ip meta_port 9 | ``` 10 | 11 | #### 建表 12 | ``` 13 | > create $table_name $paritition_count 14 | ``` 15 | 例： 16 | ``` 17 | > create test 16 # 创建名为test的Table，其中分配16个partition 18 | ``` 19 | 20 | #### 查看元信息 21 | ``` 22 | > pull $table_name 23 | ``` 24 | 例： 25 | ``` 26 | > pull test 27 | reset table:test 28 | OK 29 | current table info: 30 | epoch:4 31 | name: test 32 | partition: 3 33 | partition: 0 master: xxx.xxx.xxx.xxx : 6669 34 | partition: 1 master: xxx.xxx.xxx.xxx : 6666 35 | partition: 2 master: xxx.xxx.xxx.xxx : 6667 36 | ``` 37 | 38 | #### 查看Node状态 39 | ``` 40 | > listnode 41 | ``` 42 | 例： 43 | ``` 44 | >listnode 45 | xxx.xxx.xxx.xxx:6667 up 46 | xxx.xxx.xxx.xxx:6666 up 47 | xxx.xxx.xxx.xxx:6668 up 48 | xxx.xxx.xxx.xxx:6669 up 49 | OK 50 | ``` 51 | 52 | #### 查看Meta信息 53 | ``` 54 | > listmeta 55 | ``` 56 | 例： 57 | ``` 58 | >listmeta 59 | master:xxx.xxx.xxx.xxx 15223 60 | slave: 61 | xxx.xxx.xxx.xxx:15221 62 | xxx.xxx.xxx.xxx:15222 63 | OK 64 | ``` 65 | 66 | #### 查看Table 67 | ``` 68 | > listtable 69 | ``` 70 | 例： 71 | ``` 72 | >listtable 73 | test 74 | OK 75 | ``` 76 | 77 | #### 查看指定key的映射信息 78 | ``` 79 | > locate $table_name $key_name 80 | ``` 81 | 例： 82 | ``` 83 | > locate test wk 84 | partition_id: 0 85 | master: xxx.xxx.xxx.xxx:6669 86 | slave: xxx.xxx.xxx.xxx:6666 87 | slave: xxx.xxx.xxx.xxx:6667 88 | ``` 89 | 90 | #### 切主 91 | ``` 92 | > setmaster $table_name $patition_num $ip $port 93 | ``` 94 | 例： 95 | ``` 96 | > setmaster test 0 xxx.xxx.xxx.xxx 6669 97 | OK 98 | ``` 99 | 100 | #### 加从 101 | ``` 102 | > addslave $table_num $parititon_num $ip $port 103 | ``` 104 | 例： 105 | ``` 106 | > addslave test 0 xxx.xxx.xxx.xxx 6669 107 | OK 108 | ``` 109 | 110 | #### 删从 111 | ``` 112 | > removeslave $table_num $parititon_num $ip $port 113 | ``` 114 | 例： 115 | ``` 116 | > removeslave test 0 xxx.xxx.xxx.xxx 6669 117 | OK 118 | ``` 119 | 120 | #### 查看容量 121 | ``` 122 | > space $table_num 123 | ``` 124 | 例： 125 | ``` 126 | > space test 127 | node: xxx.xxx.xxx.xxx 6667 128 | used:255321586 bytes 129 | remain:107791286272 bytes 130 | ``` 131 | 132 | #### 查看QPS 133 | ``` 134 | > qps $table_num 135 | ``` 136 | 例： 137 | ``` 138 | > space test 139 | qps:0 140 | total query:6093572 141 | ``` 142 | 143 | #### 查看某个节点的binlog位置 144 | ``` 145 | > offset $table_num $node_ip $node_port 146 | ``` 147 | 例： 148 | ``` 149 | > offset test xxx.xxx.xxx.xxx 6667 150 | partition:0 151 | filenum:0 152 | offset:77067506 153 | partition:1 154 | filenum:0 155 | offset:77200811 156 | partition:2 157 | filenum:0 158 | offset:77016777 159 | ``` 160 | 161 | ### zp_benchmark 162 | 163 | Zeppelin Performance Test Tool 164 | 165 | ### zp_expansion 166 | 167 | Zeppelin Migration or Expansion Tool 168 | 169 | ### zp_parade 170 | 171 | Execute Zeppelin Client Operation One by One 172 | -------------------------------------------------------------------------------- /manager/utils/linenoise.h: -------------------------------------------------------------------------------- 1 | /* linenoise.h -- VERSION 1.0 2 | * 3 | * Guerrilla line editing library against the idea that a line editing lib 4 | * needs to be 20,000 lines of C code. 5 | * 6 | * See linenoise.c for more information. 7 | * 8 | * ------------------------------------------------------------------------ 9 | * 10 | * Copyright (c) 2010-2014, Salvatore Sanfilippo 11 | * Copyright (c) 2010-2013, Pieter Noordhuis 12 | * 13 | * All rights reserved. 14 | * 15 | * Redistribution and use in source and binary forms, with or without 16 | * modification, are permitted provided that the following conditions are 17 | * met: 18 | * 19 | * * Redistributions of source code must retain the above copyright 20 | * notice, this list of conditions and the following disclaimer. 21 | * 22 | * * Redistributions in binary form must reproduce the above copyright 23 | * notice, this list of conditions and the following disclaimer in the 24 | * documentation and/or other materials provided with the distribution. 25 | * 26 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30 | * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37 | */ 38 | 39 | #ifndef __LINENOISE_H 40 | #define __LINENOISE_H 41 | 42 | #ifdef __cplusplus 43 | extern "C" { 44 | #endif 45 | 46 | typedef struct linenoiseCompletions { 47 | size_t len; 48 | char **cvec; 49 | } linenoiseCompletions; 50 | 51 | typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *); 52 | typedef char*(linenoiseHintsCallback)(const char *, int *color, int *bold); 53 | typedef void(linenoiseFreeHintsCallback)(void *); 54 | void linenoiseSetCompletionCallback(linenoiseCompletionCallback *); 55 | void linenoiseSetHintsCallback(linenoiseHintsCallback *); 56 | void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *); 57 | void linenoiseAddCompletion(linenoiseCompletions *, const char *); 58 | 59 | char *linenoise(const char *prompt); 60 | void linenoiseFree(void *ptr); 61 | int linenoiseHistoryAdd(const char *line); 62 | int linenoiseHistorySetMaxLen(int len); 63 | int linenoiseHistorySave(const char *filename); 64 | int linenoiseHistoryLoad(const char *filename); 65 | void linenoiseClearScreen(void); 66 | void linenoiseSetMultiLine(int ml); 67 | void linenoisePrintKeyCodes(void); 68 | 69 | #ifdef __cplusplus 70 | } 71 | #endif 72 | 73 | #endif /* __LINENOISE_H */ 74 | -------------------------------------------------------------------------------- /manager/zp_benchmark.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright [2016] qihoo" 3 | */ 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "libzp/include/zp_cluster.h" 13 | 14 | 15 | void usage() { 16 | std::cout << "usage:\n" 17 | << " ./zp_benchmark host port table clientnum requrestnum\n"; 18 | } 19 | 20 | void Run(const libzp::Options &option, 21 | const std::string &table, const std::string &prefix, int rnum) { 22 | libzp::Cluster *cluster = new libzp::Cluster(option); 23 | libzp::Status s = cluster->Connect(); 24 | if (!s.ok()) { 25 | std::cout << "client " << std::this_thread::get_id() 26 | << " connect server failed: " << s.ToString() << std::endl; 27 | delete cluster; 28 | return; 29 | } 30 | s = cluster->Pull(table); 31 | if (!s.ok()) { 32 | std::cout << "client " << std::this_thread::get_id() 33 | << " pull table " << table << " failed: " << s.ToString() << std::endl; 34 | delete cluster; 35 | return; 36 | } 37 | 38 | // Set 39 | std::string key; 40 | std::string value = "value"; 41 | for (int i = 0; i < rnum; i++) { 42 | key = prefix + std::to_string(i); 43 | s= cluster->Set(table, key, value); 44 | if (!s.ok()) { 45 | std::cout << "client " << std::this_thread::get_id() << " set key " 46 | << key << " failed: " << s.ToString() << std::endl; 47 | delete cluster; 48 | return; 49 | } 50 | } 51 | 52 | // Get 53 | std::string v; 54 | for (int i = 0; i < rnum; i++) { 55 | key = prefix + std::to_string(i); 56 | s= cluster->Get(table, key, &v); 57 | if (!s.ok()) { 58 | std::cout << "client " << std::this_thread::get_id() 59 | << " get key " << key << " failed: " << s.ToString() << std::endl; 60 | delete cluster; 61 | return; 62 | } 63 | } 64 | 65 | // Mget 66 | std::map kvs; 67 | for (int i = 0; i < rnum; i+=10) { 68 | std::vector keys; 69 | for (int j = 10; j > 0; --j) { 70 | keys.push_back(prefix + std::to_string(i + j)); 71 | } 72 | s = cluster->Mget(table, keys, &kvs); 73 | if (!s.ok()) { 74 | std::cout << "client " << std::this_thread::get_id() 75 | << " mget 10 key start with " << keys[0] << " failed: " << s.ToString() << std::endl; 76 | delete cluster; 77 | return; 78 | } 79 | } 80 | std::cout << "client " << std::this_thread::get_id() << " is done! " 81 | << "Processed " <= 4 32 | # define PHP_ZEPPELIN_API __attribute__ ((visibility("default"))) 33 | #else 34 | # define PHP_ZEPPELIN_API 35 | #endif 36 | 37 | extern "C" 38 | { 39 | #ifdef ZTS 40 | #include "TSRM.h" 41 | #endif 42 | } 43 | 44 | PHP_MINIT_FUNCTION(zeppelin); 45 | PHP_MSHUTDOWN_FUNCTION(zeppelin); 46 | PHP_RINIT_FUNCTION(zeppelin); 47 | PHP_RSHUTDOWN_FUNCTION(zeppelin); 48 | PHP_MINFO_FUNCTION(zeppelin); 49 | 50 | PHP_FUNCTION(confirm_zeppelin_compiled); /* For testing, remove later. */ 51 | 52 | /* method for Zeppelin */ 53 | PHP_METHOD(Zeppelin, __construct); 54 | PHP_METHOD(Zeppelin, __destruct); 55 | PHP_METHOD(Zeppelin, set); 56 | PHP_METHOD(Zeppelin, get); 57 | PHP_METHOD(Zeppelin, delete); 58 | PHP_METHOD(Zeppelin, mget); 59 | 60 | /* 61 | Declare any global variables you may need between the BEGIN 62 | and END macros here: 63 | */ 64 | ZEND_BEGIN_MODULE_GLOBALS(zeppelin) 65 | void* g_zp_cli; 66 | ZEND_END_MODULE_GLOBALS(zeppelin) 67 | 68 | /* In every utility function you add that needs to use variables 69 | in php_zeppelin_globals, call TSRMLS_FETCH(); after declaring other 70 | variables used by that function, or better yet, pass in TSRMLS_CC 71 | after the last function argument and declare your utility function 72 | with TSRMLS_DC after the last declared argument. Always refer to 73 | the globals in your function as ZEPPELIN_G(variable). You are 74 | encouraged to rename these macros something shorter, see 75 | examples in any other php module directory. 76 | */ 77 | 78 | #ifdef ZTS 79 | #define ZEPPELIN_G(v) TSRMG(zeppelin_globals_id, zend_zeppelin_globals *, v) 80 | #else 81 | #define ZEPPELIN_G(v) (zeppelin_globals.v) 82 | #endif 83 | 84 | #endif /* PHP_ZEPPELIN_H */ 85 | 86 | 87 | /* 88 | * Local variables: 89 | * tab-width: 4 90 | * c-basic-offset: 4 91 | * End: 92 | * vim600: noet sw=4 ts=4 fdm=marker 93 | * vim<600: noet sw=4 ts=4 94 | */ 95 | -------------------------------------------------------------------------------- /manager/utils/json.h: -------------------------------------------------------------------------------- 1 | #ifndef MJSON_H 2 | #define MJSON_H 3 | 4 | #include 5 | #include 6 | 7 | namespace mjson { 8 | 9 | enum JsonElementType { 10 | kEnone = 0, 11 | kStr, /* value is a string */ 12 | kInt, /* value is a integer */ 13 | kJson, /* value is json */ 14 | }; 15 | 16 | enum JsonType { 17 | kJnone = 0, 18 | kSingle, 19 | kArray 20 | }; 21 | 22 | class Json; 23 | 24 | struct JsonElement { 25 | JsonElement(const JsonElementType t = JsonElementType::kEnone); 26 | JsonElement(const JsonElement& json_ele); 27 | JsonElement& operator=(const JsonElement& json_ele); 28 | virtual ~JsonElement(); 29 | 30 | void Clear(); 31 | 32 | std::string field; 33 | JsonElementType type; 34 | union { 35 | std::string *v_str; 36 | int64_t v_int; 37 | Json* v_json; 38 | } value; 39 | }; 40 | 41 | typedef std::vector SingleJson; 42 | typedef std::vector ArrayJson; 43 | 44 | class Json { 45 | static const char TAB_C = '\t'; 46 | public: 47 | Json(const JsonType t = JsonType::kJnone); 48 | Json(const Json& json); 49 | Json& operator=(const Json& json); 50 | virtual ~Json(); 51 | 52 | /* 53 | * for single json 54 | */ 55 | int32_t AddJsonElement(const JsonElement& json_ele); 56 | int32_t AddStr(const std::string& field, const std::string& value); 57 | int32_t AddInt(const std::string& field, const int64_t value); 58 | int32_t AddJson(const std::string& field, const Json& json); 59 | /* 60 | * for json array 61 | */ 62 | int32_t PushJson(const Json& json); 63 | 64 | void Clear(); 65 | std::string GetHstr(uint32_t indent = 0) const; 66 | std::string Encode() const; 67 | int32_t GetJsonValue(const std::string& field, Json* j_v) const; 68 | int32_t GetStrValue(const std::string& field, std::string* s_v) const; 69 | int32_t GetIntValue(const std::string& field, int64_t* i_v) const; 70 | 71 | JsonType type() const { 72 | return type_; 73 | } 74 | const SingleJson* single_json() const { 75 | return value_.s_json; 76 | } 77 | const ArrayJson* array_json() const { 78 | return value_.a_json; 79 | } 80 | 81 | private: 82 | std::string GetArrayJsonHstr(const ArrayJson& a_json, uint32_t indent = 0) const; 83 | std::string GetSingleJsonHstr(const SingleJson& s_json, uint32_t indent = 0) const; 84 | 85 | std::string EncodeArrayJson() const; 86 | std::string EncodeSingleJson() const; 87 | 88 | const JsonElement* GetEle(const std::string& field, JsonElementType ele_type) const; 89 | 90 | JsonType type_; 91 | union { 92 | SingleJson* s_json; 93 | ArrayJson* a_json; 94 | } value_; 95 | }; 96 | 97 | class JsonInterpreter { 98 | public: 99 | const static char LEFT_LARGE_BRACE_C = '{'; 100 | const static char RIGHT_LARGE_BRACE_C = '}'; 101 | const static char LEFT_MID_BRACE_C = '['; 102 | const static char RIGHT_MID_BRACE_C = ']'; 103 | const static char SPACE_C = ' '; 104 | const static char QUOTE_C = '\"'; 105 | const static char SEPERATE_C = ':'; 106 | const static char COMMA_C = ','; 107 | const static char LF_C = '\n'; 108 | 109 | int32_t Decode(const std::string& str, Json* json); 110 | int32_t Encode(const Json& json, std::string* str); 111 | 112 | private: 113 | Json* DecodeArrayJson(const std::string& str, 114 | std::string::const_iterator& iter); 115 | Json* DecodeJson(const std::string& str, 116 | std::string::const_iterator& iter); 117 | 118 | int32_t ValidCheck(const std::string& str); 119 | }; 120 | 121 | } /* namespace mjson */ 122 | #endif 123 | -------------------------------------------------------------------------------- /libzp/libzp/example/zp_table.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "libzp/include/zp_cluster.h" 7 | #include "libzp/include/zp_client.h" 8 | #include "slash/include/slash_status.h" 9 | 10 | int main() { 11 | libzp::Options options; 12 | options.meta_addr.push_back(libzp::Node("127.0.0.1", 9221)); 13 | options.op_timeout = 1000; 14 | std::string primary_key("testkey"); 15 | std::string table_name("test"); 16 | std::unique_ptr client(new libzp::Client(options, table_name)); 17 | slash::Status s; 18 | 19 | // PutRow, insert new 20 | printf("PutRow, insert new row\n"); 21 | std::map columns { 22 | {"col1", "value1"}, 23 | {"col2", "value2"}, 24 | {"col3", "value3"}, 25 | }; 26 | s = client->PutRow(primary_key, columns); 27 | if (!s.ok()) { 28 | std::cout << "PutRow Error: " << s.ToString() << std::endl; 29 | return -1; 30 | } 31 | 32 | // PutRow, update exist row 33 | printf("PutRow, update exist row\n"); 34 | std::map columns1 { 35 | {"col4", "value4"}, 36 | {"col5", "value5"}, 37 | }; 38 | s = client->PutRow(primary_key, columns1); 39 | if (!s.ok()) { 40 | std::cout << "PutRow Error: " << s.ToString() << std::endl; 41 | return -1; 42 | } 43 | 44 | // GetRow, get all columns 45 | printf("GetRow, get all columns\n"); 46 | std::vector empty_col; 47 | std::map results; 48 | s = client->GetRow(primary_key, empty_col, &results); 49 | if (!s.ok()) { 50 | std::cout << "GetRow Error: " << s.ToString() << std::endl; 51 | return -1; 52 | } 53 | printf("Get all columns result: \n"); 54 | for (auto& kv : results) { 55 | printf("key: %s, value: %s\n", kv.first.c_str(), kv.second.c_str()); 56 | } 57 | 58 | // GetRow, get specified columns 59 | printf("GetRow, get specified columns\n"); 60 | std::vector col_to_get{"col1", "col2"}; 61 | results.clear(); 62 | s = client->GetRow(primary_key, col_to_get, &results); 63 | if (!s.ok()) { 64 | std::cout << "GetRow Error: " << s.ToString() << std::endl; 65 | return -1; 66 | } 67 | printf("Get specified columns result: \n"); 68 | for (auto& kv : results) { 69 | printf("key: %s, value: %s\n", kv.first.c_str(), kv.second.c_str()); 70 | } 71 | 72 | // DeleteRow, delete specified columns 73 | printf("DeleteRow, delete specified columns\n"); 74 | s = client->DeleteRow(primary_key, col_to_get); 75 | if (!s.ok()) { 76 | std::cout << "DeleteRow Error: " << s.ToString() << std::endl; 77 | return -1; 78 | } 79 | results.clear(); 80 | s = client->GetRow(primary_key, empty_col, &results); 81 | if (!s.ok()) { 82 | std::cout << "GetRow Error: " << s.ToString() << std::endl; 83 | return -1; 84 | } 85 | printf("Delete specified columns result: \n"); 86 | for (auto& kv : results) { 87 | printf("key: %s, value: %s\n", kv.first.c_str(), kv.second.c_str()); 88 | } 89 | 90 | // DeleteRow, delete all columns 91 | printf("DeleteRow, delete all columns\n"); 92 | s = client->DeleteRow(primary_key, empty_col); 93 | if (!s.ok()) { 94 | std::cout << "DeleteRow Error: " << s.ToString() << std::endl; 95 | return -1; 96 | } 97 | results.clear(); 98 | s = client->GetRow(primary_key, empty_col, &results); 99 | if (!s.ok()) { 100 | std::cout << "GetRow Error: " << s.ToString() << std::endl; 101 | return -1; 102 | } 103 | printf("results size: %d\n", results.size()); 104 | 105 | return 0; 106 | } 107 | -------------------------------------------------------------------------------- /libzp/libzp/include/zp_entity.h: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright [2016] qihoo" 3 | */ 4 | #ifndef CLIENT_INCLUDE_ZP_ENTITY_H_ 5 | #define CLIENT_INCLUDE_ZP_ENTITY_H_ 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "libzp/include/zp_option.h" 14 | 15 | namespace client { 16 | class CmdResponse; 17 | class PartitionState; 18 | class CmdResponse_InfoServer; 19 | } 20 | 21 | namespace ZPMeta { 22 | class Table; 23 | class Partitions; 24 | } 25 | 26 | namespace libzp { 27 | 28 | class Partition { 29 | public: 30 | enum State { 31 | kActive, 32 | kStuck, 33 | kSlowDown, 34 | kUnknow, 35 | }; 36 | 37 | explicit Partition(const ZPMeta::Partitions& partition_info); 38 | 39 | void DebugDump() const; 40 | 41 | Node master() const { return master_; } 42 | 43 | int id() const { return id_; } 44 | 45 | std::vector slaves() const { return slaves_; } 46 | 47 | State state() const { return state_; } 48 | 49 | void SetMaster(const Node& new_master); 50 | 51 | private: 52 | std::vector slaves_; 53 | Node master_; 54 | int id_; 55 | State state_; 56 | }; 57 | 58 | class Table { 59 | public: 60 | Table(); 61 | explicit Table(const ZPMeta::Table& table_info); 62 | virtual ~Table(); 63 | std::string table_name() { return table_name_; } 64 | 65 | int partition_num() { return partition_num_; } 66 | 67 | std::map partitions() { return partitions_; } 68 | 69 | const Partition* GetPartition(const std::string& key) const; 70 | const Partition* GetPartitionById(int id) const; 71 | Status UpdatePartitionMaster(const std::string& key, const Node& target); 72 | Status UpdatePartitionMasterById(int partition_id, const Node& target); 73 | void GetAllMasters(std::set* nodes) const; 74 | void GetAllNodes(std::set* nodes) const; 75 | void GetNodesLoads(std::map>* loads) const; 76 | void DebugDump(int partition_id = -1) const; 77 | 78 | private: 79 | std::string table_name_; 80 | int partition_num_; 81 | std::map partitions_; 82 | }; 83 | 84 | struct BinlogOffset { 85 | uint32_t filenum; 86 | uint64_t offset; 87 | BinlogOffset() 88 | : filenum(0), offset(0) {} 89 | 90 | BinlogOffset(uint32_t num, uint64_t off) 91 | : filenum(num), offset(off) {} 92 | 93 | bool operator== (const BinlogOffset& rhs) const { 94 | return (filenum == rhs.filenum && offset == rhs.offset); 95 | } 96 | bool operator!= (const BinlogOffset& rhs) const { 97 | return (filenum != rhs.filenum || offset != rhs.offset); 98 | } 99 | bool operator< (const BinlogOffset& rhs) const { 100 | return (filenum < rhs.filenum || 101 | (filenum == rhs.filenum && offset < rhs.offset)); 102 | } 103 | bool operator> (const BinlogOffset& rhs) const { 104 | return (filenum > rhs.filenum || 105 | (filenum == rhs.filenum && offset > rhs.offset)); 106 | } 107 | }; 108 | 109 | extern std::ostream& operator<< (std::ostream& out, const BinlogOffset& bo); 110 | 111 | struct PartitionView { 112 | std::string role; 113 | std::string repl_state; 114 | Node master; 115 | std::vector slaves; 116 | BinlogOffset offset; 117 | uint64_t fallback_time; 118 | BinlogOffset fallback_before; 119 | BinlogOffset fallback_after; 120 | PartitionView(const client::PartitionState& state); 121 | }; 122 | 123 | struct ServerState { 124 | int64_t epoch; 125 | std::vector table_names; 126 | Node cur_meta; 127 | bool meta_renewing; 128 | ServerState() 129 | : epoch(-1), 130 | meta_renewing(false) { 131 | } 132 | ServerState(const client::CmdResponse_InfoServer & state); 133 | }; 134 | 135 | struct SpaceInfo { 136 | int64_t used; 137 | int64_t remain; 138 | }; 139 | 140 | } // namespace libzp 141 | #endif // CLIENT_INCLUDE_ZP_ENTITY_H_ 142 | -------------------------------------------------------------------------------- /javazeppelin/zeppelin_jar/src/net/qihoo/zeppelin/Zeppelin.java: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Qihoo 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:// www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package net.qihoo.zeppelin; 15 | import java.io.File; 16 | import java.io.BufferedInputStream; 17 | import java.io.FileOutputStream; 18 | import java.io.IOException; 19 | import java.io.InputStream; 20 | import java.util.TreeMap; 21 | import net.qihoo.zeppelin.ZeppelinException; 22 | 23 | public class Zeppelin { 24 | static { 25 | try{ 26 | Zeppelin.loadLib(); 27 | } catch(IOException e) { 28 | System.out.println("load qconf library failed"); 29 | System.exit(1); 30 | } 31 | } 32 | public Zeppelin(String ip, String port, String table) { 33 | try{ 34 | createZeppelin(ip, port, table); 35 | } catch(ZeppelinException e) { 36 | e.printStackTrace(); 37 | System.out.println("init zeppelin failed"); 38 | System.exit(1); 39 | } 40 | } 41 | private long ptr = 0; 42 | /** 43 | * create zeppelin client instance 44 | * 45 | * @param ip meta ip 46 | * @param port meta port 47 | * @param table table name 48 | * @exception ZeppelinException 49 | */ 50 | private native void createZeppelin(String ip, String port, String table) throws ZeppelinException; 51 | /** 52 | * delete zeppelin client instance 53 | * @exception ZeppelinException 54 | */ 55 | public native void removeZeppelin() throws ZeppelinException; 56 | /** 57 | * set key value through initialized zeppelin instance 58 | * 59 | * @param key key 60 | * @param value value 61 | * @return if set success 62 | * @exception ZeppelinException 63 | */ 64 | public native boolean set(String key, String value) throws ZeppelinException; 65 | /** 66 | * get key value through initialized zeppelin instance 67 | * 68 | * @param key key 69 | * @return value according to the corresponding key 70 | * @exception ZeppelinException 71 | */ 72 | public native String get(String key) throws ZeppelinException; 73 | /** 74 | * get multiple keys' values 75 | * 76 | * @param keys string array of keys 77 | * @return a map of keys and values 78 | * @exception ZeppelinException 79 | */ 80 | public native TreeMap mget(String[] keys) throws ZeppelinException; 81 | /** 82 | * delete key 83 | * 84 | * @param key key value 85 | * @return if delete success 86 | * @exception ZeppelinException 87 | */ 88 | public native boolean delete(String key) throws ZeppelinException; 89 | 90 | private static void exceptionCallback(String errmsg) throws ZeppelinException { 91 | throw new ZeppelinException(errmsg); 92 | } 93 | 94 | private synchronized static void loadLib() throws IOException { 95 | String libFullName = "libzeppelin.so"; 96 | String tmpLibFilePath = System.getProperty("java.io.tmpdir") + File.separator + libFullName; 97 | InputStream in = null; 98 | BufferedInputStream reader = null; 99 | FileOutputStream writer = null; 100 | 101 | File extractedLibFile = File.createTempFile("libzeppelin",".so"); 102 | try { 103 | in = Zeppelin.class.getResourceAsStream(File.separator + libFullName); 104 | Zeppelin.class.getResource(libFullName); 105 | reader = new BufferedInputStream(in); 106 | writer = new FileOutputStream(extractedLibFile); 107 | byte[] buffer = new byte[1024]; 108 | while (reader.read(buffer) > 0) { 109 | writer.write(buffer); 110 | buffer = new byte[1024]; 111 | } 112 | } catch (IOException e) { 113 | e.printStackTrace(); 114 | } finally { 115 | if(in!=null) 116 | in.close(); 117 | if(writer!=null) 118 | writer.close(); 119 | } 120 | System.load(extractedLibFile.toString()); 121 | if (extractedLibFile.exists()) { 122 | extractedLibFile.delete(); 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /libzp/libzp/src/zp_meta.proto: -------------------------------------------------------------------------------- 1 | package ZPMeta; 2 | 3 | enum StatusCode { 4 | OK = 0; 5 | NOTFOUND = 1; 6 | ERROR = 2; 7 | } 8 | 9 | enum Type { 10 | PING = 1; 11 | PULL = 2; 12 | INIT = 3; 13 | SETMASTER = 4; 14 | ADDSLAVE = 5; 15 | REMOVESLAVE = 6; 16 | LISTTABLE = 7; 17 | LISTNODE = 8; 18 | LISTMETA = 9; 19 | DROPTABLE = 10; 20 | METASTATUS = 11; 21 | MIGRATE = 12; 22 | CANCELMIGRATE = 13; 23 | REMOVENODES = 14; 24 | ADDMETANODE = 15; 25 | REMOVEMETANODE = 16; 26 | } 27 | 28 | enum PState { 29 | ACTIVE = 1; 30 | STUCK = 2; 31 | SLOWDOWN = 3; 32 | } 33 | 34 | enum NodeState { 35 | UP = 0; 36 | DOWN = 1; 37 | } 38 | 39 | message Node { 40 | required string ip = 1; 41 | required int32 port = 2; 42 | } 43 | 44 | message NodeStatus { 45 | required Node node = 1; 46 | required NodeState status = 2; 47 | } 48 | 49 | message Nodes { 50 | repeated NodeStatus nodes = 1; 51 | } 52 | 53 | message MetaNodes { 54 | repeated Node followers = 1; 55 | optional Node leader = 2; 56 | } 57 | 58 | message MetaLeader { 59 | required Node leader = 1; 60 | required int64 last_active = 2; 61 | } 62 | 63 | message Partitions { 64 | required int32 id = 1; 65 | required PState state = 2; 66 | required Node master = 3; 67 | repeated Node slaves = 4; 68 | } 69 | 70 | message TableName { 71 | repeated string name = 1; 72 | } 73 | 74 | message Table { 75 | required string name = 1; 76 | repeated Partitions partitions = 2; 77 | } 78 | 79 | message BasicCmdUnit { 80 | required string name = 1; 81 | required int32 partition = 2; 82 | required Node node = 3; 83 | } 84 | 85 | message RelationCmdUnit { 86 | required string table = 1; 87 | required int32 partition = 2; 88 | required Node left = 3; 89 | required Node right = 4; 90 | } 91 | 92 | message SyncOffset { 93 | required string table_name = 1; 94 | required int32 partition = 2; 95 | optional int32 filenum = 3; 96 | optional int64 offset = 4; 97 | } 98 | 99 | message MigrateStatus { 100 | required int64 begin_time = 1; 101 | required int32 complete_proportion = 2; 102 | } 103 | 104 | // Internal use by meta 105 | message MigrateHead { 106 | required int64 begin_time = 1; 107 | required int32 init_size = 2; 108 | repeated string diff_name = 3; 109 | } 110 | 111 | message MetaCmd { 112 | 113 | required Type type = 1; 114 | 115 | // Ping 116 | message Ping { 117 | required int32 version = 1; 118 | required Node node = 2; 119 | repeated SyncOffset offset = 3; 120 | } 121 | optional Ping ping = 2; 122 | 123 | // Pull 124 | message Pull { 125 | optional Node node = 1; 126 | optional string name = 2; 127 | } 128 | optional Pull pull = 3; 129 | 130 | // Init 131 | message Init { 132 | required Table table = 1; 133 | } 134 | optional Init init = 4; 135 | 136 | message SetMaster{ 137 | required BasicCmdUnit basic = 1; 138 | } 139 | optional SetMaster set_master = 5; 140 | 141 | message AddSlave { 142 | required BasicCmdUnit basic = 1; 143 | } 144 | optional AddSlave add_slave = 6; 145 | 146 | message RemoveSlave { 147 | required BasicCmdUnit basic = 1; 148 | } 149 | optional RemoveSlave remove_slave = 7; 150 | 151 | message DropTable { 152 | required string name = 1; 153 | } 154 | optional DropTable drop_table = 8; 155 | 156 | message Migrate { 157 | required int32 origin_epoch = 1; 158 | repeated RelationCmdUnit diff = 2; 159 | } 160 | optional Migrate migrate = 9; 161 | 162 | message RemoveNodes { 163 | repeated Node nodes = 1; 164 | } 165 | optional RemoveNodes remove_nodes = 10; 166 | 167 | optional Node add_meta_node = 11; 168 | 169 | optional Node remove_meta_node = 12; 170 | } 171 | 172 | message MetaCmdResponse { 173 | 174 | required Type type = 1; 175 | required StatusCode code = 2; 176 | optional string msg = 3; 177 | 178 | // Ping 179 | message Ping { 180 | required int32 version = 1; 181 | } 182 | optional Ping ping = 4; 183 | 184 | // Pull 185 | message Pull { 186 | required int32 version = 1; 187 | repeated Table info = 2; 188 | repeated Node meta_members = 3; 189 | } 190 | optional Pull pull = 5; 191 | 192 | // ListTable 193 | message ListTable { 194 | optional TableName tables = 1; 195 | } 196 | optional ListTable list_table = 6; 197 | 198 | // ListNode 199 | message ListNode { 200 | optional Nodes nodes = 1; 201 | } 202 | optional ListNode list_node = 7; 203 | 204 | // ListMeta 205 | message ListMeta { 206 | required MetaNodes nodes = 1; 207 | } 208 | optional ListMeta list_meta = 8; 209 | 210 | // MetaStatus 211 | message MetaStatus { 212 | required int32 version = 1; 213 | required string consistency_stautus = 2; 214 | optional MigrateStatus migrate_status = 3; // has means is migrating 215 | } 216 | optional MetaStatus meta_status = 9; 217 | } 218 | -------------------------------------------------------------------------------- /libzp/libzp/src/zp_client.cc: -------------------------------------------------------------------------------- 1 | #include "libzp/include/zp_client.h" 2 | #include "libzp/include/zp_cluster.h" 3 | 4 | namespace libzp { 5 | 6 | RawClient::RawClient(const Options& options) 7 | : cluster_(new Cluster(options)) { 8 | } 9 | 10 | RawClient::RawClient(const std::string& ip, const int port) 11 | : cluster_(new Cluster(ip, port)) { 12 | } 13 | 14 | RawClient::~RawClient() { 15 | delete cluster_; 16 | } 17 | 18 | Status RawClient::Set(const std::string& table_name, 19 | const std::string& key, 20 | const std::string& value, 21 | int32_t ttl) { 22 | return cluster_->Set(table_name, key, value, ttl); 23 | } 24 | 25 | Status RawClient::Get(const std::string& table_name, 26 | const std::string& key, 27 | std::string* value) { 28 | return cluster_->Get(table_name, key, value); 29 | } 30 | 31 | Status RawClient::Mget(const std::string& table_name, 32 | const std::vector& keys, 33 | std::map* values) { 34 | return cluster_->Mget(table_name, keys, values); 35 | } 36 | 37 | Status RawClient::Delete(const std::string& table_name, 38 | const std::string& key) { 39 | return cluster_->Delete(table_name, key); 40 | } 41 | 42 | Status RawClient::Aset(const std::string& table_name, 43 | const std::string& key, 44 | const std::string& value, 45 | zp_completion_t complietion, 46 | void* data, 47 | int32_t ttl) { 48 | return cluster_->Aset(table_name, key, value, complietion, data, ttl); 49 | } 50 | 51 | Status RawClient::Adelete(const std::string& table_name, 52 | const std::string& key, 53 | zp_completion_t complietion, 54 | void* data) { 55 | return cluster_->Adelete(table_name, key, complietion, data); 56 | } 57 | 58 | Status RawClient::Aget(const std::string& table_name, 59 | const std::string& key, 60 | zp_completion_t complietion, 61 | void* data) { 62 | return cluster_->Aget(table_name, key, complietion, data); 63 | } 64 | 65 | Status RawClient::Amget(const std::string& table_name, 66 | const std::vector& keys, 67 | zp_completion_t complietion, 68 | void* data) { 69 | return cluster_->Amget(table_name, keys, complietion, data); 70 | } 71 | 72 | const std::string kTableTag = "###"; 73 | 74 | Status RawClient::PutRow(const std::string& table_name, 75 | const std::string& primary_key, 76 | const std::map& columns) { 77 | if (columns.empty()) { 78 | return Status::InvalidArgument("Empty columns"); 79 | } 80 | std::string hash_tag = kLBrace + kTableTag + primary_key + kRBrace; 81 | Cluster::Batch batch(hash_tag); 82 | for (auto& item : columns) { 83 | batch.Write(item.first, item.second); 84 | } 85 | return cluster_->WriteBatch(table_name, batch); 86 | } 87 | 88 | Status RawClient::GetRow(const std::string& table_name, 89 | const std::string& primary_key, 90 | const std::vector& col_to_get, 91 | std::map* columns) { 92 | std::string hash_tag = kLBrace + kTableTag + primary_key + kRBrace; 93 | std::vector keys; 94 | std::map results; 95 | 96 | if (col_to_get.empty()) { 97 | Status s = cluster_->ListbyTag(table_name, hash_tag, &results); 98 | if (!s.ok()) { 99 | return s; 100 | } 101 | } else { 102 | // Build Mget request 103 | for (auto& c : col_to_get) { 104 | keys.emplace_back(hash_tag + c); 105 | } 106 | 107 | Status s = cluster_->Mget(table_name, keys, &results); 108 | if (!s.ok()) { 109 | return s; 110 | } 111 | } 112 | // Trim hash_tag from original key 113 | for (auto& kv : results) { 114 | std::string key = kv.first.substr(hash_tag.size()); 115 | columns->insert(std::make_pair(key, kv.second)); 116 | } 117 | return Status::OK(); 118 | } 119 | 120 | Status RawClient::DeleteRow(const std::string& table_name, 121 | const std::string& primary_key, 122 | const std::vector& col_to_delete) { 123 | std::string hash_tag = kLBrace + kTableTag + primary_key + kRBrace; 124 | if (col_to_delete.empty()) { 125 | return cluster_->DeletebyTag(table_name, hash_tag); 126 | } 127 | Cluster::Batch batch(hash_tag); 128 | for (auto& c : col_to_delete) { 129 | batch.Delete(c); 130 | } 131 | return cluster_->WriteBatch(table_name, batch); 132 | } 133 | 134 | } // namespace libzp 135 | -------------------------------------------------------------------------------- /manager/Makefile: -------------------------------------------------------------------------------- 1 | CXX=g++ 2 | CC=gcc 3 | LDFLAGS= -L/usr/local/protobuf/lib -lpthread -lrt -lprotobuf 4 | CXXFLAGS= -g -std=c++11 -fno-builtin-memcmp -msse -msse4.2 5 | CFLAGS= -g -fno-builtin-memcmp -msse -msse4.2 6 | PROFILING_FLAGS=-pg 7 | ARFLAGS = rs 8 | OPT= 9 | 10 | # Set the default DEBUG_LEVEL to 0 11 | DEBUG_LEVEL?=0 12 | 13 | ifeq ($(MAKECMDGOALS),dbg) 14 | DEBUG_LEVEL=2 # compatible with rocksdb 15 | endif 16 | 17 | # compile with -O2 if for release 18 | # if we're compiling for release, compile without debug code (-DNDEBUG) and 19 | # don't treat warnings as errors 20 | ifeq ($(DEBUG_LEVEL),0) 21 | DISABLE_WARNING_AS_ERROR=1 22 | OPT += -O2 -fno-omit-frame-pointer -DNDEBUG 23 | else 24 | $(warning Warning: Compiling in debug mode. Don't use the resulting binary in production) 25 | OPT += -O0 -D__XDEBUG__ $(PROFILING_FLAGS) 26 | DEBUG_SUFFIX = "_debug" 27 | endif 28 | 29 | # ----------------Dependences------------------- 30 | ifndef ZP_PATH 31 | ZP_PATH = $(CURDIR)/../libzp 32 | endif 33 | ZP_INCLUDE_DIR=$(ZP_PATH) 34 | ZP_LIBRARY=$(ZP_PATH)/libzp/lib/libzp$(DEBUG_SUFFIX).a 35 | 36 | ifndef PINK_PATH 37 | PINK_PATH=$(CURDIR)/../third/pink 38 | endif 39 | PINK_INCLUDE_DIR=$(PINK_PATH) 40 | PINK_LIBRARY=$(PINK_PATH)/pink/lib/libpink$(DEBUG_SUFFIX).a 41 | 42 | ifndef SLASH_PATH 43 | SLASH_PATH=$(CURDIR)/../third/slash 44 | endif 45 | SLASH_INCLUDE_DIR=$(SLASH_PATH) 46 | SLASH_LIBRARY=$(SLASH_PATH)/slash/lib/libslash$(DEBUG_SUFFIX).a 47 | 48 | ifeq ($(MAKECMDGOALS),rocksdb_to_zp) 49 | ifndef ROCKSDB_PATH 50 | $(error Error: missing rocksdb path!) 51 | endif 52 | ROCKSDB_INCLUDE_DIR=$(ROCKSDB_PATH)/include 53 | ROCKSDB_LIBRARY=$(ROCKSDB_PATH)/librocksdb$(DEBUG_SUFFIX).a 54 | LDFLAGS += -lz -lbz2 -lsnappy 55 | endif 56 | # ---------------------------------------------- 57 | 58 | CXXFLAGS+= -I$(PINK_INCLUDE_DIR) -I$(SLASH_INCLUDE_DIR) -I$(ZP_INCLUDE_DIR) 59 | DEP_LIBS = $(ZP_LIBRARY) $(PINK_LIBRARY) $(SLASH_LIBRARY) 60 | 61 | AM_DEFAULT_VERBOSITY = 0 62 | 63 | AM_V_at = $(am__v_at_$(V)) 64 | am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY)) 65 | am__v_at_0 = @ 66 | am__v_at_1 = 67 | AM_V_CC = $(am__v_CC_$(V)) 68 | am__v_CC_ = $(am__v_CC_$(AM_DEFAULT_VERBOSITY)) 69 | am__v_CC_0 = @echo " CC " $@; 70 | am__v_CC_1 = 71 | AM_V_LD = $(am__v_LD_$(V)) 72 | am__v_LD_ = $(am__v_LD_$(AM_DEFAULT_VERBOSITY)) 73 | am__v_LD_0 = @echo " LD " $@; 74 | am__v_LD_1 = 75 | 76 | # This (the first rule) must depend on "all". 77 | default: all 78 | 79 | WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare \ 80 | -Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings -Wpointer-arith 81 | 82 | ifndef DISABLE_WARNING_AS_ERROR 83 | WARNING_FLAGS += -Werror 84 | endif 85 | 86 | CXXFLAGS += $(WARNING_FLAGS) $(OPT) -Wreorder -Wswitch -Wsign-promo -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers 87 | CFLAGS += $(WARNING_FLAGS) $(OPT) 88 | 89 | .PHONY: clean distclean dbg all deps 90 | 91 | all: zp_benchmark zp_checkup zp_expansion zp_info zp_manager zp_parade zp_loopset 92 | 93 | dbg: zp_benchmark zp_checkup zp_expansion zp_info zp_manager zp_parade zp_loopset 94 | 95 | $(DEP_LIBS): 96 | $(AM_V_at)make -C $(SLASH_PATH)/slash DEBUG_LEVEL=$(DEBUG_LEVEL) 97 | $(AM_V_at)make -C $(PINK_PATH)/pink DEBUG_LEVEL=$(DEBUG_LEVEL) SLASH_PATH=$(SLASH_PATH) 98 | $(AM_V_at)make -C $(ZP_PATH)/libzp DEBUG_LEVEL=$(DEBUG_LEVEL) SLASH_PATH=$(SLASH_PATH) PINK_PATH=$(PINK_PATH) 99 | 100 | zp_benchmark: zp_benchmark.cc $(DEP_LIBS) 101 | $(AM_V_LD)$(CXX) $^ -o $@ $(CXXFLAGS) $(LDFLAGS) 102 | 103 | zp_loopset: zp_loopset.cc $(DEP_LIBS) 104 | $(AM_V_LD)$(CXX) $^ -o $@ $(CXXFLAGS) $(LDFLAGS) 105 | 106 | zp_checkup: zp_checkup.cc utils/json.cc utils/json.h $(DEP_LIBS) 107 | $(AM_V_LD)$(CXX) $^ -o $@ $(CXXFLAGS) $(LDFLAGS) 108 | 109 | zp_expansion: zp_expansion.cc $(DEP_LIBS) 110 | $(AM_V_LD)$(CXX) $^ -o $@ $(CXXFLAGS) $(LDFLAGS) 111 | 112 | zp_info: zp_info.cc utils/json.cc utils/json.h $(DEP_LIBS) 113 | $(AM_V_LD)$(CXX) $^ -o $@ $(CXXFLAGS) $(LDFLAGS) 114 | 115 | zp_manager: zp_manager.cc utils/linenoise.o utils/distribution.o $(DEP_LIBS) 116 | $(AM_V_LD)$(CXX) $^ -o $@ $(CXXFLAGS) $(LDFLAGS) 117 | 118 | utils/linenoise.o: utils/linenoise.c utils/linenoise.h 119 | $(AM_V_CC)$(CC) $(CFLAGS) -c $< -o $@ 120 | 121 | utils/distribution.o: utils/distribution.cc 122 | $(AM_V_CC)$(CXX) $(CXXFLAGS) -c $< -o $@ 123 | 124 | zp_parade: zp_parade.cc $(DEP_LIBS) 125 | $(AM_V_LD)$(CXX) $^ -o $@ $(CXXFLAGS) $(LDFLAGS) 126 | 127 | rocksdb_to_zp: rocksdb_to_zp.cc $(DEP_LIBS) $(ROCKSDB_LIBRARY) 128 | $(AM_V_LD)$(CXX) $^ -o $@ $(CXXFLAGS) -I$(ROCKSDB_INCLUDE_DIR) $(LDFLAGS) 129 | 130 | clean: 131 | rm -f zp_benchmark zp_loopset zp_checkup zp_expansion zp_info zp_manager zp_parade rocksdb_to_zp 132 | find . -name "*.[oda]*" ! -exec rm -f {} \; 133 | find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \; 134 | 135 | distclean: clean 136 | $(AM_V_at)make -C $(SLASH_PATH)/slash clean 137 | $(AM_V_at)make -C $(PINK_PATH)/pink clean 138 | $(AM_V_at)make -C $(ZP_PATH)/libzp clean 139 | -------------------------------------------------------------------------------- /libzp/libzp/Makefile: -------------------------------------------------------------------------------- 1 | CLEAN_FILES = # deliberately empty, so we can append below. 2 | CXX = g++ 3 | PROTOC = protoc 4 | CXXFLAGS = -g -std=c++11 -fno-builtin-memcmp -msse -msse4.2 -pipe -fPIC 5 | PROFILING_FLAGS = -pg 6 | ARFLAGS = rs 7 | OPT= 8 | 9 | # Set the default DEBUG_LEVEL to 0 10 | DEBUG_LEVEL?=0 11 | 12 | ifeq ($(MAKECMDGOALS),dbg) 13 | DEBUG_LEVEL=2 # compatible with rocksdb 14 | endif 15 | 16 | # compile with -O2 if for release 17 | # if we're compiling for release, compile without debug code (-DNDEBUG) and 18 | # don't treat warnings as errors 19 | ifeq ($(DEBUG_LEVEL),0) 20 | DISABLE_WARNING_AS_ERROR=1 21 | OPT += -O2 -fno-omit-frame-pointer -DNDEBUG 22 | else 23 | $(warning Warning: Compiling in debug mode. Don't use the resulting binary in production) 24 | OPT += -O0 -D__XDEBUG__ $(PROFILING_FLAGS) 25 | endif 26 | 27 | #----------------------------------------------- 28 | 29 | SRC_DIR=src 30 | PROTOS = $(wildcard $(SRC_DIR)/*.proto) 31 | PROTO_GENS = $(PROTOS:.proto=.pb.cc) $(PROTOS:.proto=.pb.h) 32 | CLEAN_FILES += $(PROTO_GENS) 33 | PROTO_OBJS = $(PROTOS:.proto=.pb.o) 34 | VERSION_CC=$(SRC_DIR)/build_version.cc 35 | LIB_SOURCES := $(VERSION_CC) \ 36 | $(filter-out $(VERSION_CC), $(wildcard $(SRC_DIR)/*.cc)) 37 | 38 | ifndef PINK_PATH 39 | PINK_PATH=$(CURDIR)/../../third/pink 40 | endif 41 | PINK_INCLUDE_DIR=$(PINK_PATH) 42 | 43 | ifndef SLASH_PATH 44 | SLASH_PATH=$(CURDIR)/../../third/slash 45 | endif 46 | SLASH_INCLUDE_DIR=$(SLASH_PATH) 47 | 48 | AM_DEFAULT_VERBOSITY = 0 49 | 50 | AM_V_GEN = $(am__v_GEN_$(V)) 51 | am__v_GEN_ = $(am__v_GEN_$(AM_DEFAULT_VERBOSITY)) 52 | am__v_GEN_0 = @echo " GEN " $@; 53 | am__v_GEN_1 = 54 | AM_V_at = $(am__v_at_$(V)) 55 | am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY)) 56 | am__v_at_0 = @ 57 | am__v_at_1 = 58 | 59 | AM_V_CXX = $(am__v_CXX_$(V)) 60 | am__v_CXX_ = $(am__v_CXX_$(AM_DEFAULT_VERBOSITY)) 61 | am__v_CXX_0 = @echo " CXX " $@; 62 | am__v_CXX_1 = 63 | LD = $(CXX) 64 | AM_V_LD = $(am__v_LD_$(V)) 65 | am__v_LD_ = $(am__v_LD_$(AM_DEFAULT_VERBOSITY)) 66 | am__v_LD_0 = @echo " LD " $@; 67 | am__v_LD_1 = 68 | AM_V_AR = $(am__v_AR_$(V)) 69 | am__v_AR_ = $(am__v_AR_$(AM_DEFAULT_VERBOSITY)) 70 | am__v_AR_0 = @echo " AR " $@; 71 | am__v_AR_1 = 72 | 73 | # This (the first rule) must depend on "all". 74 | default: all 75 | 76 | WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare \ 77 | -Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings \ 78 | -Wpointer-arith -Wreorder -Wswitch -Wsign-promo \ 79 | -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers 80 | 81 | ifndef DISABLE_WARNING_AS_ERROR 82 | WARNING_FLAGS += -Werror 83 | endif 84 | 85 | CXXFLAGS += $(WARNING_FLAGS) -I.. -I$(PINK_INCLUDE_DIR) -I$(SLASH_INCLUDE_DIR) $(OPT) 86 | 87 | date := $(shell date +%F) 88 | git_sha := $(shell git rev-parse HEAD 2>/dev/null) 89 | gen_build_version = sed -e s/@@GIT_SHA@@/$(git_sha)/ -e s/@@GIT_DATE_TIME@@/$(date)/ $(SRC_DIR)/build_version.cc.in 90 | # Record the version of the source that we are compiling. 91 | # We keep a record of the git revision in this file. It is then built 92 | # as a regular source file as part of the compilation process. 93 | # One can run "strings executable_filename | grep _build_" to find 94 | # the version of the source that we used to build the executable file. 95 | CLEAN_FILES += $(SRC_DIR)/build_version.cc 96 | 97 | $(SRC_DIR)/build_version.cc: FORCE 98 | $(AM_V_GEN)rm -f $@-t 99 | $(AM_V_at)$(gen_build_version) > $@-t 100 | $(AM_V_at)if test -f $@; then \ 101 | cmp -s $@-t $@ && rm -f $@-t || mv -f $@-t $@; \ 102 | else mv -f $@-t $@; fi 103 | FORCE: 104 | 105 | LIBOBJECTS = $(LIB_SOURCES:.cc=.o) 106 | 107 | # if user didn't config LIBNAME, set the default 108 | ifeq ($(LIBNAME),) 109 | # we should only run pink in production with DEBUG_LEVEL 0 110 | ifeq ($(DEBUG_LEVEL),0) 111 | LIBNAME=libzp 112 | else 113 | LIBNAME=libzp_debug 114 | endif 115 | endif 116 | LIBOUTPUT = ./lib 117 | dummy := $(shell mkdir -p $(LIBOUTPUT)) 118 | LIBRARY = $(LIBOUTPUT)/${LIBNAME}.a 119 | 120 | .PHONY: clean dbg static_lib all example 121 | 122 | all: $(LIBRARY) 123 | 124 | example: 125 | $(AM_V_at)make -C example ZP_PATH=$(CURDIR)/.. SLASH_PATH=$(SLASH_PATH) PINK_PATH=$(PINK_PATH) 126 | 127 | static_lib: $(LIBRARY) 128 | 129 | dbg: $(LIBRARY) 130 | 131 | proto: $(PROTO_GENS) 132 | 133 | $(LIBRARY): $(PROTO_OBJS) $(LIBOBJECTS) 134 | $(AM_V_AR)rm -f $@ 135 | $(AM_V_at)$(AR) $(ARFLAGS) $@ $^ 136 | 137 | clean: 138 | $(AM_V_at)echo "Cleaning..." 139 | $(AM_V_at)rm -f $(LIBRARY) 140 | $(AM_V_at)rm -rf $(CURDIR)/output 141 | $(AM_V_at)rm -rf $(CLEAN_FILES) 142 | $(AM_V_at)rm -rf $(LIBOUTPUT) 143 | $(AM_V_at)find . -name "*.[oda]" ! -path "./third/*" -exec rm -f {} \; 144 | $(AM_V_at)find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \; 145 | 146 | %.pb.cc %.pb.h: %.proto 147 | $(AM_V_GEN) 148 | $(AM_V_at)$(PROTOC) -I$(dir $@) --cpp_out=$(dir $@) $< 149 | 150 | %.o: %.cc $(PROTO_GENS) 151 | $(AM_V_CXX)$(CXX) $(CXXFLAGS) -c $< -o $@ 152 | -------------------------------------------------------------------------------- /libzp/libzp/src/zp_conn.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright [2016] qihoo" 3 | */ 4 | #include "libzp/src/zp_conn.h" 5 | 6 | #include 7 | #include 8 | #include "slash/include/env.h" 9 | #include "pink/include/pink_cli.h" 10 | 11 | namespace libzp { 12 | 13 | const int kConnKeepalive = 20000000; 14 | 15 | static uint64_t NowMicros() { 16 | struct timeval tv; 17 | gettimeofday(&tv, NULL); 18 | return static_cast(tv.tv_sec)*1000000 + 19 | static_cast(tv.tv_usec); 20 | } 21 | 22 | ZpCli::ZpCli(const Node& node) 23 | : node(node), 24 | lastchecktime(NowMicros()) { 25 | cli = pink::NewPbCli(); 26 | assert(cli); 27 | } 28 | 29 | ZpCli::~ZpCli() { 30 | cli->Close(); 31 | delete cli; 32 | } 33 | 34 | bool ZpCli::TryKeepalive() { 35 | uint64_t now = NowMicros(); 36 | if ((now - lastchecktime) > kConnKeepalive) { 37 | return false; 38 | } 39 | lastchecktime = now; 40 | return true; 41 | } 42 | 43 | Status ZpCli::SetTimeout(uint64_t deadline, TimeoutOptType type) { 44 | if (!deadline) { 45 | return Status::OK(); // no limit 46 | } 47 | 48 | int timeout = deadline - slash::NowMicros() / 1000; 49 | if (timeout <= 0) { 50 | return Status::Timeout("timeout in SetTimeout"); 51 | } 52 | 53 | switch (type) { 54 | case TimeoutOptType::CONNECT: 55 | cli->set_connect_timeout(timeout); 56 | break; 57 | case TimeoutOptType::SEND: 58 | cli->set_send_timeout(timeout); 59 | break; 60 | case TimeoutOptType::RECV: 61 | cli->set_recv_timeout(timeout); 62 | break; 63 | default: 64 | return Status::InvalidArgument("unknow TimeoutOptType"); 65 | } 66 | return Status::OK(); 67 | } 68 | 69 | ConnectionPool::ConnectionPool(size_t capacity) : 70 | capacity_(capacity), lru_head_(nullptr) { 71 | lru_head_.next = &lru_head_; 72 | lru_head_.prev = &lru_head_; 73 | } 74 | 75 | ConnectionPool::~ConnectionPool() { 76 | for (auto& item : conn_pool_) { 77 | delete item.second; 78 | } 79 | } 80 | 81 | std::shared_ptr ConnectionPool::GetConnection( 82 | const Node& node, uint64_t deadline, Status* sptr) { 83 | *sptr = Status::OK(); 84 | slash::MutexLock l(&pool_mu_); 85 | auto it = conn_pool_.find(node); 86 | if (it != conn_pool_.end()) { 87 | if (it->second->cli->TryKeepalive()) { 88 | MoveToLRUHead(it->second); 89 | return it->second->cli; 90 | } 91 | RemoveFromLRU(it->second); 92 | conn_pool_.erase(it); 93 | } 94 | 95 | // Not found or timeout, create new one 96 | std::shared_ptr zp_cli(new ZpCli(node)); 97 | ZpCliItem* item = new ZpCliItem(zp_cli); 98 | *sptr = zp_cli->SetTimeout(deadline, TimeoutOptType::CONNECT); 99 | if (!sptr->ok()) { 100 | delete item; 101 | return nullptr; 102 | } 103 | *sptr = zp_cli->cli->Connect(node.ip, node.port); 104 | if (!sptr->ok()) { 105 | delete item; 106 | return nullptr; 107 | } 108 | 109 | // Remove rear item 110 | if (conn_pool_.size() == capacity_) { 111 | ZpCliItem* rear = lru_head_.prev; 112 | conn_pool_.erase(rear->cli->node); 113 | RemoveFromLRU(rear); 114 | } 115 | 116 | // Add new item 117 | conn_pool_.insert(std::make_pair(node, item)); 118 | InsertLRU(item); 119 | return item->cli; 120 | } 121 | 122 | void ConnectionPool::RemoveConnection(std::shared_ptr conn) { 123 | slash::MutexLock l(&pool_mu_); 124 | auto iter = conn_pool_.find(conn->node); 125 | assert(iter != conn_pool_.end()); 126 | RemoveFromLRU(iter->second); 127 | conn_pool_.erase(iter); 128 | } 129 | 130 | // Only for meta node connection pool 131 | std::shared_ptr ConnectionPool::GetExistConnection() { 132 | Status s; 133 | std::map::iterator first_conn; 134 | slash::MutexLock l(&pool_mu_); 135 | while (!conn_pool_.empty()) { 136 | first_conn = conn_pool_.begin(); 137 | if (!first_conn->second->cli->TryKeepalive()) { 138 | // Expire connection 139 | RemoveFromLRU(first_conn->second); 140 | conn_pool_.erase(first_conn); 141 | continue; 142 | } 143 | return first_conn->second->cli; 144 | } 145 | return nullptr; 146 | } 147 | 148 | void ConnectionPool::MoveToLRUHead(ZpCliItem* item) { 149 | assert(item != nullptr && 150 | item->next != nullptr && 151 | item->prev != nullptr); 152 | item->prev->next = item->next; 153 | item->next->prev = item->prev; 154 | 155 | item->next = lru_head_.next; 156 | item->prev = &lru_head_; 157 | lru_head_.next->prev = item; 158 | lru_head_.next = item; 159 | } 160 | 161 | void ConnectionPool::InsertLRU(ZpCliItem* item) { 162 | assert(item != nullptr); 163 | item->next = lru_head_.next; 164 | item->prev = &lru_head_; 165 | 166 | lru_head_.next->prev = item; 167 | lru_head_.next = item; 168 | } 169 | 170 | void ConnectionPool::RemoveFromLRU(ZpCliItem* item) { 171 | assert(item != nullptr && 172 | item->next != nullptr && 173 | item->prev != nullptr); 174 | item->prev->next = item->next; 175 | item->next->prev = item->prev; 176 | delete item; 177 | } 178 | 179 | } // namespace libzp 180 | -------------------------------------------------------------------------------- /manager/zp_expansion.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "slash/include/slash_status.h" 6 | #include "libzp/include/zp_cluster.h" 7 | 8 | 9 | void usage() { 10 | std::cout << "usage:\n" 11 | << " zp_expansion meta_host meta_port table [ADD/REMOVE] host port\n"; 12 | } 13 | 14 | void error(const std::string& msg) { 15 | std::cout << msg << std::endl; 16 | usage(); 17 | exit(-1); 18 | } 19 | 20 | int main(int argc, char* argv[]) { 21 | if (argc < 7) { 22 | error("Too few arguments"); 23 | } 24 | 25 | libzp::Options option; 26 | libzp::Node meta_node(argv[1], atoi(argv[2])); 27 | option.meta_addr.push_back(meta_node); 28 | 29 | std::string table = argv[3]; 30 | std::string type = argv[4]; 31 | if (type != "ADD" && type != "REMOVE") { 32 | error("type only can be ADD or REMOVE"); 33 | } 34 | 35 | std::vector target_nodes; 36 | for (int i = 6; i < argc; ++i) { 37 | target_nodes.push_back(libzp::Node(argv[5], atoi(argv[i]))); 38 | } 39 | if (target_nodes.empty()) { 40 | error("No target node"); 41 | return -1; 42 | } 43 | 44 | std::cout << "Expending cluster(" << meta_node << ")." << std::endl; 45 | std::cout << type << "nodes: " << std::endl; 46 | for (auto& n : target_nodes) { 47 | std::cout << " " << n << std::endl; 48 | } 49 | std::cout << "Confirm? (y or n) : "; 50 | char confirm; 51 | std::cin >> confirm; 52 | if (confirm != 'y') { 53 | return 0; 54 | } 55 | 56 | std::cout << "Create cluster, " << meta_node << std::endl; 57 | libzp::Cluster *cluster = new libzp::Cluster(option); 58 | assert(cluster); 59 | 60 | std::cout << "Connect to meta" << std::endl; 61 | slash::Status s = cluster->Connect(); 62 | if (!s.ok()) { 63 | std::cout << "Connect failed: " << s.ToString() << std::endl; 64 | return -1; 65 | } 66 | 67 | std::cout << "Pull meta info from meta node" << std::endl; 68 | libzp::Table table_info; 69 | s = cluster->FetchMetaInfo(table, &table_info); 70 | if (!s.ok()) { 71 | std::cout << "Failed to pull table: " << table << s.ToString() << std::endl; 72 | return -1; 73 | } 74 | 75 | std::cout << "Check target node" << std::endl; 76 | std::set cur_nodes, master_nodes; 77 | table_info.GetAllNodes(&cur_nodes); 78 | table_info.GetAllMasters(&master_nodes); 79 | for (auto& target : target_nodes) { 80 | if (cur_nodes.find(target) != cur_nodes.end() && type == "ADD") { 81 | std::cout << "ADD Target node " << target << " already in cluster." << std::endl; 82 | return -1; 83 | } else if (cur_nodes.find(target) == cur_nodes.end() && type == "REMOVE") { 84 | std::cout << "REMOVE Target node " << target << " is not in cluster." << std::endl; 85 | return -1; 86 | } else if (master_nodes.find(target) != master_nodes.end()) { 87 | std::cout << "REMOVE Target node " << target << " is master in cluster." << std::endl; 88 | return -1; 89 | } 90 | } 91 | 92 | if (type == "ADD") { 93 | std::cout << "Begin ADD nodes into cluster" << std::endl; 94 | for (int i = 0, ti = 0; i < table_info.partition_num(); ++i) { 95 | const libzp::Partition* part = table_info.GetPartitionById(i); 96 | std::vector slaves = part->slaves(); 97 | int slave_num = slaves.size(); 98 | if (slave_num > 1) { 99 | continue; 100 | } 101 | s = cluster->AddSlave(table, i, target_nodes[ti]); 102 | std::cout << (s.ok() ? "Success" : "Failed") 103 | << " to add slave(" << target_nodes[ti] << ") into table:" << table 104 | << ", partition:" << i << " " << (s.ok() ? "" : s.ToString()) << std::endl; 105 | ti = (ti + 1) % target_nodes.size(); 106 | } 107 | std::cout << "Finish ADD nodes into cluster" << std::endl; 108 | } else { 109 | std::cout << "Begin REMOVE nodes into cluster" << std::endl; 110 | std::set target_set; 111 | for (auto t : target_nodes) { 112 | target_set.insert(t); 113 | } 114 | for (int i = 0; i < table_info.partition_num(); ++i) { 115 | const libzp::Partition* part = table_info.GetPartitionById(i); 116 | assert(part); 117 | std::vector slaves = part->slaves(); 118 | int slave_num = slaves.size(); 119 | for (auto& slave : slaves) { 120 | auto iter = target_set.find(slave); 121 | if (iter != target_set.end()) { 122 | if (slave_num <= 1) { 123 | std::cout << "REMOVE Target node " << *iter 124 | << "is the last one of table:" << table << ", partition:" << i << std::endl; 125 | return -1; 126 | } 127 | s = cluster->RemoveSlave(table, i, *iter); 128 | slave_num--; 129 | std::cout << (s.ok() ? "Success" : "Failed") 130 | << " to remove slave(" << *iter << ") from table:"<< table 131 | << ", partition:" << i << " " << (s.ok() ? "" : s.ToString()) << std::endl; 132 | } 133 | } 134 | } 135 | std::cout << "Finish REMOVE nodes from cluster" << std::endl; 136 | } 137 | 138 | delete cluster; 139 | } 140 | -------------------------------------------------------------------------------- /gozeppelin/src/zeppelin/zeppelin.go: -------------------------------------------------------------------------------- 1 | package zeppelin 2 | 3 | /* 4 | #cgo CFLAGS: -I${SRCDIR}/include 5 | #cgo LDFLAGS: -L${SRCDIR}/lib -lzp -lm -lpthread -lprotobuf -lstdc++ 6 | #include 7 | #include 8 | #include "libzp/include/zp_client_c.h" 9 | */ 10 | import "C" 11 | 12 | import "fmt" 13 | import "runtime" 14 | import "unsafe" 15 | 16 | type ZpClient struct { 17 | zp_client *C.struct_zp_client_t 18 | } 19 | 20 | type ZpNode struct { 21 | IP string 22 | Port int 23 | } 24 | 25 | func NewZpClient(table_name string, op_timeout int, metas ...ZpNode) (client *ZpClient) { 26 | client = &ZpClient{} 27 | 28 | // Init zp_node_vec_t of metas 29 | zp_metas := C.zp_nodevec_create() 30 | defer C.zp_nodevec_destroy(zp_metas) 31 | 32 | for _, v := range metas { 33 | c_ip_str := C.CString(v.IP) 34 | zp_meta := C.zp_node_create1(c_ip_str, C.int(v.Port)) 35 | C.zp_nodevec_pushback(zp_metas, zp_meta) 36 | defer C.free(unsafe.Pointer(c_ip_str)) 37 | } 38 | 39 | // Create zp_option_t 40 | zp_option := C.zp_option_create(zp_metas, C.int(op_timeout)) 41 | defer C.zp_option_destroy(zp_option) 42 | 43 | // Create zp_client 44 | c_table_name_str := C.CString(table_name) 45 | defer C.free(unsafe.Pointer(c_table_name_str)) 46 | client.zp_client = C.zp_client_create(zp_option, c_table_name_str) 47 | 48 | runtime.SetFinalizer(client, DeleteZpClient) 49 | return client 50 | } 51 | 52 | func DeleteZpClient(client *ZpClient) { 53 | fmt.Println("DeleteZpClient") 54 | C.zp_client_destroy(client.zp_client); 55 | } 56 | 57 | func (client *ZpClient) Set(key *[]byte, value *[]byte, ttl int) (bool, string) { 58 | zp_key := C.zp_string_create1((*C.char)(unsafe.Pointer(&(*key)[0])), C.int(len(*key))) 59 | zp_value := C.zp_string_create1((*C.char)(unsafe.Pointer(&(*value)[0])), C.int(len(*value))) 60 | defer C.zp_string_destroy(zp_key) 61 | defer C.zp_string_destroy(zp_value) 62 | 63 | status := C.zp_set(client.zp_client, zp_key, zp_value, C.int(ttl)) 64 | defer C.zp_status_destroy(status) 65 | 66 | if int(C.zp_status_ok(status)) != 1 { 67 | status_str := C.zp_status_tostring(status) 68 | reason := C.GoString(C.zp_string_data(status_str)) 69 | C.zp_string_destroy(status_str) 70 | return false, reason 71 | } 72 | return true, "OK" 73 | } 74 | 75 | func (client *ZpClient) Get(key *[]byte) (bool, string, []byte) { 76 | zp_key := C.zp_string_create1((*C.char)(unsafe.Pointer(&(*key)[0])), C.int(len(*key))) 77 | zp_value := C.zp_string_create() 78 | defer C.zp_string_destroy(zp_key) 79 | defer C.zp_string_destroy(zp_value) 80 | 81 | status := C.zp_get(client.zp_client, zp_key, zp_value) 82 | defer C.zp_status_destroy(status) 83 | 84 | if int(C.zp_status_ok(status)) != 1 { 85 | status_str := C.zp_status_tostring(status) 86 | reason := C.GoString(C.zp_string_data(status_str)) 87 | C.zp_string_destroy(status_str) 88 | null := make([]byte, 0) 89 | return false, reason, null 90 | } 91 | 92 | v := C.GoBytes(unsafe.Pointer(C.zp_string_data(zp_value)), C.zp_string_length(zp_value)) 93 | return true, "OK", v 94 | } 95 | 96 | func (client *ZpClient) Mget(keys []string, values *map[string]string) (bool, string) { 97 | zp_keys := C.zp_strvec_create() 98 | zp_res_keys := C.zp_strvec_create() 99 | zp_res_values := C.zp_strvec_create() 100 | defer C.zp_strvec_destroy(zp_keys) 101 | defer C.zp_strvec_destroy(zp_res_keys) 102 | defer C.zp_strvec_destroy(zp_res_values) 103 | 104 | for _, v := range keys { 105 | zp_key := C.CString(v) 106 | defer C.free(unsafe.Pointer(zp_key)) 107 | zp_str_key := C.zp_string_create1(zp_key, C.int(len(v))) 108 | C.zp_strvec_pushback(zp_keys, zp_str_key) 109 | } 110 | 111 | status := C.zp_mget(client.zp_client, zp_keys, zp_res_keys, zp_res_values) 112 | defer C.zp_status_destroy(status) 113 | if int(C.zp_status_ok(status)) != 1 { 114 | status_str := C.zp_status_tostring(status) 115 | reason := C.GoString(C.zp_string_data(status_str)) 116 | C.zp_string_destroy(status_str) 117 | return false, reason 118 | } 119 | 120 | for k := C.zp_strvec_popback(zp_res_keys); k != nil; k = 121 | C.zp_strvec_popback(zp_res_keys) { 122 | v := C.zp_strvec_popback(zp_res_values) 123 | defer C.zp_string_destroy(k); 124 | defer C.zp_string_destroy(v); 125 | 126 | gk := C.GoStringN(C.zp_string_data(k), C.zp_string_length(k)) 127 | gv := C.GoStringN(C.zp_string_data(v), C.zp_string_length(v)) 128 | (*values)[gk] = gv 129 | } 130 | 131 | return true, "OK" 132 | } 133 | 134 | func (client *ZpClient) Delete(key string) (bool, string) { 135 | c_key_str := C.CString(key) 136 | defer C.free(unsafe.Pointer(c_key_str)) 137 | 138 | zp_key := C.zp_string_create1(c_key_str, C.int(len(key))) 139 | defer C.zp_string_destroy(zp_key) 140 | 141 | status := C.zp_delete(client.zp_client, zp_key) 142 | defer C.zp_status_destroy(status) 143 | if int(C.zp_status_ok(status)) != 1 { 144 | status_str := C.zp_status_tostring(status) 145 | reason := C.GoString(C.zp_string_data(status_str)) 146 | C.zp_string_destroy(status_str) 147 | return false, reason 148 | } 149 | 150 | return true, "OK" 151 | } 152 | -------------------------------------------------------------------------------- /manager/zp_parade.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "slash/include/env.h" 7 | #include "slash/include/slash_status.h" 8 | #include "libzp/include/zp_client.h" 9 | 10 | 11 | void usage() { 12 | std::cout << "usage:\n" 13 | << " zp_parade host port [count_base]\n"; 14 | } 15 | 16 | enum Op { 17 | kSet = 0, 18 | kGet, 19 | kDelete, 20 | kMget, 21 | }; 22 | 23 | void SingleKeyOp(libzp::Client* client, int count, int vlen, Op op) { 24 | uint64_t start_time = 0, end_time = 0; 25 | std::string value(vlen, 'a'), slen = std::to_string(vlen); 26 | slash::Status s; 27 | std::cout << "Begin Op " << count << " items ..." << std::endl; 28 | start_time = slash::NowMicros(); 29 | for(int i = 0; i < count; ++i) { 30 | std::string si = std::to_string(i); 31 | std::string key = "Key_" + slen + "_" + si; 32 | switch (op) { 33 | case kSet: 34 | s = client->Set(key, value + si); 35 | break; 36 | case kGet: 37 | s = client->Get(key, &value); 38 | break; 39 | case kDelete: 40 | s = client->Delete(key); 41 | break; 42 | default: 43 | break; 44 | } 45 | if (!s.ok()) { 46 | std::cout << "[ERROR]Op failed, len" << vlen 47 | << ", " << s.ToString() << std::endl; 48 | return; 49 | } 50 | } 51 | end_time = slash::NowMicros(); 52 | std::cout << "Success Op" << count << " items, time(us): " 53 | << (end_time - start_time) << std::endl; 54 | 55 | std::cout << std::endl; 56 | } 57 | 58 | void MultiKeyOp(libzp::Client* client, int count, int vlen, Op op) { 59 | uint64_t start_time = 0, end_time = 0; 60 | std::string slen = std::to_string(vlen); 61 | slash::Status s; 62 | 63 | std::string key; 64 | std::vector keys; 65 | std::map values; 66 | for(int i = 0; i < count; ++i) { 67 | key = "Key_" + slen + "_" + std::to_string(i); 68 | keys.push_back(key); 69 | } 70 | 71 | std::cout << "Begin Op " << count << " items ..." << std::endl; 72 | start_time = slash::NowMicros(); 73 | switch (op) { 74 | case kMget: 75 | s = client->Mget(keys, &values); 76 | break; 77 | default: 78 | break; 79 | } 80 | if (!s.ok()) { 81 | std::cout << "[ERROR]Op failed, len" << s.ToString() << std::endl; 82 | return; 83 | } 84 | end_time = slash::NowMicros(); 85 | std::cout << "Success Op" << count << " items, time(us): " 86 | << (end_time - start_time) << std::endl; 87 | 88 | //for (auto& v : values) { 89 | // std::cout << v.first << "->" << v.second << std::endl; 90 | //} 91 | std::cout << std::endl; 92 | } 93 | 94 | int main(int argc, char* argv[]) { 95 | if (argc != 3 && argc != 4) { 96 | usage(); 97 | return -1; 98 | } 99 | 100 | std::cout << "Create client" << std::endl; 101 | libzp::Client* client = new libzp::Client(argv[1], atoi(argv[2]), "parade"); 102 | 103 | std::cout << "Connect cluster" << std::endl; 104 | slash::Status s; 105 | 106 | int count_base = 1; 107 | if (argc == 4) { 108 | count_base = atoi(argv[3]); 109 | } 110 | 111 | std::cout << "--------- SET " << 100 * count_base << " 10B Value ----------" << std::endl; 112 | SingleKeyOp(client, 100 * count_base, 10, Op::kSet); 113 | 114 | std::cout << "--------- SET " << 100 * count_base << " 100B Value ----------" << std::endl; 115 | SingleKeyOp(client, 100 * count_base, 100, Op::kSet); 116 | 117 | std::cout << "--------- SET " << 100 * count_base << " 1KB Value ----------" << std::endl; 118 | SingleKeyOp(client, 100 * count_base, 1000, Op::kSet); 119 | 120 | std::cout << "--------- GET " << 100 * count_base << " 10B Value ----------" << std::endl; 121 | SingleKeyOp(client, 100 * count_base, 10, Op::kGet); 122 | 123 | std::cout << "--------- GET " << 100 * count_base << " 100B Value ----------" << std::endl; 124 | SingleKeyOp(client, 100 * count_base, 100, Op::kGet); 125 | 126 | std::cout << "--------- GET " << 100 * count_base << " 1KB Value ----------" << std::endl; 127 | SingleKeyOp(client, 100 * count_base, 1000, Op::kGet); 128 | 129 | std::cout << "--------- MGET " << 100 * count_base << " 10B Value ----------" << std::endl; 130 | MultiKeyOp(client, 100 * count_base, 10, Op::kMget); 131 | 132 | std::cout << "--------- MGET " << 100 * count_base << " 100B Value ----------" << std::endl; 133 | MultiKeyOp(client, 100 * count_base, 100, Op::kMget); 134 | 135 | std::cout << "--------- MGET " << 100 * count_base << " 1KB Value ----------" << std::endl; 136 | MultiKeyOp(client, 100 * count_base, 1000, Op::kMget); 137 | 138 | std::cout << "--------- DELETE " << 100 * count_base << " 10B Value ----------" << std::endl; 139 | SingleKeyOp(client, 100 * count_base, 10, Op::kDelete); 140 | 141 | std::cout << "--------- DELETE " << 100 * count_base << " 100B Value ----------" << std::endl; 142 | SingleKeyOp(client, 100 * count_base, 100, Op::kDelete); 143 | 144 | std::cout << "--------- DELETE " << 100 * count_base << " 1KB Value ----------" << std::endl; 145 | SingleKeyOp(client, 100 * count_base, 1000, Op::kDelete); 146 | 147 | delete client; 148 | } 149 | -------------------------------------------------------------------------------- /libzp/libzp/src/client.proto: -------------------------------------------------------------------------------- 1 | package client; 2 | 3 | enum Type { 4 | SYNC = 0; 5 | SET = 1; 6 | GET = 2; 7 | DEL = 3; 8 | INFOSTATS = 4; 9 | INFOCAPACITY = 5; 10 | INFOREPL= 6; 11 | MGET = 7; 12 | INFOSERVER = 8; 13 | FLUSHDB = 9; 14 | WRITEBATCH = 10; 15 | LISTBYTAG = 11; 16 | DELETEBYTAG = 12; 17 | } 18 | 19 | enum SyncType { 20 | CMD = 0; 21 | SKIP = 1; 22 | LEASE = 2; 23 | } 24 | 25 | enum StatusCode { 26 | kOk = 0; 27 | kNotFound = 1; 28 | kWait = 2; 29 | kError = 3; 30 | kFallback = 4; 31 | kMove = 5; 32 | } 33 | 34 | message Node { 35 | required string ip = 1; 36 | required int32 port = 2; 37 | } 38 | 39 | message SyncOffset { 40 | required int32 filenum = 1; 41 | required int64 offset = 2; 42 | optional int32 partition = 3; 43 | } 44 | 45 | message KeyExpire { 46 | optional int32 base = 1; 47 | required int32 ttl = 2; 48 | } 49 | 50 | message SlaveFallback { 51 | required int64 time = 1; 52 | required SyncOffset before = 2; 53 | required SyncOffset after = 3; 54 | } 55 | 56 | message PartitionState { 57 | required int32 partition_id = 1; 58 | required string role = 2; 59 | required string repl_state = 3; 60 | required Node master = 4; 61 | repeated Node slaves = 5; 62 | required SyncOffset sync_offset = 6; 63 | optional SlaveFallback fallback = 7; 64 | } 65 | 66 | message CmdRequest { 67 | required Type type = 1; 68 | 69 | // Sync 70 | message Sync { 71 | required Node node = 1; 72 | required string table_name = 2; 73 | required SyncOffset sync_offset = 3; 74 | required int64 epoch = 4; 75 | } 76 | optional Sync sync = 2; 77 | 78 | message Set { 79 | required string table_name = 1; 80 | required string key = 2; 81 | required bytes value = 3; 82 | optional string uuid = 4; 83 | optional KeyExpire expire = 5; 84 | } 85 | optional Set set = 3; 86 | 87 | message Get { 88 | required string table_name = 1; 89 | required string key = 2; 90 | optional string uuid = 3; 91 | } 92 | optional Get get = 4; 93 | 94 | // Delete 95 | message Del { 96 | required string table_name = 1; 97 | required string key = 2; 98 | optional string uuid = 3; 99 | } 100 | optional Del del = 5; 101 | 102 | message Info { 103 | optional string table_name = 1; 104 | } 105 | optional Info info = 6; 106 | 107 | message Mget { 108 | required string table_name = 1; 109 | repeated string keys = 2; 110 | } 111 | optional Mget mget = 7; 112 | 113 | message FlushDB { 114 | required string table_name = 1; 115 | required int32 partition_id = 2; 116 | } 117 | optional FlushDB flushdb = 8; 118 | 119 | message WriteBatch { 120 | required string table_name = 1; 121 | repeated string keys_to_add = 2; 122 | repeated string values_to_add = 3; 123 | repeated string keys_to_delete = 4; 124 | required string hash_tag = 5; 125 | } 126 | optional WriteBatch write_batch = 9; 127 | 128 | message ListbyTag { 129 | required string table_name = 1; 130 | required string hash_tag = 2; 131 | } 132 | optional ListbyTag listby_tag = 10; 133 | 134 | message DeletebyTag { 135 | required string table_name = 1; 136 | required string hash_tag = 2; 137 | } 138 | optional DeletebyTag deleteby_tag = 11; 139 | } 140 | 141 | message CmdResponse { 142 | required Type type = 1; 143 | required StatusCode code = 2; 144 | optional string msg = 3; 145 | 146 | message Sync { 147 | required string table_name = 1; 148 | required SyncOffset sync_offset = 2; 149 | } 150 | optional Sync sync = 4; 151 | 152 | message Get { 153 | optional bytes value = 1; 154 | } 155 | optional Get get = 5; 156 | 157 | optional Node redirect = 6; 158 | 159 | // InfoStats 160 | message InfoStats { 161 | required string table_name = 1; 162 | required int64 total_querys = 2; 163 | required int32 qps = 3; 164 | required string latency_info = 4; 165 | } 166 | repeated InfoStats info_stats = 7; 167 | 168 | // InfoCapacity 169 | message InfoCapacity { 170 | required string table_name = 1; 171 | required int64 used = 2; 172 | required int64 remain = 3; 173 | } 174 | repeated InfoCapacity info_capacity = 8; 175 | 176 | // InfoRepl 177 | message InfoRepl { 178 | required string table_name = 1; 179 | required int64 partition_cnt = 2; 180 | repeated PartitionState partition_state = 3; 181 | } 182 | repeated InfoRepl info_repl = 9; 183 | 184 | // Mget 185 | message Mget { 186 | required string key = 1; 187 | required bytes value = 2; 188 | } 189 | repeated Mget mget = 10; 190 | 191 | // InfoServer 192 | message InfoServer { 193 | required int64 epoch = 1; 194 | repeated string table_names = 2; 195 | required Node cur_meta = 3; 196 | required bool meta_renewing = 4; 197 | } 198 | optional InfoServer info_server = 11; 199 | 200 | repeated Mget listby_tag = 12; 201 | } 202 | 203 | message BinlogSkip { 204 | required string table_name = 1; 205 | required int32 partition_id = 2; 206 | required int64 gap = 3; 207 | } 208 | 209 | message SyncLease { 210 | required string table_name = 1; 211 | required int32 partition_id = 2; 212 | required int64 lease = 3; // s 213 | } 214 | 215 | message SyncRequest { 216 | required SyncType sync_type = 1; 217 | required int64 epoch = 2; 218 | required Node from = 3; 219 | optional SyncOffset sync_offset = 4; 220 | optional CmdRequest request = 5; 221 | optional BinlogSkip binlog_skip = 6; 222 | optional SyncLease sync_lease = 7; 223 | } 224 | -------------------------------------------------------------------------------- /libzp/libzp/include/zp_client.h: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright [2016] qihoo" 3 | */ 4 | #ifndef CLIENT_INCLUDE_ZP_CLIENT_H_ 5 | #define CLIENT_INCLUDE_ZP_CLIENT_H_ 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include "slash/include/slash_status.h" 12 | #include "libzp/include/zp_option.h" 13 | 14 | namespace libzp { 15 | 16 | using slash::Status; 17 | 18 | class Cluster; 19 | 20 | class RawClient { 21 | public : 22 | RawClient(const Options& options); 23 | RawClient(const std::string& ip, const int port); 24 | 25 | virtual ~RawClient(); 26 | 27 | // data cmd 28 | Status Set(const std::string& table_name, 29 | const std::string& key, 30 | const std::string& value, 31 | int32_t ttl = -1); 32 | Status Get(const std::string& table_name, 33 | const std::string& key, 34 | std::string* value); 35 | Status Mget(const std::string& table_name, 36 | const std::vector& keys, 37 | std::map* values); 38 | Status Delete(const std::string& table_name, 39 | const std::string& key); 40 | 41 | // async data cmd 42 | Status Aset(const std::string& table_name, 43 | const std::string& key, 44 | const std::string& value, 45 | zp_completion_t complietion, 46 | void* data, 47 | int32_t ttl = -1); 48 | Status Adelete(const std::string& table_name, 49 | const std::string& key, 50 | zp_completion_t complietion, 51 | void* data); 52 | Status Aget(const std::string& table_name, 53 | const std::string& key, 54 | zp_completion_t complietion, 55 | void* data); 56 | Status Amget(const std::string& table_name, 57 | const std::vector& keys, 58 | zp_completion_t complietion, 59 | void* data); 60 | 61 | Status PutRow(const std::string& table_name, 62 | const std::string& primary_key, 63 | const std::map& columns); 64 | 65 | // Get all columns if col_to_delete is empty 66 | Status GetRow(const std::string& table_name, 67 | const std::string& primary_key, 68 | const std::vector& col_to_get, 69 | std::map* columns); 70 | 71 | // Delete all columns if col_to_delete is empty 72 | Status DeleteRow(const std::string& table_name, 73 | const std::string& primary_key, 74 | const std::vector& col_to_delete); 75 | 76 | private : 77 | Cluster* cluster_; 78 | }; 79 | 80 | class Client { 81 | public : 82 | Client(const Options& options, const std::string& table_name) 83 | : raw_client_(options), 84 | table_(table_name) { 85 | } 86 | Client(const std::string& ip, const int port, const std::string& table_name) 87 | : raw_client_(ip, port), 88 | table_(table_name) { 89 | } 90 | virtual ~Client() {} 91 | 92 | // data cmd 93 | Status Set(const std::string& key, 94 | const std::string& value, 95 | int32_t ttl = -1) { 96 | return raw_client_.Set(table_, key, value, ttl); 97 | } 98 | Status Get(const std::string& key, 99 | std::string* value) { 100 | return raw_client_.Get(table_, key, value); 101 | } 102 | Status Mget(const std::vector& keys, 103 | std::map* values) { 104 | return raw_client_.Mget(table_, keys, values); 105 | } 106 | Status Delete(const std::string& key) { 107 | return raw_client_.Delete(table_, key); 108 | } 109 | 110 | // async data cmd 111 | Status Aset(const std::string& key, 112 | const std::string& value, 113 | zp_completion_t complietion, 114 | void* data, 115 | int32_t ttl = -1) { 116 | return raw_client_.Aset(table_, key, value, complietion, data, ttl); 117 | } 118 | Status Adelete(const std::string& key, 119 | zp_completion_t complietion, 120 | void* data) { 121 | return raw_client_.Adelete(table_, key, complietion, data); 122 | } 123 | Status Aget(const std::string& key, 124 | zp_completion_t complietion, 125 | void* data) { 126 | return raw_client_.Aget(table_, key, complietion, data); 127 | } 128 | Status Amget(const std::vector& keys, 129 | zp_completion_t complietion, 130 | void* data) { 131 | return raw_client_.Amget(table_, keys, complietion, data); 132 | } 133 | 134 | Status PutRow(const std::string& primary_key, 135 | const std::map& columns) { 136 | return raw_client_.PutRow(table_, primary_key, columns); 137 | } 138 | 139 | // Get all columns if col_to_delete is empty 140 | Status GetRow(const std::string& primary_key, 141 | const std::vector& col_to_get, 142 | std::map* columns) { 143 | return raw_client_.GetRow(table_, primary_key, col_to_get, columns); 144 | } 145 | 146 | // Delete all columns if col_to_delete is empty 147 | Status DeleteRow(const std::string& primary_key, 148 | const std::vector& col_to_delete) { 149 | return raw_client_.DeleteRow(table_, primary_key, col_to_delete); 150 | } 151 | 152 | private : 153 | RawClient raw_client_; 154 | const std::string table_; 155 | }; 156 | 157 | } // namespace libzp 158 | 159 | #endif // CLIENT_INCLUDE_ZP_CLIENT_H_ 160 | -------------------------------------------------------------------------------- /pyzeppelin/pyzeppelin.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "libzp/include/zp_client.h" 3 | 4 | static void zeppelin_free(void *ptr) 5 | { 6 | libzp::Client *b = static_cast(ptr); 7 | delete b; 8 | return; 9 | } 10 | 11 | static PyObject *create_client(PyObject *, PyObject* args) 12 | { 13 | char* table = NULL; 14 | char* hostname = NULL; 15 | int port = 9866; 16 | 17 | if (!PyArg_ParseTuple(args, "sis", &hostname, &port, &table)) { 18 | return NULL; 19 | } 20 | 21 | libzp::Client *b = new libzp::Client(hostname, port, table); 22 | if (b == NULL) { 23 | return NULL; 24 | } 25 | 26 | return PyCObject_FromVoidPtr(b, zeppelin_free); 27 | } 28 | 29 | static PyObject *remove_client(PyObject *, PyObject* args) 30 | { 31 | PyObject *pyb = NULL; 32 | if (!PyArg_ParseTuple(args, "O", &pyb)) { 33 | return Py_BuildValue("(is)", -1, "ParseTuple Failed"); 34 | } 35 | 36 | void *vb = PyCObject_AsVoidPtr(pyb); 37 | libzp::Client *b = static_cast(vb); 38 | if (b) { 39 | delete b; 40 | } 41 | return NULL; 42 | } 43 | 44 | static PyObject *set(PyObject *, PyObject* args) 45 | { 46 | PyObject *pyb = NULL; 47 | char* key = NULL; 48 | char* val = NULL; 49 | int kl=0, vl=0; 50 | 51 | if (!PyArg_ParseTuple(args, "Os#s#", &pyb, &key, &kl, &val, &vl)) { 52 | return Py_BuildValue("(is)", -1, "ParseTuple Failed"); 53 | } 54 | 55 | void *vb = PyCObject_AsVoidPtr(pyb); 56 | libzp::Client *b = static_cast(vb); 57 | libzp::Status s = b->Set(std::string(key, kl), std::string(val, vl)); 58 | if (!s.ok()) { 59 | return Py_BuildValue("(is#)", -2, s.ToString().data(), s.ToString().size()); 60 | } 61 | 62 | return Py_BuildValue("(is)", 0, "Set OK"); 63 | } 64 | 65 | static PyObject *get(PyObject *, PyObject* args) 66 | { 67 | PyObject *pyb = NULL; 68 | char* key = NULL; 69 | int kl=0; 70 | 71 | if (!PyArg_ParseTuple(args, "Os#", &pyb, &key, &kl)) { 72 | return Py_BuildValue("(is#)", -1, "ParseTuple Failed"); 73 | } 74 | 75 | void *vb = PyCObject_AsVoidPtr(pyb); 76 | libzp::Client *b = static_cast(vb); 77 | std::string val; 78 | libzp::Status s = b->Get(std::string(key, kl), &val); 79 | if (s.IsNotFound()) { 80 | return Py_BuildValue("(is)", 1, NULL); 81 | } else if (!s.ok()) { 82 | return Py_BuildValue("(is#)", -2, s.ToString().data(), s.ToString().size()); 83 | } 84 | 85 | return Py_BuildValue("(is#)", 0, val.data(), val.size()); 86 | } 87 | 88 | static PyObject *mget(PyObject *, PyObject* args) 89 | { 90 | PyObject *pyb = NULL; 91 | PyObject *keylist = NULL; 92 | 93 | if (!PyArg_ParseTuple(args, "OO", &pyb, &keylist)) { 94 | return Py_BuildValue("(is#)", -1, "ParseTuple Failed"); 95 | } 96 | if (!PyList_Check(keylist)) { 97 | return Py_BuildValue("(is#)", -1, "ParseTuple Failed"); 98 | } 99 | 100 | std::vector keys; 101 | char* key = NULL; 102 | int kl = 0; 103 | for(Py_ssize_t i = 0; i < PyList_Size(keylist); i++) { 104 | PyObject *k = PyList_GetItem(keylist, i); 105 | if (!PyArg_Parse(k, "s#", &key, &kl)) { 106 | return Py_BuildValue("(is#)", -1, "Parse key Failed"); 107 | } 108 | keys.push_back(std::string(key, kl)); 109 | } 110 | 111 | std::map values; 112 | 113 | void *vb = PyCObject_AsVoidPtr(pyb); 114 | libzp::Client *b = static_cast(vb); 115 | std::string val; 116 | libzp::Status s = b->Mget(keys, &values); 117 | if (s.IsNotFound()) { 118 | return Py_BuildValue("(is)", 1, NULL); 119 | } else if (!s.ok()) { 120 | return Py_BuildValue("(is#)", -2, s.ToString().data(), s.ToString().size()); 121 | } 122 | 123 | PyObject* res_list = PyList_New(0); 124 | for (auto& kv : values) { 125 | if (kv.second.empty()) { 126 | continue; 127 | } 128 | PyObject* item = Py_BuildValue("(s#s#)", 129 | kv.first.data(), kv.first.size(), 130 | kv.second.data(), kv.second.size()); 131 | int ret = PyList_Append(res_list, item); 132 | if (ret != 0) { 133 | return Py_BuildValue("(is)", ret, "SetItem Failed"); 134 | } 135 | } 136 | 137 | return Py_BuildValue("(iO)", 0, res_list); 138 | } 139 | 140 | static PyObject *zeppelin_delete(PyObject *, PyObject* args) 141 | { 142 | PyObject *pyb = NULL; 143 | char* key = NULL; 144 | int kl=0; 145 | 146 | if (!PyArg_ParseTuple(args, "Os#", &pyb, &key, &kl)) { 147 | return Py_BuildValue("(is#)", -1, "ParseTuple Failed"); 148 | } 149 | 150 | void *vb = PyCObject_AsVoidPtr(pyb); 151 | libzp::Client *b = static_cast(vb); 152 | libzp::Status s = b->Delete(std::string(key, kl)); 153 | if (!s.ok()) { 154 | return Py_BuildValue("(is#)", -2, s.ToString().data(), s.ToString().size()); 155 | } 156 | 157 | return Py_BuildValue("(is)", 0, "Delete OK"); 158 | } 159 | 160 | static PyMethodDef pyzeppelin_methods[] = { 161 | {"create_client", create_client, METH_VARARGS}, 162 | {"set", set, METH_VARARGS}, 163 | {"get", get, METH_VARARGS}, 164 | {"mget", mget, METH_VARARGS}, 165 | {"delete", zeppelin_delete, METH_VARARGS}, 166 | {"remove_client", remove_client, METH_VARARGS}, 167 | {NULL, NULL} 168 | }; 169 | 170 | PyMODINIT_FUNC initpyzeppelin (void) 171 | { 172 | Py_InitModule("pyzeppelin", pyzeppelin_methods); 173 | } 174 | 175 | -------------------------------------------------------------------------------- /libzp/libzp/include/zp_cluster_c.h: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright [2016] qihoo" 3 | */ 4 | #ifndef CLIENT_INCLUDE_ZP_CLUSTER_C_H_ 5 | #define CLIENT_INCLUDE_ZP_CLUSTER_C_H_ 6 | 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | 11 | typedef struct zp_status_t zp_status_t; 12 | typedef struct zp_option_t zp_option_t; 13 | typedef struct zp_cluster_t zp_cluster_t; 14 | typedef struct zp_node_t zp_node_t; 15 | typedef struct zp_node_vec_t zp_node_vec_t; 16 | typedef struct zp_string_t zp_string_t; 17 | typedef struct zp_string_vec_t zp_string_vec_t; 18 | 19 | typedef struct zp_binlog_offset_t { 20 | unsigned int filenum; 21 | unsigned long offset; // 64bit ? 22 | } zp_binlog_offset_t; 23 | 24 | typedef struct zp_partition_view_t { 25 | zp_string_t* role; 26 | zp_string_t* repl_state; 27 | zp_node_t* master; 28 | zp_node_vec_t* slaves; 29 | zp_binlog_offset_t offset; 30 | unsigned long fallback_time; 31 | zp_binlog_offset_t fallback_before; 32 | zp_binlog_offset_t fallback_after; 33 | } zp_partition_view_t; 34 | 35 | typedef struct zp_space_info_t{ 36 | long used; 37 | long remain; 38 | } zp_space_info_t; 39 | 40 | typedef struct zp_server_state_t { 41 | long epoch; 42 | zp_string_vec_t* table_names; 43 | zp_node_t* cur_meta; 44 | int meta_renewing; 45 | } zp_server_state_t; 46 | 47 | 48 | // struct's constructor and destructor 49 | extern int zp_status_ok(const zp_status_t* s); 50 | extern zp_string_t* zp_status_tostring(const zp_status_t* s); 51 | extern void zp_status_destroy(zp_status_t* s); 52 | 53 | extern zp_option_t* zp_option_create(zp_node_vec_t* metas, int op_timeout); 54 | extern void zp_option_destroy(zp_option_t* option); 55 | 56 | extern zp_cluster_t* zp_cluster_create(const zp_option_t* options); 57 | extern void zp_cluster_destroy(zp_cluster_t* cluster); 58 | 59 | extern zp_node_t* zp_node_create1(const char* ip, int port); 60 | extern zp_node_t* zp_node_create(); 61 | extern void zp_node_destroy(zp_node_t* node); 62 | extern char* zp_node_ip(zp_node_t* node); 63 | extern int zp_node_port(zp_node_t* node); 64 | 65 | extern zp_node_vec_t* zp_nodevec_create(); 66 | extern void zp_nodevec_destroy(zp_node_vec_t* vec); 67 | extern void zp_nodevec_pushback(zp_node_vec_t* nodevec, const zp_node_t* node); 68 | extern zp_node_t* zp_nodevec_popback(zp_node_vec_t* nodevec); 69 | extern int zp_nodevec_length(zp_node_vec_t* vec); 70 | extern zp_node_t* zp_nodevec_get(zp_node_vec_t* vec, unsigned int index); 71 | 72 | extern zp_string_t* zp_string_create1(const char* data, int length); 73 | extern zp_string_t* zp_string_create(); 74 | extern void zp_string_destroy(zp_string_t* str); 75 | extern char* zp_string_data(zp_string_t* str); 76 | extern int zp_string_length(zp_string_t* str); 77 | 78 | extern zp_string_vec_t* zp_strvec_create(); 79 | extern void zp_strvec_destroy(zp_string_vec_t* vec); 80 | extern void zp_strvec_pushback(zp_string_vec_t* nodevec, zp_string_t* str); 81 | extern zp_string_t* zp_strvec_popback(zp_string_vec_t* strvec); 82 | extern int zp_strvec_length(zp_string_vec_t* vec); 83 | extern zp_string_t* zp_strvec_get(zp_string_vec_t* vec, unsigned int index); 84 | 85 | extern void zp_partition_view_destroy(zp_partition_view_t* var); 86 | extern void zp_server_state_destroy(zp_server_state_t* var); 87 | 88 | // Zeppelin cluster interface 89 | 90 | // extern zp_status_t* zp_create_table( 91 | // const zp_cluster_t* cluster, 92 | // const char* table_name, 93 | // int partition_num); 94 | 95 | extern zp_status_t* zp_drop_table( 96 | const zp_cluster_t* cluster, 97 | const char* table_name); 98 | 99 | extern zp_status_t* zp_pull( 100 | const zp_cluster_t* cluster, 101 | const char* table); 102 | 103 | // statistical cmd 104 | extern zp_status_t* zp_list_table( 105 | const zp_cluster_t* cluster, 106 | zp_string_vec_t* tables); 107 | 108 | // extern zp_status_t* zp_list_meta( 109 | // const zp_cluster_t* cluster, 110 | // zp_node_t* master, 111 | // zp_node_vec_t* slaves); 112 | 113 | // extern zp_status_t* zp_metastatus( 114 | // const zp_cluster_t* cluster, 115 | // zp_string_t* status); 116 | 117 | extern zp_status_t* zp_list_node( 118 | const zp_cluster_t* cluster, 119 | zp_node_vec_t* nodes, 120 | zp_string_vec_t* status); 121 | 122 | // info cmds 123 | extern zp_status_t* zp_info_qps( 124 | const zp_cluster_t* cluster, 125 | const char* table_name, 126 | int* qps, 127 | long* total_query); 128 | 129 | extern zp_status_t* zp_info_repl( 130 | const zp_cluster_t* cluster, 131 | const zp_node_t* node, 132 | const char* table_name, 133 | int* res_count, 134 | int** p_ids, 135 | zp_partition_view_t** views); 136 | 137 | extern zp_status_t* zp_info_space( 138 | const zp_cluster_t* cluster, 139 | const char* table_name, 140 | int* res_count, 141 | zp_node_vec_t* nodes, 142 | zp_space_info_t** info); 143 | 144 | extern zp_status_t* zp_info_server( 145 | const zp_cluster_t* cluster, 146 | const zp_node_t* node, 147 | zp_server_state_t* state); 148 | 149 | // data operation 150 | extern zp_status_t* zp_cluster_set( 151 | const zp_cluster_t* cluster, 152 | const char* table_name, 153 | const zp_string_t* key, 154 | const zp_string_t* value, 155 | int ttl); 156 | 157 | extern zp_status_t* zp_cluster_get( 158 | const zp_cluster_t* cluster, 159 | const char* table_name, 160 | const zp_string_t* key, 161 | zp_string_t* value); 162 | 163 | extern zp_status_t* zp_cluster_mget( 164 | const zp_cluster_t* cluster, 165 | const char* table_name, 166 | zp_string_vec_t* keys, 167 | zp_string_vec_t* res_keys, 168 | zp_string_vec_t* res_values); 169 | 170 | extern zp_status_t* zp_cluster_delete( 171 | const zp_cluster_t* cluster, 172 | const char* table_name, 173 | const zp_string_t* key); 174 | 175 | 176 | #ifdef __cplusplus 177 | } // extern "C" 178 | #endif 179 | 180 | #endif // CLIENT_INCLUDE_ZP_CLUSTER_H_ 181 | -------------------------------------------------------------------------------- /libzp/libzp/src/zp_entity.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright [2016] qihoo" 3 | */ 4 | #include "libzp/src/zp_meta.pb.h" 5 | #include "libzp/src/client.pb.h" 6 | #include "slash/include/slash_string.h" 7 | #include "libzp/include/zp_entity.h" 8 | 9 | namespace libzp { 10 | 11 | PartitionView::PartitionView(const client::PartitionState& state) 12 | : role(state.role()), 13 | repl_state(state.repl_state()), 14 | master(state.master().ip(), state.master().port()), 15 | offset(state.sync_offset().filenum(), state.sync_offset().offset()), 16 | fallback_time(0) { 17 | for (auto& s : state.slaves()) { 18 | slaves.push_back(Node(s.ip(), s.port())); 19 | } 20 | if (state.has_fallback()) { 21 | fallback_time = state.fallback().time(); 22 | fallback_before = BinlogOffset(state.fallback().before().filenum(), 23 | state.fallback().before().offset()); 24 | fallback_after = BinlogOffset(state.fallback().after().filenum(), 25 | state.fallback().after().offset()); 26 | } 27 | } 28 | 29 | ServerState::ServerState(const client::CmdResponse::InfoServer& state) 30 | : epoch(state.epoch()), 31 | cur_meta(Node(state.cur_meta().ip(), state.cur_meta().port())), 32 | meta_renewing(state.meta_renewing()) { 33 | for (auto& s : state.table_names()) { 34 | table_names.push_back(s); 35 | } 36 | } 37 | 38 | Partition::Partition(const ZPMeta::Partitions& partition_info) 39 | : master_(partition_info.master().ip(), partition_info.master().port()), 40 | id_(partition_info.id()) { 41 | for (int i = 0; i < partition_info.slaves_size(); i++) { 42 | slaves_.push_back(Node(partition_info.slaves(i).ip(), 43 | partition_info.slaves(i).port())); 44 | } 45 | id_ = partition_info.id(); 46 | 47 | switch (partition_info.state()) { 48 | case ZPMeta::PState::ACTIVE: 49 | state_ = kActive; 50 | break; 51 | case ZPMeta::PState::STUCK: 52 | state_ = kStuck; 53 | break; 54 | case ZPMeta::PState::SLOWDOWN: 55 | state_ = kSlowDown; 56 | break; 57 | default: 58 | state_ = kUnknow; 59 | break; 60 | } 61 | } 62 | 63 | void Partition::DebugDump() const { 64 | std::string state_str; 65 | switch (state_) { 66 | case kActive: 67 | state_str.assign("Active"); 68 | break; 69 | case kStuck: 70 | state_str.assign("Stuck"); 71 | break; 72 | case kSlowDown: 73 | state_str.assign("SlowDown"); 74 | break; 75 | default: 76 | state_str.assign("Unknow"); 77 | break; 78 | } 79 | printf(" -%d,\t state: %5s, master: %s:%d\t", 80 | id_, state_str.c_str(), master_.ip.c_str(), master_.port); 81 | for (auto& s : slaves_) { 82 | printf("slave: %s:%d\t", s.ip.c_str(), s.port); 83 | } 84 | printf("\n"); 85 | } 86 | 87 | void Partition::SetMaster(const Node& new_master) { 88 | master_.ip = new_master.ip; 89 | master_.port = new_master.port; 90 | } 91 | 92 | Table::Table() 93 | : partition_num_(0) { 94 | 95 | } 96 | 97 | Table::Table(const ZPMeta::Table& table_info) { 98 | table_name_ = table_info.name(); 99 | partition_num_ = table_info.partitions_size(); 100 | ZPMeta::Partitions partition_info; 101 | for (int i = 0; i < table_info.partitions_size(); i++) { 102 | partition_info = table_info.partitions(i); 103 | partitions_.insert(std::make_pair(partition_info.id(), 104 | Partition(partition_info))); 105 | } 106 | } 107 | 108 | Table::~Table() { 109 | } 110 | 111 | const Partition* Table::GetPartition(const std::string& key) const { 112 | if (partitions_.empty()) { 113 | return NULL; 114 | } 115 | 116 | // key := hash_tag 117 | std::string hash_tag(key); 118 | 119 | size_t l_brace = key.find(kLBrace); 120 | if (l_brace == 0) { 121 | // key := ...{hash_tag}... 122 | size_t r_brace = key.find(kRBrace, l_brace + kLBrace.size()); 123 | if (r_brace != std::string::npos) { 124 | hash_tag.assign(key.begin() + kLBrace.size(), key.begin() + r_brace); 125 | } 126 | } 127 | 128 | int par_num = std::hash()(hash_tag) % partitions_.size(); 129 | 130 | return GetPartitionById(par_num); 131 | } 132 | 133 | const Partition* Table::GetPartitionById(int par_num) const { 134 | if (partitions_.empty()) { 135 | return NULL; 136 | } 137 | auto iter = partitions_.find(par_num); 138 | if (iter != partitions_.end()) { 139 | return &(iter->second); 140 | } else { 141 | return NULL; 142 | } 143 | } 144 | 145 | void Table::DebugDump(int partition_id) const { 146 | std::cout << " -table name: "<< table_name_ <second.DebugDump(); 152 | } else { 153 | std::cout << " -partition: "<< partition_id << ", not exist" <()(key) % partitions_.size(); 170 | return UpdatePartitionMasterById(par_num, target); 171 | } 172 | 173 | Status Table::UpdatePartitionMasterById(int partition_id, 174 | const Node& target) { 175 | if (partitions_.empty()) { 176 | return Status::InvalidArgument("no partition yet"); 177 | } 178 | 179 | auto iter = partitions_.find(partition_id); 180 | if (iter == partitions_.end()) { 181 | return Status::InvalidArgument("not fount partition"); 182 | } 183 | partitions_.at(partition_id).SetMaster(target); 184 | return Status::OK(); 185 | } 186 | 187 | void Table::GetAllMasters(std::set* nodes) const { 188 | for (auto& par : partitions_) { 189 | nodes->insert(par.second.master()); 190 | } 191 | } 192 | 193 | void Table::GetAllNodes(std::set* nodes) const { 194 | for (auto& par : partitions_) { 195 | nodes->insert(par.second.master()); 196 | for (auto& s : par.second.slaves()) { 197 | nodes->insert(s); 198 | } 199 | } 200 | } 201 | 202 | void Table::GetNodesLoads( 203 | std::map>* loads) const { 204 | for (auto& p : partitions_) { 205 | const Partition* p_ptr = &p.second; 206 | // Master 207 | auto mn = p.second.master(); 208 | auto m_iter = loads->find(mn); 209 | if (m_iter == loads->end()) { 210 | loads->insert(std::make_pair(mn, std::vector{p_ptr})); 211 | } else { 212 | m_iter->second.push_back(p_ptr); 213 | } 214 | // Slaves 215 | for (auto& s : p.second.slaves()) { 216 | auto s_iter = loads->find(s); 217 | if (s_iter == loads->end()) { 218 | loads->insert(std::make_pair(s, std::vector{p_ptr})); 219 | } else { 220 | s_iter->second.push_back(p_ptr); 221 | } 222 | } 223 | } 224 | } 225 | 226 | std::ostream& operator<< (std::ostream& out, const BinlogOffset& bo) { 227 | out << bo.filenum << "_" << bo.offset; 228 | return out; 229 | } 230 | 231 | } // namespace libzp 232 | -------------------------------------------------------------------------------- /libzp/libzp/example/zp_parallel.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright [2016] qihoo" 3 | */ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "libzp/include/zp_cluster.h" 11 | #include "libzp/include/zp_client.h" 12 | 13 | static std::string user_data = "ASYNC"; 14 | 15 | libzp::Client* client_ptr = NULL; 16 | libzp::Cluster* cluster_ptr = NULL; 17 | libzp::Status s; 18 | 19 | void usage() { 20 | std::cout << "usage:\n" 21 | << " zp_parallel host port table cnt\n"; 22 | } 23 | 24 | void set_callback(const struct libzp::Result& stat, void* data) { 25 | std::cout << "set " << stat.key << " callback called. Result: "<< stat.ret.ToString() 26 | << ", usr data: "<< *(static_cast(data)) << std::endl; 27 | } 28 | 29 | void get_callback(const struct libzp::Result& stat, void* data) { 30 | std::cout << "get " << stat.key << " callback called. Result: "<< stat.ret.ToString() 31 | << ", value:" << *(stat.value) 32 | << ", usr data: "<< *(static_cast(data)) << std::endl; 33 | } 34 | 35 | void delete_callback(const struct libzp::Result& stat, void* data) { 36 | std::cout << "delete " << stat.key << " callback called. Result: "<< stat.ret.ToString() 37 | << ", usr data: "<< *(static_cast(data)) << std::endl; 38 | } 39 | 40 | void mget_callback(const struct libzp::Result& stat, void* data) { 41 | std::cout << "mget callback called. Result: "<< stat.ret.ToString() << std::endl; 42 | for (auto& get : *stat.kvs) { 43 | std::cout << get.first << " <-> " << get.second << std::endl; 44 | } 45 | std::cout << "usr data: "<< *(static_cast(data)) << std::endl; 46 | } 47 | 48 | 49 | void AsyncSet() { 50 | for(int i = 0; i < 10; i++) { 51 | std::string key = "async_key" + std::to_string(i); 52 | std::string value(30, 'a'); 53 | client_ptr->Aset(key, value, set_callback, &user_data); 54 | } 55 | } 56 | 57 | void AsyncGet() { 58 | for(int i = 0; i < 10; i++) { 59 | std::string key = "async_key" + std::to_string(i); 60 | client_ptr->Aget(key, get_callback, &user_data); 61 | } 62 | } 63 | 64 | void AsyncMget() { 65 | std::vector keys; 66 | for(int i = 0; i < 10; i = i + 10) { 67 | keys.clear(); 68 | for (int j = 0; j < 10; ++j) { 69 | std::string key = "async_key" + std::to_string(i + j); 70 | keys.push_back(key); 71 | } 72 | client_ptr->Amget(keys, mget_callback, &user_data); 73 | } 74 | } 75 | 76 | void AsyncDelete() { 77 | for(int i = 0; i < 10; i++) { 78 | std::string key = "async_key" + std::to_string(i); 79 | client_ptr->Adelete(key, delete_callback, &user_data); 80 | } 81 | } 82 | 83 | void Set() { 84 | libzp::Status s; 85 | for(int i = 0; i < 10; i++) { 86 | std::string key = "key" + std::to_string(i); 87 | std::string value(30, 'a'); 88 | s = client_ptr->Set(key, value); 89 | if (!s.ok()) { 90 | std::cout << "Set key:" << key << " failed, " << s.ToString() << std::endl; 91 | } else { 92 | std::cout << "Set key:" << key << " value: " << value << std::endl; 93 | } 94 | } 95 | } 96 | 97 | void Get() { 98 | for(int i = 0; i < 10; i++) { 99 | std::string key = "key" + std::to_string(i); 100 | std::string value; 101 | s = client_ptr->Get(key, &value); 102 | if (!s.ok()) { 103 | std::cout << "Get key:" << key << " failed, " << s.ToString() << std::endl; 104 | } else { 105 | std::cout << "Get key:" << key << " value: " << value << std::endl; 106 | } 107 | } 108 | } 109 | 110 | void Mget() { 111 | std::vector keys; 112 | std::map kvs; 113 | for(int i = 0; i < 10; i = i + 10) { 114 | keys.clear(); 115 | kvs.clear(); 116 | for (int j = 0; j < 10; ++j) { 117 | std::string key = "key" + std::to_string(i + j); 118 | keys.push_back(key); 119 | } 120 | client_ptr->Mget(keys, &kvs); 121 | } 122 | } 123 | 124 | void Delete() { 125 | for(int i = 0; i < 10; i++) { 126 | std::string key = "key" + std::to_string(i); 127 | s = client_ptr->Delete(key); 128 | if (!s.ok()) { 129 | std::cout << "Delete key:" << key << " failed, " << s.ToString() << std::endl; 130 | } else { 131 | std::cout << "Delete key:" << key << std::endl; 132 | } 133 | } 134 | } 135 | 136 | void CreateTable() { 137 | for(int i = 0; i < 10; i++) { 138 | std::string table = "table" + std::to_string(i); 139 | s = cluster_ptr->CreateTable(table, 3); 140 | if (!s.ok()) { 141 | std::cout << "Create table " << table << " failed, " << s.ToString() << std::endl; 142 | continue; 143 | } else { 144 | std::cout << "Create table " << table << " success" << std::endl; 145 | } 146 | } 147 | } 148 | 149 | void DropTable() { 150 | for(int i = 0; i < 10; i++) { 151 | std::string table = "table" + std::to_string(i); 152 | s = cluster_ptr->DropTable(table); 153 | if (!s.ok()) { 154 | std::cout << "Drop table " << table << " failed, " << s.ToString() << std::endl; 155 | continue; 156 | } else { 157 | std::cout << "Drop table " << table << " success" << std::endl; 158 | } 159 | } 160 | } 161 | 162 | void Pull() { 163 | for(int i = 0; i < 10; i++) { 164 | std::string table = "table" + std::to_string(i); 165 | s = cluster_ptr->Pull(table); 166 | if (!s.ok()) { 167 | std::cout << "Pull table " << table << " failed, " << s.ToString() << std::endl; 168 | continue; 169 | } else { 170 | std::cout << "Pull table " << table << " success" << std::endl; 171 | } 172 | } 173 | } 174 | 175 | 176 | void InfoQps() { 177 | for(int i = 0; i < 10; i++) { 178 | std::string table = "table" + std::to_string(i); 179 | int32_t qps = 0; int64_t total_query = 0; 180 | s = cluster_ptr->InfoQps(table, &qps, &total_query); 181 | if (!s.ok()) { 182 | std::cout << "InfoQps " << table << " failed, " << s.ToString() << std::endl; 183 | continue; 184 | } else { 185 | std::cout << table << " QPS: " << qps << ", TOTAL_QUERY: " << total_query << std::endl; 186 | } 187 | } 188 | } 189 | 190 | int main(int argc, char* argv[]) { 191 | if (argc != 5) { 192 | usage(); 193 | return -1; 194 | } 195 | libzp::Cluster cluster_obj(argv[1], atoi(argv[2])); 196 | cluster_ptr = &cluster_obj; 197 | 198 | // connect to client 199 | libzp::Client client_obj(argv[1], atoi(argv[2]), argv[3]); 200 | client_ptr = &client_obj; 201 | 202 | srand(time(NULL)); 203 | for(int i = 0; i < atoi(argv[4]); i++) { 204 | int random = rand() % 12; 205 | switch(random) { 206 | // Asynchronous command 207 | case 0: 208 | AsyncSet(); 209 | break; 210 | case 1: 211 | AsyncGet(); 212 | break; 213 | case 2: 214 | AsyncMget(); 215 | break; 216 | case 3: 217 | AsyncDelete(); 218 | break; 219 | // Data command 220 | case 4: 221 | Set(); 222 | break; 223 | case 5: 224 | Get(); 225 | break; 226 | case 6: 227 | Delete(); 228 | break; 229 | case 7: 230 | Mget(); 231 | break; 232 | // Meta command 233 | case 8: 234 | CreateTable(); 235 | break; 236 | case 9: 237 | DropTable(); 238 | break; 239 | case 10: 240 | Pull(); 241 | break; 242 | // Statistical cmd 243 | case 11: 244 | InfoQps(); 245 | break; 246 | default: 247 | break; 248 | } 249 | } 250 | std::cout << "Test complete." << std::endl; 251 | sleep(3); 252 | } 253 | -------------------------------------------------------------------------------- /libzp/libzp/include/zp_cluster.h: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright [2016] qihoo" 3 | */ 4 | #ifndef CLIENT_INCLUDE_ZP_CLUSTER_H_ 5 | #define CLIENT_INCLUDE_ZP_CLUSTER_H_ 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "slash/include/slash_status.h" 14 | #include "slash/include/slash_mutex.h" 15 | 16 | #include "libzp/include/zp_entity.h" 17 | 18 | namespace pink { 19 | class BGThread; 20 | } 21 | 22 | namespace client { 23 | class CmdRequest; 24 | class CmdResponse; 25 | } 26 | 27 | namespace ZPMeta { 28 | class MetaCmd; 29 | class MetaCmdResponse; 30 | class MetaCmdResponse_Pull; 31 | } 32 | 33 | namespace libzp { 34 | 35 | class ZpCli; 36 | class ConnectionPool; 37 | struct CmdContext; 38 | 39 | class Cluster { 40 | public: 41 | explicit Cluster(const Options& options); 42 | Cluster(const std::string& ip, int port); 43 | virtual ~Cluster(); 44 | Status Connect(); 45 | 46 | uint64_t op_timeout() { 47 | return options_.op_timeout; 48 | } 49 | 50 | // data cmd 51 | Status Set(const std::string& table, const std::string& key, 52 | const std::string& value, int32_t ttl = -1); 53 | Status Delete(const std::string& table, const std::string& key); 54 | Status Get(const std::string& table, const std::string& key, 55 | std::string* value); 56 | Status Mget(const std::string& table, const std::vector& keys, 57 | std::map* values); 58 | Status FlushTable(const std::string& table); 59 | 60 | // async data cmd 61 | Status Aset(const std::string& table, const std::string& key, 62 | const std::string& value, zp_completion_t complietion, void* data, 63 | int32_t ttl = -1); 64 | Status Adelete(const std::string& table, const std::string& key, 65 | zp_completion_t complietion, void* data); 66 | Status Aget(const std::string& table, const std::string& key, 67 | zp_completion_t complietion, void* data); 68 | Status Amget(const std::string& table, const std::vector& keys, 69 | zp_completion_t complietion, void* data); 70 | 71 | // The keys in a batch will be written into same partition hashed by 'tag' 72 | struct Batch { 73 | explicit Batch(const std::string& hash_tag) : tag(hash_tag) {} 74 | void Delete(const std::string& key) { 75 | keys_tobe_deleted.push_back(tag + key); 76 | } 77 | void Write(const std::string& key, const std::string& value) { 78 | keys_tobe_added.emplace_back(std::make_pair(tag + key, value)); 79 | } 80 | 81 | std::string tag; 82 | std::vector keys_tobe_deleted; 83 | std::vector> keys_tobe_added; 84 | }; 85 | 86 | Status WriteBatch(const std::string& table, const Batch& batch); 87 | Status ListbyTag( 88 | const std::string& table, 89 | const std::string& hash_tag, 90 | std::map* kvs); 91 | Status DeletebyTag(const std::string& table, const std::string& hash_tag); 92 | 93 | // meta cmd 94 | int64_t epoch() { return epoch_; } 95 | Status AddMetaNode(const Node& node); 96 | Status RemoveMetaNode(const Node& node); 97 | Status CreateTable(const std::string& table_name, 98 | const std::vector>& distribution); 99 | Status DropTable(const std::string& table_name); 100 | Status Pull(const std::string& table); 101 | Status FetchMetaInfo(const std::string& table, Table* table_meta); 102 | Status SetMaster(const std::string& table, const int partition, 103 | const Node& ip_port); 104 | Status AddSlave(const std::string& table, const int partition, 105 | const Node& ip_port); 106 | Status RemoveSlave(const std::string& table, const int partition, 107 | const Node& ip_port); 108 | Status RemoveNodes(const std::vector& nodes); 109 | 110 | struct MigrateCmd; 111 | MigrateCmd* GetMigrateCmd(); // Would not lead to memoryleak 112 | Status Expand(const std::string& table, 113 | const std::vector& ip_ports, 114 | MigrateCmd* cmd); 115 | Status Migrate(const std::string& table, 116 | const Node& src_node, 117 | int partition_id, 118 | const Node& dst_node, 119 | MigrateCmd* cmd); 120 | Status Shrink(const std::string& table, 121 | const std::vector& ip_ports, 122 | MigrateCmd* cmd); 123 | Status ReplaceNode(const Node& ori_node, 124 | const Node& dst_node, 125 | MigrateCmd* cmd); 126 | void DumpMigrateCmd(const MigrateCmd* cmd); 127 | Status SubmitMigrateCmd(); 128 | Status CancelMigrate(); 129 | 130 | // statistical cmd 131 | Status ListTable(std::vector* tables); 132 | Status MetaStatus(Node* leader, std::map* meta_status, 133 | int32_t* version, std::string* consistency_stautus); 134 | Status MigrateStatus( 135 | int64_t* migrate_begin_time, int32_t* complete_proportion); 136 | Status ListNode(std::vector* nodes, 137 | std::vector* status); 138 | 139 | Status InfoQps(const std::string& table, int32_t* qps, int64_t* total_query); 140 | Status InfoLatency( 141 | const std::string& table, std::map* latency_info); 142 | Status InfoRepl(const Node& node, 143 | const std::string& table, std::map* partitions); 144 | Status InfoSpace(const std::string& table, 145 | std::vector >* nodes); 146 | Status InfoServer(const Node& node, ServerState* state); 147 | 148 | // local cmd 149 | Status DebugDumpPartition( 150 | const std::string& table, int partition_id = -1, bool dump_nodes = false); 151 | int LocateKey(const std::string& table, const std::string& key); 152 | 153 | std::unordered_map tables(); 154 | 155 | private: 156 | 157 | void Init(); 158 | Status PullInternal(const std::string& table, uint64_t deadline); 159 | Status ListMeta(Node* leader, std::vector* nodes); 160 | Status MetaStatusInternal(Node* leader, std::map* meta_status, 161 | int32_t* version, std::string* consistency_stautus, 162 | int64_t* migrate_begin_time, int32_t* complete_proportion); 163 | bool DeliverMget(CmdContext* context); 164 | bool Deliver(CmdContext* context); 165 | void DeliverAndPull(CmdContext* context); 166 | static void DoAsyncTask(void* arg); 167 | void AddAsyncTask(CmdContext* context); 168 | static void DoNodeTask(void* arg); 169 | void AddNodeTask(const Node& node, CmdContext* context); 170 | Status SubmitDataCmd(const Node& master, 171 | client::CmdRequest& req, client::CmdResponse *res, 172 | uint64_t deadline, int attempt = 0); 173 | Status SubmitMetaCmd(ZPMeta::MetaCmd& req, ZPMeta::MetaCmdResponse *res, 174 | uint64_t deadline, int attempt = 0, const Node* specific_meta = nullptr); 175 | std::shared_ptr GetMetaConnection( 176 | uint64_t deadline, Status* s, const Node* specific_meta = nullptr); 177 | 178 | // options 179 | Options options_; 180 | 181 | // Protect meta info include epoch, tables 182 | // Which may be changed when meta updated 183 | pthread_rwlock_t meta_rw_; 184 | int64_t epoch_; 185 | std::unordered_map tables_; 186 | void ResetMetaInfo(const std::string& table_name, 187 | const ZPMeta::MetaCmdResponse_Pull& pull); 188 | void PointUpdateMetaInfo(const std::string& table_name, 189 | const std::string& sample_key, const Node& target); 190 | 191 | Status GetTableMasters(const std::string& table_name, 192 | std::set* related_nodes); 193 | Status GetDataMasterById(const std::string& table, 194 | int partition_id, Node* master); 195 | Status GetDataMaster(const std::string& table, 196 | const std::string& key, Node* master); 197 | Status UpdateDataMaster(const std::string& table, 198 | const std::string& key, const Node& target); 199 | Status UpdateDataMasterById(const std::string& table_name, 200 | int partition_id, const Node& target); 201 | 202 | // connection pool 203 | ConnectionPool* meta_pool_; 204 | ConnectionPool* data_pool_; 205 | 206 | // Pb command for communication 207 | ZPMeta::MetaCmd *meta_cmd_; 208 | ZPMeta::MetaCmdResponse *meta_res_; 209 | CmdContext* context_; 210 | 211 | // BG worker 212 | slash::Mutex peer_mu_; 213 | std::map peer_workers_; 214 | pink::BGThread* async_worker_; 215 | }; 216 | 217 | } // namespace libzp 218 | 219 | #endif // CLIENT_INCLUDE_ZP_CLUSTER_H_ 220 | -------------------------------------------------------------------------------- /javazeppelin/lib/zeppelin.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Qihoo 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:// www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include "net_qihoo_zeppelin_Zeppelin.h" 15 | 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | 22 | #include "zp_client.h" 23 | 24 | #define INVALID_PARAM "Invalid parameter" 25 | 26 | static void print_error_message(JNIEnv *env, jobject obj, const std::string& msg); 27 | static libzp::Client* get_client(JNIEnv *env, jobject obj); 28 | static jobject convert_to_java_map(JNIEnv *env, const std::map& values); 29 | 30 | JNIEXPORT void JNICALL Java_net_qihoo_zeppelin_Zeppelin_createZeppelin( 31 | JNIEnv * env, jobject obj, jstring ip, jstring port, jstring table) { 32 | const char *ip_c_str = env->GetStringUTFChars(ip, NULL); 33 | if (NULL == ip_c_str) { 34 | print_error_message(env, obj, INVALID_PARAM); 35 | return; 36 | } 37 | const char *port_c_str = env->GetStringUTFChars(port, NULL); 38 | if (NULL == port_c_str) { 39 | env->ReleaseStringUTFChars(ip, ip_c_str); 40 | print_error_message(env, obj, INVALID_PARAM); 41 | return; 42 | } 43 | const char *table_c_str = env->GetStringUTFChars(table, NULL); 44 | if (NULL == table_c_str) { 45 | env->ReleaseStringUTFChars(ip, ip_c_str); 46 | env->ReleaseStringUTFChars(port, port_c_str); 47 | print_error_message(env, obj, INVALID_PARAM); 48 | return; 49 | } 50 | 51 | std::string ip_str(ip_c_str); 52 | std::string port_str(port_c_str); 53 | std::string table_str(table_c_str); 54 | env->ReleaseStringUTFChars(ip, ip_c_str); 55 | env->ReleaseStringUTFChars(port, port_c_str); 56 | env->ReleaseStringUTFChars(table, table_c_str); 57 | libzp::Client* client = new libzp::Client(ip_str, atoi(port_str.c_str()), table_str); 58 | 59 | jclass cls = env->GetObjectClass(obj); 60 | jfieldID fid_ptr = env->GetFieldID(cls, "ptr", "J"); 61 | env->DeleteLocalRef(cls); 62 | if (NULL == fid_ptr) { 63 | print_error_message(env, obj, INVALID_PARAM); 64 | return; 65 | } 66 | jlong ptr = env->GetLongField(obj, fid_ptr); 67 | ptr = (jlong)client; 68 | env->SetLongField(obj, fid_ptr, ptr); 69 | } 70 | 71 | JNIEXPORT void JNICALL Java_net_qihoo_zeppelin_Zeppelin_removeZeppelin( 72 | JNIEnv * env, jobject obj) { 73 | libzp::Client* client = get_client(env, obj); 74 | if (NULL == client) { 75 | print_error_message(env, obj, INVALID_PARAM); 76 | return; 77 | } 78 | 79 | delete client; 80 | 81 | jclass cls = env->GetObjectClass(obj); 82 | jfieldID fid_ptr = env->GetFieldID(cls, "ptr", "J"); 83 | env->DeleteLocalRef(cls); 84 | if (NULL == fid_ptr) { 85 | print_error_message(env, obj, INVALID_PARAM); 86 | return; 87 | } 88 | jlong ptr = 0; 89 | env->SetLongField(obj, fid_ptr, ptr); 90 | } 91 | 92 | JNIEXPORT jboolean JNICALL Java_net_qihoo_zeppelin_Zeppelin_set(JNIEnv * env, 93 | jobject obj, jstring key, jstring value) { 94 | libzp::Client* client = get_client(env, obj); 95 | if (NULL == client) { 96 | print_error_message(env, obj, INVALID_PARAM); 97 | return false; 98 | } 99 | 100 | const char *key_c_str = env->GetStringUTFChars(key, NULL); 101 | if (NULL == key_c_str) { 102 | print_error_message(env, obj, INVALID_PARAM); 103 | return false; 104 | } 105 | const char *value_c_str = env->GetStringUTFChars(value, NULL); 106 | if (NULL == value_c_str) { 107 | env->ReleaseStringUTFChars(key, key_c_str); 108 | print_error_message(env, obj, INVALID_PARAM); 109 | return false; 110 | } 111 | std::string key_str(key_c_str); 112 | std::string value_str(value_c_str); 113 | env->ReleaseStringUTFChars(key, key_c_str); 114 | env->ReleaseStringUTFChars(value, value_c_str); 115 | 116 | libzp::Status s = client->Set(key_str, value_str); 117 | if (!s.ok()) { 118 | print_error_message(env, obj, s.ToString()); 119 | return false; 120 | } 121 | return true; 122 | } 123 | 124 | JNIEXPORT jstring JNICALL Java_net_qihoo_zeppelin_Zeppelin_get(JNIEnv *env, 125 | jobject obj, jstring key) { 126 | libzp::Client* client = get_client(env, obj); 127 | if (NULL == client) { 128 | print_error_message(env, obj, INVALID_PARAM); 129 | return NULL; 130 | } 131 | const char *key_c_str = env->GetStringUTFChars(key, NULL); 132 | if (NULL == key_c_str) { 133 | print_error_message(env, obj, INVALID_PARAM); 134 | return NULL; 135 | } 136 | 137 | std::string key_str(key_c_str); 138 | env->ReleaseStringUTFChars(key, key_c_str); 139 | std::string value_str; 140 | libzp::Status s = client->Get(key_str, &value_str); 141 | if (!s.ok() && !s.IsNotFound()) { 142 | print_error_message(env, obj, s.ToString()); 143 | return NULL; 144 | } 145 | return env->NewStringUTF(value_str.c_str()); 146 | } 147 | 148 | JNIEXPORT jobject JNICALL Java_net_qihoo_zeppelin_Zeppelin_mget(JNIEnv *env, 149 | jobject obj, jobjectArray string_array) { 150 | libzp::Client* client = get_client(env, obj); 151 | if (NULL == client) { 152 | print_error_message(env, obj, INVALID_PARAM); 153 | return NULL; 154 | } 155 | 156 | std::vector keys; 157 | jsize length = env->GetArrayLength(string_array); 158 | for (int i = 0; i < length; ++i) { 159 | jstring key = (jstring)env->GetObjectArrayElement(string_array, i); 160 | const char *key_c_str = env->GetStringUTFChars(key, NULL); 161 | std::string key_str(key_c_str); 162 | env->ReleaseStringUTFChars(key, key_c_str); 163 | env->DeleteLocalRef(key); 164 | keys.push_back(key_str); 165 | } 166 | 167 | std::map values; 168 | libzp::Status s = client->Mget(keys, &values); 169 | if (!s.ok()) { 170 | print_error_message(env, obj, s.ToString()); 171 | return NULL; 172 | } 173 | return convert_to_java_map(env, values); 174 | } 175 | 176 | JNIEXPORT jboolean JNICALL Java_net_qihoo_zeppelin_Zeppelin_delete(JNIEnv *env, 177 | jobject obj, jstring key) { 178 | libzp::Client* client = get_client(env, obj); 179 | if (NULL == client) { 180 | print_error_message(env, obj, INVALID_PARAM); 181 | return false; 182 | } 183 | const char *key_c_str = env->GetStringUTFChars(key, NULL); 184 | if (NULL == key_c_str) { 185 | print_error_message(env, obj, INVALID_PARAM); 186 | return false; 187 | } 188 | 189 | std::string key_str(key_c_str); 190 | env->ReleaseStringUTFChars(key, key_c_str); 191 | libzp::Status s = client->Delete(key_str); 192 | if (!s.ok() && !s.IsNotFound()) { 193 | print_error_message(env, obj, s.ToString()); 194 | return false; 195 | } 196 | return true; 197 | } 198 | 199 | static void print_error_message(JNIEnv *env, jobject obj, 200 | const std::string& msg) { 201 | jclass cls = env->GetObjectClass(obj); 202 | jmethodID mid = env->GetStaticMethodID(cls, "exceptionCallback", 203 | "(Ljava/lang/String;)V"); 204 | if (NULL == mid) { 205 | env->DeleteLocalRef(cls); 206 | return; 207 | } 208 | jstring msg_j = env->NewStringUTF(msg.c_str()); 209 | env->CallStaticVoidMethod(cls, mid, msg_j); 210 | env->DeleteLocalRef(msg_j); 211 | env->DeleteLocalRef(cls); 212 | } 213 | 214 | static libzp::Client* get_client(JNIEnv *env, jobject obj) { 215 | jclass cls = env->GetObjectClass(obj); 216 | jfieldID fid_ptr = env->GetFieldID(cls, "ptr", "J"); 217 | env->DeleteLocalRef(cls); 218 | if (NULL == fid_ptr) { 219 | print_error_message(env, obj, INVALID_PARAM); 220 | return NULL; 221 | } 222 | jlong ptr = env->GetLongField(obj, fid_ptr); 223 | return (libzp::Client*)ptr; 224 | } 225 | 226 | static jobject convert_to_java_map(JNIEnv *env, 227 | const std::map& values) { 228 | if (values.empty()) { 229 | return NULL; 230 | } 231 | jclass cls_treemap = env->FindClass("java/util/TreeMap"); 232 | jmethodID mid_init = env->GetMethodID(cls_treemap, "", "()V"); 233 | if (NULL == mid_init) { 234 | env->DeleteLocalRef(cls_treemap); 235 | return NULL; 236 | } 237 | jobject obj_treemap = env->NewObject(cls_treemap, mid_init, ""); 238 | jmethodID mid_put = env->GetMethodID(cls_treemap, "put", 239 | "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;"); 240 | env->DeleteLocalRef(cls_treemap); 241 | std::map::const_iterator iter = values.begin(); 242 | for (; iter != values.end(); ++iter) { 243 | const std::string& key_str = iter->first; 244 | const std::string& value_str = iter->second; 245 | jstring key_j = env->NewStringUTF(key_str.c_str()); 246 | jstring value_j = env->NewStringUTF(value_str.c_str()); 247 | env->CallObjectMethod(obj_treemap, mid_put, key_j, value_j); 248 | env->DeleteLocalRef(key_j); 249 | env->DeleteLocalRef(value_j); 250 | } 251 | return obj_treemap; 252 | } 253 | 254 | -------------------------------------------------------------------------------- /php7_zeppelin/zeppelin.cc: -------------------------------------------------------------------------------- 1 | /* 2 | +----------------------------------------------------------------------+ 3 | | PHP Version 7 | 4 | +----------------------------------------------------------------------+ 5 | | Copyright (c) 1997-2017 The PHP Group | 6 | +----------------------------------------------------------------------+ 7 | | This source file is subject to version 3.01 of the PHP license, | 8 | | that is bundled with this package in the file LICENSE, and is | 9 | | available through the world-wide-web at the following url: | 10 | | http://www.php.net/license/3_01.txt | 11 | | If you did not receive a copy of the PHP license and are unable to | 12 | | obtain it through the world-wide-web, please send a note to | 13 | | license@php.net so we can mail you a copy immediately. | 14 | +----------------------------------------------------------------------+ 15 | | Author: | 16 | +----------------------------------------------------------------------+ 17 | */ 18 | 19 | /* $Id$ */ 20 | extern "C" 21 | { 22 | #ifdef HAVE_CONFIG_H 23 | #include "config.h" 24 | #endif 25 | 26 | #include "php.h" 27 | #include "php_ini.h" 28 | #include "ext/standard/info.h" 29 | #include "php_zeppelin.h" 30 | } 31 | 32 | #include 33 | #include 34 | 35 | #include "libzp/include/zp_client.h" 36 | #include "slash/include/slash_status.h" 37 | 38 | /* If you declare any globals in php_zeppelin.h uncomment this:*/ 39 | ZEND_DECLARE_MODULE_GLOBALS(zeppelin) 40 | 41 | /* True global resources - no need for thread safety here */ 42 | static int le_zeppelin; 43 | 44 | /* Every user-visible function in PHP should document itself in the source */ 45 | /* {{{ proto string confirm_zeppelin_compiled(string arg) 46 | Return a string to confirm that the module is compiled in */ 47 | PHP_FUNCTION(confirm_zeppelin_compiled) 48 | { 49 | char *arg = NULL; 50 | size_t arg_len, len; 51 | zend_string *strg; 52 | 53 | if (zend_parse_parameters(ZEND_NUM_ARGS(), "s", &arg, &arg_len) == FAILURE) { 54 | return; 55 | } 56 | 57 | strg = strpprintf(0, "Congratulations! You have successfully modified ext/%.78s/config.m4. Module %.78s is now compiled into PHP.", "zeppelin", arg); 58 | 59 | RETURN_STR(strg); 60 | } 61 | /* }}} */ 62 | /* The previous line is meant for vim and emacs, so it can correctly fold and 63 | unfold functions in source code. See the corresponding marks just before 64 | function definition, where the functions purpose is also documented. Please 65 | follow this convention for the convenience of others editing your code. 66 | */ 67 | 68 | struct ClientInfo { 69 | ClientInfo() : timeout(-1), zp_cli(NULL) {} 70 | ~ClientInfo() { delete zp_cli; } 71 | std::string addrs; 72 | std::string table; 73 | int timeout; 74 | libzp::Client* zp_cli; 75 | }; 76 | 77 | static bool need_reconnect(ClientInfo* info, std::string new_addrs, 78 | std::string new_table, int new_timeout) { 79 | if (info->addrs != new_addrs || 80 | info->table != new_table || 81 | info->timeout != new_timeout) { 82 | return true; 83 | } 84 | return false; 85 | } 86 | 87 | zend_class_entry *zeppelin_ce; 88 | 89 | PHP_METHOD(Zeppelin, __construct) 90 | { 91 | char *ip = NULL; 92 | int ip_len = 0; 93 | int port = 0; 94 | char *table = NULL; 95 | int table_len = 0; 96 | int id; 97 | int timeout = -1; 98 | libzp::Options options; 99 | char * addrs = NULL; 100 | int addrs_len = 0; 101 | 102 | if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ss|l", 103 | &addrs, &addrs_len, &table, &table_len, 104 | &timeout) == SUCCESS) { 105 | std::vector addr_v; 106 | char *addr = strtok(addrs, ";"); 107 | while(addr != NULL) { 108 | addr_v.push_back(std::string(addr)); 109 | addr = strtok(NULL,";"); 110 | } 111 | 112 | std::string ip; 113 | int port = 0; 114 | // Split address into ip and port 115 | for (size_t i = 0; i < addr_v.size(); i++) { 116 | if(!slash::ParseIpPortString(addr_v[i], ip, port)) { 117 | RETURN_FALSE; 118 | } 119 | libzp::Node node(ip, port); 120 | options.meta_addr.push_back(node); 121 | } 122 | // Timeout 123 | if (timeout != -1) { 124 | options.op_timeout = timeout; 125 | } 126 | // Connect TODO (gaodq) thread safe ? 127 | ClientInfo* info = reinterpret_cast(ZEPPELIN_G(g_zp_cli)); 128 | 129 | if (info->zp_cli == NULL || 130 | need_reconnect(info, addrs, table, timeout)) { 131 | delete info->zp_cli; 132 | info->zp_cli = new libzp::Client(options, std::string(table, table_len)); 133 | info->addrs = addrs; 134 | info->table = table; 135 | info->timeout = timeout; 136 | } 137 | 138 | RETVAL_TRUE; 139 | } else { 140 | RETURN_FALSE; 141 | } 142 | } 143 | 144 | PHP_METHOD(Zeppelin, __destruct) 145 | { 146 | // Do nothing 147 | } 148 | 149 | PHP_METHOD(Zeppelin, set) 150 | { 151 | char *key = NULL; 152 | char *value = NULL; 153 | int key_len = 0; 154 | int value_len = 0; 155 | int ttl = -1; 156 | if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ss|l", 157 | &key, &key_len, &value, &value_len, &ttl) == FAILURE) { 158 | RETVAL_FALSE; 159 | } 160 | 161 | ClientInfo* info = reinterpret_cast(ZEPPELIN_G(g_zp_cli)); 162 | libzp::Client *zp = info->zp_cli; 163 | if (zp != NULL) { 164 | slash::Status s = zp->Set(std::string(key, key_len), std::string(value, value_len), ttl); 165 | if (s.ok()) { 166 | RETVAL_TRUE; // Won't return 167 | } else { 168 | RETURN_FALSE; 169 | } 170 | } else { 171 | RETURN_FALSE; 172 | } 173 | } 174 | 175 | PHP_METHOD(Zeppelin, get) 176 | { 177 | struct timeval tv; 178 | char *key = NULL; 179 | int key_len = 0; 180 | zval *object; 181 | if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s", 182 | &key, &key_len) == FAILURE) { 183 | RETURN_FALSE; 184 | } 185 | 186 | ClientInfo* info = reinterpret_cast(ZEPPELIN_G(g_zp_cli)); 187 | libzp::Client *zp = info->zp_cli; 188 | if (zp != NULL) { 189 | std::string val; 190 | libzp::Status s = zp->Get(std::string(key, key_len), &val); 191 | if (s.ok()) { 192 | RETVAL_STRINGL((char *)val.data(), val.size()); 193 | } else { 194 | RETVAL_FALSE; 195 | } 196 | } else { 197 | RETVAL_FALSE; 198 | } 199 | } 200 | 201 | PHP_METHOD(Zeppelin, mget) 202 | { 203 | zval *keys = NULL; 204 | zval *object; 205 | 206 | if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "a", &keys) == FAILURE) { 207 | RETURN_FALSE; 208 | } 209 | 210 | ClientInfo* info = reinterpret_cast(ZEPPELIN_G(g_zp_cli)); 211 | libzp::Client *zp = info->zp_cli; 212 | if (zp != NULL) { 213 | // keys value 214 | std::vector vkeys; 215 | HashTable *keys_arr = Z_ARRVAL_P(keys); 216 | zval* val; 217 | ZEND_HASH_FOREACH_VAL(keys_arr, val) { 218 | vkeys.push_back(std::string(Z_STR_P(val)->val, Z_STR_P(val)->len)); 219 | } ZEND_HASH_FOREACH_END(); 220 | 221 | std::map result; 222 | libzp::Status s = zp->Mget(vkeys, &result); 223 | if (s.ok()) { 224 | array_init(return_value); 225 | std::map::iterator r_it = result.begin(); 226 | for (; r_it != result.end(); ++r_it) { 227 | add_assoc_stringl_ex(return_value, (char *) r_it->first.data(), r_it->first.size(), 228 | (char *) r_it->second.data(), r_it->second.size()); 229 | } 230 | } else { 231 | RETVAL_FALSE; 232 | } 233 | } else { 234 | RETVAL_FALSE; 235 | } 236 | } 237 | 238 | PHP_METHOD(Zeppelin, delete) 239 | { 240 | char *key = NULL; 241 | int key_len = 0; 242 | zval *object; 243 | if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s", 244 | &key, &key_len) == FAILURE) { 245 | RETURN_FALSE; 246 | } 247 | 248 | ClientInfo* info = reinterpret_cast(ZEPPELIN_G(g_zp_cli)); 249 | libzp::Client *zp = info->zp_cli; 250 | if (zp != NULL) { 251 | libzp::Status s = zp->Delete(std::string(key, key_len)); 252 | if (s.ok()) { 253 | RETVAL_TRUE; 254 | } else { 255 | RETVAL_FALSE; 256 | } 257 | } else { 258 | RETVAL_FALSE; 259 | } 260 | } 261 | 262 | /* {{{ PHP_MSHUTDOWN_FUNCTION 263 | */ 264 | PHP_MSHUTDOWN_FUNCTION(zeppelin) 265 | { 266 | /* uncomment this line if you have INI entries 267 | UNREGISTER_INI_ENTRIES(); 268 | */ 269 | delete reinterpret_cast(ZEPPELIN_G(g_zp_cli)); 270 | return SUCCESS; 271 | } 272 | /* }}} */ 273 | 274 | /* Remove if there's nothing to do at request start */ 275 | /* {{{ PHP_RINIT_FUNCTION 276 | */ 277 | PHP_RINIT_FUNCTION(zeppelin) 278 | { 279 | #if defined(COMPILE_DL_ZEPPELIN) && defined(ZTS) 280 | ZEND_TSRMLS_CACHE_UPDATE(); 281 | #endif 282 | return SUCCESS; 283 | } 284 | /* }}} */ 285 | 286 | /* Remove if there's nothing to do at request end */ 287 | /* {{{ PHP_RSHUTDOWN_FUNCTION 288 | */ 289 | PHP_RSHUTDOWN_FUNCTION(zeppelin) 290 | { 291 | return SUCCESS; 292 | } 293 | /* }}} */ 294 | 295 | /* {{{ PHP_MINFO_FUNCTION 296 | */ 297 | PHP_MINFO_FUNCTION(zeppelin) 298 | { 299 | php_info_print_table_start(); 300 | php_info_print_table_header(2, "zeppelin support", "enabled"); 301 | php_info_print_table_header(2, "zeppelin version", PHP_ZEPPELIN_VERSION); 302 | php_info_print_table_end(); 303 | 304 | /* Remove comments if you have entries in php.ini 305 | DISPLAY_INI_ENTRIES(); 306 | */ 307 | } 308 | /* }}} */ 309 | 310 | /* {{{ zeppelin_functions[] 311 | * 312 | * Every user visible function must have an entry in zeppelin_functions[]. 313 | */ 314 | const zend_function_entry zeppelin_functions[] = { 315 | PHP_FE(confirm_zeppelin_compiled, NULL) /* For testing, remove later. */ 316 | PHP_ME(Zeppelin, __construct, NULL, ZEND_ACC_CTOR | ZEND_ACC_PUBLIC) 317 | PHP_ME(Zeppelin, __destruct, NULL, ZEND_ACC_DTOR | ZEND_ACC_PUBLIC) 318 | PHP_ME(Zeppelin, set, NULL, ZEND_ACC_PUBLIC) 319 | PHP_ME(Zeppelin, get, NULL, ZEND_ACC_PUBLIC) 320 | PHP_ME(Zeppelin, delete, NULL, ZEND_ACC_PUBLIC) 321 | PHP_ME(Zeppelin, mget, NULL, ZEND_ACC_PUBLIC) 322 | PHP_FE_END /* Must be the last line in zeppelin_functions[] */ 323 | }; 324 | /* }}} */ 325 | 326 | /* {{{ PHP_MINIT_FUNCTION 327 | */ 328 | PHP_MINIT_FUNCTION(zeppelin) 329 | { 330 | /* If you have INI entries, uncomment these lines 331 | REGISTER_INI_ENTRIES(); 332 | */ 333 | 334 | zend_class_entry ce; 335 | INIT_CLASS_ENTRY(ce, "Zeppelin", zeppelin_functions); 336 | zeppelin_ce = zend_register_internal_class(&ce TSRMLS_CC); 337 | 338 | ZEPPELIN_G(g_zp_cli) = reinterpret_cast(new ClientInfo()); 339 | 340 | return SUCCESS; 341 | } 342 | /* }}} */ 343 | 344 | /* {{{ zeppelin_module_entry 345 | */ 346 | zend_module_entry zeppelin_module_entry = { 347 | STANDARD_MODULE_HEADER, 348 | "zeppelin", 349 | zeppelin_functions, 350 | PHP_MINIT(zeppelin), 351 | PHP_MSHUTDOWN(zeppelin), 352 | PHP_RINIT(zeppelin), /* Replace with NULL if there's nothing to do at request start */ 353 | PHP_RSHUTDOWN(zeppelin), /* Replace with NULL if there's nothing to do at request end */ 354 | PHP_MINFO(zeppelin), 355 | PHP_ZEPPELIN_VERSION, 356 | STANDARD_MODULE_PROPERTIES 357 | }; 358 | /* }}} */ 359 | 360 | #ifdef COMPILE_DL_ZEPPELIN 361 | #ifdef ZTS 362 | ZEND_TSRMLS_CACHE_DEFINE() 363 | #endif 364 | ZEND_GET_MODULE(zeppelin) 365 | #endif 366 | 367 | /* 368 | * Local variables: 369 | * tab-width: 4 370 | * c-basic-offset: 4 371 | * End: 372 | * vim600: noet sw=4 ts=4 fdm=marker 373 | * vim<600: noet sw=4 ts=4 374 | */ 375 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------