├── Makefile.am ├── libeasy ├── include │ ├── .gitignore │ ├── Makefile.am │ ├── easy_define.h │ ├── easy_list.h │ └── easy_atomic.h ├── .gitignore ├── io │ ├── .gitignore │ ├── easy_file.h │ ├── easy_request.h │ ├── easy_client.h │ ├── easy_message.h │ ├── easy_socket.h │ ├── Makefile.am │ ├── easy_connection.h │ ├── easy_baseth_pool.h │ ├── easy_log.h │ ├── easy_log.c │ ├── easy_file.c │ ├── easy_baseth_pool.c │ ├── easy_message.c │ ├── easy_socket.c │ └── ev_wrap.h ├── memory │ ├── .gitignore │ ├── Makefile.am │ ├── easy_mem_page.h │ ├── easy_mem_slab.h │ └── easy_mem_pool.h ├── packet │ ├── .gitignore │ ├── Makefile.am │ ├── easy_simple_handler.h │ └── http │ │ └── easy_http_handler.h ├── thread │ ├── .gitignore │ ├── Makefile.am │ └── easy_uthread.h ├── util │ ├── .gitignore │ ├── easy_time.h │ ├── Makefile.am │ ├── easy_string.h │ ├── easy_inet.h │ ├── easy_array.h │ ├── easy_array.c │ ├── easy_pool.h │ ├── easy_hash.h │ ├── easy_time.c │ ├── easy_buf.h │ ├── easy_string.c │ ├── easy_inet.c │ ├── easy_buf.c │ ├── easy_hash.c │ └── easy_pool.c └── Makefile.am ├── NEWS ├── doc ├── structure.png ├── TDH_Socket_Compare_zh_CN.pdf ├── TDH_Socket_benchmark_report_zh_CN.xlsx ├── design_zh_CN.txt ├── about-TDH_Socket_zh_CN.txt ├── status_zh_CN.txt └── installation_zh_CN.txt ├── TDHSocket ├── .gitignore ├── tdh_socket_monitor.hpp ├── tdh_socket_table_balance.hpp ├── tdh_socket_decode_request_binary.hpp ├── tdh_socket_decode_request_binary_v2.hpp ├── tdh_socket_request_thread.hpp ├── tdh_socket_optimize.hpp ├── tdh_socket_handler.hpp ├── tdh_socket_statistic.cpp ├── tdh_socket_connection_context.hpp ├── tdh_socket_statistic.hpp ├── Makefile.am ├── mysql_inc.hpp ├── tdh_socket_share.hpp ├── tdh_socket_config.cpp ├── tdh_socket_dbcontext.hpp ├── tdh_socket_config.hpp ├── default_mysql_sysvar_for_tdh_socket.hpp ├── tdh_socket_table_balance.cpp ├── tdh_socket_thd.hpp └── tdh_socket_connection_context.cpp ├── libtdhs ├── .gitignore ├── Makefile.am ├── debug_util.hpp ├── tdh_socket_bloom_filter.h ├── tdh_socket_error.cpp ├── tdh_socket_error.hpp ├── tdh_socket_bloom_filter.c ├── thread_and_lock.hpp ├── tdh_socket_time.hpp ├── tdh_socket_protocol.hpp ├── tdh_socket_time.cpp ├── util.hpp └── tdh_socket_protocol.cpp ├── AUTHORS ├── ChangeLog ├── .gitignore ├── bootstrap.sh ├── python ├── src │ ├── exception.py │ ├── logger.py │ ├── response.py │ ├── block_tdhs_client.py │ ├── statement.py │ ├── tdhs_common.py │ └── batch_statement.py └── test │ ├── test_client_now.py │ ├── test_client_del.py │ ├── test_client_2.py │ ├── test_client_insert.py │ ├── test_client_update.py │ ├── test_client_3.py │ ├── test_client_batch.py │ ├── test_client_count.py │ ├── test_client_test.py │ └── test_client.py ├── README.md └── row_cache └── Installation.txt /Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS=@TDHSOCKET_SUBDIRS@ 2 | -------------------------------------------------------------------------------- /libeasy/include/.gitignore: -------------------------------------------------------------------------------- 1 | /Makefile 2 | /*.in 3 | -------------------------------------------------------------------------------- /NEWS: -------------------------------------------------------------------------------- 1 | No NEWS for TDH_Socket_Plugin_for_MySQL project. 2 | -------------------------------------------------------------------------------- /libeasy/.gitignore: -------------------------------------------------------------------------------- 1 | /.libs/ 2 | /*.la 3 | /Makefile 4 | /*.in 5 | -------------------------------------------------------------------------------- /doc/structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/TDH_Socket/HEAD/doc/structure.png -------------------------------------------------------------------------------- /libeasy/include/Makefile.am: -------------------------------------------------------------------------------- 1 | noinst_HEADERS=easy_atomic.h easy_define.h easy_list.h 2 | -------------------------------------------------------------------------------- /TDHSocket/.gitignore: -------------------------------------------------------------------------------- 1 | /.deps/ 2 | /.libs/ 3 | /Makefile 4 | /*.in 5 | /*.la 6 | /*.lo 7 | /*.o 8 | -------------------------------------------------------------------------------- /libeasy/io/.gitignore: -------------------------------------------------------------------------------- 1 | /.deps/ 2 | /.libs/ 3 | /*.o 4 | /*.lo 5 | /*.la 6 | /Makefile 7 | /*.in 8 | -------------------------------------------------------------------------------- /libtdhs/.gitignore: -------------------------------------------------------------------------------- 1 | /.deps/ 2 | /.libs/ 3 | /*.lo 4 | /*.o 5 | /*.la 6 | /Makefile 7 | /*.in 8 | -------------------------------------------------------------------------------- /libeasy/memory/.gitignore: -------------------------------------------------------------------------------- 1 | /.deps/ 2 | /.libs/ 3 | /*.lo 4 | /*.o 5 | /*.la 6 | /Makefile 7 | /*.in 8 | -------------------------------------------------------------------------------- /libeasy/packet/.gitignore: -------------------------------------------------------------------------------- 1 | /.deps/ 2 | /.libs/ 3 | /*.lo 4 | /*.o 5 | /*.la 6 | /Makefile 7 | /*.in 8 | -------------------------------------------------------------------------------- /libeasy/thread/.gitignore: -------------------------------------------------------------------------------- 1 | /.deps/ 2 | /.libs/ 3 | /*.lo 4 | /*.o 5 | /*.la 6 | /Makefile 7 | /*.in 8 | -------------------------------------------------------------------------------- /libeasy/util/.gitignore: -------------------------------------------------------------------------------- 1 | /.deps/ 2 | /.libs/ 3 | /*.lo 4 | /*.o 5 | /*.la 6 | /Makefile 7 | /*.in 8 | -------------------------------------------------------------------------------- /doc/TDH_Socket_Compare_zh_CN.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/TDH_Socket/HEAD/doc/TDH_Socket_Compare_zh_CN.pdf -------------------------------------------------------------------------------- /doc/TDH_Socket_benchmark_report_zh_CN.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alibaba/TDH_Socket/HEAD/doc/TDH_Socket_benchmark_report_zh_CN.xlsx -------------------------------------------------------------------------------- /libeasy/thread/Makefile.am: -------------------------------------------------------------------------------- 1 | AM_CFLAGS=$(EASY_INCLUDES) 2 | noinst_LTLIBRARIES=libeasy_thread.la 3 | noinst_HEADERS=easy_uthread.h 4 | libeasy_thread_la_SOURCES=easy_uthread.c 5 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Kevin.Huang = {"Real Name": "Huang Ruohui", 2 | "Gmail": "zephyrleaves@gmail.com" , 3 | "Company mail": "wentong@taobao.com" , 4 | "Github": "https://github.com/zephyrleaves"} 5 | -developed TDH_Socket plugin , Java Client , Python Client 6 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | 0.5 - 2013-03-01 2 | * Balance io thread 3 | * Support TC Malloc 4 | * Support between 5 | 0.3.3 - 2012-07-13 6 | * New Hash Code 7 | * Add tdhs_group_commit_limits for limit group max number 8 | 0.3.2 - 2012-06-28 9 | * Add request thread task count limit 10 | 0.3.1 - 2012-03-27 11 | * Initial public release 12 | -------------------------------------------------------------------------------- /libeasy/memory/Makefile.am: -------------------------------------------------------------------------------- 1 | AM_CFLAGS=$(EASY_INCLUDES) 2 | noinst_LTLIBRARIES = libeasy_memory.la 3 | noinst_HEADERS = \ 4 | easy_mem_pool.h \ 5 | easy_mem_page.h \ 6 | easy_mem_slab.h 7 | 8 | libeasy_memory_la_SOURCES = \ 9 | easy_mem_pool.c \ 10 | easy_mem_page.c \ 11 | easy_mem_slab.c 12 | -------------------------------------------------------------------------------- /libeasy/packet/Makefile.am: -------------------------------------------------------------------------------- 1 | AM_CFLAGS=$(EASY_INCLUDES) 2 | noinst_HEADERS= \ 3 | http/easy_http_handler.h \ 4 | http/http_parser.h \ 5 | easy_simple_handler.h 6 | 7 | noinst_LTLIBRARIES=libeasy_packet.la 8 | libeasy_packet_la_SOURCES = \ 9 | http/easy_http_handler.c \ 10 | http/http_parser.c 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /autom4te.cache/ 2 | /Makefile 3 | /*.am.user 4 | /Makefile.in 5 | /TDH_Socket.creator.user 6 | /*.m4 7 | /ar-lib 8 | /build.sh 9 | /config.guess 10 | /config.log 11 | /config.status 12 | /config.sub 13 | /configure 14 | /depcomp 15 | /install-sh 16 | /libtool 17 | /ltmain.sh 18 | /missing 19 | /TDH_Socket.config 20 | /TDH_Socket.creator 21 | /TDH_Socket.files 22 | /TDH_Socket.includes 23 | *.pyc 24 | *.iml 25 | -------------------------------------------------------------------------------- /libeasy/Makefile.am: -------------------------------------------------------------------------------- 1 | LIBDIRS = memory util thread io packet 2 | #SUBDIRS = include $(LIBDIRS) 3 | SUBDIRS = include memory util thread io packet 4 | 5 | noinst_LTLIBRARIES=libeasy.la 6 | libeasy_la_SOURCES= 7 | 8 | #libeasy_la_LIBADD= $(foreach sub,$(LIBDIRS),$(sub)/libeasy_$(sub).la) 9 | libeasy_la_LIBADD= memory/libeasy_memory.la \ 10 | util/libeasy_util.la \ 11 | thread/libeasy_thread.la \ 12 | io/libeasy_io.la \ 13 | packet/libeasy_packet.la 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$1" == "clean" ]; then 4 | make distclean 5 | rm -rf aclocal.m4 autom4te.cache config.guess config.sub configure depcomp INSTALL install-sh ltmain.sh missing config.log config.status libtool 6 | find . -name 'Makefile.in' -exec rm -f {} \; 7 | exit; 8 | fi 9 | 10 | if [ -x "/usr/bin/libtoolize" ]; then 11 | libtoolize --force 12 | else 13 | glibtoolize --force 14 | fi 15 | aclocal 16 | autoconf 17 | automake --add-missing --force 18 | 19 | -------------------------------------------------------------------------------- /python/src/exception.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License version 2 as 6 | # published by the Free Software Foundation. 7 | # 8 | # Authors: 9 | # wentong 10 | 11 | # wentong@taobao.com 12 | # 11-11-22 13 | # 14 | 15 | class request_exception(Exception): 16 | pass 17 | 18 | 19 | class response_exception(Exception): 20 | pass -------------------------------------------------------------------------------- /libeasy/io/easy_file.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_FILE_H_ 11 | #define EASY_FILE_H_ 12 | 13 | #include 14 | #include "easy_io_struct.h" 15 | 16 | /** 17 | * 对文件操作 18 | */ 19 | 20 | EASY_CPP_START 21 | 22 | #define EASY_MAX_FILE_BUFFER 65536 23 | 24 | EASY_CPP_END 25 | 26 | #endif 27 | 28 | -------------------------------------------------------------------------------- /libeasy/util/easy_time.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_TIME_H_ 11 | #define EASY_TIME_H_ 12 | 13 | /** 14 | * time的通用函数 15 | */ 16 | #include "easy_define.h" 17 | 18 | EASY_CPP_START 19 | 20 | int easy_localtime(const time_t *t, struct tm *tp); 21 | int64_t easy_time_now(); 22 | 23 | EASY_CPP_END 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /libeasy/util/Makefile.am: -------------------------------------------------------------------------------- 1 | AM_CFLAGS=$(EASY_INCLUDES) 2 | noinst_LTLIBRARIES=libeasy_util.la 3 | noinst_HEADERS = \ 4 | easy_buf.h \ 5 | easy_hash.h \ 6 | easy_pool.h \ 7 | easy_string.h \ 8 | easy_inet.h \ 9 | easy_time.h \ 10 | easy_array.h 11 | 12 | libeasy_util_la_SOURCES = \ 13 | easy_buf.c \ 14 | easy_hash.c \ 15 | easy_pool.c \ 16 | easy_string.c \ 17 | easy_inet.c \ 18 | easy_time.c \ 19 | easy_array.c 20 | -------------------------------------------------------------------------------- /doc/design_zh_CN.txt: -------------------------------------------------------------------------------- 1 | 设计文档 2 | 3 | 结构图可见:structure.png 4 | 5 | TDH_Socket采用动态IO策略 6 | IO线程处理的处理逻辑 主要源码:tdh_socket_handler.cpp 7 | IO线程处理 8 | 1.解码和编码包,以及相关的网络操作 源码:tdh_socket_protocol.cpp 9 | 2.握手请求 源码:tdh_socket_connection_context.cpp 10 | 3.认证 源码:tdh_socket_share.cpp 11 | 4.请求的分派 源码: tdh_socket_handler.cpp 12 | a)读写请求的分派 13 | b)物理读和逻辑读请求的分派 具体源码:tdh_socket_optimize.cpp 14 | c)对物理读请求的流控 具体源码:tdh_socket_share.cpp 15 | Request线程处理的逻辑 16 | 1.Group Commit逻辑 源码:tdh_socket_request_thread.cpp 17 | 2.具体的数据库操作 源码:tdh_socket_dbcontext.cpp 18 | 19 | 会有一个监控线程来判断当前数据库的状态 源码:tdh_socket_monitor.cpp -------------------------------------------------------------------------------- /libeasy/io/easy_request.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_REQUEST_H_ 11 | #define EASY_REQUEST_H_ 12 | 13 | #include 14 | #include "easy_io_struct.h" 15 | 16 | /** 17 | * 一个request对象 18 | */ 19 | 20 | EASY_CPP_START 21 | 22 | void easy_request_server_done(easy_request_t *r); 23 | void easy_request_client_done(easy_request_t *r); 24 | 25 | EASY_CPP_END 26 | 27 | #endif 28 | 29 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_monitor.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_monitor.hpp 12 | * 13 | * Created on: 2011-11-8 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef TDH_SOCKET_MONITOR_HPP_ 18 | #define TDH_SOCKET_MONITOR_HPP_ 19 | 20 | namespace taobao { 21 | 22 | extern void start_monitor_thd(); 23 | 24 | extern void cancel_monitor_thd(); 25 | 26 | } // namespace taobao 27 | 28 | #endif /* TDH_SOCKET_MONITOR_HPP_ */ 29 | -------------------------------------------------------------------------------- /libeasy/io/easy_client.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef easy_client_H_ 11 | #define easy_client_H_ 12 | 13 | #include 14 | #include "easy_io_struct.h" 15 | 16 | /** 17 | * 主动连接管理 18 | */ 19 | 20 | EASY_CPP_START 21 | 22 | #define EASY_CLIENT_DEFAULT_TIMEOUT 5000 23 | 24 | void *easy_client_list_find(easy_hash_t *table, easy_addr_t *addr); 25 | int easy_client_list_add(easy_hash_t *table, easy_addr_t *addr, easy_hash_list_t *list); 26 | 27 | EASY_CPP_END 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /libeasy/io/easy_message.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_MESSAGE_H_ 11 | #define EASY_MESSAGE_H_ 12 | 13 | #include 14 | #include "easy_io_struct.h" 15 | 16 | /** 17 | * 接收message 18 | */ 19 | 20 | EASY_CPP_START 21 | 22 | easy_message_t *easy_message_create(easy_connection_t *c); 23 | void easy_message_destroy(easy_message_t *m, int del); 24 | int easy_session_process(easy_session_t *s, int stop); 25 | void easy_message_cleanup(easy_buf_t *b, void *args); 26 | 27 | EASY_CPP_END 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /libeasy/util/easy_string.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_STRING_H_ 11 | #define EASY_STRING_H_ 12 | 13 | /** 14 | * inet的通用函数 15 | */ 16 | #include "easy_define.h" 17 | #include "easy_pool.h" 18 | 19 | EASY_CPP_START 20 | 21 | extern char *easy_strncpy(char *dst, const char *src, size_t n); 22 | extern char *easy_string_tohex(const char *str, int n, char *result, int size); 23 | extern char *easy_string_toupper(char *str); 24 | extern char *easy_string_tolower(char *str); 25 | 26 | EASY_CPP_END 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /libtdhs/Makefile.am: -------------------------------------------------------------------------------- 1 | noinst_HEADERS = util.hpp tdh_socket_atomic.hpp thread_and_lock.hpp \ 2 | tdh_socket_protocol.hpp debug_util.hpp \ 3 | tdh_socket_define.hpp tdh_socket_time.hpp \ 4 | tdh_socket_bloom_filter.h tdh_socket_error.hpp 5 | noinst_LTLIBRARIES = libtdhs.la 6 | libtdhs_la_SOURCES = tdh_socket_protocol.cpp tdh_socket_time.cpp tdh_socket_bloom_filter.c \ 7 | tdh_socket_error.cpp 8 | libtdhs_la_LDFLAGS = -static -L../libeasy -module ../libeasy/libeasy.la 9 | libtdhs_la_CFLAGS = $(AM_CFLAGS) $(EASY_INCLUDES) 10 | libtdhs_la_CXXFLAGS = ${libtdhs_la_CFLAGS} 11 | 12 | am__DEPENDENCIES_1=../libeasy/.libs/libeasy.a 13 | ../libeasy/.libs/libeasy.a: 14 | rm -f ${top_srcdir}/libeasy/libeasy.la 15 | cd ${top_srcdir}/libeasy && make 16 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_table_balance.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_table_balance.hpp 12 | * 13 | * Created on: 2011-12-9 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef TDH_SOCKET_TABLE_BALANCE_HPP_ 18 | #define TDH_SOCKET_TABLE_BALANCE_HPP_ 19 | 20 | #include "tdh_socket_define.hpp" 21 | 22 | namespace taobao { 23 | //返回需要分配的hash值 24 | extern uint64_t table_need_balance(tdhs_request_t& req, tdhs_optimize_t type, 25 | uint32_t custom_hash); 26 | 27 | } // namespace taobao 28 | 29 | #endif /* TDH_SOCKET_TABLE_BALANCE_HPP_ */ 30 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_decode_request_binary.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_decode_request_binary.hpp 12 | * 13 | * Created on: 2011-11-21 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef TDH_SOCKET_DECODE_REQUEST_BINARY_HPP_ 18 | #define TDH_SOCKET_DECODE_REQUEST_BINARY_HPP_ 19 | 20 | #include "tdh_socket_dbcontext.hpp" 21 | 22 | namespace taobao { 23 | 24 | extern int decode_request_by_binary(tdhs_request_t &req, tdhs_packet_t& packet); 25 | 26 | } // namespace taobao 27 | 28 | 29 | #endif /* TDH_SOCKET_DECODE_REQUEST_BINARY_HPP_ */ 30 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_decode_request_binary_v2.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_decode_request_binary.hpp 12 | * 13 | * Created on: 2011-11-21 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef TDH_SOCKET_DECODE_REQUEST_BINARY_V2_HPP_ 18 | #define TDH_SOCKET_DECODE_REQUEST_BINARY_V2_HPP_ 19 | 20 | #include "tdh_socket_dbcontext.hpp" 21 | 22 | namespace taobao { 23 | 24 | extern int decode_request_by_binary_v2(tdhs_request_t &req, tdhs_packet_t& packet); 25 | 26 | } // namespace taobao 27 | 28 | 29 | #endif /* TDH_SOCKET_DECODE_REQUEST_BINARY_V2_HPP_ */ 30 | -------------------------------------------------------------------------------- /libeasy/io/easy_socket.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_SOCKET_H_ 11 | #define EASY_SOCKET_H_ 12 | 13 | #include 14 | #include "easy_io_struct.h" 15 | #include "easy_log.h" 16 | 17 | /** 18 | * socket处理 19 | */ 20 | 21 | EASY_CPP_START 22 | 23 | int easy_socket_listen(easy_addr_t *address); 24 | int easy_socket_write(int fd, easy_list_t *l); 25 | 26 | int easy_socket_non_blocking(int fd); 27 | int easy_socket_set_tcpopt(int fd, int option, int value); 28 | int easy_socket_set_opt(int fd, int option, int value); 29 | 30 | EASY_CPP_END 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /libeasy/util/easy_inet.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_INET_H_ 11 | #define EASY_INET_H_ 12 | 13 | /** 14 | * inet的通用函数 15 | */ 16 | #include "easy_define.h" 17 | 18 | EASY_CPP_START 19 | 20 | extern char *easy_inet_addr_to_str(easy_addr_t *addr, char *buffer, int len); 21 | extern easy_addr_t easy_inet_str_to_addr(const char *host, int port); 22 | extern int easy_inet_parse_host(easy_addr_t *address, const char *host, int port); 23 | extern int easy_inet_is_ipaddr(const char *host); 24 | extern int easy_inet_hostaddr(uint64_t *address, int size); 25 | 26 | EASY_CPP_END 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /libtdhs/debug_util.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * debug_util.hpp 12 | * 13 | * Created on: 2011-8-26 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef DEBUG_UTIL_HPP_ 18 | #define DEBUG_UTIL_HPP_ 19 | 20 | #include "util.hpp" 21 | 22 | namespace taobao { 23 | 24 | #define tb_assert(S) 25 | #define TDHS_INLINE inline 26 | 27 | #ifdef TDHS_DEBUG 28 | 29 | #undef TDHS_INLINE 30 | #define TDHS_INLINE 31 | 32 | #undef tb_assert 33 | #define tb_assert(S) \ 34 | if(!(S)){ \ 35 | fatal_abort("tb_assert failed!") \ 36 | } 37 | 38 | #endif 39 | 40 | } // namespace taobao 41 | 42 | #endif /* DEBUG_UTIL_HPP_ */ 43 | -------------------------------------------------------------------------------- /python/test/test_client_now.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License version 2 as 6 | # published by the Free Software Foundation. 7 | # 8 | # Authors: 9 | # wentong 10 | # 11-9-28 11 | # 12 | from block_tdhs_client import * 13 | 14 | con_num = 1 15 | connect_pool = [] 16 | 17 | for i in xrange(con_num): 18 | connect_pool.append(ConnectionManager("t-wentong")) 19 | 20 | for i in xrange(1): 21 | connect = connect_pool[i % con_num] 22 | field_types, records = connect.insert(u"test", u"test", [u"data"], [[TDHS_UPDATE_NOW, u"123"]]) 23 | print field_types 24 | print len(records) 25 | for r in records: 26 | print r 27 | 28 | for c in connect_pool: 29 | c.close() 30 | 31 | -------------------------------------------------------------------------------- /libeasy/util/easy_array.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_ARRAY_H_ 11 | #define EASY_ARRAY_H_ 12 | 13 | /** 14 | * 固定长度数组分配 15 | */ 16 | #include "easy_pool.h" 17 | #include "easy_list.h" 18 | 19 | EASY_CPP_START 20 | 21 | typedef struct easy_array_t { 22 | easy_pool_t *pool; 23 | easy_list_t list; 24 | int object_size; 25 | int count; 26 | } easy_array_t; 27 | 28 | easy_array_t *easy_array_create(int object_size); 29 | void easy_array_destroy(easy_array_t *array); 30 | void *easy_array_alloc(easy_array_t *array); 31 | void easy_array_free(easy_array_t *array, void *ptr); 32 | 33 | EASY_CPP_END 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /python/test/test_client_del.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License version 2 as 6 | # published by the Free Software Foundation. 7 | # 8 | # Authors: 9 | # wentong 10 | # 11-9-28 11 | # 12 | from block_tdhs_client import * 13 | 14 | con_num = 1 15 | connect_pool = [] 16 | 17 | for i in xrange(con_num): 18 | connect_pool.append(ConnectionManager("t-wentong")) 19 | 20 | for i in xrange(1): 21 | connect = connect_pool[i % con_num] 22 | field_types, records = connect.get(u"test", u"s", None,[u"id",u"data"], [[u"3"],[u"4",u"1"]], 23 | TDHS_IN, 0, 0) 24 | print field_types 25 | print len(records) 26 | for r in records: 27 | print r 28 | 29 | for c in connect_pool: 30 | c.close() 31 | 32 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_request_thread.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_request_thread.hpp 12 | * 13 | * Created on: 2011-11-25 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef TDH_SOCKET_REQUEST_THREAD_HPP_ 18 | #define TDH_SOCKET_REQUEST_THREAD_HPP_ 19 | 20 | #include "easy_baseth_pool.h" 21 | 22 | namespace taobao { 23 | extern easy_thread_pool_t *tdhs_thread_pool_create_ex(easy_io_t *eio, int cnt, 24 | easy_baseth_on_start_pt *start, easy_request_process_pt *cb, 25 | void *args); 26 | 27 | extern void wakeup_request_thread(easy_thread_pool_t * thread_pool); 28 | 29 | } // namespace taobao 30 | 31 | #endif /* TDH_SOCKET_REQUEST_THREAD_HPP_ */ 32 | -------------------------------------------------------------------------------- /python/test/test_client_2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License version 2 as 6 | # published by the Free Software Foundation. 7 | # 8 | # Authors: 9 | # wentong 10 | # 11-9-28 11 | # 12 | from block_tdhs_client import * 13 | 14 | con_num = 1 15 | connect_pool = [] 16 | 17 | for i in xrange(con_num): 18 | connect_pool.append(ConnectionManager("t-wentong")) 19 | 20 | for i in xrange(1): 21 | connect = connect_pool[i % con_num] 22 | field_types, records = connect.get(u"test", u"test", u"|id", [u"id", u"data"], [[u"1"],[u"asdasda"],[u"4"]], 23 | TDHS_IN, 0, 0) 24 | print field_types 25 | print len(records) 26 | for r in records: 27 | print r 28 | 29 | for c in connect_pool: 30 | c.close() 31 | 32 | -------------------------------------------------------------------------------- /python/test/test_client_insert.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License version 2 as 6 | # published by the Free Software Foundation. 7 | # 8 | # Authors: 9 | # wentong 10 | # 11-9-28 11 | # 12 | from block_tdhs_client import * 13 | 14 | con_num = 1 15 | connect_pool = [] 16 | 17 | for i in xrange(con_num): 18 | connect_pool.append(ConnectionManager("t-wentong" ,time_out=10000000)) 19 | 20 | for i in xrange(1000000): 21 | connect = connect_pool[i % con_num] 22 | field_types, records = connect.get_statement().insert(u"test", u"test", [u"data"], [[TDHS_UPDATE_SET, u"123"]]) 23 | print field_types 24 | print len(records) 25 | for r in records: 26 | print r 27 | 28 | for c in connect_pool: 29 | c.close() 30 | 31 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_optimize.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_optimize.hpp 12 | * 13 | * Created on: 2011-11-10 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef TDH_SOCKET_OPTIMIZE_HPP_ 18 | #define TDH_SOCKET_OPTIMIZE_HPP_ 19 | 20 | #include "tdh_socket_define.hpp" 21 | 22 | namespace taobao { 23 | 24 | extern int init_optimize(); 25 | 26 | extern tdhs_optimize_t optimize(const tdhs_request_t &request); 27 | 28 | extern void add_optimize(const tdhs_request_t &request); 29 | 30 | extern void destory_optimize(); 31 | 32 | extern void show_optimize_status(char* status, const size_t n); 33 | } // namespace taobao 34 | 35 | #endif /* TDH_SOCKET_OPTIMIZE_HPP_ */ 36 | -------------------------------------------------------------------------------- /python/test/test_client_update.py: -------------------------------------------------------------------------------- 1 | #coding=UTF8 2 | #!/usr/bin/python 3 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License version 2 as 7 | # published by the Free Software Foundation. 8 | # 9 | # Authors: 10 | # wentong 11 | # 11-9-28 12 | from block_tdhs_client import * 13 | 14 | con_num = 1 15 | connect_pool = [] 16 | 17 | for i in xrange(con_num): 18 | connect_pool.append(ConnectionManager("t-wentong")) 19 | 20 | for i in xrange(1): 21 | connect = connect_pool[i % con_num] 22 | field_types, records = connect.update(u"test", u"test", None, [u"data"], 23 | [[TDHS_UPDATE_NOW, u"你们都是坏人"]], [[u"2"],[u"3"]], 24 | TDHS_IN, 0, 0) 25 | print field_types 26 | print len(records) 27 | for r in records: 28 | print r 29 | 30 | for c in connect_pool: 31 | c.close() 32 | 33 | -------------------------------------------------------------------------------- /libeasy/io/Makefile.am: -------------------------------------------------------------------------------- 1 | AM_CFLAGS=-I$(top_srcdir)/src/include -I$(top_srcdir)/src/util \ 2 | -I$(top_srcdir)/src/thread -I$(top_srcdir)/src/memory $(EASY_INCLUDES) 3 | 4 | EXTRA_DIST = ev_vars.h ev_wrap.h ev_epoll.c 5 | noinst_HEADERS=easy_connection.h \ 6 | easy_io.h \ 7 | easy_file.h \ 8 | easy_message.h \ 9 | easy_request.h \ 10 | easy_socket.h \ 11 | easy_client.h \ 12 | easy_io_struct.h \ 13 | easy_baseth_pool.h \ 14 | easy_log.h \ 15 | ev.h 16 | 17 | noinst_LTLIBRARIES=libeasy_io.la 18 | libeasy_io_la_SOURCES = easy_socket.c \ 19 | easy_io.c \ 20 | easy_file.c \ 21 | ev.c \ 22 | easy_connection.c \ 23 | easy_message.c \ 24 | easy_request.c \ 25 | easy_baseth_pool.c \ 26 | easy_log.c \ 27 | easy_client.c 28 | -------------------------------------------------------------------------------- /python/src/logger.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License version 2 as 6 | # published by the Free Software Foundation. 7 | # 8 | # Authors: 9 | # wentong 10 | # 11-1-21 11 | # 12 | 13 | import logging 14 | 15 | class logger: 16 | logger = None 17 | 18 | def __init__(self, handler=None, level=logging.ERROR): 19 | import logging.handlers 20 | 21 | self.logger = logging.getLogger() 22 | if handler is None: 23 | handler = logging.StreamHandler() 24 | fmt = logging.Formatter(logging.BASIC_FORMAT) 25 | handler.setFormatter(fmt) 26 | self.logger.addHandler(handler) 27 | self.logger.setLevel(level) 28 | 29 | def set_level(self, level): 30 | self.logger.setLevel(level) 31 | 32 | def info(self, msg): 33 | self.logger.info(msg) 34 | 35 | def debug(self, msg): 36 | self.logger.debug(msg) -------------------------------------------------------------------------------- /python/test/test_client_3.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License version 2 as 6 | # published by the Free Software Foundation. 7 | # 8 | # Authors: 9 | # wentong 10 | # 11-9-28 11 | # 12 | from block_tdhs_client import * 13 | from exception import response_exception 14 | 15 | con_num = 1 16 | connect_pool = [] 17 | 18 | for i in xrange(con_num): 19 | connect_pool.append(ConnectionManager("t-wentong", read_code="ab", write_code="cd")) 20 | 21 | for i in xrange(100000): 22 | connect = connect_pool[i % con_num] 23 | try: 24 | field_types, records = connect.get(u"tEsT", u"test", None, [u"id", u"data"], [["1"]], 25 | TDHS_EQ, 0, 0) 26 | print field_types 27 | print len(records) 28 | for r in records: 29 | print r 30 | except response_exception, e: 31 | print e 32 | 33 | for c in connect_pool: 34 | c.close() 35 | 36 | -------------------------------------------------------------------------------- /python/test/test_client_batch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License version 2 as 6 | # published by the Free Software Foundation. 7 | # 8 | # Authors: 9 | # wentong 10 | # 11-9-28 11 | # 12 | from block_tdhs_client import * 13 | 14 | con_num = 1 15 | connect_pool = [] 16 | 17 | for i in xrange(con_num): 18 | connect_pool.append(ConnectionManager("t-wentong",time_out=10000000)) 19 | 20 | for i in xrange(1): 21 | connect = connect_pool[i % con_num] 22 | batch = connect.get_batch_statement() 23 | batch.add_insert(u"test", u"test", [u"data"], [[TDHS_UPDATE_SET,u"1"]]) 24 | batch.add_insert(u"test", u"test", [u"data"], [ [TDHS_UPDATE_SET,u"2"]]) 25 | # batch.add_insert(u"test", u"t", [u"a", u"b"], [[TDHS_UPDATE_SET,u"2222"], [TDHS_UPDATE_SET,u"3"]]) 26 | ret = batch.commit() 27 | 28 | for r in ret: 29 | print r 30 | 31 | for c in connect_pool: 32 | c.close() 33 | 34 | -------------------------------------------------------------------------------- /python/test/test_client_count.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License version 2 as 6 | # published by the Free Software Foundation. 7 | # 8 | # Authors: 9 | # wentong 10 | # 11-9-28 11 | # 12 | from block_tdhs_client import * 13 | from exception import response_exception 14 | 15 | con_num = 1 16 | connect_pool = [] 17 | 18 | for i in xrange(con_num): 19 | connect_pool.append(ConnectionManager("t-wentong", read_code="ab", write_code="cd", time_out=10000000)) 20 | 21 | for i in xrange(1): 22 | connect = connect_pool[i % con_num] 23 | try: 24 | field_types, records = connect.count(u"jdbc_test", u"orders", None, [["8"]], 25 | TDHS_LE, 0, 0) 26 | print field_types 27 | print len(records) 28 | for r in records: 29 | print r 30 | except response_exception, e: 31 | print e 32 | 33 | for c in connect_pool: 34 | c.close() 35 | 36 | -------------------------------------------------------------------------------- /python/test/test_client_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License version 2 as 6 | # published by the Free Software Foundation. 7 | # 8 | # Authors: 9 | # wentong 10 | # 11-9-28 11 | # 12 | from block_tdhs_client import * 13 | from exception import response_exception 14 | 15 | con_num = 1 16 | connect_pool = [] 17 | 18 | for i in xrange(con_num): 19 | connect_pool.append(ConnectionManager("t-wentong-u.local" ,time_out=10000000)) 20 | 21 | for i in xrange(1): 22 | connect = connect_pool[i % con_num] 23 | try: 24 | field_types, records = connect.get(u"test", u"test", u"idx_i", [u"id", u"v", u"i"], [["2"],["5"]], 25 | TDHS_BETWEEN, 0, 0) 26 | print field_types 27 | print len(records) 28 | for r in records: 29 | print r 30 | except response_exception, e: 31 | print e 32 | 33 | for c in connect_pool: 34 | c.close() 35 | 36 | -------------------------------------------------------------------------------- /python/test/test_client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License version 2 as 6 | # published by the Free Software Foundation. 7 | # 8 | # Authors: 9 | # wentong 10 | # 11-9-28 11 | # 12 | from block_tdhs_client import * 13 | from exception import response_exception 14 | 15 | con_num = 1 16 | connect_pool = [] 17 | 18 | for i in xrange(con_num): 19 | connect_pool.append(ConnectionManager("t-wentong", read_code="ab", write_code="cd" ,time_out=10000000)) 20 | 21 | for i in xrange(1): 22 | connect = connect_pool[i % con_num] 23 | try: 24 | field_types, records = connect.get(u"teSt", u"t",u"Ab", [u"ID", u"a", u"B"], [["2222"]], 25 | TDHS_DEQ, 0, 0) 26 | print field_types 27 | print len(records) 28 | for r in records: 29 | print r 30 | except response_exception, e: 31 | print e 32 | 33 | for c in connect_pool: 34 | c.close() 35 | 36 | -------------------------------------------------------------------------------- /libtdhs/tdh_socket_bloom_filter.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef TDH_SOCKET_BLOOM_FILTER_H 11 | #define TDH_SOCKET_BLOOM_FILTER_H 12 | 13 | #ifdef __cplusplus 14 | extern "C" { 15 | #endif 16 | 17 | #include 18 | #include 19 | 20 | #define SETBIT(a, n) (a[n/CHAR_BIT] |= (1<<(n%CHAR_BIT))) 21 | #define GETBIT(a, n) (a[n/CHAR_BIT] & (1<<(n%CHAR_BIT))) 22 | #define FSIZE(size) (((size + CHAR_BIT - 1) / CHAR_BIT) * sizeof(char)) 23 | 24 | typedef struct { 25 | size_t asize; 26 | unsigned char *a; 27 | size_t fsize; 28 | } bloom_filter; 29 | 30 | bloom_filter* create_bfilter(size_t size); 31 | void destroy_bfilter(bloom_filter* bFilter); 32 | 33 | void bfilter_add(const bloom_filter* bFilter, const uint64_t* input); 34 | 35 | int bfilter_check(const bloom_filter* bFilter, const uint64_t* input); 36 | 37 | #ifdef __cplusplus 38 | } 39 | #endif 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_handler.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_handler.hpp 12 | * 13 | * Created on: 2011-9-2 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef TDH_SOCKET_HANDLER_HPP_ 18 | #define TDH_SOCKET_HANDLER_HPP_ 19 | 20 | #include 21 | 22 | namespace taobao { 23 | extern int on_server_io_process(easy_request_t *r); 24 | 25 | extern int on_server_process(easy_request_t *r, void *args); 26 | 27 | extern int on_server_connect(easy_connection_t *c); 28 | 29 | extern int on_server_disconnect(easy_connection_t *c); 30 | 31 | extern void *request_thread_start(void* args); 32 | 33 | extern void *slow_request_thread_start(void* args); 34 | 35 | extern void *write_request_thread_start(void* args); 36 | 37 | extern int cleanup_request(easy_request_t *r, void *apacket); 38 | 39 | } // namespace taobao 40 | 41 | #endif /* TDH_SOCKET_HANDLER_HPP_ */ 42 | -------------------------------------------------------------------------------- /libtdhs/tdh_socket_error.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_error.cpp 12 | * 13 | * Created on: 2011-10-25 14 | * Author: wentong 15 | */ 16 | 17 | #include "tdh_socket_error.hpp" 18 | 19 | namespace taobao { 20 | 21 | const char* error_msg[] = { "nothing", "TDH_SOCKET failed to open table!", 22 | "TDH_SOCKET failed to open index!", "TDH_SOCKET field is missing!", 23 | "TDH_SOCKET request can't match the key number!", 24 | "TDH_SOCKET failed to lock table!", 25 | "TDH_SOCKET server don't have enough memory!", 26 | "TDH_SOCKET server can't decode your request!", 27 | "TDH_SOCKET field is missing in filter or use blob!", 28 | "TDH_SOCKET failed to commit, will be rollback!", 29 | "TDH_SOCKET not implemented!", "TDH_SOCKET request time out!", 30 | "TDH_SOCKET request is unauthentication!", 31 | "TDH_SOCKET request is killed!", "TDH_SOCKET request is throttled!" }; 32 | 33 | } // namespace taobao 34 | 35 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_statistic.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_statistic.cpp 12 | * 13 | * Created on: 2011-10-26 14 | * Author: wentong 15 | */ 16 | 17 | #include "tdh_socket_statistic.hpp" 18 | 19 | namespace taobao { 20 | 21 | unsigned long long int open_tables_count = 0; 22 | 23 | unsigned long long int lock_tables_count = 0; 24 | 25 | unsigned long long int unlock_tables_count = 0; 26 | 27 | unsigned long long int close_tables_count = 0; 28 | 29 | unsigned long long int get_count = 0; 30 | 31 | unsigned long long int count_count = 0; 32 | 33 | unsigned long long int update_count = 0; 34 | 35 | unsigned long long int delete_count = 0; 36 | 37 | unsigned long long int insert_count = 0; 38 | 39 | unsigned long long int batch_count = 0; 40 | 41 | long long int last_io_read_per_second = 0; 42 | 43 | unsigned long long int optimize_lv3_assign_to_quick_count = 0; 44 | 45 | unsigned long long int optimize_lv3_assign_to_slow_count = 0; 46 | 47 | unsigned long long int throttle_count = 0; 48 | 49 | } // namespace taobao 50 | 51 | -------------------------------------------------------------------------------- /libtdhs/tdh_socket_error.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_error.hpp 12 | * 13 | * Created on: 2011-10-25 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef TDH_SOCKET_ERROR_HPP_ 18 | #define TDH_SOCKET_ERROR_HPP_ 19 | 20 | namespace taobao { 21 | 22 | typedef enum { 23 | CLIENT_ERROR_CODE_FAILED_TO_OPEN_TABLE = 1, 24 | CLIENT_ERROR_CODE_FAILED_TO_OPEN_INDEX, 25 | CLIENT_ERROR_CODE_FAILED_TO_MISSING_FIELD, 26 | CLIENT_ERROR_CODE_FAILED_TO_MATCH_KEY_NUM, 27 | CLIENT_ERROR_CODE_FAILED_TO_LOCK_TABLE, 28 | CLIENT_ERROR_CODE_NOT_ENOUGH_MEMORY, 29 | CLIENT_ERROR_CODE_DECODE_REQUEST_FAILED, 30 | CLIENT_ERROR_CODE_FAILED_TO_MISSING_FIELD_IN_FILTER_OR_USE_BLOB, 31 | CLIENT_ERROR_CODE_FAILED_TO_COMMIT, 32 | CLIENT_ERROR_CODE_NOT_IMPLEMENTED, 33 | CLIENT_ERROR_CODE_REQUEST_TIME_OUT, 34 | CLIENT_ERROR_CODE_UNAUTHENTICATION, 35 | CLIENT_ERROR_CODE_KILLED, 36 | CLIENT_ERROR_CODE_THROTTLED, 37 | CLIENT_ERROR_CODE_END 38 | } tdhs_client_error_code_t; 39 | 40 | extern const char* error_msg[]; 41 | 42 | } // namespace taobao 43 | 44 | #endif /* TDH_SOCKET_ERROR_HPP_ */ 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## About TDH_Socket_Plugin_for_MySQL 2 | 3 | TDH_Socket_Plugin_for_MySQL is a MySQL plugin which like HandlerSocket(https://github.com/DeNADev/HandlerSocket-Plugin-for-MySQL). 4 | And it provide more features and better performance. 5 | It direct access the MySQL handler to bypass sql parse for better performance, and it use thread pool and a series of strategy to have high performance. 6 | 7 | ### Features: 8 | 1. Like HandlerSocket with Full Functionality 9 | 2. Connection Multiplexing (Dynamic IOStrategy) ,use only one port 10 | 3. DDL no hang (can close opened table manually) 11 | 4. Support stream output (like dump) 12 | * Large amount of data ,less memory used 13 | 5. Easy to use 14 | * Execute command without open_table at first(it can be cached with thread) 15 | * Java Client (https://github.com/alibaba/tdhs-java-client) support JDBC 16 | 6. Support multithreading modifying operation 17 | * A table must be executed in a constant thread 18 | * One table modifying operation can be configured for concurrency (may cause deadlock) 19 | * Client can assign thead which to execute the request 20 | 7. Dynamic working thread number adjustment (more Physical Reads ,more thread working) 21 | 8. Better performance than HandlerSocket when have many Physical Reads scene 22 | 9. Can Throttling the Physical Reads 23 | 24 | More documents are available in doc/. 25 | -------------------------------------------------------------------------------- /libtdhs/tdh_socket_bloom_filter.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #include 11 | #include 12 | 13 | #include "tdh_socket_bloom_filter.h" 14 | 15 | bloom_filter* create_bfilter(size_t size) { 16 | bloom_filter* bFilter; 17 | if (!(bFilter = (bloom_filter*) malloc(sizeof(bloom_filter)))) 18 | return NULL; 19 | bFilter->fsize = FSIZE(size); 20 | if (!(bFilter->a = malloc(bFilter->fsize))) { 21 | free(bFilter); 22 | return NULL; 23 | } 24 | memset(bFilter->a, 0, bFilter->fsize); 25 | bFilter->asize = size; 26 | return bFilter; 27 | } 28 | 29 | void destroy_bfilter(bloom_filter* bFilter) { 30 | if (bFilter == NULL) { 31 | return; 32 | } 33 | if (bFilter->a) { 34 | free(bFilter->a); 35 | bFilter->a = NULL; 36 | } 37 | if (bFilter) { 38 | free(bFilter); 39 | bFilter = NULL; 40 | } 41 | } 42 | 43 | void bfilter_add(const bloom_filter* bFilter, const uint64_t* input) { 44 | SETBIT(bFilter->a, *input % bFilter->asize); 45 | } 46 | 47 | int bfilter_check(const bloom_filter* bFilter, const uint64_t* input) { 48 | return GETBIT(bFilter->a, *input % bFilter->asize); 49 | } 50 | 51 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_connection_context.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_connection_context.hpp 12 | * 13 | * Created on: 2011-9-29 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef TDH_SOCKET_CONNECTION_CONTEXT_HPP_ 18 | #define TDH_SOCKET_CONNECTION_CONTEXT_HPP_ 19 | #include "tdh_socket_protocol.hpp" 20 | #include "tdh_socket_dbcontext.hpp" 21 | #include "util.hpp" 22 | #include 23 | #include 24 | 25 | namespace taobao { 26 | 27 | class tdh_socket_connection_context: private noncopyable { 28 | public: 29 | tdh_socket_connection_context(); 30 | virtual ~tdh_socket_connection_context(); 31 | int init(easy_connection_t* c); 32 | int destory(); 33 | int decode(tdhs_packet_t* request); 34 | int shake_hands_if(tdhs_packet_t* request); 35 | uint32_t get_timeout(); 36 | bool can_read(); 37 | bool can_write(); 38 | private: 39 | bool is_shaked; 40 | easy_connection_t* connection; 41 | decode_request_t* decode_request; 42 | uint32_t time_out; 43 | char permission; 44 | }; 45 | 46 | } /* namespace taobao */ 47 | #endif /* TDH_SOCKET_CONNECTION_CONTEXT_HPP_ */ 48 | -------------------------------------------------------------------------------- /doc/about-TDH_Socket_zh_CN.txt: -------------------------------------------------------------------------------- 1 | ---------------------------------------------------------------------- 2 | TDH_Socket plugin for MySQL 3 | 4 | Copyright(C) 2011-2012 Alibaba Group Holding Limited 5 | 6 | This program is free software; you can redistribute it and/or modify 7 | it under the terms of the GNU General Public License version 2 as 8 | published by the Free Software Foundation. 9 | 10 | ---------------------------------------------------------------------- 11 | 12 | 关于TDH_Socket 13 | 14 | TDH_Socket是一个MySQL daemon plugin 类似于HandlerSocket(https://github.com/DeNADev/HandlerSocket-Plugin-for-MySQL) 15 | 现在TDH_Socket能接受客户端的TCP请求,并且直接通过MySQL的Handler层访问数据,绕开了SQL解析等一系列逻辑 16 | 17 | TDH_Socket的Java客户端可以通过在客户端解析SQL的方式提供JDBC接口来提高易用性,并且也不会降低性能 18 | 19 | TDH_Socket的一些特性和优点: 20 | 1.具有HandlerSocket的全部功能 21 | 2.连接复用,采用动态IO策略,只使用一个port进行通讯 22 | 3.进行DDL操作时不会hang住(可手动关闭被cache住的表) 23 | 4.支持流输出,对于大数量的返回,不会占用太多内存 24 | 5.易用 25 | a)不需要在一开始open_table,会在具体执行时open_table,被open的table还是会被当前线程cache住,下次请求不需要再次open 26 | b)Java客户端(https://github.com/taobao/tdhs-java-client)支持JDBC,在客户端进行SQL解析 27 | 6.支持多线程的并发写操作 28 | a)默认情况下,对一个表的写操作都会在一个固定的线程被执行,从而避免可能的死锁 29 | b)也可以通过配置使一个表的写操作被多个线程并发执行,但是可能会导致死锁而进行回滚 30 | c)客户端可以将写请求发给指定的线程执行,可以在客户端那边在逻辑上保证不会死锁 31 | 7.读线程的动态调整 32 | 8.由于采用读写分开以及物理读和逻辑读的线程分开策略使在有大量物理读的时候也能提供比较高的性能 33 | 9.能对物理读进行流控 34 | 35 | 36 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_statistic.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_statistic.hpp 12 | * 13 | * Created on: 2011-10-26 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef TDH_SOCKET_STATISTIC_HPP_ 18 | #define TDH_SOCKET_STATISTIC_HPP_ 19 | 20 | namespace taobao { 21 | 22 | extern unsigned long long int open_tables_count; 23 | 24 | extern unsigned long long int lock_tables_count; 25 | 26 | extern unsigned long long int unlock_tables_count; 27 | 28 | extern unsigned long long int close_tables_count; 29 | 30 | extern unsigned long long int get_count; 31 | 32 | extern unsigned long long int count_count; 33 | 34 | extern unsigned long long int update_count; 35 | 36 | extern unsigned long long int delete_count; 37 | 38 | extern unsigned long long int insert_count; 39 | 40 | extern unsigned long long int batch_count; 41 | 42 | extern long long int last_io_read_per_second; 43 | 44 | extern unsigned long long int optimize_lv3_assign_to_quick_count; 45 | 46 | extern unsigned long long int optimize_lv3_assign_to_slow_count; 47 | 48 | extern unsigned long long int throttle_count; 49 | 50 | } // namespace taobao 51 | 52 | #endif /* TDH_SOCKET_STATISTIC_HPP_ */ 53 | -------------------------------------------------------------------------------- /TDHSocket/Makefile.am: -------------------------------------------------------------------------------- 1 | pkgplugindir = $(PLUGIN_DIR) 2 | pkgplugin_LTLIBRARIES = tdhsocket.la 3 | noinst_HEADERS = mysql_inc.hpp tdh_socket_config.hpp tdh_socket_handler.hpp tdh_socket_connection_context.hpp \ 4 | tdh_socket_dbcontext.hpp tdh_socket_share.hpp \ 5 | tdh_socket_encode_response.hpp tdh_socket_dbutil.hpp \ 6 | tdh_socket_statistic.hpp tdh_socket_monitor.hpp tdh_socket_optimize.hpp \ 7 | tdh_socket_decode_request_binary.hpp tdh_socket_decode_request_binary_v2.hpp \ 8 | tdh_socket_request_thread.hpp \ 9 | tdh_socket_table_balance.hpp tdh_socket_thd.hpp default_mysql_sysvar_for_tdh_socket.hpp 10 | tdhsocket_la_LDFLAGS = -module ../libtdhs/libtdhs.la 11 | tdhsocket_la_CFLAGS = $(AM_CFLAGS) $(MYSQL_INC) $(MYSQL_CFLAGS) -I../libtdhs $(EASY_INCLUDES) 12 | tdhsocket_la_CXXFLAGS = ${tdhsocket_la_CFLAGS} 13 | tdhsocket_la_SOURCES = tdh_socket.cpp tdh_socket_config.cpp tdh_socket_handler.cpp \ 14 | tdh_socket_connection_context.cpp \ 15 | tdh_socket_dbcontext.cpp tdh_socket_share.cpp \ 16 | tdh_socket_statistic.cpp \ 17 | tdh_socket_monitor.cpp tdh_socket_optimize.cpp \ 18 | tdh_socket_decode_request_binary.cpp tdh_socket_decode_request_binary_v2.cpp \ 19 | tdh_socket_request_thread.cpp \ 20 | tdh_socket_table_balance.cpp 21 | 22 | am__DEPENDENCIES_1=../libtdhs/.libs/libtdhs.a 23 | ${PRESET_LDADD}: 24 | rm -f ${top_srcdir}/libtdhs/libtdhs.la 25 | cd ${top_srcdir}/libtdhs && make -------------------------------------------------------------------------------- /libtdhs/thread_and_lock.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * thread_and_lock.hpp 12 | * 13 | * Created on: 2011-8-25 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef THREAD_AND_LOCK_HPP_ 18 | #define THREAD_AND_LOCK_HPP_ 19 | 20 | #include "debug_util.hpp" 21 | #include 22 | 23 | #ifdef __linux__ 24 | #include 25 | #else 26 | //XXX Compatibility with other operating system 27 | #endif 28 | 29 | #define pthread_wait_cond(mutex, cond) \ 30 | pthread_mutex_lock((mutex)); \ 31 | pthread_cond_wait((cond), (mutex)); \ 32 | pthread_mutex_unlock((mutex)); 33 | 34 | class pthread_mutex_locker { 35 | public: 36 | pthread_mutex_locker(pthread_mutex_t *_lock) : 37 | lock(_lock) { 38 | tb_assert(lock!=NULL) 39 | pthread_mutex_lock(lock); 40 | } 41 | ~pthread_mutex_locker() { 42 | pthread_mutex_unlock(lock); 43 | } 44 | private: 45 | pthread_mutex_t *lock; 46 | }; 47 | 48 | class easy_spin_locker { 49 | public: 50 | easy_spin_locker(easy_atomic_t *_lock) : 51 | lock(_lock) { 52 | tb_assert(lock!=NULL) 53 | easy_spin_lock(lock); 54 | } 55 | ~easy_spin_locker() { 56 | easy_spin_unlock(lock); 57 | } 58 | private: 59 | easy_atomic_t *lock; 60 | }; 61 | 62 | #endif /* THREAD_AND_LOCK_HPP_ */ 63 | -------------------------------------------------------------------------------- /libeasy/util/easy_array.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /** 11 | * 固定长度数组分配 12 | */ 13 | 14 | #include "easy_array.h" 15 | 16 | easy_array_t *easy_array_create(int object_size) 17 | { 18 | easy_pool_t *pool; 19 | easy_array_t *array; 20 | 21 | if ((pool = easy_pool_create(0)) == NULL) 22 | return NULL; 23 | 24 | if ((array = (easy_array_t *)easy_pool_alloc(pool, sizeof(easy_array_t))) == NULL) 25 | return NULL; 26 | 27 | easy_list_init(&array->list); 28 | array->count = 0; 29 | array->pool = pool; 30 | array->object_size = easy_max(object_size, (int)sizeof(easy_list_t)); 31 | 32 | return array; 33 | } 34 | 35 | void easy_array_destroy(easy_array_t *array) 36 | { 37 | easy_pool_destroy(array->pool); 38 | } 39 | 40 | void *easy_array_alloc(easy_array_t *array) 41 | { 42 | if (easy_list_empty(&array->list) == 0) { 43 | array->count --; 44 | char *ptr = (char *)array->list.prev; 45 | easy_list_del((easy_list_t *)ptr); 46 | return ptr; 47 | } 48 | 49 | return easy_pool_alloc(array->pool, array->object_size); 50 | } 51 | 52 | void easy_array_free(easy_array_t *array, void *ptr) 53 | { 54 | array->count ++; 55 | easy_list_add_tail((easy_list_t *)ptr, &array->list); 56 | } 57 | -------------------------------------------------------------------------------- /libeasy/io/easy_connection.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_CONNECTION_H_ 11 | #define EASY_CONNECTION_H_ 12 | 13 | #include 14 | #include "easy_io_struct.h" 15 | 16 | /** 17 | * 连接主程序 18 | */ 19 | 20 | EASY_CPP_START 21 | 22 | #define EASY_CONNECT_ADDR 1 23 | #define EASY_DISCONNECT_ADDR 2 24 | #define EASY_MIN_INTERVAL 0.1 25 | 26 | typedef struct easy_connection_list_t { 27 | easy_connection_t *head; 28 | easy_connection_t *tail; 29 | } easy_connection_list_t; 30 | 31 | // fuction 32 | easy_listen_t *easy_connection_listen_addr(easy_io_t *eio, easy_addr_t addr, easy_io_handler_pt *handler); 33 | void easy_connection_on_wakeup(struct ev_loop *loop, ev_async *w, int revents); 34 | void easy_connection_on_listen(struct ev_loop *loop, ev_async *w, int revents); 35 | int easy_connection_write_socket(easy_connection_t *c); 36 | int easy_connection_request_process(easy_request_t *r, easy_io_process_pt *process); 37 | 38 | int easy_connection_send_session_list(easy_list_t *list); 39 | int easy_connection_session_build(easy_session_t *s); 40 | void easy_connection_wakeup_session(easy_connection_t *c); 41 | void easy_connection_destroy(easy_connection_t *c); 42 | int easy_connection_request_done(easy_request_t *c); 43 | 44 | EASY_CPP_END 45 | 46 | #endif 47 | 48 | -------------------------------------------------------------------------------- /python/src/response.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License version 2 as 6 | # published by the Free Software Foundation. 7 | # 8 | # Authors: 9 | # wentong 10 | # 11-11-22 11 | # 12 | from _struct import unpack 13 | 14 | class response: 15 | def __init__(self, rbuff, len): 16 | self.rbuff = rbuff 17 | self.len = len 18 | 19 | def parse(self): 20 | rpos = 0 21 | records = [] 22 | field_num, = unpack('!L', self.rbuff[rpos:rpos + 4]) 23 | rpos += 4 24 | field_types = [] 25 | for i in xrange(field_num): 26 | type, = unpack('!B', self.rbuff[rpos:rpos + 1]) 27 | field_types.append(type) 28 | rpos += 1 29 | while rpos < self.len: 30 | record = [] 31 | for i in xrange(field_num): 32 | data_len, = unpack('!L', self.rbuff[rpos:rpos + 4]) 33 | rpos += 4 34 | if data_len > 0: 35 | c_fmt = str(data_len) + 's' 36 | field_value, = unpack(c_fmt, self.rbuff[rpos:rpos + data_len]) 37 | rpos += data_len 38 | if data_len == 1 and field_value == '\0': 39 | record.append("") 40 | else: 41 | record.append(field_value) 42 | else: 43 | record.append(None) 44 | records.append(record) 45 | return field_types, records -------------------------------------------------------------------------------- /libtdhs/tdh_socket_time.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved. 11 | 12 | This program is free software; you can redistribute it and/or modify 13 | it under the terms of the GNU General Public License as published by 14 | the Free Software Foundation; version 2 of the License. 15 | 16 | This program is distributed in the hope that it will be useful, 17 | but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | GNU General Public License for more details. 20 | 21 | You should have received a copy of the GNU General Public License 22 | along with this program; if not, write to the Free Software 23 | Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ 24 | 25 | /* get time since epoc in 100 nanosec units */ 26 | /* thus to get the current time we should use the system function 27 | with the highest possible resolution */ 28 | 29 | /* 30 | TODO: in functions my_micro_time() and my_micro_time_and_time() there 31 | exists some common code that should be merged into a function. 32 | */ 33 | 34 | #ifndef TDH_SOCKET_TIME_HPP_ 35 | #define TDH_SOCKET_TIME_HPP_ 36 | 37 | #include "debug_util.hpp" 38 | 39 | namespace taobao { 40 | 41 | typedef unsigned long long int ullong; 42 | 43 | ullong tdhs_micro_time(); 44 | 45 | ullong tdhs_micro_time_and_time(time_t *time_arg); 46 | 47 | } // namespace taobao 48 | 49 | #endif /* TDH_SOCKET_TIME_HPP_ */ 50 | -------------------------------------------------------------------------------- /libeasy/memory/easy_mem_page.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_MEM_PAGE_H_ 11 | #define EASY_MEM_PAGE_H_ 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | /** 18 | * 简单内存分配器 19 | */ 20 | EASY_CPP_START 21 | 22 | #define EASY_MEM_PAGE_SHIFT 16 23 | #define EASY_MEM_PAGE_SIZE (1<= 50505 33 | #include 34 | #include 35 | #include "sql_class.h" 36 | #include "unireg.h" 37 | #include "lock.h" 38 | #include "key.h" // key_copy() 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include "sql_show.h" //for schema_table 44 | 45 | #define safeFree(X) my_free(X) 46 | 47 | #define tdhs_mysql_cond_timedwait mysql_cond_timedwait 48 | #define tdhs_mysql_mutex_lock mysql_mutex_lock 49 | #define tdhs_mysql_mutex_unlock mysql_mutex_unlock 50 | #define tdhs_mysql_cond_broadcast mysql_cond_broadcast 51 | 52 | #define current_stmt_binlog_row_based is_current_stmt_binlog_format_row 53 | #define clear_current_stmt_binlog_row_based clear_current_stmt_binlog_format_row 54 | 55 | #else 56 | 57 | #include "mysql_priv.h" 58 | #define tdhs_mysql_cond_timedwait pthread_cond_timedwait 59 | #define tdhs_mysql_mutex_lock pthread_mutex_lock 60 | #define tdhs_mysql_mutex_unlock pthread_mutex_unlock 61 | #define tdhs_mysql_cond_broadcast pthread_cond_broadcast 62 | 63 | #endif 64 | 65 | #undef min 66 | #undef max 67 | 68 | 69 | #endif /* MYSQL_INC_HPP_ */ 70 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_share.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_share.hpp 12 | * 13 | * Created on: 2011-10-11 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef TDH_SOCKET_SHARE_HPP_ 18 | #define TDH_SOCKET_SHARE_HPP_ 19 | 20 | #include "easy_baseth_pool.h" 21 | #include "tdh_socket_define.hpp" 22 | 23 | namespace taobao { 24 | 25 | extern easy_thread_pool_t *request_server_tp; 26 | 27 | extern easy_thread_pool_t *slow_read_request_server_tp; 28 | 29 | extern easy_thread_pool_t *write_request_server_tp; 30 | 31 | typedef struct tdhs_share_t { 32 | int shutdown; 33 | } tdhs_share_t; 34 | 35 | extern tdhs_share_t* tdhs_share; 36 | 37 | typedef enum { 38 | TDHS_THREAD_LV_1 = 1, //只有QUICK 线程池工作 39 | TDHS_THREAD_LV_2, //QUICK SLOW线程池一起工作 40 | TDHS_THREAD_LV_3 41 | //根据请求 可能走缓存的 走QUICK线程池 可能走IO的走SLOW线程池 42 | } tdhs_thread_strategy_t; 43 | 44 | extern tdhs_thread_strategy_t thread_strategy; 45 | 46 | //当前活动的slow线程数 47 | extern unsigned int active_slow_read_thread_num; 48 | 49 | //auth 50 | extern tdhs_string_t* tdhs_auth_read_code_string; 51 | 52 | extern tdhs_string_t* tdhs_auth_write_code_string; 53 | 54 | extern int init_auth(); 55 | 56 | extern int destory_auth(); 57 | 58 | extern bool tdhs_auth_read(tdhs_string_t &str); 59 | 60 | extern bool tdhs_auth_write(tdhs_string_t &str); 61 | 62 | extern void reset_auth_read_code(); 63 | 64 | extern void reset_auth_write_code(); 65 | 66 | extern void tdhs_close_cached_table(); 67 | 68 | //throttle 69 | extern unsigned long long int slow_read_io_num; 70 | /*return true meaning limited*/ 71 | extern bool slow_read_limit(); 72 | 73 | } // namespace taobao 74 | 75 | #endif /* TDH_SOCKET_SHARE_HPP_ */ 76 | -------------------------------------------------------------------------------- /doc/status_zh_CN.txt: -------------------------------------------------------------------------------- 1 | Status: 2 | ---------------------------------- 3 | tdhs_table_open 4 | 被TDH_Socket打开的table的次数 5 | ---------------------------------- 6 | tdhs_table_close 7 | 被TDH_Socket关闭的table的次数 8 | ---------------------------------- 9 | tdhs_table_lock 10 | 被TDH_Socket锁的table的次数 11 | ---------------------------------- 12 | tdhs_table_unlock 13 | 被TDH_Socket解锁的table的次数(也可以理解为事务提交数) 14 | ---------------------------------- 15 | tdhs_get_count 16 | 执行的GET请求的数目 17 | ---------------------------------- 18 | tdhs_count_count 19 | 执行的COUNT请求的数目 20 | ---------------------------------- 21 | tdhs_update_count 22 | 执行的UPDATE请求的数目 23 | ---------------------------------- 24 | tdhs_delete_count 25 | 执行的DELETE请求的数目 26 | ---------------------------------- 27 | tdhs_insert_count 28 | 执行的INSERT请求的数目 29 | ---------------------------------- 30 | tdhs_batch_count 31 | 执行的BATCH请求的数目 32 | ---------------------------------- 33 | tdhs_thread_strategy 34 | 当前的读策略是个啥? 1 ,2 or 3 35 | ---------------------------------- 36 | tdhs_last_io_read_per_second 37 | 最后一次统计的每秒的Innodb_buffer_pool_reads差值 38 | ---------------------------------- 39 | tdhs_optimize_status 40 | 当开启optimize时,记录的各组bloom filter的数目 如idx[0] num[0,0,0,0,0], idx为当前判断的bloom filter组id 41 | ---------------------------------- 42 | tdhs_optimize_lv3_assign_to_quick_count 43 | 当在LV3并开启optimize时,被分配到TDH_SOCKET_QUICK类型线程的请求数 44 | ---------------------------------- 45 | tdhs_optimize_lv3_assign_to_slow_count 46 | 当在LV3并开启optimize时,被分配到TDH_SOCKET_SLOW类型线程的请求数 47 | ---------------------------------- 48 | tdhs_active_slow_read_thread_num 49 | 当前TDH_SOCKET_SLOW类型线程活跃的线程数 50 | ---------------------------------- 51 | tdhs_throttle_count 52 | 被流控掉的请求数 53 | ---------------------------------- 54 | tdhs_slow_read_done_count 55 | 没有被流控的请求数 56 | ---------------------------------- 57 | tdhs_io_status 58 | io线程上挂载的连接数,这是一个数组如[2,2,3] 表示有3个io线程 其中第一个和第二个io线程上各有2个连接,第三个io线程上有3个连接 -------------------------------------------------------------------------------- /libeasy/include/easy_define.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_DEFINE_H_ 11 | #define EASY_DEFINE_H_ 12 | 13 | /** 14 | * 定义一些编译参数 15 | */ 16 | 17 | #ifdef __cplusplus 18 | # define EASY_CPP_START extern "C" { 19 | # define EASY_CPP_END } 20 | #else 21 | # define EASY_CPP_START 22 | # define EASY_CPP_END 23 | #endif 24 | 25 | #ifndef __STDC_FORMAT_MACROS 26 | #define __STDC_FORMAT_MACROS 27 | #endif 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | 42 | /////////////////////////////////////////////////////////////////////////////////////////////////// 43 | // define 44 | #define likely(x) __builtin_expect(!!(x), 1) 45 | #define unlikely(x) __builtin_expect(!!(x), 0) 46 | #define easy_align_ptr(p, a) (uint8_t*)(((uintptr_t)(p) + ((uintptr_t) a - 1)) & ~((uintptr_t) a - 1)) 47 | #define easy_align(d, a) (((d) + (a - 1)) & ~(a - 1)) 48 | #define easy_max(a,b) (a > b ? a : b) 49 | #define easy_min(a,b) (a < b ? a : b) 50 | 51 | #define EASY_OK 0 52 | #define EASY_ERROR (-1) 53 | #define EASY_ABORT (-2) 54 | #define EASY_ASYNC (-3) 55 | #define EASY_BREAK (-4) 56 | #define EASY_AGAIN (-EAGAIN) 57 | /////////////////////////////////////////////////////////////////////////////////////////////////// 58 | // typedef 59 | typedef struct easy_addr_t { 60 | uint64_t addr; 61 | uint64_t cidx; 62 | } easy_addr_t; 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_config.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | /* 10 | * tdh_socket_config.cpp 11 | * 12 | * Created on: 2011-8-11 13 | * Author: wentong 14 | */ 15 | 16 | #include "tdh_socket_config.hpp" 17 | 18 | namespace taobao { 19 | 20 | unsigned int tdhs_log_level = 3; 21 | 22 | unsigned int tdhs_io_thread_num = 4; 23 | 24 | unsigned int tdhs_thread_num = 16; 25 | 26 | unsigned int tdhs_slow_read_thread_num = 28; 27 | 28 | unsigned int tdhs_write_thread_num = 1; 29 | 30 | char tdhs_cache_table_on = 1; 31 | 32 | unsigned int tdhs_cache_table_num_for_thd = 3; 33 | 34 | char tdhs_group_commit = 1; 35 | 36 | int tdhs_group_commit_limits = 0; 37 | 38 | //optimize 39 | char tdhs_optimize_on = 0; 40 | 41 | unsigned int tdhs_optimize_bloom_filter_group = 5; 42 | 43 | unsigned int tdhs_optimize_bloom_filter_num_buckets = 16 * 1024; 44 | 45 | int tdhs_optimize_guess_hot_request_num = 500 * 10000; 46 | 47 | //optimize end 48 | 49 | int tdhs_listen_port = 9999; 50 | 51 | unsigned int tdhs_monitor_interval = 5; 52 | 53 | unsigned int tdhs_thread_strategy_requests_lv_1 = 1000; 54 | 55 | unsigned int tdhs_thread_strategy_requests_lv_2 = 12 * 1024; 56 | 57 | char tdhs_concurrency_insert = 0; 58 | 59 | char tdhs_concurrency_update = 0; 60 | 61 | char tdhs_concurrency_delete = 0; 62 | 63 | char tdhs_auth_on = 0; 64 | 65 | char* tdhs_auth_read_code = 0; 66 | 67 | char* tdhs_auth_write_code = 0; 68 | 69 | //throttle 70 | char tdhs_throttle_on = 0; 71 | 72 | unsigned int tdhs_slow_read_limits = 2; 73 | 74 | //extra 75 | unsigned int tdhs_write_buff_size = 8 * 1024; 76 | 77 | unsigned int tdhs_quick_request_thread_task_count_limit = 0; 78 | 79 | unsigned int tdhs_slow_request_thread_task_count_limit = 0; 80 | 81 | unsigned int tdhs_write_request_thread_task_count_limit = 0; 82 | 83 | } // namespace taobao 84 | 85 | -------------------------------------------------------------------------------- /libeasy/util/easy_pool.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_POOL_H_ 11 | #define EASY_POOL_H_ 12 | 13 | /** 14 | * 简单的内存池 15 | */ 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | EASY_CPP_START 22 | 23 | #define EASY_POOL_ALIGNMENT 512 24 | #define EASY_POOL_PAGE_SIZE 4096 25 | 26 | typedef void *(*easy_pool_realloc_pt)(void *ptr, size_t size); 27 | typedef struct easy_pool_large_t easy_pool_large_t; 28 | typedef struct easy_pool_t easy_pool_t; 29 | 30 | struct easy_pool_large_t { 31 | easy_pool_large_t *next; 32 | uint8_t data[0]; 33 | }; 34 | 35 | struct easy_pool_t { 36 | uint8_t *last; 37 | uint8_t *end; 38 | easy_pool_t *next; 39 | uint16_t failed; 40 | uint16_t flags; 41 | uint32_t max; 42 | 43 | // pool header 44 | easy_pool_t *current; 45 | easy_pool_large_t *large; 46 | easy_atomic_t ref; 47 | easy_atomic_t tlock; 48 | }; 49 | 50 | extern easy_pool_realloc_pt easy_pool_realloc; 51 | extern void *easy_pool_default_realloc (void *ptr, size_t size); 52 | 53 | extern easy_pool_t *easy_pool_create(uint32_t size); 54 | extern void easy_pool_clear(easy_pool_t *pool); 55 | extern void easy_pool_destroy(easy_pool_t *pool); 56 | extern void *easy_pool_alloc(easy_pool_t *pool, uint32_t size); 57 | extern void *easy_pool_nalloc(easy_pool_t *pool, uint32_t size); 58 | extern void *easy_pool_calloc(easy_pool_t *pool, uint32_t size); 59 | extern void easy_pool_set_allocator(easy_pool_realloc_pt alloc); 60 | extern void easy_pool_set_lock(easy_pool_t *pool); 61 | 62 | extern char *easy_pool_strdup(easy_pool_t *pool, const char *str); 63 | 64 | EASY_CPP_END 65 | #endif 66 | -------------------------------------------------------------------------------- /libeasy/io/easy_baseth_pool.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_BASETH_POOL_H 11 | #define EASY_BASETH_POOL_H 12 | 13 | #include 14 | 15 | /** 16 | * base pthread线程池 17 | */ 18 | 19 | EASY_CPP_START 20 | 21 | #include "easy_io_struct.h" 22 | 23 | #define easy_thread_pool_for_each(th, tp, offset) \ 24 | for((th) = (typeof(*(th))*)&(tp)->data[offset]; \ 25 | (char*)(th) < (tp)->last; \ 26 | th = (typeof(*th)*)(((char*)th) + (tp)->member_size)) 27 | 28 | // 第n个 29 | static inline void *easy_thread_pool_index(easy_thread_pool_t *tp, int n) 30 | { 31 | if (n < 0 || n >= tp->thread_count) 32 | return NULL; 33 | 34 | return &tp->data[n * tp->member_size]; 35 | } 36 | 37 | static inline void *easy_thread_pool_hash(easy_thread_pool_t *tp, uint64_t hv) 38 | { 39 | hv %= tp->thread_count; 40 | return &tp->data[hv * tp->member_size]; 41 | } 42 | 43 | static inline void *easy_thread_pool_rr(easy_thread_pool_t *tp, int start) 44 | { 45 | int n, t; 46 | 47 | if ((t = tp->thread_count - start) > 0) { 48 | n = easy_atomic32_add_return(&tp->last_number, 1); 49 | n %= t; 50 | n += start; 51 | } else { 52 | n = 0; 53 | } 54 | 55 | return &tp->data[n * tp->member_size]; 56 | } 57 | 58 | // baseth 59 | void *easy_baseth_on_start(void *args); 60 | void easy_baseth_on_wakeup(void *args); 61 | void easy_baseth_init(void *args, easy_thread_pool_t *tp, 62 | easy_baseth_on_start_pt *start, easy_baseth_on_wakeup_pt *wakeup); 63 | void easy_baseth_pool_on_wakeup(easy_thread_pool_t *tp); 64 | easy_thread_pool_t *easy_baseth_pool_create(easy_io_t *eio, int thread_count, int member_size); 65 | void easy_baseth_pool_destroy(easy_thread_pool_t *tp); 66 | 67 | EASY_CPP_END 68 | 69 | #endif 70 | -------------------------------------------------------------------------------- /libeasy/util/easy_hash.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_HASH_H_ 11 | #define EASY_HASH_H_ 12 | 13 | /** 14 | * 固定HASH桶的hashtable, 需要在使用的对象上定义一个easy_hash_list_t 15 | */ 16 | #include "easy_pool.h" 17 | #include "easy_list.h" 18 | 19 | EASY_CPP_START 20 | 21 | typedef struct easy_hash_t easy_hash_t; 22 | typedef struct easy_hash_list_t easy_hash_list_t; 23 | typedef int (easy_hash_cmp_pt)(const void *a, const void *b); 24 | 25 | struct easy_hash_t { 26 | easy_hash_list_t **buckets; 27 | uint32_t size; 28 | uint32_t mask; 29 | uint32_t count; 30 | int offset; 31 | 32 | uint64_t seqno; 33 | easy_list_t list; 34 | }; 35 | 36 | struct easy_hash_list_t { 37 | easy_hash_list_t *next; 38 | easy_hash_list_t **pprev; 39 | uint64_t key; 40 | }; 41 | 42 | #define easy_hash_for_each(i, node, table) \ 43 | for(i=0; isize; i++) \ 44 | for(node = table->buckets[i]; node; node = node->next) 45 | 46 | extern easy_hash_t *easy_hash_create(easy_pool_t *pool, uint32_t size, int offset); 47 | extern int easy_hash_add(easy_hash_t *table, uint64_t key, easy_hash_list_t *list); 48 | extern void *easy_hash_find(easy_hash_t *table, uint64_t key); 49 | void *easy_hash_find_ex(easy_hash_t *table, uint64_t key, easy_hash_cmp_pt cmp, const void *a); 50 | extern void *easy_hash_del(easy_hash_t *table, uint64_t key); 51 | extern int easy_hash_del_node(easy_hash_list_t *n); 52 | extern uint64_t easy_hash_key(uint64_t key); 53 | extern uint64_t easy_hash_code(const void *key, int len, unsigned int seed); 54 | 55 | extern int easy_hash_dlist_add(easy_hash_t *table, uint64_t key, easy_hash_list_t *hash, easy_list_t *list); 56 | extern void *easy_hash_dlist_del(easy_hash_t *table, uint64_t key); 57 | 58 | EASY_CPP_END 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /doc/installation_zh_CN.txt: -------------------------------------------------------------------------------- 1 | 1.构建TDH_Socket 2 | TDH_Socket现在包含libeasy,libtdhs,TDHSocket. 3 | libeasy是公用的网络库 4 | libtdhs是公用的工具库 5 | TDHSocket是MySQL的daemon plugin 6 | 构建TDH_Socket需要MySQL的源码和MySQL编译完成后的二进制文件,TDH_Socket目前支持MySQL 5.1 和 MySQL 5.5 极其衍生版本. 7 | 需要确认MySQL的源码和MySQL二进制文件能够匹配起来,否则可能会出现一些不可预知的情况(比如MySQL服务器Crash) 8 | 9 | 编译步骤: 10 | 1) 通过MySQL源码编译好MySQL 11 | a)例如源码路径/sources/mysql-5.1.48 12 | b)例如编译好的MySQL目录/usr/mysql 13 | 2)编译TDH_Socket 14 | a) $ ./bootstrap.sh 15 | b) $ ./configure --with-mysql-source=/sources/mysql-5.1.48 --with-mysql-bindir=/usr/mysql/bin --with-mysql-plugindir=/usr/mysql/lib/plugin 16 | i) --with-mysql-source 指向MySQL的源码路径 17 | ii) --with-mysql-bindir 指向MySQL的二进制文件路径,主要为了通过 ${with-mysql-bindir}/bin/mysql_config 来获取一些编译参数 18 | iii) --with-mysql-plugindir 指向生成的tdhsocket.so的安装路径,需要指向MySQL的plugin目录 19 | iv) --enable-mysql-debug 如果MySQL编译时设置DEBUG_ON,那么也需要开启起此选项来匹配编译,否则可能会出现Crash 20 | v) --enable-tdhs-debug 开启TDH_Socket的一些assert校验(默认关闭) 21 | vi) --enable-tdhs-row-cache 开启Row Cache优化,如果你的MySQL版本包含了Row Cache,那么可以开启此选项来优化性能,对于流控来说可以提高流控的准确性 关于Row Cache 可以见/row_cache 目录 22 | c) $ make 23 | d) $ sudo make install 24 | 25 | 2.TDH_Socket的使用 26 | 用root账户登陆到mysql中执行 27 | 28 | mysql> install plugin tdh_socket soname 'tdhsocket.so'; 29 | 以激活TDH_Socket 30 | 可以通过 31 | mysql> show processlist; 32 | 看到TDH_Socket的线程在工作了 33 | 34 | 在my.cnf中添加 35 | [mysqld] 36 | #指定TDH_Socket的监听端口号 37 | tdh_socket_listen_port = 8888 38 | #其他配置项见variable文件的描述 39 | 40 | 41 | 42 | 3.TDH_Socket的客户端 43 | 目前在项目中包含了Java客户端和Python客户端 44 | 1) Java客户端 (https://github.com/taobao/tdhs-java-client) 45 | a)Java客户端是一个高性能的客户端,基于Netty,并且支持JDBC,对于应用来说迁移起来很方便 46 | b)直接用法见/java/src/test/java/benchmark目录,这个目录下提供了一些常见的性能评估工具 47 | c)JDBC用法见/java/src/test/java/com/taobao/tdhs/client/jdbc/test 下的测试用例 48 | d)结合MyBatis的用法见 /java/src/test/java/com/taobao/tdhs/client/jdbc/test/mybatis 下的测试用例 49 | 2) Python客户端 见/python目录 50 | b) Python客户端主要提供方便调试用.是一个堵塞io的客户端 51 | c) 用法见 /python/test下的用例 52 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_dbcontext.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | /* 10 | * tdh_socket_dbcontext.hpp 11 | * 12 | * Created on: 2011-9-30 13 | * Author: wentong 14 | */ 15 | 16 | #ifndef TDH_SOCKET_DBCONTEXT_HPP_ 17 | #define TDH_SOCKET_DBCONTEXT_HPP_ 18 | 19 | #include "tdh_socket_protocol.hpp" 20 | #include "tdh_socket_define.hpp" 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | namespace taobao { 27 | 28 | extern const char* PRIMARY; 29 | 30 | #define PRIMARY_STRING "PRIMARY" 31 | 32 | #define PRIMARY_SIZE (sizeof(PRIMARY_STRING)) 33 | 34 | typedef int (decode_request_t)(tdhs_request_t &req, tdhs_packet_t& packet); 35 | 36 | extern decode_request_t* decode_request_array[TDHS_PROTOCOL_END]; 37 | 38 | class tdh_socket_connection_context; 39 | class tdhs_dbcontext_i { 40 | public: 41 | virtual ~tdhs_dbcontext_i() { 42 | } 43 | virtual int init(tdhs_optimize_t _type, const void *stack_bottom) = 0; 44 | virtual tdhs_optimize_t get_type() = 0; 45 | virtual unsigned long get_use_steam_count() = 0; 46 | virtual bool need_write() = 0; 47 | virtual void open_table(tdhs_request_t &req) = 0; 48 | virtual void lock_table() = 0; 49 | virtual int unlock_table() = 0; 50 | virtual void close_table() = 0; 51 | virtual void close_cached_table() = 0; 52 | virtual int destory() = 0; 53 | virtual int execute(easy_request_t *r) = 0; 54 | virtual void set_thd_info(unsigned int bulk_request_num) = 0; 55 | virtual void using_stream() = 0; 56 | virtual void set_group_commit(bool gc) = 0; 57 | virtual time_t* get_thd_time() = 0; 58 | 59 | virtual bool need_close_table() = 0; 60 | virtual void set_need_close_table(bool need) = 0; 61 | #ifdef TDHS_ROW_CACHE 62 | virtual bool is_in_cache(tdhs_request_t &req) = 0; 63 | #endif 64 | static tdhs_dbcontext_i* create(); 65 | }; 66 | 67 | } 68 | // namespace taobao 69 | 70 | #endif /* TDH_SOCKET_DBCONTEXT_HPP_ */ 71 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_config.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_config.hpp 12 | * 13 | * Created on: 2011-8-11 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef TDH_SOCKET_CONFIG_HPP_ 18 | #define TDH_SOCKET_CONFIG_HPP_ 19 | 20 | namespace taobao { 21 | 22 | extern unsigned int tdhs_log_level; 23 | 24 | extern unsigned int tdhs_io_thread_num; 25 | 26 | extern unsigned int tdhs_thread_num; 27 | 28 | extern unsigned int tdhs_slow_read_thread_num; 29 | 30 | extern unsigned int tdhs_write_thread_num; 31 | 32 | extern char tdhs_cache_table_on; 33 | 34 | extern unsigned int tdhs_cache_table_num_for_thd; 35 | 36 | extern char tdhs_group_commit; 37 | 38 | extern int tdhs_group_commit_limits; 39 | 40 | //optimize 41 | 42 | extern char tdhs_optimize_on; 43 | 44 | extern unsigned int tdhs_optimize_bloom_filter_group; 45 | 46 | extern unsigned int tdhs_optimize_bloom_filter_num_buckets; 47 | 48 | extern int tdhs_optimize_guess_hot_request_num; 49 | 50 | //optimize end 51 | 52 | extern int tdhs_listen_port; 53 | 54 | extern unsigned int tdhs_monitor_interval; 55 | 56 | extern unsigned int tdhs_thread_strategy_requests_lv_1; 57 | 58 | extern unsigned int tdhs_thread_strategy_requests_lv_2; 59 | 60 | extern char tdhs_concurrency_insert; 61 | 62 | extern char tdhs_concurrency_update; 63 | 64 | extern char tdhs_concurrency_delete; 65 | 66 | //auth 67 | extern char tdhs_auth_on; 68 | 69 | extern char* tdhs_auth_read_code; 70 | 71 | extern char* tdhs_auth_write_code; 72 | 73 | //throttle 74 | extern char tdhs_throttle_on; 75 | 76 | extern unsigned int tdhs_slow_read_limits; 77 | 78 | //extra 79 | extern unsigned int tdhs_write_buff_size; 80 | 81 | extern unsigned int tdhs_quick_request_thread_task_count_limit; 82 | 83 | extern unsigned int tdhs_slow_request_thread_task_count_limit; 84 | 85 | extern unsigned int tdhs_write_request_thread_task_count_limit; 86 | 87 | } // namespace taobao 88 | 89 | #endif /* TDH_SOCKET_CONFIG_HPP_ */ 90 | -------------------------------------------------------------------------------- /TDHSocket/default_mysql_sysvar_for_tdh_socket.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * default_mysql_sysvar_for_tdh_socket.hpp 12 | * 13 | * Created on: 2012-2-15 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef DEFAULT_MYSQL_SYSVAR_FOR_TDH_SOCKET_HPP_ 18 | #define DEFAULT_MYSQL_SYSVAR_FOR_TDH_SOCKET_HPP_ 19 | 20 | #define DEFAULT_TDHS_LISTEN_PORT 9999 21 | 22 | #define DEFAULT_TDHS_LOG_LEVEL 3 23 | 24 | #define DEFAULT_TDHS_IO_THREAD_NUM 4 25 | 26 | #define DEFAULT_TDHS_THREAD_NUM 8 27 | 28 | #define DEFAULT_TDHS_SLOW_READ_THREAD_NUM 28 29 | 30 | #define DEFAULT_TDHS_WRITE_THREAD_NUM 1 31 | 32 | #define DEFAULT_TDHS_CACHE_TABLE_ON TRUE 33 | 34 | #define DEFAULT_TDHS_CACHE_TABLE_NUM_FOR_THD 3 35 | 36 | #define DEFAULT_TDHS_OPTIMIZE_ON FALSE 37 | 38 | #define DEFAULT_TDHS_OPTIMIZE_BLOOM_FILTER_GROUP 5 39 | 40 | #define DEFAULT_TDHS_OPTIMIZE_BLOOM_FILTER_NUM_BUCKETS (16 * 1024) 41 | 42 | #define DEFAULT_TDHS_OPTIMIZE_GUESS_HOT_REQUEST_NUM (500 * 10000) 43 | 44 | #define DEFAULT_TDHS_MONITOR_INTERVAL 5 45 | 46 | #define DEFAULT_TDHS_THREAD_STRATEGY_REQUESTS_LV_1 128 47 | 48 | #define DEFAULT_TDHS_THREAD_STRATEGY_REQUESTS_LV_2 1024 49 | 50 | #define DEFAULT_TDHS_CONCURRENCY_INSERT FALSE 51 | 52 | #define DEFAULT_TDHS_CONCURRENCY_UPDATE FALSE 53 | 54 | #define DEFAULT_TDHS_CONCURRENCY_DELETE FALSE 55 | 56 | #define DEFAULT_TDHS_AUTH_ON FALSE 57 | 58 | #define DEFAULT_TDHS_AUTH_READ_CODE NULL 59 | 60 | #define DEFAULT_TDHS_AUTH_WRITE_CODE NULL 61 | 62 | #define DEFAULT_TDHS_THROTTLE_ON FALSE 63 | 64 | #define DEFAULT_TDHS_SLOW_READ_LIMITS 2 65 | 66 | #define DEFAULT_TDHS_WRITE_BUFF_SIZE (8*1024) 67 | 68 | #define DEFAULT_TDHS_GROUP_COMMIT TRUE 69 | 70 | #define DEFAULT_TDHS_GROUP_COMMIT_LIMITS 0 71 | 72 | #define DEFAULT_TDHS_QUICK_REQUEST_THREAD_TASK_COUNT_LIMIT 0 73 | 74 | #define DEFAULT_TDHS_SLOW_REQUEST_THREAD_TASK_COUNT_LIMIT 0 75 | 76 | #define DEFAULT_TDHS_WRITE_REQUEST_THREAD_TASK_COUNT_LIMIT 0 77 | 78 | #endif /* DEFAULT_MYSQL_SYSVAR_FOR_TDH_SOCKET_HPP_ */ 79 | -------------------------------------------------------------------------------- /libeasy/memory/easy_mem_slab.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_MEM_SLAB_H_ 11 | #define EASY_MEM_SLAB_H_ 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | /** 19 | * 简单内存分配器 20 | */ 21 | EASY_CPP_START 22 | 23 | #define EASY_MEM_SLAB_MIN 512 24 | typedef struct easy_mem_slab_t easy_mem_slab_t; 25 | typedef struct easy_mem_cache_t easy_mem_cache_t; 26 | typedef struct easy_mem_mgr_t easy_mem_mgr_t; 27 | typedef struct easy_mem_sizes_t easy_mem_sizes_t; 28 | 29 | struct easy_mem_slab_t { 30 | easy_list_t list; 31 | unsigned char *mem; 32 | uint16_t inuse; 33 | uint16_t free; 34 | uint16_t cache_idx; 35 | uint16_t next_pos[0]; 36 | }; 37 | 38 | struct easy_mem_cache_t { 39 | easy_list_t slabs_partial; 40 | easy_list_t slabs_full; 41 | easy_list_t slabs_free; 42 | easy_list_t next; 43 | 44 | uint32_t order; 45 | uint32_t buffer_size; 46 | uint32_t num; 47 | uint32_t offset; 48 | uint32_t free_objects; 49 | uint32_t free_limit; 50 | int idx; 51 | easy_atomic_t lock; 52 | }; 53 | 54 | struct easy_mem_mgr_t { 55 | int started; 56 | int cache_max_num; 57 | int cache_fix_num; 58 | int cache_num; 59 | int64_t max_size; 60 | 61 | easy_list_t list; 62 | easy_atomic_t lock; 63 | easy_mem_cache_t *caches; 64 | easy_mem_zone_t *zone; 65 | }; 66 | 67 | // 内存初始化 68 | int easy_mem_slab_init(int start_size, int64_t max_size); 69 | void easy_mem_slab_destroy(); 70 | // 内存分配 71 | void *easy_mem_slab_realloc(void *ptr, size_t size); 72 | // 分配 73 | void *easy_mem_cache_alloc(easy_mem_cache_t *cache); 74 | // 释放 75 | void easy_mem_cache_free(easy_mem_cache_t *cache, void *obj); 76 | // 创建一下mem_cache 77 | easy_mem_cache_t *easy_mem_cache_create(int buffer_size); 78 | 79 | EASY_CPP_END 80 | 81 | #endif 82 | -------------------------------------------------------------------------------- /libeasy/io/easy_log.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_LOG_H_ 11 | #define EASY_LOG_H_ 12 | 13 | /** 14 | * 简单的log输出 15 | */ 16 | #include 17 | #include 18 | 19 | EASY_CPP_START 20 | 21 | typedef void (*easy_log_print_pt)(const char *message); 22 | typedef enum { 23 | EASY_LOG_OFF = 1, 24 | EASY_LOG_FATAL, 25 | EASY_LOG_ERROR, 26 | EASY_LOG_WARN, 27 | EASY_LOG_INFO, 28 | EASY_LOG_DEBUG, 29 | EASY_LOG_TRACE, 30 | EASY_LOG_ALL 31 | } easy_log_level_t; 32 | 33 | #define easy_fatal_log(format, args...) if(easy_log_level>=EASY_LOG_FATAL) \ 34 | easy_log_common(__FILE__, __LINE__, format, ## args); 35 | #define easy_error_log(format, args...) if(easy_log_level>=EASY_LOG_ERROR) \ 36 | easy_log_common(__FILE__, __LINE__, format, ## args); 37 | #define easy_warn_log(format, args...) if(easy_log_level>=EASY_LOG_WARN) \ 38 | easy_log_common(__FILE__, __LINE__, format, ## args); 39 | #define easy_info_log(format, args...) if(easy_log_level>=EASY_LOG_INFO) \ 40 | easy_log_common(__FILE__, __LINE__, format, ## args); 41 | #define easy_debug_log(format, args...) if(easy_log_level>=EASY_LOG_DEBUG) \ 42 | easy_log_common(__FILE__, __LINE__, format, ## args); 43 | #define easy_trace_log(format, args...) if(easy_log_level>=EASY_LOG_TRACE) \ 44 | easy_log_common(__FILE__, __LINE__, format, ## args); 45 | 46 | // 打印backtrace 47 | #define EASY_PRINT_BT(format, args...) \ 48 | {char _buffer_stack_[256];{void *array[10];int i, idx=0, n = backtrace(array, 10); \ 49 | for (i = 0; i < n; i++) idx += snprintf(idx+_buffer_stack_, 25, "%p ", array[i]);}\ 50 | easy_log_common(__FILE__, __LINE__, "%s" format, _buffer_stack_, ## args);} 51 | 52 | extern easy_log_level_t easy_log_level; 53 | extern void easy_log_set_print(easy_log_print_pt p); 54 | extern void easy_log_common(const char *file, int line, const char *fmt, ...); 55 | extern void easy_log_print_default(const char *message); 56 | 57 | EASY_CPP_END 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /libeasy/io/easy_log.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #include 11 | #include 12 | #include 13 | #include "easy_io.h" 14 | 15 | static easy_log_print_pt easy_log_print = easy_log_print_default; 16 | easy_log_level_t easy_log_level = EASY_LOG_WARN; 17 | 18 | /** 19 | * 设置log的打印函数 20 | */ 21 | void easy_log_set_print(easy_log_print_pt p) 22 | { 23 | easy_log_print = p; 24 | ev_set_syserr_cb(easy_log_print); 25 | } 26 | 27 | /** 28 | * 加上日志 29 | */ 30 | void easy_log_common(const char *file, int line, const char *fmt, ...) 31 | { 32 | static __thread ev_tstamp oldtime = 0.0; 33 | static __thread char time_str[32]; 34 | ev_tstamp now; 35 | int len; 36 | char buffer[4096]; 37 | 38 | // 从loop中取 39 | if (easy_baseth_self && easy_baseth_self->loop) { 40 | now = ev_now(easy_baseth_self->loop); 41 | } else { 42 | now = time(NULL); 43 | } 44 | 45 | if (oldtime != now) { 46 | time_t t; 47 | struct tm tm; 48 | oldtime = now; 49 | t = (time_t) now; 50 | easy_localtime((const time_t *)&t, &tm); 51 | snprintf(time_str, 32, "[%04d-%02d-%02d %02d:%02d:%02d.%03d]", 52 | tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, 53 | tm.tm_hour, tm.tm_min, tm.tm_sec, (int)((now - t) * 1000)); 54 | } 55 | 56 | // print 57 | len = snprintf(buffer, 128, "%s %s:%d(tid:%lx) ", time_str, file, line, pthread_self()); 58 | va_list args; 59 | va_start(args, fmt); 60 | len += vsnprintf(buffer + len, 4090 - len, fmt, args); 61 | va_end(args); 62 | 63 | // 去掉最后'\n' 64 | while(buffer[len - 1] == '\n') len --; 65 | 66 | buffer[len++] = '\n'; 67 | buffer[len] = '\0'; 68 | 69 | easy_log_print(buffer); 70 | } 71 | 72 | /** 73 | * 打印出来 74 | */ 75 | void easy_log_print_default(const char *message) 76 | { 77 | write(2, message, strlen(message)); 78 | } 79 | 80 | void __attribute__((constructor)) easy_log_start_() 81 | { 82 | char *p = getenv("easy_log_level"); 83 | 84 | if (p) easy_log_level = (easy_log_level_t)atoi(p); 85 | } 86 | -------------------------------------------------------------------------------- /libtdhs/tdh_socket_protocol.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_protocol.hpp 12 | * 13 | * Created on: 2011-8-26 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef TDH_SOCKET_PROTOCOL_HPP_ 18 | #define TDH_SOCKET_PROTOCOL_HPP_ 19 | 20 | #include "tdh_socket_define.hpp" 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | namespace taobao { 28 | //===in shake hander read 29 | #define TDH_SOCKET_SHAKE_HANDS_HEADER {'T','D','H','S'} 30 | #define TDH_SOCKET_SHAKE_HANDS_HEADER_LENGTH 4 31 | #define TDH_SOCKET_PROTOCOL_VERSION_LENGTH sizeof(uint32_t) 32 | #define TDH_SOCKET_TIME_OUT_LENGTH sizeof(uint32_t) 33 | #define TDH_SOCKET_SHAKE_HANDS_TOTAL_SIZE (TDH_SOCKET_SHAKE_HANDS_HEADER_LENGTH + TDH_SOCKET_PROTOCOL_VERSION_LENGTH + TDH_SOCKET_TIME_OUT_LENGTH) 34 | 35 | //===in every command read 36 | #define TDHS_MAGIC_CODE (0xFFFFFFFF) 37 | #define TDHS_MAGIC_CODE_SIZE (4) 38 | #define TDH_SOCKET_COMAND_LENGTH sizeof(uint32_t) 39 | #define TDH_SOCKET_SEQ_ID_LENGTH sizeof(uint32_t) 40 | #define TDH_SOCKET_REVERSE_LENGTH sizeof(uint32_t) 41 | #define TDH_SOCKET_SIZE_LENGTH sizeof(uint32_t) 42 | #define TDH_SOCKET_HEADER_LENGTH (TDHS_MAGIC_CODE_SIZE+TDH_SOCKET_COMAND_LENGTH+TDH_SOCKET_SEQ_ID_LENGTH+TDH_SOCKET_REVERSE_LENGTH+TDH_SOCKET_SIZE_LENGTH) 43 | 44 | #define ERROR_OUT_OF_IN (-100) 45 | 46 | typedef enum { 47 | TDHS_PROTOCOL_BINARY = 1, TDHS_PROTOCOL_BINARY_V2 = 2,TDHS_PROTOCOL_END 48 | } tdhs_protocol_version_t; 49 | 50 | typedef struct tdhs_packet_t tdhs_packet_t; 51 | 52 | struct tdhs_packet_t { 53 | uint32_t command_id_or_response_code; 54 | uint32_t seq_id; 55 | uint32_t reserved; //当type为REQUEST_TYPE_BATCH,记录批量请求的个数 56 | uint32_t length; 57 | char *rdata; //for read 58 | easy_buf_t *wbuff; //for write 59 | char *stream_buffer; //for stream reuse 60 | easy_pool_t *pool; 61 | tdhs_request_t req; 62 | unsigned long long int start_time; 63 | tdhs_packet_t *next; //为batch请求串联用 64 | char buffer[0]; 65 | }; 66 | 67 | extern void *tdhs_decode(easy_message_t *m); 68 | 69 | extern int tdhs_encode(easy_request_t *r, void *data); 70 | 71 | extern easy_session_t* create_thds_packet(uint32_t id, char* data, 72 | uint32_t data_len, int timeout, void* args, bool need_copy = true); 73 | 74 | } // namespace taobao 75 | 76 | #endif /* TDH_SOCKET_PROTOCOL_HPP_ */ 77 | -------------------------------------------------------------------------------- /libeasy/memory/easy_mem_pool.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_MEMPOOL_H_ 11 | #define EASY_MEMPOOL_H_ 12 | 13 | /** 14 | * ring buffer 15 | */ 16 | #include "easy_define.h" 17 | 18 | EASY_CPP_START 19 | 20 | #define EASY_MEMPOOL_PAGE_SIZE 512 * 1024 21 | #define EASY_MEMPOOL_PAGE_MAX_NUM 16384 // max page num per-pool 22 | #define EASY_MEMPOOL_PAGE_FREE_NUM 16 // max free page hold per-pool 23 | #define EASY_MEMPOOL_ALIGNMENT sizeof(unsigned long) 24 | 25 | #define easy_mempool_align(d, a) (((d) + (a - 1)) & ~(a - 1)) 26 | #define easy_mempool_align_ptr(p, a) (uint8_t*)(((uintptr_t)(p) + ((uintptr_t) a - 1)) & ~((uintptr_t) a - 1)) 27 | 28 | typedef void *(*easy_mempool_memalign_pt)(size_t alignment, size_t size); 29 | typedef void (*easy_mempool_free_pt)(void *ptr); 30 | 31 | typedef struct easy_mempool_allocator_t { 32 | easy_mempool_memalign_pt memalign; 33 | easy_mempool_free_pt free; 34 | } easy_mempool_allocator_t; 35 | 36 | struct easy_mempool_t; 37 | typedef struct easy_mempool_t easy_mempool_t; 38 | 39 | //////////////////////////////////////////////////////////////////////////////////////////////////// 40 | 41 | extern easy_mempool_t *easy_mempool_create(uint32_t size); 42 | extern void easy_mempool_destroy(easy_mempool_t *pool); 43 | extern void easy_mempool_clear(easy_mempool_t *pool); 44 | 45 | // alloc align buffer 46 | extern void *easy_mempool_alloc(easy_mempool_t *pool, uint32_t size); 47 | // free buffer 48 | extern void easy_mempool_free(easy_mempool_t *pool, void *ptr); 49 | 50 | extern void easy_mempool_set_memlimit(easy_mempool_t *pool, int64_t limit); 51 | extern void easy_mempool_set_allocator(easy_mempool_t *pool, easy_mempool_allocator_t *allocator); 52 | extern int64_t easy_mempool_get_memtotal(easy_mempool_t *pool); 53 | 54 | //////////////////////////////////////////////////////////////////////////////////////////////////// 55 | 56 | // 使用全局分配器分配内存 57 | extern void easy_mempool_set_global_memlimit(int64_t limit); 58 | extern int64_t easy_mempool_get_global_memtotal(); 59 | extern void *easy_mempool_global_realloc(void *ptr, size_t size); 60 | 61 | // 使用线程缓存分配内存 62 | extern void easy_mempool_set_thread_memlimit(int64_t limit); 63 | extern int64_t easy_mempool_get_thread_memtotal(); 64 | extern void *easy_mempool_thread_realloc(void *ptr, size_t size); 65 | 66 | EASY_CPP_END 67 | #endif 68 | 69 | -------------------------------------------------------------------------------- /libeasy/io/easy_file.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #undef _XOPEN_SOURCE 11 | #define _XOPEN_SOURCE 500 12 | #include 13 | #include 14 | #include "easy_io.h" 15 | #include "easy_file.h" 16 | #include "easy_connection.h" 17 | #include "easy_message.h" 18 | #include "easy_request.h" 19 | #include "easy_file.h" 20 | #include "easy_client.h" 21 | #include "easy_socket.h" 22 | #include "easy_log.h" 23 | #undef _XOPEN_SOURCE 24 | #define _XOPEN_SOURCE 600 25 | #include 26 | 27 | easy_file_task_t *easy_file_task_create(easy_request_t *r, int fd, int bufsize) 28 | { 29 | struct stat fs; 30 | easy_file_task_t *ft; 31 | 32 | ft = (easy_file_task_t *)easy_pool_calloc(r->ms->pool, sizeof(easy_file_task_t)); 33 | 34 | if (ft == NULL) 35 | return NULL; 36 | 37 | ft->fd = fd; 38 | 39 | if (fstat(fd, &fs) == 0) 40 | ft->count = fs.st_size; 41 | 42 | if (bufsize == 0) bufsize = EASY_MAX_FILE_BUFFER; 43 | 44 | ft->bufsize = easy_min(ft->count, bufsize); 45 | ft->b = easy_buf_create(r->ms->pool, ft->bufsize); 46 | ft->buffer = ft->b->pos; 47 | 48 | if (ft->b == NULL) 49 | return NULL; 50 | 51 | return ft; 52 | } 53 | 54 | void easy_file_task_reset(easy_file_task_t *ft, int type) 55 | { 56 | easy_list_init(&ft->b->node); 57 | ft->b->pos = ft->buffer; 58 | ft->b->last = ft->b->pos; 59 | ((easy_file_buf_t *)ft->b)->flags = type; 60 | } 61 | 62 | /** 63 | * file_task process 64 | */ 65 | /* 66 | int easy_file_task_process(easy_file_task_t *ft) 67 | { 68 | int64_t rc = 0; 69 | long offset = ft->offset; 70 | 71 | do { 72 | switch(ft->type) { 73 | case EASY_FILE_WRITE: 74 | rc = pwrite(ft->fd, ft->buf, ft->count, offset); 75 | break; 76 | 77 | case EASY_FILE_READ: 78 | rc = pread(ft->fd, ft->buf, ft->count, offset); 79 | break; 80 | 81 | case EASY_FILE_SENDFILE: 82 | case EASY_FILE_WILLNEED: 83 | rc = posix_fadvise(ft->fd, offset, ft->count, POSIX_FADV_WILLNEED); 84 | break; 85 | } 86 | } while(rc == -1 && errno == EINTR); 87 | 88 | // 结果处理 89 | if (rc == -1) { 90 | ft->ret = (errno > 0) ? -errno : EASY_ERROR; 91 | } else { 92 | ft->ret = rc; 93 | } 94 | 95 | ft->done = 1; 96 | return rc; 97 | } 98 | */ 99 | -------------------------------------------------------------------------------- /libtdhs/tdh_socket_time.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved. 11 | 12 | This program is free software; you can redistribute it and/or modify 13 | it under the terms of the GNU General Public License as published by 14 | the Free Software Foundation; version 2 of the License. 15 | 16 | This program is distributed in the hope that it will be useful, 17 | but WITHOUT ANY WARRANTY; without even the implied warranty of 18 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 | GNU General Public License for more details. 20 | 21 | You should have received a copy of the GNU General Public License 22 | along with this program; if not, write to the Free Software 23 | Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ 24 | 25 | /* get time since epoc in 100 nanosec units */ 26 | /* thus to get the current time we should use the system function 27 | with the highest possible resolution */ 28 | 29 | /* 30 | TODO: in functions my_micro_time() and my_micro_time_and_time() there 31 | exists some common code that should be merged into a function. 32 | */ 33 | #include "tdh_socket_time.hpp" 34 | #include 35 | #include 36 | 37 | namespace taobao { 38 | 39 | ullong tdhs_micro_time() { 40 | #if defined(__WIN__) 41 | ullong newtime; 42 | GetSystemTimeAsFileTime((FILETIME*)&newtime); 43 | return (newtime/10); 44 | #elif defined(HAVE_GETHRTIME) 45 | return gethrtime()/1000; 46 | #else 47 | ullong newtime; 48 | struct timeval t; 49 | /* 50 | The following loop is here because gettimeofday may fail on some systems 51 | */ 52 | while (gettimeofday(&t, NULL) != 0) { 53 | } 54 | newtime = (ullong) t.tv_sec * 1000000 + t.tv_usec; 55 | return newtime; 56 | #endif /* defined(__WIN__) */ 57 | } 58 | 59 | /* Difference between GetSystemTimeAsFileTime() and now() */ 60 | #define OFFSET_TO_EPOCH 116444736000000000ULL 61 | 62 | ullong tdhs_micro_time_and_time(time_t *time_arg) 63 | { 64 | #ifdef _WIN32 65 | ullong newtime; 66 | GetSystemTimeAsFileTime((FILETIME*)&newtime); 67 | *time_arg= (time_t) ((newtime - OFFSET_TO_EPOCH) / 10000000); 68 | return (newtime/10); 69 | #else 70 | ullong newtime; 71 | struct timeval t; 72 | /* 73 | The following loop is here because gettimeofday may fail on some systems 74 | */ 75 | while (gettimeofday(&t, NULL) != 0) 76 | {} 77 | *time_arg= t.tv_sec; 78 | newtime= (ullong)t.tv_sec * 1000000 + t.tv_usec; 79 | return newtime; 80 | #endif 81 | } 82 | 83 | } // namespace taobao 84 | 85 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_table_balance.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | /* 10 | * tdh_socket_table_balance.cpp 11 | * 12 | * Created on: 2011-12-9 13 | * Author: wentong 14 | */ 15 | #include "tdh_socket_table_balance.hpp" 16 | #include "tdh_socket_config.hpp" 17 | #include "tdh_socket_share.hpp" 18 | #include "easy_define.h" 19 | 20 | #define TDHS_GUESS_TABLE_NUM (64 * 1024) 21 | 22 | namespace taobao { 23 | 24 | static unsigned long long total_execute_count; 25 | 26 | static unsigned long long table_execute_count[TDHS_GUESS_TABLE_NUM]; 27 | 28 | static uint64_t quick_hash_key = 0; 29 | 30 | static uint64_t slow_hash_key = 0; 31 | 32 | static uint64_t round_hash_key = 0; 33 | 34 | uint64_t table_need_balance(tdhs_request_t& req, tdhs_optimize_t type, 35 | uint32_t custom_hash) { 36 | unsigned int compare_num = tdhs_thread_num; //TODO 需要优化~ 37 | unsigned long long total_count; 38 | unsigned long long table_count; 39 | //MARK 针对request type 需要进行判断 40 | if (req.type == REQUEST_TYPE_GET || req.type == REQUEST_TYPE_COUNT) { 41 | uint64_t hash = req.table_info.hash_code_table(); 42 | table_count = ++table_execute_count[hash % TDHS_GUESS_TABLE_NUM]; 43 | total_count = ++total_execute_count; 44 | if (type == TDHS_QUICK && custom_hash > 0 45 | && thread_strategy == TDHS_THREAD_LV_1) { 46 | return custom_hash; 47 | } 48 | if (total_count / table_count < compare_num) { 49 | if (type == TDHS_QUICK) { 50 | hash = easy_hash_key(quick_hash_key++); 51 | } else { 52 | hash = easy_hash_key(slow_hash_key++); 53 | } 54 | } 55 | if (type == TDHS_SLOW) { 56 | //在这里控制线程数的分配 57 | hash %= active_slow_read_thread_num; 58 | //保证hash不为0 否则会rehash 59 | hash = hash == 0 ? active_slow_read_thread_num : hash; 60 | } 61 | return hash; 62 | } else if (req.type == REQUEST_TYPE_INSERT) { 63 | return tdhs_concurrency_insert ? 64 | (custom_hash > 0 ? custom_hash : easy_hash_key(round_hash_key++)) : 65 | req.table_info.hash_code_table(); 66 | 67 | } else if (req.type == REQUEST_TYPE_UPDATE) { 68 | return tdhs_concurrency_update ? 69 | (custom_hash > 0 ? custom_hash : easy_hash_key(round_hash_key++)) : 70 | req.table_info.hash_code_table(); 71 | 72 | } else if (req.type == REQUEST_TYPE_DELETE) { 73 | return tdhs_concurrency_delete ? 74 | (custom_hash > 0 ? custom_hash : easy_hash_key(round_hash_key++)) : 75 | req.table_info.hash_code_table(); 76 | 77 | } else if (req.type == REQUEST_TYPE_BATCH) { 78 | return easy_hash_key(round_hash_key++); 79 | } else { 80 | return req.table_info.hash_code_table(); 81 | } 82 | } 83 | 84 | } // namespace taobao 85 | 86 | -------------------------------------------------------------------------------- /libeasy/util/easy_time.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #include 11 | #include 12 | 13 | /** 14 | * localtime删除tzset,一起调用一次tzset 15 | */ 16 | int easy_localtime (const time_t *t, struct tm *tp) 17 | { 18 | static const unsigned short int mon_yday[2][13] = { 19 | /* Normal years. */ 20 | { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365 }, 21 | /* Leap years. */ 22 | { 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366 } 23 | }; 24 | 25 | #define SECS_PER_HOUR (60 * 60) 26 | #define SECS_PER_DAY (SECS_PER_HOUR * 24) 27 | #define ISLEAP(year) ((year) % 4 == 0 && ((year) % 100 != 0 || (year) % 400 == 0)) 28 | #define DIV(a, b) ((a) / (b) - ((a) % (b) < 0)) 29 | #define LEAPS_THRU_END_OF(y) (DIV (y, 4) - DIV (y, 100) + DIV (y, 400)) 30 | 31 | long int days, rem, y; 32 | const unsigned short int *ip; 33 | 34 | days = *t / SECS_PER_DAY; 35 | rem = *t % SECS_PER_DAY; 36 | rem -= timezone; 37 | 38 | while (rem < 0) { 39 | rem += SECS_PER_DAY; 40 | --days; 41 | } 42 | 43 | while (rem >= SECS_PER_DAY) { 44 | rem -= SECS_PER_DAY; 45 | ++days; 46 | } 47 | 48 | tp->tm_hour = rem / SECS_PER_HOUR; 49 | rem %= SECS_PER_HOUR; 50 | tp->tm_min = rem / 60; 51 | tp->tm_sec = rem % 60; 52 | /* January 1, 1970 was a Thursday. */ 53 | tp->tm_wday = (4 + days) % 7; 54 | 55 | if (tp->tm_wday < 0) 56 | tp->tm_wday += 7; 57 | 58 | y = 1970; 59 | 60 | while (days < 0 || days >= (ISLEAP (y) ? 366 : 365)) { 61 | /* Guess a corrected year, assuming 365 days per year. */ 62 | long int yg = y + days / 365 - (days % 365 < 0); 63 | 64 | /* Adjust DAYS and Y to match the guessed year. */ 65 | days -= ((yg - y) * 365 66 | + LEAPS_THRU_END_OF (yg - 1) 67 | - LEAPS_THRU_END_OF (y - 1)); 68 | y = yg; 69 | } 70 | 71 | tp->tm_year = y - 1900; 72 | 73 | if (tp->tm_year != y - 1900) { 74 | return 0; 75 | } 76 | 77 | tp->tm_yday = days; 78 | ip = mon_yday[ISLEAP(y)]; 79 | 80 | for (y = 11; days < (long int) ip[y]; --y) 81 | continue; 82 | 83 | days -= ip[y]; 84 | tp->tm_mon = y; 85 | tp->tm_mday = days + 1; 86 | return 1; 87 | } 88 | 89 | void __attribute__((constructor)) easy_time_start_() 90 | { 91 | tzset(); 92 | } 93 | 94 | int64_t easy_time_now() 95 | { 96 | struct timeval tv; 97 | gettimeofday (&tv, 0); 98 | return __INT64_C(1000000) * tv.tv_sec + tv.tv_usec; 99 | } 100 | -------------------------------------------------------------------------------- /libeasy/util/easy_buf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_BUF_H_ 11 | #define EASY_BUF_H_ 12 | 13 | /** 14 | * 网络的读写的BUFFER 15 | */ 16 | #include "easy_define.h" 17 | #include "easy_pool.h" 18 | 19 | EASY_CPP_START 20 | 21 | #define EASY_BUF_FILE 1 22 | #define EASY_BUF_CLOSE_FILE 3 23 | 24 | typedef struct easy_buf_t easy_buf_t; 25 | typedef struct easy_file_buf_t easy_file_buf_t; 26 | typedef struct easy_buf_string_t easy_buf_string_t; 27 | typedef void (easy_buf_cleanup_pt)(easy_buf_t *, void *); 28 | 29 | #define EASY_BUF_DEFINE \ 30 | easy_list_t node; \ 31 | int flags; \ 32 | easy_buf_cleanup_pt *cleanup; \ 33 | void *args; 34 | 35 | struct easy_buf_t { 36 | EASY_BUF_DEFINE; 37 | char *pos; 38 | char *last; 39 | char *end; 40 | }; 41 | 42 | struct easy_file_buf_t { 43 | EASY_BUF_DEFINE; 44 | int fd; 45 | int64_t offset; 46 | int64_t count; 47 | }; 48 | 49 | struct easy_buf_string_t { 50 | char *data; 51 | int len; 52 | }; 53 | 54 | extern easy_buf_t *easy_buf_create(easy_pool_t *pool, uint32_t size); 55 | extern void easy_buf_set_cleanup(easy_buf_t *b, easy_buf_cleanup_pt *cleanup, void *args); 56 | extern void easy_buf_set_data(easy_pool_t *pool, easy_buf_t *b, const void *data, uint32_t size); 57 | extern easy_buf_t *easy_buf_pack(easy_pool_t *pool, const void *data, uint32_t size); 58 | extern easy_file_buf_t *easy_file_buf_create(easy_pool_t *pool); 59 | extern void easy_buf_destroy(easy_buf_t *b); 60 | extern int easy_buf_check_read_space(easy_pool_t *pool, easy_buf_t *b, uint32_t size); 61 | extern easy_buf_t *easy_buf_check_write_space(easy_pool_t *pool, easy_list_t *bc, uint32_t size); 62 | extern void easy_file_buf_set_close(easy_file_buf_t *b); 63 | 64 | extern void easy_buf_chain_clear(easy_list_t *l); 65 | extern void easy_buf_chain_offer(easy_list_t *l, easy_buf_t *b); 66 | 67 | /////////////////////////////////////////////////////////////////////////////////////////////////// 68 | // easy_buf_string 69 | 70 | #define easy_buf_string_set(str, text) {(str)->len=strlen(text); (str)->data=(char*)text;} 71 | 72 | static inline char *easy_buf_string_ptr(easy_buf_string_t *s) 73 | { 74 | return s->data; 75 | } 76 | 77 | static inline void easy_buf_string_append(easy_buf_string_t *s, 78 | const char *value, int len) 79 | { 80 | s->data = (char *)(value - s->len); 81 | s->len += len; 82 | } 83 | 84 | static inline int easy_buf_len(easy_buf_t *b) 85 | { 86 | if (unlikely(b->flags & EASY_BUF_FILE)) 87 | return ((easy_file_buf_t *)b)->count; 88 | else 89 | return (b->last - b->pos); 90 | } 91 | 92 | extern int easy_buf_string_copy(easy_pool_t *pool, easy_buf_string_t *d, easy_buf_string_t *s); 93 | extern int easy_buf_string_printf(easy_pool_t *pool, easy_buf_string_t *d, const char *fmt, ...); 94 | 95 | EASY_CPP_END 96 | 97 | #endif 98 | -------------------------------------------------------------------------------- /libtdhs/util.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * util.hpp 12 | * 13 | * Created on: 2011-9-2 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef UTIL_HPP_ 18 | #define UTIL_HPP_ 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | namespace taobao { 25 | 26 | #define TAOBAO_MALLOC(s) malloc(s) 27 | #define TAOBAO_REALLOC(p,s) realloc(p,s) 28 | #define TAOBAO_FREE(s) free(s) 29 | 30 | #define CACHE_MEMORY_SIZE (512) 31 | class stack_liker { 32 | public: 33 | stack_liker(size_t s) { 34 | if (s <= CACHE_MEMORY_SIZE) { 35 | memset(cache_memory, 0, sizeof(cache_memory)); 36 | is_use_cache = true; 37 | ptr = cache_memory; 38 | } else { 39 | is_use_cache = false; 40 | if ((ptr = TAOBAO_MALLOC(s)) != NULL) { 41 | memset(ptr, 0, s); 42 | } 43 | } 44 | } 45 | ~stack_liker() { 46 | if (!is_use_cache && ptr != NULL) { 47 | TAOBAO_FREE(ptr); 48 | } 49 | } 50 | void* get_ptr() { 51 | return ptr; 52 | } 53 | private: 54 | char cache_memory[CACHE_MEMORY_SIZE]; 55 | void *ptr; 56 | bool is_use_cache; 57 | }; 58 | 59 | /* boost::noncopyable */ 60 | struct noncopyable { 61 | noncopyable() { 62 | } 63 | private: 64 | noncopyable(const noncopyable&); 65 | noncopyable& operator =(const noncopyable&); 66 | }; 67 | 68 | #define fatal_abort(M) \ 69 | easy_fatal_log((M)); \ 70 | abort(); 71 | 72 | #define fatal_exit(M) \ 73 | easy_fatal_log((M)); \ 74 | exit(-1); 75 | 76 | #define onBit(flag,bit) ((flag) |= (bit)) 77 | #define offBit(flag,bit) ((flag) &= ~(bit)) 78 | #define testFlag(flag,bit) (((flag) & (bit)) == (bit)) 79 | 80 | #define min(A,B) ((A)<(B))?(A):(B) 81 | #define max(A,B) ((A)>(B))?(A):(B) 82 | 83 | #define NULL_STRING "" 84 | 85 | #define read_uint8(POS,LEN) ((LEN) 11 | 12 | char *easy_strncpy(char *dst, const char *src, size_t n) 13 | { 14 | if (!n || !dst) 15 | return NULL; 16 | 17 | const uint64_t himagic = __UINT64_C(0x8080808080808080); 18 | const uint64_t lomagic = __UINT64_C(0x0101010101010101); 19 | const uint64_t *nsrc = (const uint64_t *)src; 20 | const uint64_t *nend = nsrc + (--n / 8); 21 | uint64_t *ndst = (uint64_t *)dst; 22 | 23 | while(nsrc != nend) { 24 | uint64_t k = *nsrc; 25 | 26 | if (((k - lomagic) & ~k & himagic) != 0) { 27 | const char *cp = (const char *) nsrc; 28 | 29 | if (cp[0] == 0) { 30 | n = 0; 31 | break; 32 | } 33 | 34 | if (cp[1] == 0) { 35 | n = 1; 36 | break; 37 | } 38 | 39 | if (cp[2] == 0) { 40 | n = 2; 41 | break; 42 | } 43 | 44 | if (cp[3] == 0) { 45 | n = 3; 46 | break; 47 | } 48 | 49 | if (cp[4] == 0) { 50 | n = 4; 51 | break; 52 | } 53 | 54 | if (cp[5] == 0) { 55 | n = 5; 56 | break; 57 | } 58 | 59 | if (cp[6] == 0) { 60 | n = 6; 61 | break; 62 | } 63 | 64 | n = 7; 65 | break; 66 | } 67 | 68 | *ndst++ = k; 69 | nsrc ++; 70 | } 71 | 72 | const char *nsrc2 = (const char *) nsrc; 73 | 74 | char *ndst2 = (char *) ndst; 75 | 76 | switch(n & 7) { 77 | case 7: 78 | *ndst2++ = *nsrc2++; 79 | 80 | case 6: 81 | *ndst2++ = *nsrc2++; 82 | 83 | case 5: 84 | *ndst2++ = *nsrc2++; 85 | 86 | case 4: 87 | *ndst2++ = *nsrc2++; 88 | 89 | case 3: 90 | *ndst2++ = *nsrc2++; 91 | 92 | case 2: 93 | *ndst2++ = *nsrc2++; 94 | 95 | case 1: 96 | *ndst2++ = *nsrc2++; 97 | }; 98 | 99 | *ndst2 = '\0'; 100 | 101 | return dst; 102 | } 103 | 104 | /** 105 | * 把char转成hex 106 | */ 107 | char *easy_string_tohex(const char *str, int n, char *result, int size) 108 | { 109 | int i, j = 0; 110 | static char hexconvtab[] = "0123456789ABCDEF"; 111 | const unsigned char *p = (const unsigned char *)str; 112 | 113 | n = easy_min((size - 1) / 2, n); 114 | 115 | for(i = 0; i < n; i++) { 116 | result[j++] = hexconvtab[p[i] >> 4]; 117 | result[j++] = hexconvtab[p[i] & 0xf]; 118 | } 119 | 120 | result[j] = '\0'; 121 | 122 | return result; 123 | } 124 | 125 | /** 126 | * 转成大写 127 | */ 128 | char *easy_string_toupper(char *str) 129 | { 130 | char *p = str; 131 | 132 | while(*p) { 133 | if ((*p) >= 'a' && (*p) <= 'z') 134 | (*p) -= 36; 135 | 136 | p ++; 137 | } 138 | 139 | return str; 140 | } 141 | 142 | /** 143 | * 转成小写 144 | */ 145 | char *easy_string_tolower(char *str) 146 | { 147 | char *p = str; 148 | 149 | while(*p) { 150 | if ((*p) >= 'A' && (*p) <= 'Z') 151 | (*p) += 36; 152 | 153 | p ++; 154 | } 155 | 156 | return str; 157 | } 158 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_thd.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_thd.hpp 12 | * 13 | * Created on: 2011-12-28 14 | * Author: wentong 15 | */ 16 | 17 | #ifndef TDH_SOCKET_THD_HPP_ 18 | #define TDH_SOCKET_THD_HPP_ 19 | 20 | #include "tdh_socket_share.hpp" 21 | #include "debug_util.hpp" 22 | #include "mysql_inc.hpp" 23 | #include "easy_log.h" 24 | 25 | namespace taobao { 26 | 27 | #define MAX_INFO_SIZE 256 28 | 29 | static TDHS_INLINE int set_thread_message(char* info, const char *fmt, ...); 30 | 31 | static TDHS_INLINE THD* init_THD(char* db, const void *stack_bottom, 32 | bool need_write); 33 | 34 | static TDHS_INLINE void destory_thd(THD *thd); 35 | 36 | static TDHS_INLINE int wait_server_to_start(THD *thd); 37 | 38 | static TDHS_INLINE int set_thread_message(char* info, const char *fmt, ...) { 39 | va_list ap; 40 | va_start(ap, fmt); 41 | const int n = vsnprintf(info, MAX_INFO_SIZE, fmt, ap); 42 | va_end(ap); 43 | return n; 44 | } 45 | 46 | static TDHS_INLINE THD* init_THD(char* db, const void *stack_bottom, 47 | bool need_write) { 48 | THD *thd = NULL; 49 | my_thread_init(); 50 | thd = new THD; 51 | if (thd == NULL) { 52 | my_thread_end(); 53 | return NULL; 54 | } 55 | thd->thread_stack = (char*) stack_bottom; 56 | easy_debug_log("TDHS:thread_stack = %p sizeof(THD)=%zu sizeof(mtx)=%zu ", 57 | thd->thread_stack, sizeof(THD), sizeof(LOCK_thread_count)); 58 | thd->store_globals(); 59 | thd->system_thread = static_cast(1 << 30UL); 60 | const NET v = { 0 }; 61 | thd->net = v; 62 | if (need_write) { 63 | //for write 64 | #if MYSQL_VERSION_ID >= 50505 65 | thd->variables.option_bits |= OPTION_BIN_LOG; 66 | #else 67 | thd->options |= OPTION_BIN_LOG; 68 | #endif 69 | } 70 | //for db 71 | safeFree(thd->db); 72 | thd->db = db; 73 | my_pthread_setspecific_ptr(THR_THD, thd); 74 | 75 | tdhs_mysql_mutex_lock(&LOCK_thread_count); 76 | thd->thread_id = thread_id++; 77 | threads.append(thd); 78 | ++thread_count; 79 | tdhs_mysql_mutex_unlock(&LOCK_thread_count); 80 | return thd; 81 | } 82 | 83 | static TDHS_INLINE void destory_thd(THD *thd) { 84 | tb_assert(thd!=NULL); 85 | my_pthread_setspecific_ptr(THR_THD, 0); 86 | 87 | tdhs_mysql_mutex_lock(&LOCK_thread_count); 88 | delete thd; 89 | --thread_count; 90 | tdhs_mysql_mutex_unlock(&LOCK_thread_count); 91 | my_thread_end(); 92 | (void) tdhs_mysql_cond_broadcast(&COND_thread_count); 93 | } 94 | 95 | static TDHS_INLINE int wait_server_to_start(THD *thd) { 96 | int r = 0; 97 | tdhs_mysql_mutex_lock(&LOCK_server_started); 98 | while (!mysqld_server_started) { 99 | timespec abstime = { }; 100 | set_timespec(abstime, 1); 101 | tdhs_mysql_cond_timedwait(&COND_server_started, &LOCK_server_started, 102 | &abstime); 103 | tdhs_mysql_mutex_unlock(&LOCK_server_started); 104 | tdhs_mysql_mutex_lock(&thd->mysys_var->mutex); 105 | THD::killed_state st = thd->killed; 106 | tdhs_mysql_mutex_unlock(&thd->mysys_var->mutex); 107 | tdhs_mysql_mutex_lock(&LOCK_server_started); 108 | if (st != THD::NOT_KILLED) { 109 | r = -1; 110 | break; 111 | } 112 | if (tdhs_share->shutdown) { 113 | r = -1; 114 | break; 115 | } 116 | } 117 | 118 | tdhs_mysql_mutex_unlock(&LOCK_server_started); 119 | return r; 120 | } 121 | 122 | } // namespace taobao 123 | 124 | #endif /* TDH_SOCKET_THD_HPP_ */ 125 | -------------------------------------------------------------------------------- /row_cache/Installation.txt: -------------------------------------------------------------------------------- 1 | 1.Installation 2 | How to install row cache for innodb. The patch is for mysql 5.1.48 or mysql 5.5.21 or percona server 5.5.18 3 | 4 | Installation Steps 5 | a) Download mysql 5.1.48 source code if you don't have. Link is http://downloads.mysql.com/archives/mysql-5.1/mysql-5.1.48.tar.gz 6 | b) Decompress the mysql source code. 7 | tar zvxf mysql-5.1.48.tar.gz 8 | c) Apply the patch in the source root directory . 9 | patch -p0 < ./row_cache_for_mysql.5.1.48_2011_05_11.diff 10 | d) Configure. 11 | CFLAGS="-O3" CXX=gcc CXXFLAGS="-O3 -felide-constructors -fno-exceptions -fno-rtti" ./configure --prefix=/usr/mysql --with-extra-charsets=all --with-plugins=partition,heap,innobase,myisam,myisammrg,csv --enable-assembler 12 | e) make && make install 13 | 14 | 2.Setting 15 | Setting added in my.cnf(The Row Cache is based on innodb_plugin): 16 | innodb_row_cache_mem_pool_size 17 | the size of row cache used in memory. 18 | default is 1M 19 | innodb_row_cache_on 20 | the switch for row cache,if you want enable row cache,it need to be ON, 21 | default is OFF. 22 | innodb_row_cache_cell_num 23 | the row cache used hash table's cell num. 24 | default is 1000 25 | innodb_row_cache_mutex_num_shift 26 | the shift for row cache mutex num, more large higher concurrency ,it can refer to innodb_thread_concurrency ,recommend (1< 14 | 15 | /** 16 | * 对TCP简单包的解析 17 | */ 18 | EASY_CPP_START 19 | 20 | #include 21 | 22 | #define EASY_SIMPLE_PACKET_HEADER_SIZE (sizeof(int)*2) 23 | typedef struct easy_simple_packet_t easy_simple_packet_t; 24 | 25 | /** 26 | * len + data 27 | */ 28 | struct easy_simple_packet_t { 29 | uint32_t len; 30 | uint32_t chid; 31 | char *data; 32 | easy_list_t list; 33 | char buffer[0]; 34 | }; 35 | 36 | /** 37 | * decode 38 | */ 39 | static inline void *easy_simple_decode(easy_message_t *m) 40 | { 41 | easy_simple_packet_t *packet; 42 | uint32_t len, datalen; 43 | 44 | // length 45 | if ((len = m->input->last - m->input->pos) < EASY_SIMPLE_PACKET_HEADER_SIZE) 46 | return NULL; 47 | 48 | // data len 49 | datalen = *((uint32_t *)m->input->pos); 50 | 51 | if (datalen > 0x4000000) { // 64M 52 | easy_error_log("data_len is invalid: %d\n", datalen); 53 | m->status = EASY_ERROR; 54 | return NULL; 55 | } 56 | 57 | // 长度不够 58 | len -= EASY_SIMPLE_PACKET_HEADER_SIZE; 59 | 60 | if (len < datalen) { 61 | m->next_read_len = datalen - len; 62 | return NULL; 63 | } 64 | 65 | // alloc packet 66 | if ((packet = (easy_simple_packet_t *)easy_pool_calloc(m->pool, 67 | sizeof(easy_simple_packet_t))) == NULL) { 68 | m->status = EASY_ERROR; 69 | return NULL; 70 | } 71 | 72 | packet->chid = *((uint32_t *)(m->input->pos + sizeof(int))); 73 | m->input->pos += EASY_SIMPLE_PACKET_HEADER_SIZE; 74 | packet->len = datalen; 75 | packet->data = (char *)m->input->pos; 76 | m->input->pos += datalen; 77 | 78 | return packet; 79 | } 80 | 81 | /** 82 | * encode 83 | */ 84 | static inline int easy_simple_encode(easy_request_t *r, void *data) 85 | { 86 | easy_simple_packet_t *packet; 87 | easy_buf_t *b, *b1; 88 | 89 | packet = (easy_simple_packet_t *) data; 90 | 91 | if ((b = easy_buf_create(r->ms->pool, EASY_SIMPLE_PACKET_HEADER_SIZE)) == NULL) 92 | return EASY_ERROR; 93 | 94 | // 加入 95 | *((uint32_t *)b->last) = packet->len; 96 | *((uint32_t *)(b->last + sizeof(int))) = packet->chid; 97 | b->last += EASY_SIMPLE_PACKET_HEADER_SIZE; 98 | easy_request_addbuf(r, b); 99 | 100 | // 加入data 101 | if (packet->data) { 102 | b1 = easy_buf_pack(r->ms->pool, packet->data, packet->len); 103 | easy_request_addbuf(r, b1); 104 | } else { 105 | easy_request_addbuf_list(r, &packet->list); 106 | } 107 | 108 | return EASY_OK; 109 | } 110 | 111 | static inline uint64_t easy_simple_packet_id(easy_connection_t *c, void *packet) 112 | { 113 | return ((easy_simple_packet_t *) packet)->chid; 114 | } 115 | 116 | static inline easy_simple_packet_t *easy_simple_rnew(easy_request_t *r, int size) 117 | { 118 | easy_simple_packet_t *packet; 119 | 120 | size += sizeof(easy_simple_packet_t); 121 | packet = (easy_simple_packet_t *)easy_pool_alloc(r->ms->pool, size); 122 | memset(packet, 0, sizeof(easy_simple_packet_t)); 123 | 124 | return packet; 125 | } 126 | 127 | #define easy_simple_new(s, size) easy_session_packet_create(easy_simple_packet_t, s, size) 128 | 129 | EASY_CPP_END 130 | 131 | #endif 132 | -------------------------------------------------------------------------------- /python/src/block_tdhs_client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License version 2 as 7 | # published by the Free Software Foundation. 8 | # 9 | # Authors: 10 | # wentong 11 | 12 | # wentong@taobao.com 13 | # 11-9-28 14 | # 15 | from _struct import pack 16 | import socket 17 | from batch_statement import batch_statement 18 | import logger 19 | from statement import statement 20 | from tdhs_common import * 21 | 22 | import threading 23 | 24 | def add_str_to_binary(s): 25 | if s: 26 | es = s.encode("GBK") 27 | return pack('!L', len(es) + 1) + pack(str(len(es)) + 's', es) + '\x00' #for c string 28 | else: 29 | return pack('!L', 0) 30 | 31 | 32 | class TDHSConnection: 33 | id = 0 34 | 35 | id_lock = threading.Lock() 36 | 37 | logger = logger.logger() 38 | 39 | def __init__(self, host, port=9999, time_out=1000, read_code=None, write_code=None): 40 | self.addr = (host, port) 41 | self.time_out = time_out 42 | self.sock = socket.socket(socket.AF_INET) 43 | self.sock.connect((host, port)) 44 | self.__shake_hands(read_code, write_code) 45 | 46 | def __shake_hands(self, read_code, write_code): 47 | shake_hands = TDHS_HEAD 48 | wbuf = shake_hands 49 | wbuf += pack('!L', TDHS_VERSION_BINARY) 50 | wbuf += pack('!L', self.time_out) 51 | wbuf += add_str_to_binary(read_code) 52 | wbuf += add_str_to_binary(write_code) 53 | self.send(pack(FMT, MAGIC_CODE, REQUEST_TYPE_SHAKE_HANDS, self.getid(), 0, len(wbuf)) + wbuf) 54 | 55 | def getid(self): 56 | try: 57 | self.id_lock.acquire() 58 | self.id += 1 59 | return self.id 60 | finally: 61 | self.id_lock.release() 62 | 63 | def send(self, wbuf): 64 | self.logger.debug("send [%s]" % wbuf) 65 | self.sock.send(wbuf) 66 | 67 | def recv(self, size): 68 | length = size 69 | rbuff = "" 70 | while length > 0: 71 | b = self.sock.recv(length) 72 | rbuff += b 73 | length -= len(b) 74 | return rbuff 75 | 76 | 77 | def get(self, db, table, index, fields, key, find_flag=TDHS_EQ, start=0, limit=0, filters=None): 78 | sm = self.get_statement() 79 | return sm.get(db, table, index, fields, key, find_flag, start, limit, filters) 80 | 81 | def delete(self, db, table, index, key, find_flag=TDHS_EQ, start=0, limit=0, filters=None): 82 | sm = self.get_statement() 83 | return sm.delete(db, table, index, key, find_flag, start, limit, filters) 84 | 85 | def count(self, db, table, index, key, find_flag=TDHS_EQ, start=0, limit=0, filters=None): 86 | sm = self.get_statement() 87 | return sm.count(db, table, index, key, find_flag, start, limit, filters) 88 | 89 | 90 | def update(self, db, table, index, fields, updates, key, find_flag=TDHS_EQ, start=0, limit=0, filters=None ): 91 | sm = self.get_statement() 92 | return sm.update(db, table, index, fields, updates, key, find_flag, start, limit, filters) 93 | 94 | def insert(self, db, table, fields, values): 95 | sm = self.get_statement() 96 | return sm.insert(db, table, fields, values) 97 | 98 | 99 | def get_statement(self, hash=0): 100 | return statement(self, hash) 101 | 102 | def get_batch_statement(self): 103 | return batch_statement(self) 104 | 105 | 106 | def close(self): 107 | self.sock.close() 108 | 109 | 110 | def ConnectionManager(host, port=9999, time_out=1000, read_code=None, write_code=None): 111 | return TDHSConnection(host, port, time_out, read_code, write_code) 112 | -------------------------------------------------------------------------------- /libeasy/util/easy_inet.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include // inet_addr 17 | #include 18 | #include 19 | 20 | /** 21 | * 把sockaddr_in转成string 22 | */ 23 | char *easy_inet_addr_to_str(easy_addr_t *ptr, char *buffer, int len) 24 | { 25 | struct sockaddr_in *addr; 26 | unsigned char *b; 27 | 28 | addr = (struct sockaddr_in *) ptr; 29 | b = (unsigned char *) &addr->sin_addr.s_addr; 30 | 31 | if (addr->sin_port) 32 | snprintf(buffer, len, "%d.%d.%d.%d:%d", b[0], b[1], b[2], b[3], ntohs(addr->sin_port)); 33 | else 34 | snprintf(buffer, len, "%d.%d.%d.%d", b[0], b[1], b[2], b[3]); 35 | 36 | return buffer; 37 | } 38 | 39 | /** 40 | * 把str转成addr(用uint64_t表示,IPV4) 41 | */ 42 | easy_addr_t easy_inet_str_to_addr(const char *host, int port) 43 | { 44 | easy_addr_t address = {0, 0}; 45 | char *p, buffer[64]; 46 | int len; 47 | 48 | if (host && (p = strchr(host, ':')) != NULL) { 49 | if ((len = p - host) > 63) 50 | return address; 51 | 52 | memcpy(buffer, host, len); 53 | buffer[len] = '\0'; 54 | host = buffer; 55 | 56 | if (!port) 57 | port = atoi(p + 1); 58 | } 59 | 60 | // parse host 61 | easy_inet_parse_host(&address, host, port); 62 | 63 | return address; 64 | } 65 | 66 | /** 67 | * 是IP地址, 如: 192.168.1.2 68 | */ 69 | int easy_inet_is_ipaddr(const char *host) 70 | { 71 | unsigned char c, *p; 72 | 73 | p = (unsigned char *)host; 74 | 75 | while ((c = (*p++)) != '\0') { 76 | if ((c != '.') && (c < '0' || c > '9')) { 77 | return 0; 78 | } 79 | } 80 | 81 | return 1; 82 | } 83 | 84 | /** 85 | * 解析host 86 | */ 87 | int easy_inet_parse_host(easy_addr_t *address, const char *host, int port) 88 | { 89 | struct sockaddr_in addr; 90 | 91 | memset(&addr, 0, sizeof(addr)); 92 | addr.sin_family = AF_INET; 93 | addr.sin_port = htons(port); 94 | 95 | if (host && host[0]) { 96 | // 如果是ip地址, 用inet_addr转一下, 否则用gethostbyname 97 | if (easy_inet_is_ipaddr(host)) { 98 | if ((addr.sin_addr.s_addr = inet_addr(host)) == INADDR_NONE) 99 | return EASY_ERROR; 100 | } else { 101 | // FIXME: gethostbyname会阻塞 102 | char buffer[1024]; 103 | struct hostent h, *hp; 104 | int rc; 105 | 106 | if (gethostbyname_r(host, &h, buffer, 1024, &hp, &rc) || hp == NULL) 107 | return EASY_ERROR; 108 | 109 | addr.sin_addr.s_addr = *((in_addr_t *) (hp->h_addr)); 110 | } 111 | } else { 112 | addr.sin_addr.s_addr = htonl(INADDR_ANY); 113 | } 114 | 115 | memcpy(address, &addr, sizeof(easy_addr_t)); 116 | 117 | return EASY_OK; 118 | } 119 | 120 | /** 121 | * 得到本机所有IP 122 | */ 123 | int easy_inet_hostaddr(uint64_t *address, int size) 124 | { 125 | int fd, ret, n; 126 | struct ifconf ifc; 127 | struct ifreq *ifr; 128 | 129 | ret = 0; 130 | 131 | if ((fd = socket(AF_INET, SOCK_DGRAM, 0)) < 0) 132 | return 0; 133 | 134 | ifc.ifc_len = sizeof(struct ifreq) * size; 135 | ifc.ifc_buf = (char *) malloc(ifc.ifc_len); 136 | 137 | if (ioctl(fd, SIOCGIFCONF, (char *)&ifc) < 0) 138 | goto out; 139 | 140 | ifr = ifc.ifc_req; 141 | 142 | for (n = 0; n < ifc.ifc_len; n += sizeof(struct ifreq)) { 143 | memcpy(&address[ret++], &(ifr->ifr_addr), sizeof(uint64_t)); 144 | ifr++; 145 | } 146 | 147 | out: 148 | free(ifc.ifc_buf); 149 | close(fd); 150 | return ret; 151 | } 152 | -------------------------------------------------------------------------------- /libeasy/include/easy_list.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_LIST_H_ 11 | #define EASY_LIST_H_ 12 | 13 | /** 14 | * 列表，参考kernel上的list.h 15 | */ 16 | #include "easy_define.h" 17 | 18 | EASY_CPP_START 19 | 20 | // from kernel list 21 | typedef struct easy_list_t easy_list_t; 22 | 23 | struct easy_list_t { 24 | easy_list_t *next, *prev; 25 | }; 26 | 27 | #define EASY_LIST_HEAD_INIT(name) {&(name), &(name)} 28 | #define easy_list_init(ptr) do { \ 29 | (ptr)->next = (ptr); \ 30 | (ptr)->prev = (ptr); \ 31 | } while (0) 32 | 33 | static inline void __easy_list_add(easy_list_t *list, 34 | easy_list_t *prev, easy_list_t *next) 35 | { 36 | next->prev = list; 37 | list->next = next; 38 | list->prev = prev; 39 | prev->next = list; 40 | } 41 | // list head to add it after 42 | static inline void easy_list_add_head(easy_list_t *list, easy_list_t *head) 43 | { 44 | __easy_list_add(list, head, head->next); 45 | } 46 | // list head to add it before 47 | static inline void easy_list_add_tail(easy_list_t *list, easy_list_t *head) 48 | { 49 | __easy_list_add(list, head->prev, head); 50 | } 51 | static inline void __easy_list_del(easy_list_t *prev, easy_list_t *next) 52 | { 53 | next->prev = prev; 54 | prev->next = next; 55 | } 56 | // deletes entry from list 57 | static inline void easy_list_del(easy_list_t *entry) 58 | { 59 | __easy_list_del(entry->prev, entry->next); 60 | easy_list_init(entry); 61 | } 62 | // tests whether a list is empty 63 | static inline int easy_list_empty(const easy_list_t *head) 64 | { 65 | return (head->next == head); 66 | } 67 | // move list to new_list 68 | static inline void easy_list_movelist(easy_list_t *list, easy_list_t *new_list) 69 | { 70 | if (!easy_list_empty(list)) { 71 | new_list->prev = list->prev; 72 | new_list->next = list->next; 73 | new_list->prev->next = new_list; 74 | new_list->next->prev = new_list; 75 | easy_list_init(list); 76 | } else { 77 | easy_list_init(new_list); 78 | } 79 | } 80 | // join list to head 81 | static inline void easy_list_join(easy_list_t *list, easy_list_t *head) 82 | { 83 | if (!easy_list_empty(list)) { 84 | easy_list_t *first = list->next; 85 | easy_list_t *last = list->prev; 86 | easy_list_t *at = head->prev; 87 | 88 | first->prev = at; 89 | at->next = first; 90 | last->next = head; 91 | head->prev = last; 92 | } 93 | } 94 | 95 | // get last 96 | #define easy_list_get_last(list, type, member) \ 97 | easy_list_empty(list) ? NULL : easy_list_entry((list)->prev, type, member) 98 | 99 | // get first 100 | #define easy_list_get_first(list, type, member) \ 101 | easy_list_empty(list) ? NULL : easy_list_entry((list)->next, type, member) 102 | 103 | #define easy_list_entry(ptr, type, member) ({ \ 104 | const typeof( ((type *)0)->member ) *__mptr = (ptr); \ 105 | (type *)( (char *)__mptr - offsetof(type,member) );}) 106 | 107 | #define easy_list_for_each_entry(pos, head, member) \ 108 | for (pos = easy_list_entry((head)->next, typeof(*pos), member); \ 109 | &pos->member != (head); \ 110 | pos = easy_list_entry(pos->member.next, typeof(*pos), member)) 111 | 112 | #define easy_list_for_each_entry_safe(pos, n, head, member) \ 113 | for (pos = easy_list_entry((head)->next, typeof(*pos), member), \ 114 | n = easy_list_entry(pos->member.next, typeof(*pos), member); \ 115 | &pos->member != (head); \ 116 | pos = n, n = easy_list_entry(n->member.next, typeof(*n), member)) 117 | 118 | EASY_CPP_END 119 | 120 | #endif 121 | -------------------------------------------------------------------------------- /python/src/statement.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License version 2 as 6 | # published by the Free Software Foundation. 7 | # 8 | # Authors: 9 | # wentong 10 | # 12-2-20 11 | # 12 | from exception import * 13 | from request import * 14 | from response import * 15 | from tdhs_common import * 16 | 17 | def encode_packet(type, id, wbuff, reserved=0): 18 | buf = pack(FMT, MAGIC_CODE, type, id, reserved, len(wbuff)) 19 | buf += wbuff 20 | return buf 21 | 22 | 23 | class statement: 24 | id = -1 25 | 26 | hash = 0 27 | 28 | def __init__(self, client, h=0): 29 | self.client = client 30 | self.hash = h 31 | 32 | def getid(self): 33 | self.id = self.client.getid() 34 | return self.id 35 | 36 | def do_reponse(self): 37 | databuff = "" 38 | data_len = 0 39 | while True: 40 | rbuff = self.client.recv(FSIZE) 41 | magic_code, resp_code, id, reverse, length = unpack(FMT, rbuff) 42 | self.client.logger.debug("return resp_code[%d] length[%d],id[%d]" % (resp_code, length, id)) 43 | if id != self.id: 44 | raise response_exception("id is error!") 45 | rbuff = self.client.recv(length) 46 | if 400 <= resp_code < 600: 47 | rpos = 0 48 | error_code, = unpack('!L', rbuff[rpos:rpos + 4]) 49 | if resp_code == CLIENT_STATUS_DB_ERROR: 50 | raise response_exception("mysql return code [%d]" % error_code) 51 | if error_code < len(ERROR_MSG): 52 | raise response_exception(ERROR_MSG[error_code]) 53 | else: 54 | raise response_exception("unkown error code [%d]" % error_code) 55 | elif resp_code == CLIENT_STATUS_ACCEPT: 56 | databuff += rbuff 57 | data_len += length 58 | elif resp_code == CLIENT_STATUS_OK: 59 | databuff += rbuff 60 | data_len += length 61 | self.client.logger.debug("return data_length[%d]" % data_len) 62 | return response(databuff, data_len).parse() 63 | else: 64 | raise response_exception("unknown response code [%d]" % resp_code) 65 | 66 | 67 | def get(self, db, table, index, fields, key, find_flag=TDHS_EQ, start=0, limit=0, filters=None): 68 | req = get(table_info(db, table, index, fields), key, find_flag, start, limit, filters) 69 | self.client.send(encode_packet(REQUEST_TYPE_GET, self.getid(), req.to_binary())) 70 | return self.do_reponse() 71 | 72 | def delete(self, db, table, index, key, find_flag=TDHS_EQ, start=0, limit=0, filters=None): 73 | info = table_info(db, table, index, []) 74 | info.need_field = False 75 | req = get(info, key, find_flag, start, limit, filters) 76 | self.client.send(encode_packet(REQUEST_TYPE_DELETE, self.getid(), req.to_binary(), self.hash)) 77 | return self.do_reponse() 78 | 79 | 80 | def count(self, db, table, index, key, find_flag=TDHS_EQ, start=0, limit=0, filters=None): 81 | info = table_info(db, table, index, []) 82 | info.need_field = False 83 | req = get(info, key, find_flag, start, limit, filters) 84 | self.client.send(encode_packet(REQUEST_TYPE_COUNT, self.getid(), req.to_binary(), self.hash)) 85 | return self.do_reponse() 86 | 87 | def update(self, db, table, index, fields, updates, key, find_flag=TDHS_EQ, start=0, limit=0, filters=None ): 88 | get_req = get(table_info(db, table, index, fields), key, find_flag, start, limit, filters) 89 | req = update(get_req, updates) 90 | self.client.send(encode_packet(REQUEST_TYPE_UPDATE, self.getid(), req.to_binary(), self.hash)) 91 | return self.do_reponse() 92 | 93 | def insert(self, db, table, fields, values): 94 | info = table_info(db, table, None, fields) 95 | req = insert(info, values) 96 | self.client.send(encode_packet(REQUEST_TYPE_INSERT, self.getid(), req.to_binary(), self.hash)) 97 | return self.do_reponse() -------------------------------------------------------------------------------- /python/src/tdhs_common.py: -------------------------------------------------------------------------------- 1 | # coding=UTF8 2 | #!/usr/bin/python 3 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License version 2 as 7 | # published by the Free Software Foundation. 8 | # 9 | # Authors: 10 | # wentong 11 | # 11-9-28 12 | 13 | 14 | 15 | TDHS_HEAD = "TDHS" 16 | TDHS_VERSION_BSON = 0 #use bson 17 | TDHS_VERSION_BINARY = 1 #use binary 18 | 19 | FMT = '!LLLLL' 20 | FSIZE = 20 # LLLL 21 | 22 | REQUEST_TYPE_GET = 0 23 | REQUEST_TYPE_COUNT = 1 24 | REQUEST_TYPE_UPDATE = 10 25 | REQUEST_TYPE_DELETE = 11 26 | REQUEST_TYPE_INSERT = 12 27 | REQUEST_TYPE_BATCH = 20 28 | REQUEST_TYPE_SHAKE_HANDS = 0xFFFF 29 | 30 | MAGIC_CODE = 0xFFFFFFFF 31 | 32 | #find_flag 33 | TDHS_EQ = 0 # = for asc 34 | TDHS_GE = 1 # >= 35 | TDHS_LE = 2 # <= 36 | TDHS_GT = 3 # > 37 | TDHS_LT = 4 # < 38 | TDHS_IN = 5 # in 39 | TDHS_DEQ = 6 # = for desc 40 | TDHS_BETWEEN = 7 # between 41 | 42 | #filter flag 43 | TDHS_FILTER_EQ = 0 # = 44 | TDHS_FILTER_GE = 1 # >= 45 | TDHS_FILTER_LE = 2 # <= 46 | TDHS_FILTER_GT = 3 # > 47 | TDHS_FILTER_LT = 4 # < 48 | TDHS_FILTER_NOT = 5 # ! 49 | 50 | 51 | #update flag 52 | TDHS_UPDATE_SET = 0 # = 53 | TDHS_UPDATE_ADD = 1 # + 54 | TDHS_UPDATE_SUB = 2 # - 55 | TDHS_UPDATE_NOW = 3 # now() 56 | TDHS_UPDATE_END = 4 57 | 58 | #return to client status 59 | #正确返回status 60 | CLIENT_STATUS_OK = 200 #完成所有数据的返回 61 | CLIENT_STATUS_ACCEPT = 202 #对于流的处理,还有未返回的数据 62 | CLIENT_STATUS_MULTI_STATUS = 207 #对于batch请求的返回,表示后面跟了多个请求 63 | #请求导致的错误信息 64 | CLIENT_STATUS_BAD_REQUEST = 400 65 | CLIENT_STATUS_FORBIDDEN = 403 #没权限 66 | CLIENT_STATUS_NOT_FOUND = 404 #没有找到资源,如 db/table/index 等 67 | CLIENT_STATUS_REQUEST_TIME_OUT = 408 #超时 68 | #服务器导致的错误信息 69 | CLIENT_STATUS_SERVER_ERROR = 500 #server无法处理的错误,比如内存不够 70 | CLIENT_STATUS_NOT_IMPLEMENTED = 501 #server没实现这个功能 71 | CLIENT_STATUS_DB_ERROR = 502 #handler返回的错误码 72 | CLIENT_STATUS_SERVICE_UNAVAILABLE = 503 #负载过重 73 | 74 | #error code 75 | CLIENT_ERROR_CODE_FAILED_TO_OPEN_TABLE = 1 76 | CLIENT_ERROR_CODE_FAILED_TO_OPEN_INDEX = 2 77 | CLIENT_ERROR_CODE_FAILED_TO_MISSING_FIELD = 3 78 | CLIENT_ERROR_CODE_FAILED_TO_MATCH_KEY_NUM = 4 79 | CLIENT_ERROR_CODE_FAILED_TO_LOCK_TABLE = 5 80 | CLIENT_ERROR_CODE_NOT_ENOUGH_MEMORY = 6 81 | CLIENT_ERROR_CODE_DECODE_REQUEST_FAILED = 7 82 | CLIENT_ERROR_CODE_FAILED_TO_MISSING_FIELD_IN_FILTER_OR_USE_BLOB = 8 83 | CLIENT_ERROR_CODE_FAILED_TO_COMMIT = 9 84 | CLIENT_ERROR_CODE_NOT_IMPLEMENTED = 10 85 | CLIENT_ERROR_CODE_REQUEST_TIME_OUT = 11 86 | CLIENT_ERROR_CODE_UNAUTHENTICATION = 12 87 | CLIENT_ERROR_CODE_KILLED = 13 88 | CLIENT_ERROR_CODE_THROTTLED = 14 89 | 90 | 91 | #error code message 92 | ERROR_MSG = [ 93 | "nothing", 94 | "TDH_SOCKET failed to open table!", 95 | "TDH_SOCKET failed to open index!", 96 | "TDH_SOCKET field is missing!", 97 | "TDH_SOCKET request can't match the key number!", 98 | "TDH_SOCKET failed to lock table!", 99 | "TDH_SOCKET server don't have enough memory!", 100 | "TDH_SOCKET server can't decode your request!", 101 | "TDH_SOCKET field is missing in filter or use blob!", 102 | "TDH_SOCKET failed to commit, will be rollback!", 103 | "TDH_SOCKET not implemented!", 104 | "TDH_SOCKET request time out!", 105 | "TDH_SOCKET request is unauthentication!", 106 | "TDH_SOCKET request is killed!", 107 | "TDH_SOCKET request is throttled!" 108 | ] 109 | 110 | #field type 111 | MYSQL_TYPE_DECIMAL = 0 112 | MYSQL_TYPE_TINY = 1 113 | MYSQL_TYPE_SHORT = 2 114 | MYSQL_TYPE_LONG = 3 115 | MYSQL_TYPE_FLOAT = 4 116 | MYSQL_TYPE_DOUBLE = 5 117 | MYSQL_TYPE_NULL = 6 118 | MYSQL_TYPE_TIMESTAMP = 7 119 | MYSQL_TYPE_LONGLONG = 8 120 | MYSQL_TYPE_INT24 = 9 121 | MYSQL_TYPE_DATE = 10 122 | MYSQL_TYPE_TIME = 11 123 | MYSQL_TYPE_DATETIME = 12 124 | MYSQL_TYPE_YEAR = 13 125 | MYSQL_TYPE_NEWDATE = 14 126 | MYSQL_TYPE_VARCHAR = 15 127 | MYSQL_TYPE_BIT = 16 128 | MYSQL_TYPE_NEWDECIMAL = 246 129 | MYSQL_TYPE_ENUM = 247 130 | MYSQL_TYPE_SET = 248 131 | MYSQL_TYPE_TINY_BLOB = 249 132 | MYSQL_TYPE_MEDIUM_BLOB = 250 133 | MYSQL_TYPE_LONG_BLOB = 251 134 | MYSQL_TYPE_BLOB = 252 135 | MYSQL_TYPE_VAR_STRING = 253 136 | MYSQL_TYPE_STRING = 254 137 | MYSQL_TYPE_GEOMETRY = 255 138 | 139 | 140 | 141 | 142 | -------------------------------------------------------------------------------- /libeasy/include/easy_atomic.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_LOCK_ATOMIC_H_ 11 | #define EASY_LOCK_ATOMIC_H_ 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | /** 18 | * 原子操作 19 | */ 20 | 21 | EASY_CPP_START 22 | 23 | #define EASY_SMP_LOCK "lock;" 24 | #define easy_atomic_set(v,i) ((v) = (i)) 25 | typedef volatile int32_t easy_atomic32_t; 26 | 27 | // 32bit 28 | static __inline__ void easy_atomic32_add(easy_atomic32_t *v, int i) 29 | { 30 | __asm__ __volatile__( 31 | EASY_SMP_LOCK "addl %1,%0" 32 | : "=m" ((*v)) : "r" (i), "m" ((*v))); 33 | } 34 | static __inline__ int32_t easy_atomic32_add_return(easy_atomic32_t *value, int32_t diff) 35 | { 36 | int32_t old = diff; 37 | __asm__ volatile ( 38 | EASY_SMP_LOCK "xaddl %0, %1" 39 | :"+r" (diff), "+m" (*value) : : "memory"); 40 | return diff + old; 41 | } 42 | static __inline__ void easy_atomic32_inc(easy_atomic32_t *v) 43 | { 44 | __asm__ __volatile__(EASY_SMP_LOCK "incl %0" : "=m" (*v) :"m" (*v)); 45 | } 46 | static __inline__ void easy_atomic32_dec(easy_atomic32_t *v) 47 | { 48 | __asm__ __volatile__(EASY_SMP_LOCK "decl %0" : "=m" (*v) :"m" (*v)); 49 | } 50 | 51 | // 64bit 52 | #if __WORDSIZE == 64 53 | typedef volatile int64_t easy_atomic_t; 54 | static __inline__ void easy_atomic_add(easy_atomic_t *v, int64_t i) 55 | { 56 | __asm__ __volatile__( 57 | EASY_SMP_LOCK "addq %1,%0" 58 | : "=m" ((*v)) : "r" (i), "m" ((*v))); 59 | } 60 | static __inline__ int64_t easy_atomic_add_return(easy_atomic_t *value, int64_t diff) 61 | { 62 | int64_t old = diff; 63 | __asm__ volatile ( 64 | EASY_SMP_LOCK "xaddq %0, %1" 65 | :"+r" (diff), "+m" (*value) : : "memory"); 66 | return diff + old; 67 | } 68 | static __inline__ int64_t easy_atomic_cmp_set(easy_atomic_t *lock, int64_t old, int64_t set) 69 | { 70 | uint8_t res; 71 | __asm__ volatile ( 72 | EASY_SMP_LOCK "cmpxchgq %3, %1; sete %0" 73 | : "=a" (res) : "m" (*lock), "a" (old), "r" (set) : "cc", "memory"); 74 | return res; 75 | } 76 | static __inline__ void easy_atomic_inc(easy_atomic_t *v) 77 | { 78 | __asm__ __volatile__(EASY_SMP_LOCK "incq %0" : "=m" (*v) :"m" (*v)); 79 | } 80 | static __inline__ void easy_atomic_dec(easy_atomic_t *v) 81 | { 82 | __asm__ __volatile__(EASY_SMP_LOCK "decq %0" : "=m" (*v) :"m" (*v)); 83 | } 84 | #else 85 | typedef volatile int32_t easy_atomic_t; 86 | #define easy_atomic_add(v,i) easy_atomic32_add(v,i) 87 | #define easy_atomic_add_return(v,diff) easy_atomic32_add_return(v,diff) 88 | #define easy_atomic_inc(v) easy_atomic32_inc(v) 89 | #define easy_atomic_dec(v) easy_atomic32_dec(v) 90 | static __inline__ int32_t easy_atomic_cmp_set(easy_atomic_t *lock, int32_t old, int32_t set) 91 | { 92 | uint8_t res; 93 | __asm__ volatile ( 94 | EASY_SMP_LOCK "cmpxchgl %3, %1; sete %0" 95 | : "=a" (res) : "m" (*lock), "a" (old), "r" (set) : "cc", "memory"); 96 | return res; 97 | } 98 | #endif 99 | 100 | #define easy_trylock(lock) (*(lock) == 0 && easy_atomic_cmp_set(lock, 0, 1)) 101 | #define easy_unlock(lock) {__asm__ ("" ::: "memory"); *(lock) = 0;} 102 | #define easy_spin_unlock easy_unlock 103 | 104 | static __inline__ void easy_spin_lock(easy_atomic_t *lock) 105 | { 106 | int i, n; 107 | 108 | for ( ; ; ) { 109 | if (*lock == 0 && easy_atomic_cmp_set(lock, 0, 1)) { 110 | return; 111 | } 112 | 113 | for (n = 1; n < 1024; n <<= 1) { 114 | 115 | for (i = 0; i < n; i++) { 116 | __asm__ (".byte 0xf3, 0x90"); 117 | } 118 | 119 | if (*lock == 0 && easy_atomic_cmp_set(lock, 0, 1)) { 120 | return; 121 | } 122 | } 123 | 124 | sched_yield(); 125 | } 126 | } 127 | 128 | static __inline__ void easy_clear_bit(unsigned long nr, volatile void *addr) 129 | { 130 | int8_t *m = ((int8_t *) addr) + (nr >> 3); 131 | *m &= ~(1 << (nr & 7)); 132 | } 133 | static __inline__ void easy_set_bit(unsigned long nr, volatile void *addr) 134 | { 135 | int8_t *m = ((int8_t *) addr) + (nr >> 3); 136 | *m |= 1 << (nr & 7); 137 | } 138 | 139 | EASY_CPP_END 140 | 141 | #endif 142 | -------------------------------------------------------------------------------- /TDHSocket/tdh_socket_connection_context.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_connection_context.cpp 12 | * 13 | * Created on: 2011-9-29 14 | * Author: wentong 15 | */ 16 | 17 | #include "tdh_socket_connection_context.hpp" 18 | #include "tdh_socket_encode_response.hpp" 19 | #include "tdh_socket_config.hpp" 20 | #include "tdh_socket_share.hpp" 21 | #include "debug_util.hpp" 22 | 23 | #include 24 | 25 | namespace taobao { 26 | 27 | #define PERMISSION_READ 1 28 | #define PERMISSION_WRITE (1<<1) 29 | 30 | tdh_socket_connection_context::tdh_socket_connection_context() { 31 | 32 | } 33 | 34 | tdh_socket_connection_context::~tdh_socket_connection_context() { 35 | } 36 | 37 | int tdh_socket_connection_context::init(easy_connection_t* c) { 38 | is_shaked = false; 39 | connection = c; 40 | decode_request = NULL; 41 | time_out = 0; 42 | permission = 0; 43 | return EASY_OK; 44 | } 45 | 46 | int tdh_socket_connection_context::destory() { 47 | return EASY_OK; 48 | } 49 | 50 | int tdh_socket_connection_context::decode(tdhs_packet_t* request) { 51 | tb_assert(decode_request!=NULL); 52 | int state = decode_request(request->req, *request); 53 | if (state) { 54 | easy_warn_log("TDHS: decode failed!"); 55 | request->req.status = TDHS_DECODE_FAILED; 56 | if(state == ERROR_OUT_OF_IN) 57 | return ERROR_OUT_OF_IN; 58 | else 59 | return EASY_ERROR; 60 | } 61 | request->req.status = TDHS_DECODE_DONE; 62 | return EASY_OK; 63 | } 64 | 65 | int tdh_socket_connection_context::shake_hands_if(tdhs_packet_t *request) { 66 | if (!is_shaked) { 67 | char header[TDH_SOCKET_SHAKE_HANDS_HEADER_LENGTH] = 68 | TDH_SOCKET_SHAKE_HANDS_HEADER; 69 | if (request->command_id_or_response_code != REQUEST_TYPE_SHAKE_HANDS 70 | || request->length < TDH_SOCKET_SHAKE_HANDS_TOTAL_SIZE 71 | || strncmp(request->rdata, header, 72 | TDH_SOCKET_SHAKE_HANDS_HEADER_LENGTH) != 0) { 73 | char info[TDH_SOCKET_SHAKE_HANDS_TOTAL_SIZE + 1]; 74 | memcpy(info, request->rdata, TDH_SOCKET_SHAKE_HANDS_TOTAL_SIZE); 75 | info[TDH_SOCKET_SHAKE_HANDS_TOTAL_SIZE] = 0; 76 | easy_warn_log( 77 | "TDHS: shake_hands failed! request length is [%d],head is [%s]", 78 | request->length, info); 79 | return EASY_ERROR; 80 | } 81 | uint32_t protocol_version = ntohl(*((uint32_t*) (request->rdata 82 | + TDH_SOCKET_SHAKE_HANDS_HEADER_LENGTH))); 83 | if (protocol_version >= TDHS_PROTOCOL_END) { 84 | easy_warn_log("TDHS: shark_hands failed! protocol_version is [%d]", 85 | protocol_version); 86 | return EASY_ERROR; 87 | } 88 | decode_request = decode_request_array[protocol_version]; 89 | if (decode_request == NULL) { 90 | easy_warn_log( 91 | "TDHS: shark_hands failed! can't find decode_request,protocol_version is [%d]", 92 | protocol_version); 93 | return EASY_ERROR; 94 | } 95 | time_out = ntohl(*((uint32_t*) (request->rdata 96 | + TDH_SOCKET_SHAKE_HANDS_HEADER_LENGTH 97 | + TDH_SOCKET_PROTOCOL_VERSION_LENGTH))); 98 | easy_debug_log("TDHS: timeout is [%d]", time_out); 99 | 100 | //for auth 101 | char* pos = request->rdata + TDH_SOCKET_SHAKE_HANDS_TOTAL_SIZE; 102 | uint32_t len = request->length - TDH_SOCKET_SHAKE_HANDS_TOTAL_SIZE; 103 | tdhs_string_t read_code; 104 | tdhs_string_t write_code; 105 | read_uint32_ref(read_code.len, pos, len); 106 | read_str_ref(read_code.str, pos, len, read_code.len); 107 | read_uint32_ref(write_code.len, pos, len); 108 | read_str_ref(write_code.str, pos, len, write_code.len); 109 | easy_debug_log("TDHS: read_code [%s] write_code [%s]", 110 | read_code.str_print(), write_code.str_print()); 111 | if (tdhs_auth_read(read_code)) { 112 | onBit(permission, PERMISSION_READ); 113 | } 114 | if (tdhs_auth_write(write_code)) { 115 | onBit(permission, PERMISSION_WRITE); 116 | } 117 | is_shaked = true; 118 | return EASY_AGAIN; 119 | } else { 120 | return EASY_OK; 121 | } 122 | } 123 | 124 | uint32_t tdh_socket_connection_context::get_timeout() { 125 | return time_out; 126 | } 127 | 128 | bool tdh_socket_connection_context::can_read() { 129 | if (tdhs_auth_on) { 130 | return testFlag(permission, PERMISSION_READ); 131 | } 132 | return true; 133 | } 134 | bool tdh_socket_connection_context::can_write() { 135 | if (tdhs_auth_on) { 136 | return testFlag(permission, PERMISSION_WRITE); 137 | } 138 | return true; 139 | } 140 | 141 | } /* namespace taobao */ 142 | -------------------------------------------------------------------------------- /libeasy/io/easy_baseth_pool.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #include 11 | #include "easy_io_struct.h" 12 | #include "easy_log.h" 13 | #include "easy_baseth_pool.h" 14 | #include "easy_connection.h" 15 | #include "easy_message.h" 16 | 17 | __thread easy_baseth_t *easy_baseth_self; 18 | static void easy_baseth_pool_invoke(struct ev_loop *loop); 19 | 20 | /** 21 | * start 22 | */ 23 | void *easy_baseth_on_start(void *args) 24 | { 25 | easy_baseth_t *th; 26 | th = (easy_baseth_t *) args; 27 | easy_baseth_self = th; 28 | ev_run(th->loop, 0); 29 | easy_baseth_self = NULL; 30 | 31 | easy_debug_log("pthread exit: %lx.\n", pthread_self()); 32 | 33 | return (void *)NULL; 34 | } 35 | 36 | /** 37 | * wakeup 38 | */ 39 | void easy_baseth_on_wakeup(void *args) 40 | { 41 | easy_baseth_t *th = (easy_baseth_t *)args; 42 | 43 | easy_spin_lock(&th->thread_lock); 44 | ev_async_send(th->loop, &th->thread_watcher); 45 | easy_spin_unlock(&th->thread_lock); 46 | } 47 | 48 | void easy_baseth_init(void *args, easy_thread_pool_t *tp, 49 | easy_baseth_on_start_pt *start, easy_baseth_on_wakeup_pt *wakeup) 50 | { 51 | easy_baseth_t *th = (easy_baseth_t *)args; 52 | th->idx = (((char *)(th)) - (&(tp)->data[0])) / (tp)->member_size; 53 | th->on_start = start; 54 | 55 | th->loop = ev_loop_new(0); 56 | th->thread_lock = 0; 57 | 58 | ev_async_init (&th->thread_watcher, wakeup); 59 | th->thread_watcher.data = th; 60 | ev_async_start (th->loop, &th->thread_watcher); 61 | 62 | ev_set_userdata(th->loop, th); 63 | ev_set_invoke_pending_cb(th->loop, easy_baseth_pool_invoke); 64 | } 65 | 66 | /////////////////////////////////////////////////////////////////////////////////////////////////// 67 | /** 68 | * 创建一个thread pool 69 | */ 70 | easy_thread_pool_t *easy_baseth_pool_create(easy_io_t *eio, int thread_count, int member_size) 71 | { 72 | easy_baseth_t *th; 73 | easy_thread_pool_t *tp; 74 | int size; 75 | 76 | size = sizeof(easy_thread_pool_t) + member_size * thread_count; 77 | 78 | if ((tp = (easy_thread_pool_t *) easy_pool_calloc(eio->pool, size)) == NULL) 79 | return NULL; 80 | 81 | tp->thread_count = thread_count; 82 | tp->member_size = member_size; 83 | tp->last = &tp->data[0] + member_size * thread_count; 84 | easy_list_add_tail(&tp->list_node, &eio->thread_pool_list); 85 | easy_thread_pool_for_each(th, tp, 0) { 86 | th->eio = eio; 87 | } 88 | 89 | return tp; 90 | } 91 | 92 | /** 93 | * wakeup pool 94 | */ 95 | void easy_baseth_pool_on_wakeup(easy_thread_pool_t *tp) 96 | { 97 | easy_baseth_t *th; 98 | easy_thread_pool_for_each(th, tp, 0) { 99 | easy_baseth_on_wakeup(th); 100 | } 101 | } 102 | 103 | /** 104 | * destroy pool 105 | */ 106 | void easy_baseth_pool_destroy(easy_thread_pool_t *tp) 107 | { 108 | easy_baseth_t *th; 109 | easy_thread_pool_for_each(th, tp, 0) { 110 | ev_loop_destroy(th->loop); 111 | } 112 | } 113 | 114 | static void easy_baseth_pool_wakeup_session(easy_baseth_t *th) 115 | { 116 | if (th->iot == 0) 117 | return; 118 | 119 | easy_connection_t *c, *c1; 120 | easy_session_t *s, *s1; 121 | easy_io_thread_t *ioth = (easy_io_thread_t *) th; 122 | 123 | // session at ioth 124 | easy_spin_lock(&ioth->thread_lock); 125 | 126 | easy_list_for_each_entry_safe(s, s1, &ioth->session_list, session_list_node) { 127 | easy_list_del(&s->session_list_node); 128 | 129 | if (s->status == 0) easy_session_process(s, 0); 130 | } 131 | // connection at ioth 132 | easy_list_for_each_entry_safe(c, c1, &ioth->conn_list, conn_list_node) { 133 | easy_connection_wakeup_session(c); 134 | } 135 | // foreach connected_list 136 | easy_list_for_each_entry_safe(c, c1, &ioth->connected_list, conn_list_node) { 137 | easy_connection_wakeup_session(c); 138 | } 139 | easy_spin_unlock(&ioth->thread_lock); 140 | } 141 | 142 | 143 | /** 144 | * 判断是否退出 145 | */ 146 | static void easy_baseth_pool_invoke(struct ev_loop *loop) 147 | { 148 | easy_baseth_t *th = (easy_baseth_t *) ev_userdata (loop); 149 | 150 | if (th->eio->stoped) { 151 | easy_baseth_pool_wakeup_session(th); 152 | ev_break(loop, EVBREAK_ALL); 153 | easy_error_log("ev_break: eio=%p\n", th->eio); 154 | return; 155 | } 156 | 157 | ev_invoke_pending(loop); 158 | } 159 | -------------------------------------------------------------------------------- /libeasy/thread/easy_uthread.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_UTHREAD_H 11 | #define EASY_UTHREAD_H 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | /** 18 | * 创建一用户态线程 19 | */ 20 | 21 | EASY_CPP_START 22 | 23 | #define EASY_UTHREAD_STACK (65536-sizeof(easy_pool_t)) 24 | 25 | typedef void (easy_uthread_start_pt)(void *args); 26 | typedef struct easy_uthread_t easy_uthread_t; 27 | typedef struct easy_uthread_control_t easy_uthread_control_t; 28 | 29 | struct easy_uthread_t { 30 | easy_list_t runqueue_node; 31 | easy_list_t thread_list_node; 32 | easy_pool_t *pool; 33 | easy_uthread_start_pt *startfn; 34 | void *startargs; 35 | 36 | uint32_t id; 37 | int8_t exiting; 38 | int8_t ready; 39 | int8_t errcode; 40 | uint32_t stksize; 41 | unsigned char *stk; 42 | ucontext_t context; 43 | }; 44 | 45 | struct easy_uthread_control_t { 46 | int gid; 47 | int nswitch; 48 | int16_t stoped; 49 | int16_t thread_count; 50 | int exit_value; 51 | easy_list_t runqueue; 52 | easy_list_t thread_list; 53 | easy_uthread_t *running; 54 | ucontext_t context; 55 | }; 56 | 57 | // 函数 58 | void easy_uthread_init(easy_uthread_control_t *control); 59 | void easy_uthread_destroy(); 60 | easy_uthread_t *easy_uthread_create(easy_uthread_start_pt *start, void *args, int stack_size); 61 | easy_uthread_t *easy_uthread_current(); 62 | int easy_uthread_yield(); 63 | int easy_uthread_scheduler(); 64 | void easy_uthread_stop(); 65 | void easy_uthread_ready(easy_uthread_t *t); 66 | void easy_uthread_switch(); 67 | void easy_uthread_needstack(int n); 68 | void easy_uthread_ready(easy_uthread_t *t); 69 | void easy_uthread_print(int sig); 70 | int easy_uthread_get_errcode(); 71 | void easy_uthread_set_errcode(easy_uthread_t *t, int errcode); 72 | 73 | ////////////////////// 74 | // 下面对main的处理 75 | #define EASY_UTHREAD_RUN_MAIN(main_name) \ 76 | static int easy_uthread_stacksize = 0; \ 77 | static int easy_uthread_argc; \ 78 | static char **easy_uthread_argv; \ 79 | static void easy_uthread_mainstart(void *v) \ 80 | { \ 81 | main_name(easy_uthread_argc, easy_uthread_argv); \ 82 | } \ 83 | int main(int argc, char **argv) \ 84 | { \ 85 | int ret; struct sigaction sa, osa; \ 86 | easy_uthread_control_t control; \ 87 | memset(&sa, 0, sizeof sa); \ 88 | sa.sa_handler = easy_uthread_print; \ 89 | sa.sa_flags = SA_RESTART; \ 90 | sigaction(SIGQUIT, &sa, &osa); \ 91 | easy_uthread_argc = argc; \ 92 | easy_uthread_argv = argv; \ 93 | if (easy_uthread_stacksize == 0) easy_uthread_stacksize = 256*1024; \ 94 | easy_uthread_init(&control); \ 95 | easy_uthread_create(easy_uthread_mainstart, NULL, easy_uthread_stacksize); \ 96 | ret = easy_uthread_scheduler(); \ 97 | easy_uthread_destroy(); \ 98 | return ret; \ 99 | } 100 | 101 | EASY_CPP_END 102 | 103 | #endif 104 | -------------------------------------------------------------------------------- /python/src/batch_statement.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | #Copyright(C) 2011-2012 Alibaba Group Holding Limited 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License version 2 as 7 | # published by the Free Software Foundation. 8 | # 9 | # Authors: 10 | # wentong 11 | 12 | # wentong@taobao.com 13 | # 12-2-20 14 | # 15 | from _struct import unpack 16 | from exception import response_exception 17 | from request import * 18 | from response import response 19 | from statement import encode_packet 20 | 21 | class batch_statement: 22 | wbuff = "" 23 | id = -1 24 | ids = [] 25 | 26 | def __init__(self, client): 27 | self.client = client 28 | 29 | def append_request(self, type, req): 30 | _id = self.client.getid() 31 | self.wbuff += encode_packet(type, _id, req.to_binary()) 32 | self.ids.append(_id) 33 | 34 | def do_single_reponse(self,idx): 35 | databuff = "" 36 | data_len = 0 37 | while True: 38 | rbuff = self.client.recv(FSIZE) 39 | magic_code, resp_code, id, reverse, length = unpack(FMT, rbuff) 40 | self.client.logger.debug("return resp_code[%d] length[%d],id[%d]" % (resp_code, length, id)) 41 | if id != self.ids[idx]: 42 | raise response_exception("id is error!") 43 | rbuff = self.client.recv(length) 44 | if 400 <= resp_code < 600: 45 | rpos = 0 46 | error_code, = unpack('!L', rbuff[rpos:rpos + 4]) 47 | if resp_code == CLIENT_STATUS_DB_ERROR: 48 | raise response_exception("mysql return code [%d]" % error_code) 49 | if error_code < len(ERROR_MSG): 50 | raise response_exception(ERROR_MSG[error_code]) 51 | else: 52 | raise response_exception("unkown error code [%d]" % error_code) 53 | elif resp_code == CLIENT_STATUS_ACCEPT: 54 | databuff += rbuff 55 | data_len += length 56 | elif resp_code == CLIENT_STATUS_OK: 57 | databuff += rbuff 58 | data_len += length 59 | self.client.logger.debug("return data_length[%d]" % data_len) 60 | return response(databuff, data_len).parse() 61 | else: 62 | raise response_exception("unknown response code [%d]" % resp_code) 63 | 64 | def do_response(self): 65 | rbuff = self.client.recv(FSIZE) 66 | magic_code, resp_code, id, reverse, length = unpack(FMT, rbuff) 67 | self.client.logger.debug("return resp_code[%d] length[%d],id[%d]" % (resp_code, length, id)) 68 | if id != self.id: 69 | raise response_exception("id is error!") 70 | if length > 0: 71 | rbuff = self.client.recv(length) 72 | if 400 <= resp_code < 600: 73 | rpos = 0 74 | error_code, = unpack('!L', rbuff[rpos:rpos + 4]) 75 | if resp_code == CLIENT_STATUS_DB_ERROR: 76 | raise response_exception("mysql return code [%d]" % error_code) 77 | if error_code < len(ERROR_MSG): 78 | raise response_exception(ERROR_MSG[error_code]) 79 | else: 80 | raise response_exception("unkown error code [%d]" % error_code) 81 | elif resp_code == CLIENT_STATUS_MULTI_STATUS: 82 | ret = [] 83 | for i in xrange(len(self.ids)): 84 | ret.append(self.do_single_reponse(i)) 85 | return ret 86 | else: 87 | raise response_exception("unknown response code [%d]" % resp_code) 88 | 89 | 90 | def commit(self): 91 | self.id = self.client.getid() 92 | batch_buff = encode_packet(REQUEST_TYPE_BATCH, self.id, self.wbuff, len(self.ids)) 93 | self.client.send(batch_buff) 94 | return self.do_response() 95 | 96 | 97 | def add_delete(self, db, table, index, key, find_flag=TDHS_EQ, start=0, limit=0, filters=None): 98 | info = table_info(db, table, index, []) 99 | info.need_field = False 100 | req = get(info, key, find_flag, start, limit, filters) 101 | self.append_request.append(REQUEST_TYPE_DELETE, req) 102 | 103 | 104 | def add_update(self, db, table, index, fields, updates, key, find_flag=TDHS_EQ, start=0, limit=0, filters=None ): 105 | get_req = get(table_info(db, table, index, fields), key, find_flag, start, limit, filters) 106 | req = update(get_req, updates) 107 | self.append_request(REQUEST_TYPE_UPDATE, req) 108 | 109 | def add_insert(self, db, table, fields, values): 110 | info = table_info(db, table, None, fields) 111 | req = insert(info, values) 112 | self.append_request(REQUEST_TYPE_INSERT, req) 113 | 114 | 115 | -------------------------------------------------------------------------------- /libeasy/io/easy_message.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #include "easy_io.h" 11 | #include "easy_message.h" 12 | #include "easy_connection.h" 13 | #include "easy_request.h" 14 | #include "easy_baseth_pool.h" 15 | 16 | easy_message_t *easy_message_create(easy_connection_t *c) 17 | { 18 | easy_pool_t *pool; 19 | easy_message_t *m; 20 | easy_buf_t *input; 21 | int size; 22 | 23 | if ((pool = easy_pool_create(c->default_message_len)) == NULL) 24 | return NULL; 25 | 26 | // 新建一个message 27 | m = (easy_message_t *)easy_pool_calloc(pool, sizeof(easy_message_t)); 28 | size = pool->end - pool->last - sizeof(easy_buf_t); 29 | size = ((size < 0) ? EASY_POOL_ALIGNMENT : size); 30 | input = easy_buf_create(pool, size); 31 | 32 | if (m == NULL || input == NULL) { 33 | easy_pool_destroy(pool); 34 | return NULL; 35 | } 36 | 37 | pool->ref = 1; 38 | m->pool = pool; 39 | m->c = c; 40 | m->next_read_len = size; 41 | m->input = input; 42 | m->type = EASY_TYPE_MESSAGE; 43 | m->request_list_count = 0; 44 | easy_list_init(&m->request_list); 45 | easy_list_init(&m->request_done_list); 46 | easy_list_add_tail(&m->message_list_node, &c->message_list); 47 | 48 | return m; 49 | } 50 | 51 | /** 52 | * destroy掉easy_message_t对象 53 | * 54 | * @param m - easy_message_t对象 55 | */ 56 | void easy_message_destroy(easy_message_t *m, int del) 57 | { 58 | easy_request_t *r, *n; 59 | 60 | // delete from message_list 61 | if (del) { 62 | m->status = EASY_MESG_DESTROY; 63 | easy_list_del(&m->message_list_node); 64 | } 65 | 66 | if (easy_atomic_add_return(&m->pool->ref, -1) == 0) { 67 | // server done 68 | easy_list_for_each_entry_safe(r, n, &m->request_list, request_list_node) { 69 | easy_list_del(&r->request_list_node); 70 | easy_request_server_done(r); 71 | } 72 | easy_list_for_each_entry_safe(r, n, &m->request_done_list, request_list_node) { 73 | easy_list_del(&r->request_list_node); 74 | easy_request_server_done(r); 75 | } 76 | 77 | easy_list_del(&m->message_list_node); 78 | 79 | if (m->input) easy_buf_destroy(m->input); 80 | 81 | easy_pool_destroy(m->pool); 82 | } 83 | } 84 | 85 | /** 86 | * 用于回调 87 | */ 88 | void easy_message_cleanup(easy_buf_t *b, void *args) 89 | { 90 | easy_message_t *m = (easy_message_t *) args; 91 | easy_message_destroy(m, 0); 92 | } 93 | 94 | /** 95 | * 新建一个session_t 96 | */ 97 | easy_session_t *easy_session_create(int size) 98 | { 99 | easy_pool_t *pool; 100 | easy_session_t *s; 101 | 102 | // 新建一个pool 103 | size += sizeof(easy_session_t); 104 | 105 | if ((pool = easy_pool_create(size)) == NULL) 106 | return NULL; 107 | 108 | // 新建一个message 109 | if ((s = (easy_session_t *)easy_pool_alloc(pool, size)) == NULL) { 110 | easy_pool_destroy(pool); 111 | return NULL; 112 | } 113 | 114 | memset(s, 0, sizeof(easy_session_t)); 115 | s->pool = pool; 116 | s->r.ms = (easy_message_session_t *)s; 117 | s->type = EASY_TYPE_SESSION; 118 | easy_list_init(&s->session_list_node); 119 | 120 | return s; 121 | } 122 | 123 | /** 124 | * destroy掉easy_session_t对象 125 | * 126 | * @param s - easy_session_t对象 127 | */ 128 | void easy_session_destroy(void *data) 129 | { 130 | easy_message_t *m; 131 | easy_session_t *s; 132 | 133 | s = (easy_session_t *) data; 134 | 135 | if (s->cleanup) 136 | (s->cleanup)(&s->r, NULL); 137 | 138 | // 如果存在 139 | if (s->async && (m = (easy_message_t *)s->r.request_list_node.next)) { 140 | s->r.request_list_node.next = NULL; 141 | easy_message_destroy(m, 0); 142 | } 143 | 144 | easy_pool_destroy(s->pool); 145 | } 146 | 147 | int easy_session_process(easy_session_t *s, int stop) 148 | { 149 | if (stop) { 150 | ev_timer_stop(s->c->loop, &s->timeout_watcher); 151 | easy_list_del(&s->session_list_node); 152 | easy_request_client_done(&s->r); 153 | s->c->pool->ref --; 154 | } 155 | 156 | // timeout, 把output里的clear掉 157 | if (s->r.ipacket == NULL && s->nextb) { 158 | easy_buf_t *b, *b2; 159 | easy_list_for_each_entry_safe(b, b2, s->nextb, node) { 160 | if (b->args != s->pool) break; 161 | 162 | easy_list_del(&b->node); 163 | } 164 | easy_list_del(s->nextb); 165 | } 166 | 167 | if (s->process) { 168 | return (s->process)(&s->r); 169 | } else { 170 | easy_error_log("session process is null, s = %p\n", s); 171 | easy_session_destroy(s); 172 | return EASY_ERROR; 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /libeasy/util/easy_buf.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #include 11 | #include "easy_buf.h" 12 | 13 | /** 14 | * 创建一个新的easy_buf_t 15 | */ 16 | easy_buf_t *easy_buf_create(easy_pool_t *pool, uint32_t size) 17 | { 18 | easy_buf_t *b; 19 | 20 | if ((b = (easy_buf_t *)easy_pool_calloc(pool, sizeof(easy_buf_t))) == NULL) 21 | return NULL; 22 | 23 | // 一个page大小 24 | if (size == 0) 25 | size = pool->end - pool->last; 26 | 27 | if ((b->pos = (char *)easy_pool_alloc(pool, size)) == NULL) 28 | return NULL; 29 | 30 | b->last = b->pos; 31 | b->end = b->last + size; 32 | b->cleanup = NULL; 33 | b->args = pool; 34 | easy_list_init(&b->node); 35 | 36 | return b; 37 | } 38 | 39 | /** 40 | * 把data包成easy_buf_t 41 | */ 42 | easy_buf_t *easy_buf_pack(easy_pool_t *pool, const void *data, uint32_t size) 43 | { 44 | easy_buf_t *b; 45 | 46 | if ((b = (easy_buf_t *)easy_pool_calloc(pool, sizeof(easy_buf_t))) == NULL) 47 | return NULL; 48 | 49 | easy_buf_set_data(pool, b, data, size); 50 | 51 | return b; 52 | } 53 | 54 | /** 55 | * 设置数据到b里 56 | */ 57 | void easy_buf_set_data(easy_pool_t *pool, easy_buf_t *b, const void *data, uint32_t size) 58 | { 59 | b->pos = (char *)data; 60 | b->last = b->pos + size; 61 | b->end = b->last; 62 | b->cleanup = NULL; 63 | b->args = pool; 64 | b->flags = 0; 65 | easy_list_init(&b->node); 66 | } 67 | 68 | /** 69 | * 创建一个easy_file_buf_t, 用于sendfile等 70 | */ 71 | easy_file_buf_t *easy_file_buf_create(easy_pool_t *pool) 72 | { 73 | easy_file_buf_t *b; 74 | 75 | b = (easy_file_buf_t *)easy_pool_calloc(pool, sizeof(easy_file_buf_t)); 76 | b->flags = EASY_BUF_FILE; 77 | b->cleanup = NULL; 78 | b->args = pool; 79 | easy_list_init(&b->node); 80 | 81 | return b; 82 | } 83 | 84 | void easy_file_buf_set_close(easy_file_buf_t *b) 85 | { 86 | if ((b->flags & EASY_BUF_FILE)) 87 | b->flags = EASY_BUF_CLOSE_FILE; 88 | } 89 | 90 | void easy_buf_set_cleanup(easy_buf_t *b, easy_buf_cleanup_pt *cleanup, void *args) 91 | { 92 | b->cleanup = cleanup; 93 | b->args = args; 94 | } 95 | 96 | void easy_buf_destroy(easy_buf_t *b) 97 | { 98 | easy_buf_cleanup_pt *cleanup; 99 | easy_list_del(&b->node); 100 | 101 | if ((b->flags & EASY_BUF_CLOSE_FILE) == EASY_BUF_CLOSE_FILE) 102 | close(((easy_file_buf_t *)b)->fd); 103 | 104 | // cleanup 105 | if ((cleanup = b->cleanup)) { 106 | b->cleanup = NULL; 107 | (*cleanup)(b, b->args); 108 | } 109 | } 110 | 111 | /** 112 | * 空间不够,分配出一块来,保留之前的空间 113 | */ 114 | int easy_buf_check_read_space(easy_pool_t *pool, easy_buf_t *b, uint32_t size) 115 | { 116 | int dsize; 117 | char *ptr; 118 | 119 | if ((b->end - b->last) >= (int)size) 120 | return EASY_OK; 121 | 122 | // 需要大小 123 | dsize = (b->last - b->pos); 124 | size = easy_align(size + dsize, EASY_POOL_PAGE_SIZE); 125 | 126 | // alloc 127 | if ((ptr = (char *)easy_pool_alloc(pool, size)) == NULL) 128 | return EASY_ERROR; 129 | 130 | // copy old buf to new buf 131 | if (dsize > 0) 132 | memcpy(ptr, b->pos, dsize); 133 | 134 | b->pos = ptr; 135 | b->last = b->pos + dsize; 136 | b->end = b->pos + size; 137 | 138 | return EASY_OK; 139 | } 140 | 141 | /** 142 | * 空间不够,分配出一块来,保留之前的空间 143 | */ 144 | easy_buf_t *easy_buf_check_write_space(easy_pool_t *pool, easy_list_t *bc, uint32_t size) 145 | { 146 | easy_buf_t *b = easy_list_get_last(bc, easy_buf_t, node); 147 | 148 | if (b != NULL && (b->end - b->last) >= (int)size) 149 | return b; 150 | 151 | // 重新生成一个buf,放入buf_chain_t中 152 | size = easy_align(size, EASY_POOL_PAGE_SIZE); 153 | 154 | if ((b = easy_buf_create(pool, size)) == NULL) 155 | return NULL; 156 | 157 | easy_list_add_tail(&b->node, bc); 158 | 159 | return b; 160 | } 161 | 162 | /** 163 | * 清除掉 164 | */ 165 | void easy_buf_chain_clear(easy_list_t *l) 166 | { 167 | easy_buf_t *b, *b1; 168 | 169 | easy_list_for_each_entry_safe(b, b1, l, node) { 170 | easy_buf_destroy(b); 171 | } 172 | easy_list_init(l); 173 | } 174 | 175 | /** 176 | * 加到后面 177 | */ 178 | void easy_buf_chain_offer(easy_list_t *l, easy_buf_t *b) 179 | { 180 | if (!l->next) easy_list_init(l); 181 | 182 | easy_list_add_tail(&b->node, l); 183 | } 184 | 185 | /** 186 | * 把s复制到d上 187 | */ 188 | int easy_buf_string_copy(easy_pool_t *pool, easy_buf_string_t *d, easy_buf_string_t *s) 189 | { 190 | if (s->len > 0) { 191 | d->data = (char *)easy_pool_alloc(pool, s->len); 192 | memcpy(d->data, s->data, s->len); 193 | d->len = s->len; 194 | } 195 | 196 | return s->len; 197 | } 198 | 199 | int easy_buf_string_printf(easy_pool_t *pool, easy_buf_string_t *d, const char *fmt, ...) 200 | { 201 | int len; 202 | char buffer[2048]; 203 | 204 | va_list args; 205 | va_start(args, fmt); 206 | len = vsnprintf(buffer, 2048, fmt, args); 207 | va_end(args); 208 | d->data = (char *)easy_pool_alloc(pool, len + 1); 209 | memcpy(d->data, buffer, len); 210 | d->data[len] = '\0'; 211 | d->len = len; 212 | return len; 213 | } 214 | 215 | -------------------------------------------------------------------------------- /libeasy/util/easy_hash.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #include "easy_hash.h" 11 | 12 | /** 13 | * 创建一easy_hash_t 14 | */ 15 | easy_hash_t *easy_hash_create(easy_pool_t *pool, uint32_t size, int offset) 16 | { 17 | easy_hash_t *table; 18 | easy_hash_list_t **buckets; 19 | uint32_t n; 20 | 21 | // 2 ^ m 22 | n = 4; 23 | size &= 0x7fffffff; 24 | 25 | while(size > n) n <<= 1; 26 | 27 | // alloc 28 | buckets = (easy_hash_list_t **)easy_pool_calloc(pool, n * sizeof(easy_hash_list_t *)); 29 | table = (easy_hash_t *)easy_pool_alloc(pool, sizeof(easy_hash_t)); 30 | 31 | if (table == NULL || buckets == NULL) 32 | return NULL; 33 | 34 | table->buckets = buckets; 35 | table->size = n; 36 | table->mask = n - 1; 37 | table->count = 0; 38 | table->offset = offset; 39 | table->seqno = 1; 40 | easy_list_init(&table->list); 41 | 42 | return table; 43 | } 44 | 45 | int easy_hash_add(easy_hash_t *table, uint64_t key, easy_hash_list_t *list) 46 | { 47 | uint64_t n; 48 | easy_hash_list_t *first; 49 | 50 | n = easy_hash_key(key); 51 | n &= table->mask; 52 | 53 | // init 54 | list->key = key; 55 | table->count ++; 56 | table->seqno ++; 57 | 58 | // add to list 59 | first = table->buckets[n]; 60 | list->next = first; 61 | 62 | if (first) 63 | first->pprev = &list->next; 64 | 65 | table->buckets[n] = (easy_hash_list_t *)list; 66 | list->pprev = &(table->buckets[n]); 67 | 68 | return EASY_OK; 69 | } 70 | 71 | void *easy_hash_find(easy_hash_t *table, uint64_t key) 72 | { 73 | uint64_t n; 74 | easy_hash_list_t *list; 75 | 76 | n = easy_hash_key(key); 77 | n &= table->mask; 78 | list = table->buckets[n]; 79 | 80 | // foreach 81 | while(list) { 82 | if (list->key == key) { 83 | return ((char *)list - table->offset); 84 | } 85 | 86 | list = list->next; 87 | } 88 | 89 | return NULL; 90 | } 91 | 92 | void *easy_hash_find_ex(easy_hash_t *table, uint64_t key, easy_hash_cmp_pt cmp, const void *a) 93 | { 94 | uint64_t n; 95 | easy_hash_list_t *list; 96 | 97 | n = easy_hash_key(key); 98 | n &= table->mask; 99 | list = table->buckets[n]; 100 | 101 | // foreach 102 | while(list) { 103 | if (list->key == key) { 104 | if (cmp(a, ((char *)list - table->offset)) == 0) 105 | return ((char *)list - table->offset); 106 | } 107 | 108 | list = list->next; 109 | } 110 | 111 | return NULL; 112 | } 113 | 114 | void *easy_hash_del(easy_hash_t *table, uint64_t key) 115 | { 116 | uint64_t n; 117 | easy_hash_list_t *list; 118 | 119 | n = easy_hash_key(key); 120 | n &= table->mask; 121 | list = table->buckets[n]; 122 | 123 | // foreach 124 | while(list) { 125 | if (list->key == key) { 126 | easy_hash_del_node(list); 127 | table->count --; 128 | 129 | return ((char *)list - table->offset); 130 | } 131 | 132 | list = list->next; 133 | } 134 | 135 | return NULL; 136 | } 137 | 138 | int easy_hash_del_node(easy_hash_list_t *node) 139 | { 140 | easy_hash_list_t *next, **pprev; 141 | 142 | if (!node->pprev) 143 | return 0; 144 | 145 | next = node->next; 146 | pprev = node->pprev; 147 | *pprev = next; 148 | 149 | if (next) next->pprev = pprev; 150 | 151 | node->next = NULL; 152 | node->pprev = NULL; 153 | 154 | return 1; 155 | } 156 | 157 | int easy_hash_dlist_add(easy_hash_t *table, uint64_t key, easy_hash_list_t *hash, easy_list_t *list) 158 | { 159 | easy_list_add_tail(list, &table->list); 160 | return easy_hash_add(table, key, hash); 161 | } 162 | 163 | void *easy_hash_dlist_del(easy_hash_t *table, uint64_t key) 164 | { 165 | char *object; 166 | 167 | if ((object = (char *)easy_hash_del(table, key)) != NULL) { 168 | easy_list_del((easy_list_t *)(object + table->offset + sizeof(easy_hash_list_t))); 169 | } 170 | 171 | return object; 172 | } 173 | 174 | /////////////////////////////////////////////////////////////////////////////////////////////////// 175 | // hash 64 bit 176 | uint64_t easy_hash_key(volatile uint64_t key) 177 | { 178 | void *ptr = (void *) &key; 179 | return easy_hash_code(ptr, sizeof(uint64_t), 5); 180 | } 181 | 182 | uint64_t easy_hash_code(const void *key, int len, unsigned int seed) 183 | { 184 | const uint64_t m = __UINT64_C(0xc6a4a7935bd1e995); 185 | const int r = 47; 186 | 187 | uint64_t h = seed ^ (len * m); 188 | 189 | const uint64_t *data = (const uint64_t *)key; 190 | const uint64_t *end = data + (len / 8); 191 | 192 | while(data != end) { 193 | uint64_t k = *data++; 194 | 195 | k *= m; 196 | k ^= k >> r; 197 | k *= m; 198 | 199 | h ^= k; 200 | h *= m; 201 | } 202 | 203 | const unsigned char *data2 = (const unsigned char *)data; 204 | 205 | switch(len & 7) { 206 | case 7: 207 | h ^= (uint64_t)(data2[6]) << 48; 208 | 209 | case 6: 210 | h ^= (uint64_t)(data2[5]) << 40; 211 | 212 | case 5: 213 | h ^= (uint64_t)(data2[4]) << 32; 214 | 215 | case 4: 216 | h ^= (uint64_t)(data2[3]) << 24; 217 | 218 | case 3: 219 | h ^= (uint64_t)(data2[2]) << 16; 220 | 221 | case 2: 222 | h ^= (uint64_t)(data2[1]) << 8; 223 | 224 | case 1: 225 | h ^= (uint64_t)(data2[0]); 226 | h *= m; 227 | }; 228 | 229 | h ^= h >> r; 230 | 231 | h *= m; 232 | 233 | h ^= h >> r; 234 | 235 | return h; 236 | } 237 | -------------------------------------------------------------------------------- /libeasy/io/easy_socket.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #include "easy_socket.h" 11 | #include "easy_io.h" 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | static int easy_socket_chain_writev(int fd, easy_list_t *l, struct iovec *iovs, int cnt, int *again); 20 | static int easy_socket_sendfile(int fd, easy_file_buf_t *fb, int *again); 21 | 22 | /** 23 | * 打开监听端口 24 | */ 25 | int easy_socket_listen(easy_addr_t *address) 26 | { 27 | int fd = -1; 28 | struct sockaddr_in addr; 29 | 30 | if ((fd = socket(AF_INET, SOCK_STREAM, 0)) < 0) { 31 | easy_trace_log("create socket error.\n"); 32 | goto error_exit; 33 | } 34 | 35 | easy_socket_non_blocking(fd); 36 | easy_socket_set_opt(fd, SO_REUSEADDR, 1); 37 | easy_socket_set_tcpopt(fd, TCP_DEFER_ACCEPT, 1); 38 | memset(&addr, 0, sizeof(struct sockaddr_in)); 39 | memcpy(&addr, address, sizeof(uint64_t)); 40 | 41 | if (bind(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)) < 0) { 42 | easy_trace_log("bind socket error: %d\n", errno); 43 | goto error_exit; 44 | } 45 | 46 | if (listen(fd, 1024) < 0) { 47 | easy_trace_log("listen error. %d\n", errno); 48 | goto error_exit; 49 | } 50 | 51 | return fd; 52 | 53 | error_exit: 54 | 55 | if (fd >= 0) 56 | close(fd); 57 | 58 | return -1; 59 | } 60 | 61 | /** 62 | * 把buf_chain_t上的内容通过writev写到socket上 63 | */ 64 | #define EASY_SOCKET_RET_CHECK(ret, size, again) if (ret<0) return ret; else size += ret; if (again) return size; 65 | int easy_socket_write(int fd, easy_list_t *l) 66 | { 67 | easy_buf_t *b, *b1; 68 | easy_file_buf_t *fb; 69 | struct iovec iovs[EASY_IOV_MAX]; 70 | int sended, size, cnt, ret, wbyte, again; 71 | 72 | wbyte = cnt = sended = again = 0; 73 | 74 | // foreach 75 | easy_list_for_each_entry_safe(b, b1, l, node) { 76 | // sendfile 77 | if ((b->flags & EASY_BUF_FILE)) { 78 | // 先writev出去 79 | if (cnt > 0) { 80 | ret = easy_socket_chain_writev(fd, l, iovs, cnt, &again); 81 | EASY_SOCKET_RET_CHECK(ret, wbyte, again); 82 | cnt = 0; 83 | } 84 | 85 | fb = (easy_file_buf_t *)b; 86 | sended += fb->count; 87 | ret = easy_socket_sendfile(fd, fb, &again); 88 | EASY_SOCKET_RET_CHECK(ret, wbyte, again); 89 | } else { 90 | size = b->last - b->pos; 91 | iovs[cnt].iov_base = b->pos; 92 | iovs[cnt].iov_len = size; 93 | cnt ++; 94 | sended += size; 95 | } 96 | 97 | // 跳出 98 | if (cnt >= EASY_IOV_MAX || sended >= EASY_IOV_SIZE) 99 | break; 100 | } 101 | 102 | // writev 103 | if (cnt > 0) { 104 | ret = easy_socket_chain_writev(fd, l, iovs, cnt, &again); 105 | EASY_SOCKET_RET_CHECK(ret, wbyte, again); 106 | } 107 | 108 | return wbyte; 109 | } 110 | 111 | /** 112 | * writev 113 | */ 114 | static int easy_socket_chain_writev(int fd, easy_list_t *l, struct iovec *iovs, int cnt, int *again) 115 | { 116 | int ret, sended, size; 117 | easy_buf_t *b, *b1; 118 | 119 | do { 120 | ret = writev(fd, iovs, cnt); 121 | } while(ret == -1 && errno == EINTR); 122 | 123 | // 结果处理 124 | if (ret >= 0) { 125 | sended = ret; 126 | easy_debug_log("writev: %d, fd: %d\n", ret, fd); 127 | easy_atomic_add(&EASY_IOTH_SELF->eio->send_byte, ret); 128 | 129 | easy_list_for_each_entry_safe(b, b1, l, node) { 130 | size = b->last - b->pos; 131 | 132 | if (easy_log_level >= EASY_LOG_TRACE) { 133 | char btmp[64]; 134 | easy_trace_log("write: %d,%d => %s", size, sended, easy_string_tohex(b->pos, size, btmp, 64)); 135 | } 136 | 137 | b->pos += sended; 138 | sended -= size; 139 | 140 | if (sended >= 0) { 141 | cnt --; 142 | easy_buf_destroy(b); 143 | } 144 | 145 | if (sended <= 0) 146 | break; 147 | } 148 | *again = (cnt > 0); 149 | } else { 150 | ret = ((errno == EAGAIN) ? EASY_AGAIN : EASY_ERROR); 151 | } 152 | 153 | return ret; 154 | } 155 | 156 | /** 157 | * sendfile 158 | */ 159 | static int easy_socket_sendfile(int fd, easy_file_buf_t *fb, int *again) 160 | { 161 | int ret; 162 | 163 | do { 164 | ret = sendfile(fd, fb->fd, (off_t *)&fb->offset, fb->count); 165 | } while(ret == -1 && errno == EINTR); 166 | 167 | // 结果处理 168 | if (ret >= 0) { 169 | easy_debug_log("sendfile: %d, fd: %d\n", ret, fd); 170 | easy_atomic_add(&EASY_IOTH_SELF->eio->send_byte, ret); 171 | 172 | if (ret < fb->count) { 173 | fb->count -= ret; 174 | *again = 1; 175 | } else { 176 | easy_buf_destroy((easy_buf_t *)fb); 177 | } 178 | } else { 179 | ret = ((errno == EAGAIN) ? EASY_AGAIN : EASY_ERROR); 180 | } 181 | 182 | return ret; 183 | } 184 | 185 | // 非阻塞 186 | int easy_socket_non_blocking(int fd) 187 | { 188 | int flags = 1; 189 | return ioctl(fd, FIONBIO, &flags); 190 | } 191 | 192 | // TCP 193 | int easy_socket_set_tcpopt(int fd, int option, int value) 194 | { 195 | return setsockopt(fd, IPPROTO_TCP, option, (const void *) &value, sizeof(value)); 196 | } 197 | 198 | // SOCKET 199 | int easy_socket_set_opt(int fd, int option, int value) 200 | { 201 | return setsockopt(fd, SOL_SOCKET, option, (void *)&value, sizeof(value)); 202 | } 203 | -------------------------------------------------------------------------------- /libeasy/io/ev_wrap.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* DO NOT EDIT, automatically generated by update_ev_wrap */ 11 | #ifndef EV_WRAP_H 12 | #define EV_WRAP_H 13 | #define now_floor ((loop)->now_floor) 14 | #define mn_now ((loop)->mn_now) 15 | #define rtmn_diff ((loop)->rtmn_diff) 16 | #define io_blocktime ((loop)->io_blocktime) 17 | #define timeout_blocktime ((loop)->timeout_blocktime) 18 | #define backend ((loop)->backend) 19 | #define activecnt ((loop)->activecnt) 20 | #define loop_done ((loop)->loop_done) 21 | #define backend_fd ((loop)->backend_fd) 22 | #define backend_fudge ((loop)->backend_fudge) 23 | #define backend_modify ((loop)->backend_modify) 24 | #define backend_poll ((loop)->backend_poll) 25 | #define anfds ((loop)->anfds) 26 | #define anfdmax ((loop)->anfdmax) 27 | #define pendings ((loop)->pendings) 28 | #define pendingmax ((loop)->pendingmax) 29 | #define pendingcnt ((loop)->pendingcnt) 30 | #define pending_w ((loop)->pending_w) 31 | #define rfeeds ((loop)->rfeeds) 32 | #define rfeedmax ((loop)->rfeedmax) 33 | #define rfeedcnt ((loop)->rfeedcnt) 34 | #define evfd ((loop)->evfd) 35 | #define evpipe ((loop)->evpipe) 36 | #define pipe_w ((loop)->pipe_w) 37 | #define curpid ((loop)->curpid) 38 | #define postfork ((loop)->postfork) 39 | #define vec_ri ((loop)->vec_ri) 40 | #define vec_ro ((loop)->vec_ro) 41 | #define vec_wi ((loop)->vec_wi) 42 | #define vec_wo ((loop)->vec_wo) 43 | #define vec_eo ((loop)->vec_eo) 44 | #define vec_max ((loop)->vec_max) 45 | #define polls ((loop)->polls) 46 | #define pollmax ((loop)->pollmax) 47 | #define pollcnt ((loop)->pollcnt) 48 | #define pollidxs ((loop)->pollidxs) 49 | #define pollidxmax ((loop)->pollidxmax) 50 | #define epoll_events ((loop)->epoll_events) 51 | #define epoll_eventmax ((loop)->epoll_eventmax) 52 | #define kqueue_changes ((loop)->kqueue_changes) 53 | #define kqueue_changemax ((loop)->kqueue_changemax) 54 | #define kqueue_changecnt ((loop)->kqueue_changecnt) 55 | #define kqueue_events ((loop)->kqueue_events) 56 | #define kqueue_eventmax ((loop)->kqueue_eventmax) 57 | #define port_events ((loop)->port_events) 58 | #define port_eventmax ((loop)->port_eventmax) 59 | #define iocp ((loop)->iocp) 60 | #define fdchanges ((loop)->fdchanges) 61 | #define fdchangemax ((loop)->fdchangemax) 62 | #define fdchangecnt ((loop)->fdchangecnt) 63 | #define timers ((loop)->timers) 64 | #define timermax ((loop)->timermax) 65 | #define timercnt ((loop)->timercnt) 66 | #define periodics ((loop)->periodics) 67 | #define periodicmax ((loop)->periodicmax) 68 | #define periodiccnt ((loop)->periodiccnt) 69 | #define idles ((loop)->idles) 70 | #define idlemax ((loop)->idlemax) 71 | #define idlecnt ((loop)->idlecnt) 72 | #define idleall ((loop)->idleall) 73 | #define prepares ((loop)->prepares) 74 | #define preparemax ((loop)->preparemax) 75 | #define preparecnt ((loop)->preparecnt) 76 | #define checks ((loop)->checks) 77 | #define checkmax ((loop)->checkmax) 78 | #define checkcnt ((loop)->checkcnt) 79 | #define forks ((loop)->forks) 80 | #define forkmax ((loop)->forkmax) 81 | #define forkcnt ((loop)->forkcnt) 82 | #define cleanups ((loop)->cleanups) 83 | #define cleanupmax ((loop)->cleanupmax) 84 | #define cleanupcnt ((loop)->cleanupcnt) 85 | #define async_pending ((loop)->async_pending) 86 | #define asyncs ((loop)->asyncs) 87 | #define asyncmax ((loop)->asyncmax) 88 | #define asynccnt ((loop)->asynccnt) 89 | #define fs_fd ((loop)->fs_fd) 90 | #define fs_w ((loop)->fs_w) 91 | #define fs_2625 ((loop)->fs_2625) 92 | #define fs_hash ((loop)->fs_hash) 93 | #define sig_pending ((loop)->sig_pending) 94 | #define sigfd ((loop)->sigfd) 95 | #define sigfd_w ((loop)->sigfd_w) 96 | #define sigfd_set ((loop)->sigfd_set) 97 | #define loop_count ((loop)->loop_count) 98 | #define loop_depth ((loop)->loop_depth) 99 | #define userdata ((loop)->userdata) 100 | #define release_cb ((loop)->release_cb) 101 | #define acquire_cb ((loop)->acquire_cb) 102 | #define invoke_cb ((loop)->invoke_cb) 103 | #else 104 | #undef EV_WRAP_H 105 | #undef now_floor 106 | #undef mn_now 107 | #undef rtmn_diff 108 | #undef io_blocktime 109 | #undef timeout_blocktime 110 | #undef backend 111 | #undef activecnt 112 | #undef loop_done 113 | #undef backend_fd 114 | #undef backend_fudge 115 | #undef backend_modify 116 | #undef backend_poll 117 | #undef anfds 118 | #undef anfdmax 119 | #undef pendings 120 | #undef pendingmax 121 | #undef pendingcnt 122 | #undef pending_w 123 | #undef rfeeds 124 | #undef rfeedmax 125 | #undef rfeedcnt 126 | #undef evfd 127 | #undef evpipe 128 | #undef pipe_w 129 | #undef curpid 130 | #undef postfork 131 | #undef vec_ri 132 | #undef vec_ro 133 | #undef vec_wi 134 | #undef vec_wo 135 | #undef vec_eo 136 | #undef vec_max 137 | #undef polls 138 | #undef pollmax 139 | #undef pollcnt 140 | #undef pollidxs 141 | #undef pollidxmax 142 | #undef epoll_events 143 | #undef epoll_eventmax 144 | #undef kqueue_changes 145 | #undef kqueue_changemax 146 | #undef kqueue_changecnt 147 | #undef kqueue_events 148 | #undef kqueue_eventmax 149 | #undef port_events 150 | #undef port_eventmax 151 | #undef iocp 152 | #undef fdchanges 153 | #undef fdchangemax 154 | #undef fdchangecnt 155 | #undef timers 156 | #undef timermax 157 | #undef timercnt 158 | #undef periodics 159 | #undef periodicmax 160 | #undef periodiccnt 161 | #undef idles 162 | #undef idlemax 163 | #undef idlecnt 164 | #undef idleall 165 | #undef prepares 166 | #undef preparemax 167 | #undef preparecnt 168 | #undef checks 169 | #undef checkmax 170 | #undef checkcnt 171 | #undef forks 172 | #undef forkmax 173 | #undef forkcnt 174 | #undef cleanups 175 | #undef cleanupmax 176 | #undef cleanupcnt 177 | #undef async_pending 178 | #undef asyncs 179 | #undef asyncmax 180 | #undef asynccnt 181 | #undef fs_fd 182 | #undef fs_w 183 | #undef fs_2625 184 | #undef fs_hash 185 | #undef sig_pending 186 | #undef sigfd 187 | #undef sigfd_w 188 | #undef sigfd_set 189 | #undef loop_count 190 | #undef loop_depth 191 | #undef userdata 192 | #undef release_cb 193 | #undef acquire_cb 194 | #undef invoke_cb 195 | #endif 196 | -------------------------------------------------------------------------------- /libeasy/packet/http/easy_http_handler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #ifndef EASY_HTTP_HANDLER_H_ 11 | #define EASY_HTTP_HANDLER_H_ 12 | 13 | #include 14 | 15 | /** 16 | * 对ＨＴＴＰ的流解析 17 | */ 18 | EASY_CPP_START 19 | 20 | #include 21 | #include 22 | 23 | #define EASY_HTTP_HDR_SIZE 32 24 | #define EASY_HTTP_HDR_HSEED 5 25 | #define EASY_HTTP_CRLF "\r\n" 26 | 27 | // http status code 28 | #define EASY_HTTP_STATUS_200 "200 OK" 29 | #define EASY_HTTP_STATUS_201 "201 Created" 30 | #define EASY_HTTP_STATUS_202 "202 Accepted" 31 | #define EASY_HTTP_STATUS_204 "204 No Content" 32 | #define EASY_HTTP_STATUS_206 "206 Partial Content" 33 | #define EASY_HTTP_STATUS_301 "301 Moved Permanently" 34 | #define EASY_HTTP_STATUS_302 "302 Moved Temporarily" 35 | #define EASY_HTTP_STATUS_303 "303 See Other" 36 | #define EASY_HTTP_STATUS_304 "304 Not Modified" 37 | #define EASY_HTTP_STATUS_400 "400 Bad Request" 38 | #define EASY_HTTP_STATUS_401 "401 Unauthorized" 39 | #define EASY_HTTP_STATUS_402 "402 Payment Required" 40 | #define EASY_HTTP_STATUS_403 "403 Forbidden" 41 | #define EASY_HTTP_STATUS_404 "404 Not Found" 42 | #define EASY_HTTP_STATUS_405 "405 Not Allowed" 43 | #define EASY_HTTP_STATUS_406 "406 Not Acceptable" 44 | #define EASY_HTTP_STATUS_408 "408 Request Time-out" 45 | #define EASY_HTTP_STATUS_409 "409 Conflict" 46 | #define EASY_HTTP_STATUS_410 "410 Gone" 47 | #define EASY_HTTP_STATUS_411 "411 Length Required" 48 | #define EASY_HTTP_STATUS_412 "412 Precondition Failed" 49 | #define EASY_HTTP_STATUS_413 "413 Request Entity Too Large" 50 | #define EASY_HTTP_STATUS_415 "415 Unsupported Media Type" 51 | #define EASY_HTTP_STATUS_416 "416 Requested Range Not Satisfiable" 52 | #define EASY_HTTP_STATUS_500 "500 Internal Server Error" 53 | #define EASY_HTTP_STATUS_501 "501 Method Not Implemented" 54 | #define EASY_HTTP_STATUS_502 "502 Bad Gateway" 55 | #define EASY_HTTP_STATUS_503 "503 Service Temporarily Unavailable" 56 | #define EASY_HTTP_STATUS_504 "504 Gateway Time-out" 57 | #define EASY_HTTP_STATUS_507 "507 Insufficient Storage" 58 | 59 | typedef struct easy_http_request_t easy_http_request_t; 60 | typedef struct easy_http_header_t easy_http_header_t; 61 | typedef struct easy_http_header_hash_t easy_http_header_hash_t; 62 | typedef struct easy_http_packet_t easy_http_packet_t; 63 | 64 | struct easy_http_header_t { 65 | easy_buf_string_t name; 66 | easy_buf_string_t value; 67 | easy_http_header_t *next; 68 | }; 69 | 70 | struct easy_http_header_hash_t { 71 | easy_http_header_t *buckets[EASY_HTTP_HDR_SIZE]; 72 | uint32_t count; 73 | }; 74 | 75 | #define easy_http_header_hash_foreach(i, header, table) \ 76 | for(i = 0; i < EASY_HTTP_HDR_SIZE; i++) \ 77 | for(header = (table)->buckets[i]; header; header = header->next) 78 | 79 | struct easy_http_request_t { 80 | easy_message_t *m; 81 | http_parser parser; 82 | 83 | easy_buf_string_t str_path; 84 | easy_buf_string_t str_query_string; 85 | easy_buf_string_t str_fragment; 86 | easy_buf_string_t str_body; 87 | 88 | easy_http_header_hash_t headers_in; 89 | easy_http_header_t *last_header; 90 | 91 | // response 92 | easy_buf_string_t status_line; 93 | easy_http_header_hash_t headers_out; 94 | easy_list_t output; 95 | easy_buf_string_t content_type; 96 | int64_t content_length; 97 | 98 | // flags 99 | unsigned int message_begin_called : 1; 100 | unsigned int header_complete_called : 1; 101 | unsigned int message_complete_called : 1; 102 | unsigned int last_was_value : 1; 103 | unsigned int keep_alive : 1; 104 | unsigned int is_raw_header : 1; 105 | 106 | int parsed_byte; 107 | }; 108 | 109 | struct easy_http_packet_t { 110 | easy_buf_string_t str_query_string; 111 | easy_buf_string_t str_path; 112 | easy_http_header_hash_t headers_out; 113 | easy_list_t output; 114 | 115 | unsigned int is_raw_header : 1; 116 | unsigned int post_method : 1; 117 | unsigned int keep_alive : 1; 118 | }; 119 | 120 | void easy_http_add_header(easy_pool_t *pool, easy_http_header_hash_t *table, const char *name, const char *value); 121 | char *easy_http_del_header(easy_http_header_hash_t *table, const char *name); 122 | char *easy_http_get_header(easy_http_header_hash_t *table, const char *name); 123 | 124 | void easy_http_header_hash_add(easy_http_header_hash_t *table, easy_http_header_t *header); 125 | easy_http_header_t *easy_http_header_hash_get(easy_http_header_hash_t *table, const char *key, int len); 126 | easy_http_header_t *easy_http_header_hash_del(easy_http_header_hash_t *table, const char *key, int len); 127 | 128 | void easy_http_handler_init(easy_io_handler_pt *handler, easy_io_process_pt *process); 129 | void *easy_http_server_on_decode(easy_message_t *m); 130 | int easy_http_server_on_encode(easy_request_t *r, void *data); 131 | void *easy_http_client_on_decode(easy_message_t *m); 132 | int easy_http_client_on_encode(easy_request_t *r, void *data); 133 | easy_http_packet_t *easy_http_packet_create (easy_pool_t *pool); 134 | 135 | int easy_url_decode(char *str, int len); 136 | void easy_http_header_string_end(easy_http_request_t *p); 137 | int easy_http_merge_path(char *newpath, int len, const char *rootpath, const char *addpath); 138 | int easy_http_request_printf(easy_http_request_t *r, const char *fmt, ...); 139 | 140 | EASY_CPP_END 141 | 142 | #endif 143 | -------------------------------------------------------------------------------- /libeasy/util/easy_pool.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | #include "easy_pool.h" 11 | #include 12 | #include 13 | 14 | /** 15 | * 简单的内存池 16 | */ 17 | 18 | static void *easy_pool_alloc_block(easy_pool_t *pool, uint32_t size); 19 | static void *easy_pool_alloc_large(easy_pool_t *pool, uint32_t size); 20 | easy_pool_realloc_pt easy_pool_realloc = easy_pool_default_realloc; 21 | 22 | easy_pool_t *easy_pool_create(uint32_t size) 23 | { 24 | easy_pool_t *p; 25 | 26 | // 对齐 27 | size = easy_align(size + sizeof(easy_pool_t), EASY_POOL_ALIGNMENT); 28 | 29 | if ((p = (easy_pool_t *)easy_pool_realloc(NULL, size)) == NULL) 30 | return NULL; 31 | 32 | memset(p, 0, sizeof(easy_pool_t)); 33 | p->last = (uint8_t *) p + sizeof(easy_pool_t); 34 | p->end = (uint8_t *) p + size; 35 | p->max = size - sizeof(easy_pool_t); 36 | p->current = p; 37 | 38 | return p; 39 | } 40 | 41 | // clear 42 | void easy_pool_clear(easy_pool_t *pool) 43 | { 44 | easy_pool_t *p, *n; 45 | easy_pool_large_t *l, *nl; 46 | 47 | // large 48 | l = pool->large; 49 | 50 | while(l) { 51 | nl = l->next; 52 | easy_pool_realloc(l, 0); 53 | l = nl; 54 | } 55 | 56 | // other page 57 | p = pool->next; 58 | 59 | while(p) { 60 | n = p->next; 61 | easy_pool_realloc(p, 0); 62 | p = n; 63 | } 64 | 65 | pool->large = NULL; 66 | pool->next = NULL; 67 | pool->current = pool; 68 | pool->failed = 0; 69 | pool->last = (uint8_t *) pool + sizeof(easy_pool_t); 70 | } 71 | 72 | void easy_pool_destroy(easy_pool_t *pool) 73 | { 74 | easy_pool_clear(pool); 75 | assert(pool->ref == 0); 76 | easy_pool_realloc(pool, 0); 77 | } 78 | 79 | void *easy_pool_alloc(easy_pool_t *pool, uint32_t size) 80 | { 81 | uint8_t *m; 82 | easy_pool_t *p; 83 | int flags = pool->flags; 84 | 85 | if (unlikely(flags)) easy_spin_lock(&pool->tlock); 86 | 87 | if (size <= pool->max) { 88 | p = pool->current; 89 | 90 | do { 91 | m = easy_align_ptr(p->last, sizeof(unsigned long)); 92 | 93 | if (m + size <= p->end) { 94 | p->last = m + size; 95 | break; 96 | } 97 | 98 | p = p->next; 99 | } while (p); 100 | 101 | // 重新分配一块出来 102 | if (p == NULL) { 103 | m = (uint8_t *)easy_pool_alloc_block(pool, size); 104 | } 105 | } else { 106 | m = (uint8_t *)easy_pool_alloc_large(pool, size); 107 | } 108 | 109 | if (unlikely(flags)) easy_spin_unlock(&pool->tlock); 110 | 111 | return m; 112 | } 113 | 114 | void *easy_pool_nalloc(easy_pool_t *pool, uint32_t size) 115 | { 116 | uint8_t *m; 117 | easy_pool_t *p; 118 | 119 | if (size <= pool->max) { 120 | 121 | p = pool->current; 122 | 123 | do { 124 | m = p->last; 125 | 126 | if (m + size <= p->end) { 127 | p->last = m + size; 128 | break; 129 | } 130 | 131 | p = p->next; 132 | } while (p); 133 | 134 | // 重新分配一块出来 135 | if (p == NULL) { 136 | m = (uint8_t *)easy_pool_alloc_block(pool, size); 137 | } 138 | } else { 139 | m = (uint8_t *)easy_pool_alloc_large(pool, size); 140 | } 141 | 142 | return m; 143 | } 144 | 145 | void *easy_pool_calloc(easy_pool_t *pool, uint32_t size) 146 | { 147 | void *p; 148 | 149 | if ((p = easy_pool_alloc(pool, size)) != NULL) 150 | memset(p, 0, size); 151 | 152 | return p; 153 | } 154 | 155 | // set lock 156 | void easy_pool_set_lock(easy_pool_t *pool) 157 | { 158 | pool->flags = 1; 159 | } 160 | 161 | // set realloc 162 | void easy_pool_set_allocator(easy_pool_realloc_pt alloc) 163 | { 164 | easy_pool_realloc = (alloc ? alloc : easy_pool_default_realloc); 165 | } 166 | 167 | void *easy_pool_default_realloc (void *ptr, size_t size) 168 | { 169 | if (size) { 170 | return realloc (ptr, size); 171 | } else if (ptr) { 172 | free (ptr); 173 | } 174 | 175 | return 0; 176 | } 177 | /////////////////////////////////////////////////////////////////////////////////////////////////// 178 | // default realloc 179 | 180 | static void *easy_pool_alloc_block(easy_pool_t *pool, uint32_t size) 181 | { 182 | uint8_t *m; 183 | uint32_t psize; 184 | easy_pool_t *p, *newpool, *current; 185 | 186 | psize = (uint32_t) (pool->end - (uint8_t *) pool); 187 | 188 | if ((m = (uint8_t *)easy_pool_realloc(NULL, psize)) == NULL) 189 | return NULL; 190 | 191 | newpool = (easy_pool_t *) m; 192 | newpool->end = m + psize; 193 | newpool->next = NULL; 194 | newpool->failed = 0; 195 | 196 | m += offsetof(easy_pool_t, current); 197 | m = easy_align_ptr(m, sizeof(unsigned long)); 198 | newpool->last = m + size; 199 | current = pool->current; 200 | 201 | for (p = current; p->next; p = p->next) { 202 | if (p->failed++ > 4) { 203 | current = p->next; 204 | } 205 | } 206 | 207 | p->next = newpool; 208 | pool->current = current ? current : newpool; 209 | 210 | return m; 211 | } 212 | 213 | static void *easy_pool_alloc_large(easy_pool_t *pool, uint32_t size) 214 | { 215 | easy_pool_large_t *large; 216 | 217 | size = easy_align(size + sizeof(easy_pool_large_t), EASY_POOL_ALIGNMENT); 218 | 219 | if ((large = (easy_pool_large_t *)easy_pool_realloc(NULL, size)) == NULL) 220 | return NULL; 221 | 222 | large->next = pool->large; 223 | pool->large = large; 224 | return &(large->data[0]); 225 | } 226 | 227 | /** 228 | * strdup 229 | */ 230 | char *easy_pool_strdup(easy_pool_t *pool, const char *str) 231 | { 232 | int sz; 233 | char *ptr; 234 | 235 | if (str == NULL) 236 | return NULL; 237 | 238 | sz = strlen(str) + 1; 239 | 240 | if ((ptr = (char *)easy_pool_alloc(pool, sz)) == NULL) 241 | return NULL; 242 | 243 | memcpy(ptr, str, sz); 244 | return ptr; 245 | } 246 | -------------------------------------------------------------------------------- /libtdhs/tdh_socket_protocol.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright(C) 2011-2012 Alibaba Group Holding Limited 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | 9 | 10 | /* 11 | * tdh_socket_protocol.cpp 12 | * 13 | * Created on: 2011-9-30 14 | * Author: wentong 15 | */ 16 | 17 | #include "tdh_socket_protocol.hpp" 18 | #include "debug_util.hpp" 19 | #include "tdh_socket_time.hpp" 20 | 21 | #include 22 | 23 | namespace taobao { 24 | 25 | #define GET_HEADER_INFO(COM,SEQ,REV,DATA,POS) \ 26 | COM = ntohl(*((uint32_t *) ((POS) + TDHS_MAGIC_CODE_SIZE))); \ 27 | SEQ = ntohl( \ 28 | *((uint32_t *) ((POS) + TDHS_MAGIC_CODE_SIZE \ 29 | + TDH_SOCKET_COMAND_LENGTH))); \ 30 | REV = \ 31 | ntohl(*((uint32_t *) ((POS) + TDHS_MAGIC_CODE_SIZE \ 32 | + TDH_SOCKET_COMAND_LENGTH + TDH_SOCKET_SEQ_ID_LENGTH))); \ 33 | DATA = \ 34 | ntohl( *((uint32_t *) ((POS) + TDHS_MAGIC_CODE_SIZE \ 35 | + TDH_SOCKET_COMAND_LENGTH + TDH_SOCKET_SEQ_ID_LENGTH \ 36 | + TDH_SOCKET_REVERSE_LENGTH))); \ 37 | 38 | void*tdhs_decode(easy_message_t *m) { 39 | tdhs_packet_t *packet; 40 | uint32_t len, magic_code, com_or_resp, seq_id, reverse, datalen; 41 | uint32_t batch_request_num = 0; 42 | unsigned long long int now; 43 | size_t i; 44 | char *pos; //解析batch请求用 45 | uint32_t left_length; //判断batch请求长度用 46 | 47 | if ((len = m->input->last - m->input->pos) < TDH_SOCKET_HEADER_LENGTH) 48 | return NULL; 49 | 50 | // packet 51 | magic_code = ntohl(*((uint32_t *) (m->input->pos))); 52 | if (magic_code != TDHS_MAGIC_CODE) { 53 | easy_error_log("TDHS:magic_code is invalid: %d\n", magic_code); 54 | m->status = EASY_ERROR; 55 | return NULL; 56 | } 57 | 58 | GET_HEADER_INFO(com_or_resp, seq_id, reverse, datalen, m->input->pos) 59 | 60 | batch_request_num = com_or_resp == REQUEST_TYPE_BATCH ? reverse : 0; 61 | 62 | if (datalen > 0x4000000) { // 64M 63 | easy_error_log("TDHS:data_len is invalid: %d\n", datalen); 64 | m->status = EASY_ERROR; 65 | return NULL; 66 | } 67 | 68 | // 长度不够 69 | len -= TDH_SOCKET_HEADER_LENGTH; 70 | 71 | if (len < datalen) { 72 | m->next_read_len = datalen - len; 73 | return NULL; 74 | } 75 | 76 | // alloc packet 77 | if ((packet = (tdhs_packet_t *) easy_pool_calloc(m->pool, 78 | sizeof(tdhs_packet_t) * (1 + batch_request_num))) == NULL) { 79 | m->status = EASY_ERROR; 80 | return NULL; 81 | } 82 | now = tdhs_micro_time(); 83 | 84 | packet->command_id_or_response_code = com_or_resp; 85 | packet->seq_id = seq_id; 86 | packet->reserved = reverse; 87 | packet->length = datalen; 88 | m->input->pos += TDH_SOCKET_HEADER_LENGTH; 89 | packet->rdata = (char *) m->input->pos; 90 | packet->pool = m->pool; //此处设置使用message的pool 91 | packet->start_time = now; 92 | //处理batch请求 93 | pos = packet->rdata; 94 | left_length = packet->length; 95 | for (i = 0; i < batch_request_num; i++) { 96 | tdhs_packet_t *batch_packet = packet + 1 + i; 97 | if (left_length < TDH_SOCKET_HEADER_LENGTH) { 98 | //长度不全导致没法完整解析batch请求的头 99 | easy_error_log( "TDHS:batch left_length is invalid: %d %d\n", 100 | left_length); 101 | m->status = EASY_ERROR; 102 | return NULL; 103 | } 104 | 105 | GET_HEADER_INFO(com_or_resp, seq_id, reverse, datalen, pos) 106 | pos += TDH_SOCKET_HEADER_LENGTH; 107 | left_length -= TDH_SOCKET_HEADER_LENGTH; 108 | if (left_length < datalen) { 109 | //长度不全导致没法完整解析batch请求 110 | easy_error_log( 111 | "TDHS:batch data_len is invalid: %d ,the left_length is %d\n", 112 | datalen, left_length); 113 | m->status = EASY_ERROR; 114 | return NULL; 115 | } 116 | batch_packet->command_id_or_response_code = com_or_resp; 117 | batch_packet->seq_id = seq_id; 118 | //batch 目前不支持嵌套 ,所以不设置reserved 119 | batch_packet->length = datalen; 120 | batch_packet->rdata = pos; 121 | batch_packet->pool = m->pool; //此处设置使用message的pool 122 | batch_packet->start_time = now; 123 | (batch_packet - 1)->next = batch_packet; //形成链 124 | pos += datalen; 125 | left_length -= datalen; 126 | } 127 | m->input->pos += packet->length; 128 | return packet; 129 | } 130 | 131 | #define WRITE_HEADER_INFO(COM,SEQ,REV,DATA,POS) \ 132 | *((uint32_t *) (POS)) = htonl(TDHS_MAGIC_CODE); \ 133 | *((uint32_t *) (POS + TDHS_MAGIC_CODE_SIZE)) = htonl(COM); \ 134 | *((uint32_t *) (POS+ TDHS_MAGIC_CODE_SIZE + TDH_SOCKET_COMAND_LENGTH)) = \ 135 | htonl(SEQ); \ 136 | *((uint32_t *) (POS + TDHS_MAGIC_CODE_SIZE + TDH_SOCKET_COMAND_LENGTH \ 137 | + TDH_SOCKET_SEQ_ID_LENGTH)) = htonl(REV); \ 138 | *((uint32_t *) (POS + TDHS_MAGIC_CODE_SIZE + TDH_SOCKET_COMAND_LENGTH \ 139 | + TDH_SOCKET_SEQ_ID_LENGTH + TDH_SOCKET_REVERSE_LENGTH)) = htonl(DATA);\ 140 | 141 | #define WRITE_PACKET_HEADER_INFO(PACKET,POS) \ 142 | WRITE_HEADER_INFO(PACKET->command_id_or_response_code, \ 143 | PACKET->seq_id,PACKET->reserved,PACKET->length,POS) 144 | 145 | int tdhs_encode(easy_request_t *r, void *data) { 146 | tdhs_packet_t *packet; 147 | easy_buf_t *b; 148 | packet = (tdhs_packet_t *) data; 149 | tb_assert(packet!=NULL); 150 | 151 | while (packet) { 152 | tb_assert(packet->wbuff!=NULL); 153 | tb_assert((packet->command_id_or_response_code==CLIENT_STATUS_MULTI_STATUS)? \ 154 | packet->length==0:packet->length>0); 155 | b = packet->wbuff; 156 | //batch请求和返回的buf只包含header 157 | tb_assert((packet->command_id_or_response_code==CLIENT_STATUS_MULTI_STATUS|| \ 158 | packet->command_id_or_response_code==REQUEST_TYPE_BATCH)? \ 159 | (uint32_t)(b->last-b->pos)==TDH_SOCKET_HEADER_LENGTH: \ 160 | packet->length==((uint32_t)(b->last-b->pos)-TDH_SOCKET_HEADER_LENGTH)); 161 | 162 | // 加入header 163 | WRITE_PACKET_HEADER_INFO(packet, b->pos) 164 | easy_request_addbuf(r, b); 165 | packet = packet->next; 166 | } 167 | return EASY_OK; 168 | } 169 | 170 | } // namespace taobao 171 | 172 | --------------------------------------------------------------------------------