├── AUTHORS
├── COPYING
├── COPYING.lfs
├── ChangeLog
├── Doxyfile
├── INSTALL
├── Makefile.in
├── NEWS
├── README.TESTS
├── README.md
├── README_ZH.md
├── THANKS
├── autogen.sh
├── bootstrap.sh
├── compile
├── config.guess
├── config.h.in
├── config.sub
├── configure
├── depcomp
├── deps
├── CMakeLists.txt
├── libevent.CMakeLists.txt
├── libevent.cmake
├── libevent.config.h.cmake
├── libevent.def
├── libevent.event-config.h.cmake
├── libevent.event.h.cmake
└── libevent.evutil.h.cmake
├── doc
├── Makefile.am
├── Makefile.in
├── architecture-overview.dot
├── architecture.dot
├── architecture.txt
├── chapter
│ ├── Makefile.am
│ ├── Makefile.in
│ ├── protocol.txt
│ └── scripting.txt
├── chassis.txt
├── core.txt
├── lifecycle.msc
├── lua-classes.dot
├── plugins.txt
├── protocol.txt
├── scripting.txt
└── tests.txt
├── doxygen-scripts
├── doxygen_version.sh
├── footer.html
└── project_number_bzr.sh
├── example.cnf
├── examples
├── CMakeLists.txt
├── Makefile.am
├── Makefile.in
├── tutorial-basic.lua
├── tutorial-constants.lua
├── tutorial-inject.lua
├── tutorial-keepalive.lua
├── tutorial-monitor.lua
├── tutorial-packets.lua
├── tutorial-prep-stmts.lua
├── tutorial-query-time.lua
├── tutorial-resultset.lua
├── tutorial-rewrite.lua
├── tutorial-routing.lua
├── tutorial-scramble.lua
├── tutorial-states.lua
├── tutorial-tokenize.lua
├── tutorial-union.lua
└── tutorial-warnings.lua
├── install-sh
├── lib
├── CMakeLists.txt
├── Makefile.am
├── Makefile.in
├── active-queries.lua
├── active-transactions.lua
├── admin-bak.lua
├── admin-sql.lua
├── admin.lua
├── analyze-query.lua
├── auditing.lua
├── chassis.c
├── chassis.def
├── commit-obfuscator.lua
├── commit-obfuscator.msc
├── crc32-lua.c
├── crc32.c
├── crc32.h
├── glib2.c
├── glib2.def
├── histogram.lua
├── lfs.c
├── lfs.def
├── load-multi.lua
├── lpeg.c
├── lpeg.def
├── mysql-password.c
├── mysql-proto.c
├── mysql.def
├── posix.c
├── posix.def
├── proxy
│ ├── CMakeLists.txt
│ ├── Makefile.am
│ ├── Makefile.in
│ ├── auth.lua
│ ├── auto-config.lua
│ ├── balance.lua
│ ├── charset.lua
│ ├── commands.lua
│ ├── crc32.lua
│ ├── filter.lua
│ ├── log.lua
│ ├── parser.lua
│ ├── split.lua
│ ├── test.lua
│ ├── ticker.lua
│ └── tokenizer.lua
├── ro-balance.lua
├── ro-pooling.lua
├── rw-splitting.lua
├── sql-tokenizer-gen.c
├── sql-tokenizer-keywords.h
├── sql-tokenizer-lua.c
├── sql-tokenizer-tokens.c
├── sql-tokenizer.h
├── sql-tokenizer.l
├── time-ticker-lua.c
└── xtab.lua
├── ltmain.sh
├── m4
├── Makefile.am
├── Makefile.in
└── sed.m4
├── missing
├── mysql-chassis.pc.in
├── mysql-proxy.pc.in
├── mysql-proxy.spec.in
├── plugins
├── CMakeLists.txt
├── Makefile.am
├── Makefile.in
├── admin
│ ├── CMakeLists.txt
│ ├── Makefile.am
│ ├── Makefile.in
│ └── admin-plugin.c
├── debug
│ ├── CMakeLists.txt
│ ├── Makefile.am
│ ├── Makefile.in
│ └── debug-plugin.c
├── proxy
│ ├── CMakeLists.txt
│ ├── Makefile.am
│ ├── Makefile.in
│ ├── proxy-plugin.c
│ └── proxy-plugin.h
└── replicant
│ ├── CMakeLists.txt
│ ├── Makefile.am
│ ├── Makefile.in
│ └── replicant-plugin.c
├── script
├── Makefile
├── create_table.sh
├── cron.sh
├── encrypt.c
├── instances
├── log.sh
└── mysql-proxyd
├── src
├── CMakeLists.txt
├── Makefile.am
├── Makefile.in
├── chassis-event-thread.c
├── chassis-event-thread.h
├── chassis-exports.h
├── chassis-filemode.c
├── chassis-filemode.h
├── chassis-frontend.c
├── chassis-frontend.h
├── chassis-gtimeval.c
├── chassis-gtimeval.h
├── chassis-keyfile.c
├── chassis-keyfile.h
├── chassis-limits.c
├── chassis-limits.h
├── chassis-log.c
├── chassis-log.h
├── chassis-mainloop.c
├── chassis-mainloop.h
├── chassis-options.c
├── chassis-options.h
├── chassis-path.c
├── chassis-path.h
├── chassis-plugin.c
├── chassis-plugin.h
├── chassis-shutdown-hooks.c
├── chassis-shutdown-hooks.h
├── chassis-stats.c
├── chassis-stats.h
├── chassis-timings.c
├── chassis-timings.h
├── chassis-unix-daemon.c
├── chassis-unix-daemon.h
├── chassis-win32-service.c
├── chassis-win32-service.h
├── disable-dtrace.h
├── glib-ext-ref.c
├── glib-ext-ref.h
├── glib-ext.c
├── glib-ext.h
├── lua-env.c
├── lua-env.h
├── lua-load-factory.c
├── lua-load-factory.h
├── lua-registry-keys.h
├── lua-scope.c
├── lua-scope.h
├── my_rdtsc.c
├── my_rdtsc.h
├── my_timer_cycles.il
├── mysql-binlog-dump.c
├── mysql-myisam-dump.c
├── mysql-proxy-cli.c
├── network-address-lua.c
├── network-address-lua.h
├── network-address.c
├── network-address.h
├── network-backend-lua.c
├── network-backend-lua.h
├── network-backend.c
├── network-backend.h
├── network-conn-pool-lua.c
├── network-conn-pool-lua.h
├── network-conn-pool.c
├── network-conn-pool.h
├── network-debug.h
├── network-exports.h
├── network-injection-lua.c
├── network-injection-lua.h
├── network-injection.c
├── network-injection.h
├── network-mysqld-binlog.c
├── network-mysqld-binlog.h
├── network-mysqld-lua.c
├── network-mysqld-lua.h
├── network-mysqld-masterinfo.c
├── network-mysqld-masterinfo.h
├── network-mysqld-packet.c
├── network-mysqld-packet.h
├── network-mysqld-proto.c
├── network-mysqld-proto.h
├── network-mysqld.c
├── network-mysqld.h
├── network-queue.c
├── network-queue.h
├── network-socket-lua.c
├── network-socket-lua.h
├── network-socket.c
├── network-socket.h
├── network_mysqld_proto_binary.c
├── network_mysqld_proto_binary.h
├── network_mysqld_type.c
├── network_mysqld_type.h
├── proxy-dtrace-provider.d
├── string-len.h
├── sys-pedantic.h
└── test-latency.c
└── ylwrap
/AUTHORS:
--------------------------------------------------------------------------------
1 | If you have questions about
2 |
3 | jan
4 |
5 | Zhu Chao(zhuchao@360.cn)
6 | Wang Chao(wangchao3@360.cn)
7 | Gui Yongzhe(guiyongzhe@360.cn)
8 | Feng Suofan(fengsuofan@360.cn)
9 |
--------------------------------------------------------------------------------
/COPYING.lfs:
--------------------------------------------------------------------------------
1 | For the test-cases we include
2 |
3 | * LuaFileSystem 1.2
4 |
5 | which is covered by the following license
6 |
7 | ---
8 | Copyright © 2004-2005 The Kepler Project.
9 |
10 | Permission is hereby granted, free of charge, to any person obtaining a copy of
11 | this software and associated documentation files (the "Software"), to deal in
12 | the Software without restriction, including without limitation the rights to
13 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
14 | of the Software, and to permit persons to whom the Software is furnished to do
15 | so, subject to the following conditions:
16 |
17 | The above copyright notice and this permission notice shall be included in all
18 | copies or substantial portions of the Software.
19 |
20 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 | THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
26 | IN THE SOFTWARE.
27 |
--------------------------------------------------------------------------------
/ChangeLog:
--------------------------------------------------------------------------------
1 | 2.1
2 | *修正配置文件里的charset不生效的bug
3 | *修正update/delete分多张子表的bug
4 | *增加对MySQL Workbench, Navicat, SQLyog的支持
5 | *SQL日志里增加端口信息
6 | *改为每个工作线程一个消息队列
7 | *去除配置文件里的admin-lua-script项
8 | *配置文件里的instance项可不设置
9 | *主库可作为一台从库使用
10 | *屏蔽不带WHERE条件的UPDATE语句
11 |
12 | 2.0.5
13 | *连接失败时将unknown状态的DB置为down
14 | *将COM_SET_OPTION优先发往从库
15 | *从admin.lua里移除uuid和connected_clients
16 |
17 | 2.0.4
18 | *修正SET CHARACTER_[CLIENT|RESULTS|CONNECTION]的bug
19 |
20 | 2.0.3
21 | *支持客户端在连接时指定默认字符集
22 |
23 | 2.0.2
24 | *检测线程只检测状态为down的DB
25 | *处于事务状态的客户端退出时,Atlas销毁该客户端使用的连接
26 |
27 | 2.0.1
28 | *Java不能接受handshake包的随机串内包含负数,将随机串的每个字节的范围限定为[1,127]
29 |
30 | 2.0.0
31 | *实现了Atlas自行向DB建立连接,而不再依赖客户端的连接动作,并且把连接池由队列结构改为栈结构,带来的好处:
32 | **支持了长连接
33 | **不再需要min-idle-connections参数
34 | **连接池内的连接数可随着客户端并发量的上升而增长,且在并发量下降后自动销毁多余连接(依靠DB的wait_timeout)
35 | *SQL日志增加了开关,可以设为关闭(OFF)、打开(ON)、实时(REALTIME)三种,默认为OFF,REALTIME用于观察调试
36 | **OFF代表不记录SQL日志
37 | **ON代表记录SQL日志,由操作系统定期写入磁盘
38 | **REALTIME代表记录SQL日志,每记录一条都实时写入磁盘
39 | *启动脚本修改,在启动失败时可以报出错误信息
40 | *去除了配置文件的mode必须低于660的限制
41 | *检测线程不再将DB状态设为down,而是只在DB恢复后将其状态设为up,以此避免网络抖动引发的误将DB下线操作,减少无谓的lost connection
42 | *修正管理接口无法识别含有多个空格的语句的bug
43 | *去除全局保存的con结构队列,提升性能
44 | *不再判断is_insert_id,提升性能
45 |
46 | 0.7.0 -
47 |
48 | chassis
49 | * added a error-msg if trailing options without dashes are specified
50 | * added config-file support that maps cmdline options to configfile options
51 | * added support for SIGHUP to reopen logfiles
52 | * added win32-service support
53 | * added --basedir
54 | * added experimental --log-backtrace-on-crash
55 | * added DTrace probes
56 | * added a angel to keep the chassis alive
57 | * added --user option to change the system user the proxy runs as
58 |
59 | proxy-core
60 | * added buffered IO on unix
61 | * added lua-scope that is shared between the plugins
62 | * added a option to disable COM_CHANGE_USER on connection reuse
63 | * added handling of the 4.0 auth packets
64 | * added parser for master.info files
65 | * replaced assertions in protocol decoders by proper error-codes
66 | * fixed assertions when COM_TIME, COM_PROCESS_INFO or COM_DEBUG are received
67 |
68 | lua layer
69 | * added unit-testing for the lua scripts
70 | * fixed tutorial-inject.lua to start indexing at 1 (#32088)
71 | * expose chassis_log_* functions to lua
72 | * added the posix and lpeg module from luaforge
73 | * use the LPEG lua-module to parse statements
74 | * added memory profiling
75 | * added { resultset_is_needed = false } as default
76 | * moved proxy.backends.* to proxy.global.backends.*
77 | * added a lua-wrapper around some glib-2.0 functions
78 | * added a script cache
79 | * removed explicit call to lua_gc() to improve the speed
80 |
81 | Proxy Plugin
82 | * track rows and bytes of a resultset
83 | * fixed quoting of SQL commands like COMMIT and ROLLBACK
84 | * added test-cases for connection-pooling, fail-over, ...
85 | * refactored the SQL tokenizer into a lua module
86 | * fixed handling -- comments in the SQL tokenizer
87 | * fixed backend state cache
88 |
89 | Admin Plugin
90 | * replaced the hardcoded admin commands by a lua script interface
91 | * added authentication support
92 |
93 | Debug Plugin
94 | * return "number" as DOUBLE or INTs
95 |
96 |
97 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | We have build another interesting project [pika](https://github.com/Qihoo360/pika). Pika is a nosql compatible with redis protocol with huge storage space. You can have a try.
2 |
3 | ### 1.Brief introduction
4 |
5 | Atlas is a MySQL protocol-based database middleware project developed and maintained by infrastructure team of the Web platform Department in QIHU 360 SOFTWARE CO. LIMITED(NYSE:QIHU). It fixed lots of bugs and added lot of new functions on the basis of MySQL-Proxy 0.8.2. Currently the project has been widely applied in QIHU, many MySQL business has connected to the Atlas platform. The number of read and write requests forwarded by Atlas has reached billions.
6 |
7 | ### 2.Major functions
8 |
9 | 1.Read/Write Splitting.
10 |
11 | 2.Load balancing and failover handling.
12 |
13 | 3.IP filtering.
14 |
15 | 4.Data sharding
16 |
17 | 5.DBA can online or offline the backend database server smoothly.
18 |
19 | 6.Remove the failed database server automatically.
20 |
21 | 7.Config file reload without downtime.
22 |
23 | ### 3.The improvement of Atlas compared with Mysql-proxy
24 |
25 | 1.Rewrite all lua code with C, Lua is only used for management interface.
26 |
27 | 2.Rewrite the network model and the threading model.
28 |
29 | 3.Implement the connection pool.
30 |
31 | 4.Optimize the locking mechanism, performance improved immensely.
32 |
33 | ### 4.Detailed descriptions about Atlas
34 |
35 | [1.The installing of Atlas](https://github.com/Qihoo360/Atlas/wiki/Installing-Atlas)
36 |
37 | [2.The Architecture Of Atlas](https://github.com/Qihoo360/Atlas/wiki/The-Architecture-Of-Atlas)
38 |
39 | [3.The FAQs about the main features of Atlas](https://github.com/Qihoo360/Atlas/wiki/The-FAQs-about-the-main-features-of-Atlas)
40 |
41 | [4.The FAQs Of Running Atlas](https://github.com/Qihoo360/Atlas/wiki/The-FAQs-Of-Running-Atlas)
42 |
43 | ### 5.Requirement and feedback
44 |
45 | If You have new functional requirements about Atlas in the production environment, or find a bug in the process of using Atlas. Welcome to send a mail to g-atlas[at]360.cn, we will reply as soon as possible. Also you can contact us in [Google group](https://groups.google.com/forum/#!forum/atlas-proxy). Enthusiastic user has established a QQ group:326544838, the developers of Atlas have also been in the QQ group.
46 |
47 | ### 6.The origin of the name
48 |
49 | In Greek mythology, Atlas was the primordial Titan who held up the celestial spheres. He is also the titan of astronomy and navigation.
50 |
51 | ### 7.Other language version
52 |
53 | [简体中文](README_ZH.md)
54 |
55 |
--------------------------------------------------------------------------------
/README_ZH.md:
--------------------------------------------------------------------------------
1 |
2 | 我们团队开发了另外一个有意思的项目 pika, pika 是一个兼容redis 协议的大容量的存储, 用来解决redis 内存不够的问题, 欢迎大家试试
3 |
4 | 由于团队精力有限, 目前Atlas 依据满足了公司大部分的需求, 因此很多Issue 我们很难再继续满足. 但是我们欢迎社区里面喜欢Atlas, 愿意给Atlas 做贡献的小伙伴可以一起来改善Atlas, 给Atlas 提交代码. 欢迎联系 g-infra@360.cn!
5 | ### 一、简介
6 |
7 |
8 | Atlas是由 Qihoo 360公司Web平台部基础架构团队开发维护的一个基于MySQL协议的数据中间层项目。它在MySQL官方推出的MySQL-Proxy 0.8.2版本的基础上,修改了大量bug,添加了很多功能特性。目前该项目在360公司内部得到了广泛应用,很多MySQL业务已经接入了Atlas平台,每天承载的读写请求数达几十亿条。同时,有超过50家公司在生产环境中部署了Atlas,超过800人已加入了我们的开发者交流群,并且这些数字还在不断增加。
9 |
10 | 主要功能:
11 |
12 | 1.读写分离
13 |
14 | 2.从库负载均衡
15 |
16 | 3.IP过滤
17 |
18 | 4.自动分表
19 |
20 | 5.DBA可平滑上下线DB
21 |
22 | 6.自动摘除宕机的DB
23 |
24 | ### 二、Atlas相对于官方MySQL-Proxy的优势
25 |
26 | 1.将主流程中所有Lua代码用C重写,Lua仅用于管理接口
27 |
28 | 2.重写网络模型、线程模型
29 |
30 | 3.实现了真正意义上的连接池
31 |
32 | 4.优化了锁机制,性能提高数十倍
33 |
34 | ### 三、Atlas详细说明
35 |
36 | [1.Atlas的安装](http://github.com/Qihoo360/Atlas/wiki/Atlas的安装)
37 |
38 | [2.Atlas的运行及常见问题](http://github.com/Qihoo360/Atlas/wiki/Atlas的运行及常见问题)
39 |
40 | [3.Atlas的分表功能简介](http://github.com/Qihoo360/Atlas/wiki/Atlas的分表功能简介)
41 |
42 | [4.Atlas部分配置参数及原理详解](http://github.com/Qihoo360/Atlas/wiki/Atlas部分配置参数及原理详解)
43 |
44 | [5.Atlas的架构](https://github.com/Qihoo360/Atlas/wiki/Atlas的架构)
45 |
46 | [6.Atlas的性能测试](https://github.com/Qihoo360/Atlas/wiki/Atlas的性能测试)
47 |
48 | [7.Atlas功能特点FAQ](https://github.com/Qihoo360/Atlas/wiki/Atlas功能特点FAQ)
49 |
50 | [8.Atlas Sharding](https://github.com/Qihoo360/Atlas/wiki/Atlas-Sharding)
51 |
52 | ### 四、Atlas的需求及Bug反馈方式
53 |
54 | 如果用户在实际的应用场景中对Atlas有新的功能需求,或者在使用Atlas的过程中发现了bug,欢迎用户发邮件至g-infra@360.cn,与我们取得联系,我们将及时回复。另外有热心网友建立了QQ群326544838,开发者也已经加入,方便讨论。
55 |
56 | ### 五、名字来源
57 |
58 | Atlas:希腊神话中双肩撑天的巨人,普罗米修斯的兄弟,最高大强壮的神之一,因反抗宙斯失败而被罚顶天。我们期望这个系统能够脚踏后端DB,为前端应用撑起一片天。
59 |
60 | ### 更多
61 |
62 | atlas、pika以及其他技术请关注:我们360私有云(HULK平台)平台微信公共号
63 |
64 |
65 |
--------------------------------------------------------------------------------
/THANKS:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/autogen.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | # $%BEGINLICENSE%$
3 | # $%ENDLICENSE%$
4 |
5 | # Run this to generate all the initial makefiles, etc.
6 |
7 | # LIBTOOLIZE=${LIBTOOLIZE:-libtoolize}
8 | LIBTOOLIZE_FLAGS="--copy --force"
9 | # ACLOCAL=${ACLOCAL:-aclocal}
10 | # AUTOHEADER=${AUTOHEADER:-autoheader}
11 | # AUTOMAKE=${AUTOMAKE:-automake}
12 | AUTOMAKE_FLAGS="--add-missing --copy"
13 | # AUTOCONF=${AUTOCONF:-autoconf}
14 |
15 | ARGV0=$0
16 | ARGS="$@"
17 |
18 |
19 | run() {
20 | echo "$ARGV0: running \`$@' $ARGS"
21 | $@ $ARGS
22 | }
23 |
24 | ## jump out if one of the programs returns 'false'
25 | set -e
26 |
27 | ## on macosx glibtoolize, others have libtool
28 | if test x$LIBTOOLIZE = x; then
29 | if test \! "x`which glibtoolize 2> /dev/null | grep -v '^no'`" = x; then
30 | LIBTOOLIZE=glibtoolize
31 | elif test \! "x`which libtoolize 2> /dev/null | grep -v '^no'`" = x; then
32 | LIBTOOLIZE=libtoolize
33 | else
34 | echo "libtoolize wasn't found, try setting LIBTOOLIZE={path-to-libtool}."; exit 0
35 | fi
36 | fi
37 |
38 | if test x$ACLOCAL = x; then
39 | if test \! "x`which aclocal 2> /dev/null | grep -v '^no'`" = x; then
40 | ACLOCAL=aclocal
41 | else
42 | echo "aclocal 1.10+ wasn't found, try setting ACLOCAL={path-to-aclocal}."; exit 0
43 | fi
44 | fi
45 |
46 | if test x$AUTOMAKE = x; then
47 | if test \! "x`which automake 2> /dev/null | grep -v '^no'`" = x; then
48 | AUTOMAKE=automake
49 | else
50 | echo "automake 1.10+ wasn't found, try setting AUTOMAKE={path-to-automake}."; exit 0
51 | fi
52 | fi
53 |
54 |
55 | if test x$AUTOCONF = x; then
56 | if test \! "x`which autoconf 2> /dev/null | grep -v '^no'`" = x; then
57 | AUTOCONF=autoconf
58 | else
59 | echo "autoconf 2.62+ wasn't found, try setting AUTOCONF={path-to-autoconf}."; exit 0
60 | fi
61 | fi
62 |
63 | if test x$AUTOHEADER = x; then
64 | if test \! "x`which autoheader 2> /dev/null | grep -v '^no'`" = x; then
65 | AUTOHEADER=autoheader
66 | else
67 | echo "autoheader 2.62+ (autoheader) wasn't found, try setting AUTOHEADER={path-to-autoheader}."; exit 0
68 | fi
69 | fi
70 |
71 |
72 | run $LIBTOOLIZE $LIBTOOLIZE_FLAGS
73 | run $ACLOCAL $ACLOCAL_FLAGS -I m4
74 | run $AUTOHEADER
75 | run $AUTOMAKE $AUTOMAKE_FLAGS
76 | run $AUTOCONF
77 | test "$ARGS" = "" && echo "Now type './configure --enable-maintainer-mode ...' and 'make' to compile."
78 |
79 |
--------------------------------------------------------------------------------
/bootstrap.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | base=$(cd "$(dirname "$0")"; pwd)
3 | cd $base
4 | export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
5 | PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./configure --with-mysql=/usr --prefix=/usr/local/mysql-proxy CFLAGS="-DHAVE_LUA_H -O2" LDFLAGS="-lm -ldl -lcrypto -ljemalloc" LUA_CFLAGS="-I/usr/local/include/" LUA_LIBS="-L/usr/local/lib -llua"
6 |
--------------------------------------------------------------------------------
/deps/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | # INCLUDE(deps/libevent.cmake)
2 |
--------------------------------------------------------------------------------
/deps/libevent.cmake:
--------------------------------------------------------------------------------
1 | IF(WIN32)
2 | SET(LIBEVENT_SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/libevent-1.4.11-stable")
3 | IF(EXISTS ${LIBEVENT_SOURCE_DIR})
4 | ## write CMake file for libevent
5 |
6 | CONFIGURE_FILE(deps/libevent.config.h.cmake ${LIBEVENT_SOURCE_DIR}/config.h.cmake COPYONLY)
7 | CONFIGURE_FILE(deps/libevent.event-config.h.cmake ${LIBEVENT_SOURCE_DIR}/event-config.h.cmake COPYONLY)
8 | CONFIGURE_FILE(deps/libevent.CMakeLists.txt ${LIBEVENT_SOURCE_DIR}/CMakeLists.txt COPYONLY)
9 | CONFIGURE_FILE(deps/libevent.def ${LIBEVENT_SOURCE_DIR}/libevent.def COPYONLY)
10 | ## CONFIGURE_FILE(deps/libevent.event.h.cmake ${LIBEVENT_SOURCE_DIR}/event.h COPYONLY)
11 | ## CONFIGURE_FILE(deps/libevent.evutil.h.cmake ${LIBEVENT_SOURCE_DIR}/evutil.h COPYONLY)
12 |
13 | ADD_SUBDIRECTORY(${LIBEVENT_SOURCE_DIR} build-libevent)
14 |
15 | SET(EVENT_INCLUDE_DIRS ${LIBEVENT_SOURCE_DIR} CACHE INTERNAL "")
16 | IF(EXISTS ${CMAKE_BINARY_DIR}/build-libevent/${CMAKE_BUILD_TYPE}/)
17 | SET(EVENT_LIBRARY_DIRS ${CMAKE_BINARY_DIR}/build-libevent/${CMAKE_BUILD_TYPE} CACHE INTERNAL "")
18 | ELSE(EXISTS ${CMAKE_BINARY_DIR}/build-libevent/${CMAKE_BUILD_TYPE}/)
19 | SET(EVENT_LIBRARY_DIRS ${CMAKE_BINARY_DIR}/build-libevent CACHE INTERNAL "")
20 | ENDIF(EXISTS ${CMAKE_BINARY_DIR}/build-libevent/${CMAKE_BUILD_TYPE}/)
21 | SET(EVENT_LIBRARIES event CACHE INTERNAL "")
22 | ELSE(EXISTS ${LIBEVENT_SOURCE_DIR})
23 | MESSAGE(FATAL_ERROR "Could not find dependency libevent-1.4.11-stable in ${LIBEVENT_SOURCE_DIR}")
24 | ENDIF(EXISTS ${LIBEVENT_SOURCE_DIR})
25 | ENDIF(WIN32)
26 |
--------------------------------------------------------------------------------
/deps/libevent.config.h.cmake:
--------------------------------------------------------------------------------
1 | #cmakedefine HAVE_STDINT_H
2 | #cmakedefine HAVE_SYS_TIME_H
3 | #cmakedefine HAVE_SYS_TREE_H
4 | #cmakedefine HAVE_FCNTL_H
5 | #cmakedefine HAVE_STDARG_H
6 | #cmakedefine HAVE_INTTYPES_H
7 | #cmakedefine HAVE_STDLIB_H
8 | #cmakedefine HAVE_POLL_H
9 | #cmakedefine HAVE_SIGNAL_H
10 | #cmakedefine HAVE_UNISTD_H
11 | #cmakedefine HAVE_SYS_DEVPOLL_H
12 | #cmakedefine HAVE_PORT_H
13 | #cmakedefine HAVE_KQUEUE_H
14 |
15 | #cmakedefine HAVE_POLL
16 | #cmakedefine HAVE_SELECT
17 | #cmakedefine HAVE_GETTIMEOFDAY
18 | #cmakedefine HAVE_VASPRINTF
19 | #cmakedefine HAVE_FCNTL
20 | #cmakedefine HAVE_CLOCK_GETTIME
21 | #cmakedefine HAVE_STRTOK_R
22 | #cmakedefine HAVE_STRSEP
23 | #cmakedefine HAVE_GETADDRINFO
24 | #cmakedefine HAVE_GETNAMEINFO
25 | #cmakedefine HAVE_STRLCPY
26 | #cmakedefine HAVE_INET_NTOP
27 | #cmakedefine HAVE_SIGTIMEDWAIT
28 | #cmakedefine HAVE_EPOLL
29 | #cmakedefine HAVE_KQUEUE
30 | #cmakedefine HAVE_TIMERADD
31 | #cmakedefine HAVE_TIMERSUB
32 | #cmakedefine HAVE_TIMERISSET
33 | #cmakedefine HAVE_TIMERCLEAR
34 |
35 | #define VERSION "libevent-1.4.11-stable"
36 |
--------------------------------------------------------------------------------
/deps/libevent.def:
--------------------------------------------------------------------------------
1 | LIBRARY event.dll
2 | EXPORTS
3 | event_base_new
4 | event_init
5 | event_reinit
6 | event_dispatch
7 | event_base_dispatch
8 | event_base_get_method
9 | event_base_free
10 | event_set_log_callback
11 | event_base_set
12 | event_loop
13 | event_base_loop
14 | event_loopexit
15 | event_base_loopexit
16 | event_loopbreak
17 | event_base_loopbreak
18 | event_set
19 | event_once
20 | event_base_once
21 | event_add
22 | event_del
23 | event_active
24 | event_pending
25 | event_get_version
26 | event_get_method
27 | bufferevent_new
28 | bufferevent_base_set
29 | bufferevent_priority_set
30 | bufferevent_free
31 | bufferevent_setcb
32 | bufferevent_setfd
33 | bufferevent_write
34 | bufferevent_write_buffer
35 | bufferevent_read
36 | bufferevent_enable
37 | bufferevent_disable
38 | bufferevent_settimeout
39 | bufferevent_setwatermark
40 | evbuffer_new
41 | evbuffer_free
42 | evbuffer_expand
43 | evbuffer_add
44 | evbuffer_remove
45 | evbuffer_readline
46 | evbuffer_add_buffer
47 | evbuffer_add_printf
48 | evbuffer_add_vprintf
49 | evbuffer_drain
50 | evbuffer_write
51 | evbuffer_read
52 | evbuffer_find
53 | evbuffer_setcb
54 | evtag_init
55 | evtag_marshal
56 | encode_int
57 | evtag_marshal_int
58 | evtag_marshal_string
59 | evtag_marshal_timeval
60 | evtag_unmarshal
61 | evtag_peek
62 | evtag_peek_length
63 | evtag_payload_length
64 | evtag_consume
65 | evtag_unmarshal_int
66 | evtag_unmarshal_fixed
67 | evtag_unmarshal_string
68 | evtag_unmarshal_timeval
69 | evutil_socketpair
70 | evutil_make_socket_nonblocking
71 |
--------------------------------------------------------------------------------
/deps/libevent.event-config.h.cmake:
--------------------------------------------------------------------------------
1 | #cmakedefine _EVENT_HAVE_STDINT_H
2 | #cmakedefine _EVENT_HAVE_SYS_TIME_H
3 | #cmakedefine _EVENT_HAVE_SYS_TREE_H
4 | #cmakedefine _EVENT_HAVE_FCNTL_H
5 | #cmakedefine _EVENT_HAVE_STDARG_H
6 | #cmakedefine _EVENT_HAVE_INTTYPES_H
7 | #cmakedefine _EVENT_HAVE_STDLIB_H
8 | #cmakedefine _EVENT_HAVE_POLL_H
9 | #cmakedefine _EVENT_HAVE_SIGNAL_H
10 | #cmakedefine _EVENT_HAVE_UNISTD_H
11 | #cmakedefine _EVENT_HAVE_SYS_DEVPOLL_H
12 | #cmakedefine _EVENT_HAVE_PORT_H
13 | #cmakedefine _EVENT_HAVE_KQUEUE_H
14 |
15 | #cmakedefine _EVENT_HAVE_POLL
16 | #cmakedefine _EVENT_HAVE_SELECT
17 | #cmakedefine _EVENT_HAVE_GETTIMEOFDAY
18 | #cmakedefine _EVENT_HAVE_VASPRINTF
19 | #cmakedefine _EVENT_HAVE_FCNTL
20 | #cmakedefine _EVENT_HAVE_CLOCK_GETTIME
21 | #cmakedefine _EVENT_HAVE_STRTOK_R
22 | #cmakedefine _EVENT_HAVE_STRSEP
23 | #cmakedefine _EVENT_HAVE_GETADDRINFO
24 | #cmakedefine _EVENT_HAVE_GETNAMEINFO
25 | #cmakedefine _EVENT_HAVE_STRLCPY
26 | #cmakedefine _EVENT_HAVE_INET_NTOP
27 | #cmakedefine _EVENT_HAVE_SIGTIMEDWAIT
28 | #cmakedefine _EVENT_HAVE_EPOLL
29 | #cmakedefine _EVENT_HAVE_KQUEUE
30 | #cmakedefine _EVENT_HAVE_TIMERADD
31 | #cmakedefine _EVENT_HAVE_TIMERSUB
32 | #cmakedefine _EVENT_HAVE_TIMERISSET
33 | #cmakedefine _EVENT_HAVE_TIMERCLEAR
34 |
35 | #define _EVENT_VERSION "libevent-1.4.11-stable"
36 |
--------------------------------------------------------------------------------
/doc/Makefile.am:
--------------------------------------------------------------------------------
1 | SUBDIRS=chapter
2 |
3 | EXTRA_DIST = \
4 | lua-classes.dot \
5 | architecture.dot \
6 | architecture-overview.dot \
7 | architecture.txt \
8 | core.txt \
9 | chassis.txt \
10 | plugins.txt \
11 | protocol.txt \
12 | tests.txt \
13 | scripting.txt \
14 | lifecycle.msc
15 |
16 | clean-local:
17 | rm -f *.html
18 |
19 | html-local: book.html protocol.html scripting.html
20 |
21 | ## we use http://docutils.sourceforge.net/rst.html to generate the docs
22 | book.html: book.txt chapter/scripting.txt chapter/protocol.txt
23 | ${RST2HTML} $< $@
24 |
25 | protocol.html: protocol.txt chapter/protocol.txt
26 | ${RST2HTML} $< $@
27 |
28 | scripting.html: scripting.txt chapter/scripting.txt
29 | ${RST2HTML} $< $@
30 |
31 |
--------------------------------------------------------------------------------
/doc/architecture-overview.dot:
--------------------------------------------------------------------------------
1 | /*
2 | the building blocks of the proxy
3 |
4 | You may use "dot" from graphviz to generate human consumable output
5 | like PNG or PDF:
6 |
7 | $ dot -T pdf -o architecture.pdf architecture.dot
8 | $ dot -T png -o architecture.png architecture.dot
9 |
10 | or you generate a image-map
11 |
12 | $ dot -T cmapx -o architecture.map -Tpng -o architecture.png architecture.dot && \
13 | echo '
' > architecture.html && \
14 | cat architecture.map >> architecture.html
15 | */
16 | digraph G {
17 | node [
18 | shape = "plaintext"
19 | fontname = "Courier"
20 | fontsize = 10
21 | ]
22 |
23 | rank = "same";
24 | rankdir = "LR";
25 | subgraph clients {
26 | Clients1 [
27 | label = <
28 |
29 |
30 |
31 | Client
32 | |
33 |
34 |
39 | |
40 | >
41 | ];
42 |
43 | Clients2 [
44 | label = <
45 |
46 |
47 |
48 | Client
49 | |
50 |
51 |
56 | |
57 | >
58 | ];
59 | }
60 |
61 | subgraph backends {
62 | Backend1 [
63 | label = <
64 |
65 |
66 |
67 | MySQL Server
68 | |
69 |
70 |
71 |
72 | 10.0.0.1:3306 |
73 |
74 |
75 | |
76 | >
77 |
78 | ];
79 |
80 | Backend2 [
81 | label = <
82 |
83 |
84 |
85 | MySQL Server
86 | |
87 |
88 |
89 |
90 | 10.0.0.2:3306 |
91 |
92 |
93 | |
94 | >
95 |
96 | ];
97 |
98 | }
99 |
100 | Blocks [
101 | label = <
102 |
103 |
104 |
105 | MySQL Proxy
106 | |
107 |
108 |
109 |
110 | script |
111 |
112 |
113 | plugin |
114 |
115 |
116 | network core |
117 |
118 |
119 | |
120 | >
121 | ]
122 |
123 |
124 | Clients1:app:e -> Blocks:w;
125 | Clients2:app:e -> Blocks:w;
126 | Blocks:e -> Backend1:port:w;
127 | Blocks:e -> Backend2:port:w;
128 | }
129 |
--------------------------------------------------------------------------------
/doc/architecture.txt:
--------------------------------------------------------------------------------
1 | /**
2 | @mainpage
3 |
4 | The MySQL Proxy is a simple program which sits between a mysql client and a mysql server and
5 | can inspect, transform and act on the data sent through it.
6 |
7 | You can use it for:
8 | @li load balancing
9 | @li fail over
10 | @li query tracking
11 | @li query analysis
12 | @li ... and much more
13 |
14 | Internally the MySQL Proxy is a stack of:
15 |
16 | @dotfile architecture-overview.dot
17 |
18 | It is based on a @subpage page-core that exposes the phases of the
19 | @subpage protocol to a @ref page-plugins.
20 |
21 | @dot
22 | digraph {
23 | connect -> auth;
24 | auth -> command;
25 | command -> disconnect;
26 | command -> command;
27 | connect -> disconnect;
28 | auth -> disconnect;
29 | }
30 |
31 | @enddot
32 |
33 | Each of the phases of the life-cycle lead to several more protocol-states. For example the auth phase is made up of at least 3 packets:
34 |
35 | @msc
36 | Client, Proxy, Server;
37 |
38 | Client -> Proxy [ label = "accept()" ];
39 | Proxy -> Proxy [ label = "script: connect_server()" ];
40 | Proxy -> Server [ label = "connect()" ];
41 | ...;
42 | Server -> Proxy [ label = "recv(auth-challenge)" ];
43 | Proxy -> Proxy [ label = "script: read_handshake()" ];
44 | Proxy -> Client [ label = "send(auth-challenge)" ];
45 | Client -> Proxy [ label = "recv(auth-response)" ];
46 | Proxy -> Proxy [ label = "script: read_auth()" ];
47 | Server -> Proxy [ label = "send(auth-response)" ];
48 | Server -> Proxy [ label = "recv(auth-result)" ];
49 | Proxy -> Proxy [ label = "script: read_auth_result()" ];
50 | Proxy -> Client [ label = "send(auth-result)" ];
51 | ...;
52 |
53 | @endmsc
54 |
55 | While the @ref page-core is scalable to a larger number of connections, the plugin/scripting
56 | layer hides the complexity from the end-users and simplifies the customization.
57 |
58 | @section section-stack-of-libs Chassis, libraries and Plugins
59 |
60 | It is built as a stack of libraries:
61 |
62 | The @subpage page-chassis provides the common functions that all commandline and daemon applications
63 | need:
64 | @li commandline and configfiles
65 | @li logging
66 | @li daemon/service support
67 | @li plugin loading
68 |
69 | The MySQL Procotol libraries which can encode and decode:
70 | @li client protocol
71 | @li binlog protocol
72 | @li myisam files
73 | @li frm files
74 | @li masterinfo files
75 |
76 | The @ref page-core and the @subpage page-plugins.
77 |
78 | @dotfile architecture.dot
79 |
80 | */
81 |
--------------------------------------------------------------------------------
/doc/chapter/Makefile.am:
--------------------------------------------------------------------------------
1 | EXTRA_DIST=\
2 | protocol.txt \
3 | scripting.txt
4 |
5 |
--------------------------------------------------------------------------------
/doc/core.txt:
--------------------------------------------------------------------------------
1 | /**
2 |
3 | @page page-core Network Core
4 |
5 | The Network core is built around the socket handling and brings a client and server connection
6 | together.
7 |
8 | @see network_socket
9 | @see network_mysqld_con
10 |
11 | @section section-lifecycle Connection Life Cycle
12 |
13 | Connections can be in one of several states which are basicly resembling the 4 basic phases
14 | of the @ref protocol :
15 |
16 | @li connect
17 | @li authentification
18 | @li query
19 | @li disconnect
20 |
21 | The plugins can change the default behaviour of the network core and impliment one of three
22 | basic plugins:
23 |
24 | @li @ref page-plugin-admin implements only the listening side
25 | @li client plugins implement only the connection side
26 | @li @ref page-plugin-proxy implements both sides
27 |
28 | @subsection section-scripting Scripting
29 |
30 | Most plugins implement a set of those callbacks and expose them to a scripting layer.
31 |
32 | For now the scriping is provided by Lua, a simple, fast and easy to embed scripting language.
33 | We expose most of the internals into the scripting layer and provide modules to operate
34 | on the data that we pass into the scripting layer
35 |
36 | @section section-network-core-layer Network Core Layer
37 |
38 | The MySQL Proxy network engine is meant to handle several thousands connections at the same time. We
39 | want to use it for load-balancing and fail-over which means we have to handle the connections for
40 | a larger group of MySQL backend servers nicely. We aim for 5k to 10k connections.
41 | Up to MySQL Proxy 0.7 we use a pure event-driven, non-blocking networking approach is described in
42 | http://kegel.com/c10k.html#nb using libevent 1.4.x.
43 | A event-driven design has a very small foot-print for idling connections: we just store the
44 | connection state and let it wait for a event.
45 |
46 | @section section-threaded-scripting Threaded Scripting
47 |
48 | Usually the scripts are small and only make simple decisions leaving most of the work to the network layer.
49 | In 0.9 we will make the scripting layer multi-threaded allow several scripting threads at the same time,
50 | working from a small pool threads.
51 |
52 | That will allow the scripting layer to call blocking or slow functions without infecting the execution of
53 | other connections.
54 |
55 | Lifting the global plugin mutex will mean we have to handle access to global structure differently. Most
56 | of the access is happening on connection-level (the way we do the event-threading) and only access to
57 | global structures like "proxy.global.*" has to synchronized. For that we will look into using Lua lanes
58 | to send data around between independent Lua-states.
59 |
60 |
61 |
62 |
63 | */
64 |
--------------------------------------------------------------------------------
/doc/lifecycle.msc:
--------------------------------------------------------------------------------
1 | #
2 | # mscgen -T png -o lifecycle.png < lifecycle.msc
3 |
4 |
5 |
6 | msc {
7 | Client, Core, Plugin, Server;
8 |
9 | # Core box Plugin [ label = "MySQL Proxy" ];
10 |
11 | --- [ label = "connect" ];
12 | Client -> Core [ label = "connect()" ];
13 | Core => Plugin [ label = "connect_server()" ];
14 | Core << Plugin [ label = "NO_DECISION" ];
15 | Core -> Server [ label = "connect()" ];
16 |
17 | --- [ label = "auth challenge" ];
18 | Server -> Core [ label = "read(auth challenge packet)" ];
19 | Core => Plugin [ label = "read_auth_handshake()" ];
20 | Core << Plugin [ label = "NO_DECISION" ];
21 | Core -> Client [ label = "write(auth challenge packet)" ];
22 | --- [ label = "auth response" ];
23 | Client -> Core [ label = "read(auth response packet)" ];
24 | Core => Plugin [ label = "read_auth()" ];
25 | Core << Plugin [ label = "NO_DECISION" ];
26 | Core -> Server [ label = "write(auth response packet)" ];
27 | --- [ label = "auth status" ];
28 | Server -> Core [ label = "read(auth response packet)" ];
29 | Core => Plugin [ label = "read_auth_result()" ];
30 | Core << Plugin [ label = "NO_DECISION" ];
31 | Core -> Client [ label = "write(auth response packet)" ];
32 | --- [ label = "query" ];
33 | Client -> Core [ label = "read(command packet)" ];
34 | Core => Plugin [ label = "read_query()" ];
35 | Core << Plugin [ label = "NO_DECISION" ];
36 | Core -> Server [ label = "write(command packet)" ];
37 | --- [ label = "query response" ];
38 | Server -> Core [ label = "read(command response packet)" ];
39 | Core => Plugin [ label = "read_query_result()" ];
40 | Core << Plugin [ label = "NO_DECISION" ];
41 | Core -> Client [ label = "write(command response packet)" ];
42 |
43 | --- [ label = "disconnect" ];
44 | Client -> Core [ label = "close()" ];
45 | Core => Plugin [ label = "disconnect_client()" ];
46 | Core << Plugin [ label = "ignored" ];
47 | Core -> Server [ label = "close()" ];
48 |
49 | }
50 |
--------------------------------------------------------------------------------
/doc/plugins.txt:
--------------------------------------------------------------------------------
1 | /**
2 |
3 | @page page-plugins Plugin and Scripting Layer
4 |
5 | What is usually being referred to as MySQL Proxy is in fact the @link page-plugin-proxy Proxy plugin@endlink.
6 |
7 | While the @ref page-chassis and @ref page-core make up an important part, it is really the plugins that make MySQL Proxy so flexible.
8 |
9 | One can describe the currently available plugins as connection lifecycle interceptors which can register callbacks for
10 | all states in the @ref protocol.
11 |
12 | Currently available plugins in the main distribution include:
13 | - @subpage page-plugin-proxy
14 | - @subpage page-plugin-admin
15 | - Replicator plugin
16 | - Debug plugin
17 | - CLI (command line) plugin
18 |
19 | @note The latter three are not documented in-depth, mainly because they are Proof Of Concept implementations that are not targeted
20 | for the MySQL Proxy 1.0 GA release.
21 |
22 | */
23 |
--------------------------------------------------------------------------------
/doc/protocol.txt:
--------------------------------------------------------------------------------
1 | ==============
2 | MySQL Protocol
3 | ==============
4 |
5 | .. contents::
6 |
7 | .. include:: chapter/protocol.txt
8 |
9 |
--------------------------------------------------------------------------------
/doc/scripting.txt:
--------------------------------------------------------------------------------
1 | ==========================
2 | Scripting the proxy plugin
3 | ==========================
4 |
5 | .. contents::
6 |
7 | .. include:: chapter/scripting.txt
8 |
9 |
--------------------------------------------------------------------------------
/doxygen-scripts/doxygen_version.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | # $%BEGINLICENSE%$
3 | # $%ENDLICENSE%$
4 |
5 | bzr log --limit 1 --line $1 | sed -n 's/^\([0-9]*\):.*$/\1/p'
6 |
--------------------------------------------------------------------------------
/doxygen-scripts/footer.html:
--------------------------------------------------------------------------------
1 |
2 | Generated on $datetime for $projectname version $projectnumber.
3 |