├── .dockerignore ├── .gitignore ├── .travis.yml ├── ChangeLog ├── Dockerfile ├── LICENSE ├── Makefile.am ├── README.md ├── README.zh_CN.md ├── TODO ├── autogen.sh ├── client ├── c │ ├── README │ └── sample │ │ ├── Makefile │ │ ├── echo2kids.c │ │ ├── loggen.c │ │ └── subscribe.c ├── go │ ├── kids.go │ └── kids_test.go └── nodejs │ ├── example │ ├── publish.js │ └── subscrible.js │ └── kids.js ├── configure.ac ├── debian ├── Dockerfile ├── after-install ├── before-remove ├── kids └── make_deb.sh ├── deps └── ae │ ├── Makefile.am │ ├── ae.c │ ├── ae.h │ ├── ae_epoll.c │ ├── ae_evport.c │ ├── ae_kqueue.c │ ├── ae_select.c │ ├── anet.c │ ├── anet.h │ ├── configure.ac │ ├── fmacros.h │ ├── libae.h │ ├── zmalloc.c │ └── zmalloc.h ├── doc ├── Makefile.am ├── config.md ├── config.zh_CN.md ├── deploy.md ├── deploy.zh_CN.md ├── image │ ├── arch.jpg │ └── thread.jpg ├── overview.md ├── overview.zh_CN.md ├── store.md └── store.zh_CN.md ├── docker └── sources.list ├── samples ├── agent.conf ├── dev.conf └── server.conf ├── src ├── Makefile.am ├── buffer.cc ├── buffer.h ├── client.cc ├── client.h ├── common.h ├── conf.cc ├── conf.h ├── constants.h ├── filesystem.cc ├── filesystem.h ├── kids.cc ├── kids.h ├── lexer.c ├── lexer.rl ├── logger.cc ├── logger.h ├── master.cc ├── master.h ├── msgqueue.cc ├── msgqueue.h ├── parser.c ├── parser.h ├── parser.y ├── pubsub.cc ├── sds.c ├── sds.h ├── store │ ├── bufferstore.cc │ ├── bufferstore.h │ ├── filestore.cc │ ├── filestore.h │ ├── multiplestore.cc │ ├── multiplestore.h │ ├── networkstore.cc │ ├── networkstore.h │ ├── prioritystore.cc │ ├── prioritystore.h │ ├── store.cc │ └── store.h ├── storer.cc ├── storer.h ├── util.cc ├── util.h ├── worker.cc └── worker.h ├── test ├── Makefile.am ├── data │ └── conf │ │ ├── agent.conf │ │ └── server.conf ├── funcational │ ├── qps.py │ └── test.py ├── gtest │ ├── gtest-all.cc │ └── gtest.h ├── test_buffer.cc ├── test_conf.cc ├── test_filesystem.cc ├── test_main.cc ├── test_message.cc ├── test_pattern.cc └── test_util.cc ├── tools └── collect_kids.py └── version.m4 /.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhihu/kids/509cd95e468b8eb98098134a63938eb27f1d0dbf/.dockerignore -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | Makefile.in 2 | aclocal.m4 3 | compile 4 | config.guess 5 | config.h.in 6 | config.sub 7 | configure 8 | depcomp 9 | aclocal.m4 10 | ar-lib 11 | install-sh 12 | missing 13 | test-driver 14 | kids-*.tar.gz 15 | 16 | debug/ 17 | src/store/.dirstamp 18 | core.* 19 | epm/kids.list 20 | client/c/sample/loggen 21 | client/c/sample/echo2kids 22 | client/c/sample/subscribe 23 | client/python/build/ 24 | client/python/dist/ 25 | *.egg-info/ 26 | .installed.cfg 27 | Makefile 28 | *.a 29 | *.heap 30 | *.la 31 | linux/ 32 | autom4te.cache/ 33 | .deps 34 | .lib 35 | *.log 36 | *.pyc 37 | *.dSYM/ 38 | *.o 39 | *~ 40 | *.out 41 | *.bak 42 | *.scan 43 | *core 44 | config.h 45 | *.status 46 | src/kids 47 | stamp-h1 48 | epm/linux-*/ 49 | *.deb 50 | debian/kids.conf 51 | *.tar.gz 52 | 53 | *.deps 54 | *.dirstamp 55 | debug/* 56 | callgrind.* 57 | !client/c/sample/Makefile 58 | \#*\# 59 | .\#* 60 | test/runtest* 61 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: objective-c 2 | 3 | xcode_sdk: iphonesimulator 4 | 5 | script: 6 | - ./autogen.sh && ./configure && make check 7 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | v 1.1.0 2 | - Add docker/sources.list. 3 | - Add support for `max_clients` in config file. 4 | - Add support for redis command `PUNSUBSCRIBE` to unsubscribe patterns. 5 | - Add Nodejs Client for kids. 6 | - Add topic wildcards in PriorityStore example. 7 | - Output deb in project directory. 8 | - All reference to version now comes from version.m4. 9 | - Change LICENSE to BSD-3-Clause in make_deb.sh. 10 | - Add contributors to LICENSE. 11 | - Add packaging for deb. 12 | - Convert README to Unix format and minor edit. 13 | - Update description for `ignore_case` in doc. 14 | - Add Travis CI. 15 | - Ask users to download releases instead. 16 | 17 | v 1.0.0 18 | - First public version. 19 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:10 2 | 3 | MAINTAINER Li Yichao 4 | 5 | 6 | RUN apt-get update && \ 7 | apt-get install -y --no-install-recommends \ 8 | build-essential \ 9 | libtool \ 10 | automake 11 | 12 | WORKDIR /kids 13 | 14 | COPY . /kids 15 | RUN ./autogen.sh && ./configure && make 16 | 17 | EXPOSE :3388 18 | 19 | CMD ["src/kids", "-c", "/kids/debian/kids.conf"] 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, Zhihu Inc. 2 | Some rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of the Kids nor the 12 | names of its contributors may be used to endorse or promote products 13 | derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | 26 | Contributor(s): 27 | Dong Chengcheng (dcc@zhihu.com) 28 | Li Yichao (lyc@zhihu.com) 29 | Chen Peng (chen@zhihu.com) 30 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = deps/ae src doc test 2 | 3 | all-am: 4 | cd deps/ae && $(MAKE) 5 | 6 | install: 7 | cd src && $(MAKE) install 8 | 9 | deb: all-am 10 | cd debian && ./make_deb.sh 11 | 12 | kids := kids_@VERSION@ 13 | 14 | bin: all-am 15 | mkdir $(kids) && cp src/kids $(kids) && cp README.md $(kids) && cp samples/{agent.conf,server.conf} $(kids) \ 16 | && cp LICENSE $(kids) && tar czf $(kids)_amd64.tar.gz $(kids) && rm -rf $(kids) 17 | test: check 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kids 2 | 3 | [![Build Status]][Travis CI] 4 | 5 | Kids is a log aggregation system. 6 | 7 | It aggregates messages like [Scribe](https://github.com/facebookarchive/scribe) and its pub/sub pattern is ported from [Redis](http://redis.io/). 8 | 9 | [中文文档](README.zh_CN.md) 10 | ## Features 11 | 12 | * Real-time subscription 13 | * Distributed collection 14 | * Message persistence 15 | * Multithreading 16 | * Redis protocol 17 | * No third-party dependencies 18 | 19 | ## Quickstart 20 | 21 | ### From Source 22 | 23 | You need a complier with C++11 support like GCC 4.7 (or later) or [Clang](http://clang.llvm.org). 24 | 25 | Download a [source release](https://github.com/zhihu/kids/releases), then: 26 | 27 | tar xzf kids-VERSION.tar.gz 28 | cd kids-VERSION 29 | ./configure 30 | make 31 | make test # optional 32 | make install 33 | 34 | By default, it will be installed to `/usr/local/bin/kids`. 35 | You can use the `--prefix` option to specify the installation location. 36 | Run `./configure --help` for more config options. 37 | 38 | Kids comes with some sample config files in `samples/`, after building, simply run: 39 | 40 | kids -c samples/dev.conf 41 | 42 | Because kids uses redis protocol, you can use `redis-cli` to play with it, open another terminal: 43 | 44 | $ redis-cli -p 3888 45 | $ 127.0.0.1:3388> PSUBSCRIBE * 46 | 47 | In yet another terminal: 48 | 49 | $ redis-cli -p 3388 50 | $ 127.0.0.1:3388> PUBLISH kids.testtopic message 51 | 52 | `redis-cli` needs `redis` to be installed. On Mac, you can run `brew install redis` to install it. On Linux, run `sudo apt-get install redis-tools` 53 | 54 | Run `kids --help` for more running options. 55 | 56 | ### Using docker 57 | 58 | Do the following: 59 | 60 | git clone https://github.com/zhihu/kids.git 61 | cd kids 62 | cp samples/dev.conf debian/kids.conf 63 | docker build -t zhihu/kids . 64 | 65 | Now you can run it like this: 66 | 67 | docker run -d -p 3388:3388 zhihu/kids 68 | 69 | You can also specify the config file like this: 70 | 71 | docker run -d -v /path/to/kids/conf:/etc/kids.conf -p 3388:3388 zhihu/kids 72 | 73 | ## Configuration 74 | 75 | See [configuration](doc/config.md). 76 | 77 | ## Run in production 78 | 79 | see [production](doc/deploy.md). 80 | 81 | ## Developer 82 | 83 | You will need 84 | 85 | * build-essential 86 | * libtool 87 | * automake 88 | * c++ compiler with c++ 11 support like gcc4.7+ or [Clang](http://clang.llvm.org) 89 | 90 | to build kids from source. Run the following to build kids: 91 | 92 | ./autogen.sh 93 | ./configure 94 | make 95 | 96 | ## License 97 | 98 | Kids Uses BSD-3, see LICENSE for more details. 99 | 100 | 101 | ## FAQ 102 | 103 | Q: What is the meaning of "kids"? 104 | A: "kids" is the recursive acronym of "Kids Is Data Stream". 105 | 106 | 107 | ## Architecture 108 | 109 | ![image](doc/image/arch.jpg) 110 | 111 | 112 | 113 | 114 | [Build Status]: https://img.shields.io/travis/zhihu/kids/master.svg?style=flat 115 | [Travis CI]: https://travis-ci.org/zhihu/kids 116 | -------------------------------------------------------------------------------- /README.zh_CN.md: -------------------------------------------------------------------------------- 1 | # kids 2 | 3 | [![Build Status]][Travis CI] 4 | 5 | 6 | Kids 是一个日志收集系统。 7 | 8 | 采用 [Scribe](https://github.com/facebookarchive/scribe) 的消息聚合模型和 [Redis](http://redis.io/) 的 pub/sub 模型。 9 | 10 | 11 | ## 特性 12 | 13 | * 实时订阅 14 | * 分布式收集，集中存储 15 | * 多线程模型 16 | * 使用 Redis 协议 17 | * 无第三方依赖 18 | 19 | 20 | ## 快速开始 21 | 22 | ### 从源码编译 23 | 24 | 编译 kids 需要 C++11 支持，如 GCC 4.7 或更高版本或 [Clang](http://clang.llvm.org) 25 | 26 | 下载 [源码发布包](https://github.com/zhihu/kids/releases)（文件名为 kids-VERSION.tar.gz），运行： 27 | 28 | tar xzf kids-VERSION.tar.gz 29 | cd kids-VERSION 30 | ./configure 31 | make 32 | 33 | Kids samples/ 文件夹带了一些示例配置，编译好后，运行： 34 | 35 | kids -c samples/dev.conf 36 | 37 | kids 使用 redis 协议，现在你可以用 `redis-cli` 和它通信： 38 | 39 | $ redis-cli -p 3388 40 | $ 127.0.0.1:3388> PSUBSCRIBE * 41 | 42 | 在另一个终端： 43 | 44 | $ redis-cli -p 3388 45 | $ 127.0.0.1:3388> PUBLISH test message 46 | 47 | `redis-cli` 命令需要安装 redis。在 MAC 上，可以用 `brew install redis` 安装它。 48 | 配置文件的具体选项详见 [配置](doc/config.zh_CN.md)。 49 | 50 | 执行 `kids --help` 查看更多选项。 51 | 52 | ### 使用 Docker 53 | 54 | 如果已经装了 Docker，使用 Docker 是最快速开始的方式。 55 | 56 | git clone https://github.com/zhihu/kids.git 57 | cd kids 58 | cp samples/dev.conf debian/kids.conf 59 | docker build -t zhihu/kids . 60 | 61 | 现在，你可以这样运行： 62 | 63 | docker run -d -p 3388:3388 zhihu/kids 64 | 65 | 你也可以指定配置文件： 66 | 67 | docker run -d -v /path/to/kids.conf:/etc/kids.conf -p 3388:3388 zhihu/kids 68 | 69 | ## 配置 70 | 71 | 请看[配置](doc/config.zh_CN.md)。 72 | 73 | ## 生产环境部署 74 | 75 | 请看[生产环境部署](doc/deploy.zh_CN.md)。 76 | 77 | ## 开发者 78 | 79 | 你需要以下东西来从源码编译 kids： 80 | 81 | * build-essential 82 | * libtool 83 | * automake 84 | * c++ compiler with c++ 11 support like gcc4.7+ or [Clang](http://clang.llvm.org) 85 | 86 | 运行以下命令编译 kids： 87 | 88 | ./autogen.sh 89 | ./configure 90 | make 91 | 92 | 93 | ## 开源协议 94 | 95 | 96 | Kids 使用 BSD-3 协议，具体内容详见 LICENSE 文件。 97 | 98 | 99 | ## FAQ 100 | 101 | Q: 为什么叫「kids」? 102 | A: 「kids」是「Kids Is Data Stream」的递归缩写。 103 | 104 | 105 | ## 架构图 106 | 107 | ![image](doc/image/arch.jpg) 108 | 109 | [Build Status]: https://img.shields.io/travis/zhihu/kids/master.svg?style=flat 110 | [Travis CI]: https://travis-ci.org/zhihu/kids 111 | 112 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhihu/kids/509cd95e468b8eb98098134a63938eb27f1d0dbf/TODO -------------------------------------------------------------------------------- /autogen.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | if [ -x "`which autoreconf 2>/dev/null`" ] ; then 4 | exec autoreconf -ivf 5 | fi 6 | 7 | if glibtoolize --version > /dev/null 2>&1; then 8 | LIBTOOLIZE='glibtoolize' 9 | else 10 | LIBTOOLIZE='libtoolize' 11 | fi 12 | 13 | $LIBTOOLIZE && \ 14 | aclocal && \ 15 | automake --add-missing --force-missing --include-deps && \ 16 | autoconf 17 | -------------------------------------------------------------------------------- /client/c/README: -------------------------------------------------------------------------------- 1 | simply use hiredis as kids client 2 | -------------------------------------------------------------------------------- /client/c/sample/Makefile: -------------------------------------------------------------------------------- 1 | all: loggen subscribe echo2kids 2 | 3 | subscribe: subscribe.c 4 | gcc -g -o subscribe subscribe.c -lhiredis 5 | 6 | loggen: loggen.c 7 | gcc -g -o loggen loggen.c -lhiredis 8 | 9 | echo2kids: echo2kids.c 10 | gcc -g -o echo2kids echo2kids.c -lhiredis 11 | 12 | clean: 13 | rm -rf loggen subscribe echo2kids 14 | -------------------------------------------------------------------------------- /client/c/sample/echo2kids.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "hiredis/hiredis.h" 8 | 9 | void Usage(const char *program_name) { 10 | printf("Usage: %s [options]\n" 11 | "options:\n" 12 | " -s, --socket unix socket of kids\n" 13 | " -h, --host host of kids (at least one of ip and socket must be specified)\n" 14 | " -p, --port port of kids (default is 3388)\n" 15 | " -t, --topic topic of log send to kids (default is \"KID_SAMPLE\")\n" 16 | " -h, --help print help and quit\n\n", 17 | program_name); 18 | exit(0); 19 | } 20 | 21 | const char* const short_options = "s:h:p:t:f:n:qH"; 22 | const struct option long_options[] = { 23 | { "socket", required_argument, NULL, 's' }, 24 | { "host", required_argument, NULL, 'h' }, 25 | { "port", required_argument, NULL, 'p' }, 26 | { "topic", required_argument, NULL, 't' }, 27 | { "help", no_argument, NULL, 'H' }, 28 | { NULL, 0, NULL, 0 }, 29 | }; 30 | 31 | int main(int argc, char **argv) { 32 | redisContext *c; 33 | redisReply *reply; 34 | const char *socket = NULL; 35 | const char *ip = "127.0.0.1"; 36 | int port = 3388; 37 | const char *topic = "loggen"; 38 | char logbuf[10240]; 39 | int opt; 40 | int daemon; 41 | struct timeval timeout = { 1, 500000 }; // 1.5 seconds 42 | 43 | while ((opt = getopt_long(argc, argv, short_options, long_options, NULL)) > 0) { 44 | switch (opt) { 45 | case 's': 46 | socket = optarg; 47 | break; 48 | case 'h': 49 | ip = optarg; 50 | break; 51 | case 'p': 52 | port = atoi(optarg); 53 | break; 54 | case 't': 55 | topic = optarg; 56 | break; 57 | case 'd': 58 | daemon = 1; 59 | break; 60 | case 'H': 61 | default: 62 | Usage(argv[0]); 63 | } 64 | } 65 | 66 | if (socket == NULL && ip == NULL) { 67 | Usage(argv[0]); 68 | } 69 | 70 | if (socket) { 71 | c = redisConnectUnixWithTimeout(socket, timeout); 72 | } else { 73 | c = redisConnectWithTimeout((char*)ip, port, timeout); 74 | } 75 | if (c->err) { 76 | printf("Connection error: %s\n", c->errstr); 77 | exit(1); 78 | } 79 | 80 | char *line = NULL; 81 | size_t linecap = 0; 82 | ssize_t linelen; 83 | while (1) { 84 | if ((linelen = getline(&line, &linecap, stdin)) <= 0) { 85 | break; 86 | } 87 | line[linelen - 1] = '\0'; 88 | reply = redisCommand(c, "PUBLISH %s %s", topic, line); 89 | if (reply) { 90 | freeReplyObject(reply); 91 | } else { 92 | printf("LOG %s : failed\n", logbuf); 93 | return -1; 94 | } 95 | 96 | } 97 | 98 | redisFree(c); 99 | } 100 | -------------------------------------------------------------------------------- /client/c/sample/loggen.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "hiredis/hiredis.h" 8 | 9 | void Usage(const char *program_name) { 10 | printf("Usage: %s [options]\n" 11 | "options:\n" 12 | " -s, --socket unix socket of kids\n" 13 | " -h, --host host of kids (at least one of ip and socket must be specified)\n" 14 | " -p, --port port of kids (default is 3388)\n" 15 | " -t, --topic topic of log send to kids (default is \"KID_SAMPLE\")\n" 16 | " -f, --file read logs from file\n" 17 | " -n, --num number of logs send to kids (default is -1 means infinite)\n" 18 | " -q, --quiet don't print logs on screen\n" 19 | " --period