├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── Modules ├── Findlibadbase.cmake ├── Findlibadbase_kafka.cmake ├── Findlibevent.cmake ├── Findlibpthread.cmake ├── Findlibrdkafka.cmake └── Findlibyaml-cpp.cmake ├── README.md ├── adbase.ini ├── cmake.sh ├── conf ├── aidt.yml └── system.ini ├── docs ├── configure.md ├── file_transfer.md ├── image │ └── aidt.png └── upgrade.md ├── rpm ├── aidt.spec ├── aidt.spec.in ├── build_rpm └── build_rpm.in └── src ├── AdServer.cpp ├── AdServer.hpp ├── AdbaseConfig.hpp ├── Aims.cpp ├── Aims.hpp ├── Aims └── Kafka │ ├── ProducerIn.cpp │ └── ProducerIn.hpp ├── App.cpp ├── App.hpp ├── App ├── Config.cpp ├── Config.hpp ├── ConfigPaser.cpp ├── ConfigPaser.hpp ├── DataStore.cpp ├── DataStore.hpp ├── Message.cpp ├── Message.hpp ├── Reader.cpp ├── Reader.hpp ├── Watcher.cpp └── Watcher.hpp ├── BootStrap.cpp ├── BootStrap.hpp ├── BootStrapCli.cpp ├── BootStrapCli.hpp ├── CMakeLists.txt ├── HeadProcessor.cpp ├── HeadProcessor.hpp ├── Http.cpp ├── Http.hpp ├── Http ├── Config.cpp ├── Config.hpp ├── HttpInterface.cpp ├── HttpInterface.hpp ├── Index.cpp ├── Index.hpp ├── Message.cpp ├── Message.hpp ├── Server.cpp ├── Server.hpp ├── Topic.cpp └── Topic.hpp ├── McProcessor.cpp ├── McProcessor.hpp ├── Timer.cpp ├── Timer.hpp ├── Version.hpp.in ├── aidt.cpp ├── aidtCli.cpp └── thridpart └── inotify ├── config.h ├── inotify-nosys.h ├── inotify.h ├── inotifytools.c ├── inotifytools.h ├── inotifytools_p.h ├── redblack.c └── redblack.h /.gitignore: -------------------------------------------------------------------------------- 1 | _build/* 2 | src/Version.hpp 3 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | PROJECT(aidt) 3 | 4 | SET (CMAKE_VERBOSE_MAKEFILE on) 5 | SET (PROJECT_VERSION_MAJOR 1) 6 | SET (PROJECT_VERSION_MINOR 1) 7 | SET (PROJECT_VERSION_PATCH 0) 8 | 9 | SET(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules) 10 | SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wl,-Bstatic -static-libstdc++ -static-libgcc") 11 | SET(LINK_FLAGS "${LINK_FLAGS}") 12 | 13 | ADD_SUBDIRECTORY(src bin) 14 | 15 | IF (NOT DEFINED CMAKE_BINARY_DIR) 16 | SET(CMAKE_BINARY_DIR ${PROJECT_SOURCE_DIR}/_build) 17 | ENDIF (NOT DEFINED CMAKE_BINARY_DIR) 18 | 19 | if (NOT DEFINED CMAKE_INSTALL_PREFIX) 20 | set(CMAKE_INSTALL_PREFIX "/usr/local/aidt") 21 | endif (NOT DEFINED CMAKE_INSTALL_PREFIX) 22 | 23 | INCLUDE (InstallRequiredSystemLibraries) 24 | SET (CPACK_RESOURCE_FILE_LICENSE 25 | "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE") 26 | IF(NOT DEFINED CPACK_SYSTEM_NAME) 27 | SET(CPACK_SYSTEM_NAME "${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}") 28 | ENDIF(NOT DEFINED CPACK_SYSTEM_NAME) 29 | 30 | SET (CPACK_PACKAGE_VERSION_MAJOR "${PROJECT_VERSION_MAJOR}") 31 | SET (CPACK_PACKAGE_VERSION_MINOR "${PROJECT_VERSION_MINOR}") 32 | SET (CPACK_PACKAGE_VERSION_PATCH "${PROJECT_VERSION_PATCH}") 33 | SET(CPACK_SOURCE_GENERATOR "TGZ;TBZ2;TZ") 34 | INCLUDE (CPack) 35 | 36 | INSTALL(DIRECTORY conf DESTINATION . PATTERN ".git" EXCLUDE) 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Modules/Findlibadbase.cmake: -------------------------------------------------------------------------------- 1 | MESSAGE(STATUS "Using bundled Findlibadbase.cmake...") 2 | FIND_PATH( 3 | LIBADBASE_INCLUDE_DIR 4 | Version.hpp 5 | /usr/local/include/adbase 6 | /usr/include/ 7 | ) 8 | 9 | FIND_LIBRARY( 10 | LIBADBASE_LIBRARIES NAMES libadbase.a 11 | PATHS /usr/lib/ /usr/local/lib/ /usr/lib64 /usr/local/lib64 12 | ) 13 | -------------------------------------------------------------------------------- /Modules/Findlibadbase_kafka.cmake: -------------------------------------------------------------------------------- 1 | MESSAGE(STATUS "Using bundled Findlibadbase_kafka.cmake...") 2 | FIND_PATH( 3 | LIBADBASE_KAFKA_INCLUDE_DIR 4 | Kafka.hpp 5 | /usr/local/include/adbase 6 | /usr/include/ 7 | ) 8 | 9 | FIND_LIBRARY( 10 | LIBADBASE_KAFKA_LIBRARIES NAMES libadbase_kafka.a 11 | PATHS /usr/lib/ /usr/local/lib/ /usr/lib64 /usr/local/lib64 12 | ) 13 | -------------------------------------------------------------------------------- /Modules/Findlibevent.cmake: -------------------------------------------------------------------------------- 1 | MESSAGE(STATUS "Using bundled Findlibevent.cmake...") 2 | FIND_PATH( 3 | LIBEVENT_INCLUDE_DIR 4 | event.h 5 | /usr/include/ 6 | /usr/local/include/event2 7 | ) 8 | 9 | FIND_LIBRARY( 10 | LIBEVENT_LIBRARIES NAMES event 11 | PATHS /usr/lib/ /usr/local/lib/ /usr/lib64 /usr/local/lib64 12 | ) 13 | FIND_LIBRARY( 14 | LIBEVENT_STATIC_LIBRARIES NAMES libevent.a 15 | PATHS /usr/lib/ /usr/local/lib/ /usr/lib64 /usr/local/lib64 16 | ) 17 | -------------------------------------------------------------------------------- /Modules/Findlibpthread.cmake: -------------------------------------------------------------------------------- 1 | MESSAGE(STATUS "Using Findlibpthread.cmake...") 2 | FIND_PATH( 3 | LIBPTHREAD_INCLUDE_DIR 4 | pthread.h 5 | /usr/include/ 6 | /usr/local/include/ 7 | ) 8 | 9 | FIND_LIBRARY( 10 | LIBPTHREAD_LIBRARIES NAMES pthread 11 | PATHS /usr/lib/ /usr/local/lib/ /usr/lib64 /usr/local/lib64 12 | ) 13 | -------------------------------------------------------------------------------- /Modules/Findlibrdkafka.cmake: -------------------------------------------------------------------------------- 1 | MESSAGE(STATUS "Using bundled Findlibrdkafka.cmake...") 2 | FIND_PATH( 3 | LIBRDKAFKA_INCLUDE_DIR 4 | librdkafka/rdkafka.h 5 | /usr/include/ 6 | /usr/local/include/ 7 | ) 8 | 9 | FIND_LIBRARY( 10 | LIBRDKAFKA_LIBRARIES NAMES librdkafka.a 11 | PATHS /usr/lib/ /usr/local/lib/ 12 | ) 13 | FIND_LIBRARY( 14 | LIBRDKAFKA_CPP_LIBRARIES NAMES librdkafka++.a 15 | PATHS /usr/lib/ /usr/local/lib/ 16 | ) 17 | -------------------------------------------------------------------------------- /Modules/Findlibyaml-cpp.cmake: -------------------------------------------------------------------------------- 1 | MESSAGE(STATUS "Using bundled Findlibyaml-cpp.cmake...") 2 | FIND_PATH( 3 | LIBYAMLCPP_INCLUDE_DIR 4 | yaml.h 5 | /usr/local/include/yaml-cpp 6 | /usr/include/ 7 | ) 8 | 9 | FIND_LIBRARY( 10 | LIBYAMLCPP_LIBRARIES NAMES libyaml-cpp.a 11 | PATHS /usr/lib/ /usr/local/lib/ /usr/lib64 /usr/local/lib64 12 | ) 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # aidt 2 | 3 | AIDT (WEIBO AD DATA TRANSFER) 是数据传输到 Kakfa 的一个代理, 统称数据传输服务, 该服务可以支持多种方式将数据传输到 Kafka 中，分别是通过 Memcache 协议方式、Http 协议方式、实时监听文件目录方式, 用户可以根据自己的业务环境来灵活选择传输方式, 服务结构如图所示： 4 | 5 | ![structure](docs/image/aidt.png) 6 | 7 | ### 特性 8 | 9 | - 支持 Memcache 协议方式发送数据 10 | - 支持 Http 协议方式发送数据 11 | - 支持文件目录方式发送数据 12 | - 支持二进制安全传输 13 | - 服务自身监控数据 14 | 15 | ### Quick Start 16 | 17 | #### 安装 18 | 19 | 目前提供 Linux 平台下主流版本的安装包 [下载](https://github.com/weiboad/aidt/releases), 安装步骤如下 20 | 21 | ``` 22 | wget https://github.com/weiboad/aidt/releases/download/v1.0.1/aidt-1.0.1-Linux-x86_64.tar.gz 23 | tar -zxf aidt-1.0.1-Linux-x86_64.tar.gz 24 | cd aidt-1.0.1-Linux-x86_64 25 | ./bin/aidt -v 26 | ``` 27 | 28 | #### 配置 29 | 30 | 在软件包中的 `conf` 目录下有两个配置文件，一个是 `system.ini` 是服务的主配置文件，`aidt.yml` 是服务使用监听文件目录方式传输数据的配置文件，如果不使用文件目录方式传输数据该文件使用默认配置即可，具体的 `aidt.yml` 配置方式参见[文件传输](docs/file_transfer.md), `system.ini` 各个配置项意义参考 [配置](docs/configure.md) 31 | 32 | 1. 修改kafka 集群的 broker list 33 | 34 | ``` 35 | // system.ini 36 | brokerListIn=127.0.0.1:9192 37 | ``` 38 | 39 | 2. 配置 Memcache 、Http Server 监听地址 40 | 41 | ``` 42 | [mc] 43 | host=0.0.0.0 44 | port=40011 45 | threadNum=4 46 | serverName=mc-server 47 | 48 | [http] 49 | host=0.0.0.0 50 | port=40010 51 | timeout=3 52 | threadNum=4 53 | serverName=adinf-adserver 54 | accessLogDir=logs/ 55 | accesslogRollSize=52428800 56 | defaultController=index 57 | defaultAction=index 58 | ``` 59 | 60 | 默认分别是 40011 和 40010 61 | 62 | 3. 配置服务日志目录 63 | 64 | ``` 65 | cd aidt-1.0.1-Linux-x86_64/bin 66 | mkdir logs 67 | ``` 68 | 69 | 4. 修改日志级别和写入方式 70 | 71 | 推荐生成环境下改为如下配置 72 | 73 | ``` 74 | [logging] 75 | logsDir=logs/ 76 | ; 日志分卷大小 77 | logRollSize=52428800 78 | ; 1: LOG_TRACE 2: LOG_DEBUG 3: LOG_INFO 79 | logLevel=3 80 | isAsync=yes 81 | ``` 82 | 83 | #### 启动 84 | 85 | ``` 86 | cd aidt-1.0.1-Linux-x86_64/bin 87 | ./aidt -c ../conf/system.ini 88 | ``` 89 | 90 | #### 测试 91 | 92 | ##### 通过 Memcache 协议发送数据 93 | 94 | 使用 Memcache 协议的 set 命令发送数据，其中key 就是要发送数据的 topic 和分区，例如有一个 test 的topic 有 [0,1] 两个分区，可以使用 test#0 表示发送到 test topic 的 0号分区，使用 test 作为key 表示分区任意， value 是要发送的数据 95 | 96 | 例如： 97 | 98 | ``` 99 | [vagrant@localhost aidt]$ telnet 127.0.0.1 40011 100 | Trying 127.0.0.1... 101 | Connected to 127.0.0.1. 102 | Escape character is '^]'. 103 | version 104 | VERSION 1.0.0 105 | set test 1 1 4 106 | data 107 | STORED 108 | set test#1 1 1 5 109 | data1 110 | STORED 111 | quit 112 | Connection closed by foreign host. 113 | ``` 114 | 115 | ##### 通过 Http 协议发送数据 116 | 117 | 通过 http 方式发送数据参数 data 是要发送的数据 118 | 119 | ``` 120 | curl -d 'data=testmessage' '127.0.0.1:40010/message?topic_name=test' 121 | {"code":0,"baseid":804904302018560,"data":"","msg":"Message send success"} 122 | ``` 123 | 124 | ##### 通过文件目录方式 125 | 126 | 参见[文件目录传输](docs/file_transfer.md) 127 | -------------------------------------------------------------------------------- /adbase.ini: -------------------------------------------------------------------------------- 1 | [project] 2 | ADINF_PROJECT_NAME=aidt 3 | ADINF_PROJECT_SUMMARY=Weibo adinf data transfer 4 | ADINF_PROJECT_URL=https://github.com/weiboad/aidt 5 | ADINF_PROJECT_VENDOR=nmred 6 | ADINF_PROJECT_PACKAGER=nmred 7 | 8 | [module] 9 | adserver=yes 10 | timer=yes 11 | kafkac=no 12 | kafkap=yes 13 | logging=yes 14 | 15 | [params] 16 | timers=syncOffset 17 | http_controllers=Index,Config,Topic,Message 18 | kafka_consumers=StatusDelete,StatusUpdate,UserUpdate 19 | kafka_consumers_topics=fa_status_delete,fa_status_update,fa_user_update 20 | kafka_consumers_groups=fr_test,fr_test,fr_test 21 | kafka_producers=In 22 | kafka_producers_topics=in 23 | 24 | [files] 25 | src/main.cpp=src/@ADINF_PROJECT_NAME@.cpp 26 | rpm/main.spec.in=rpm/@ADINF_PROJECT_NAME@.spec.in 27 | 28 | [execs] 29 | cmake.sh=1 30 | build_rpm.in=1 31 | -------------------------------------------------------------------------------- /cmake.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #{{{ options 3 | 4 | echo "Start cmake configure....." 5 | echo -e "\033[31m\033[05mCompile debug level [Debug(D)|Release(R)]: \033[0m\c" 6 | read makelevel 7 | case $makelevel in 8 | Debug | D ) 9 | CMAKE_BUILD_TYPE="Debug";; 10 | Release | R ) 11 | CMAKE_BUILD_TYPE="Release";; 12 | *) 13 | CMAKE_BUILD_TYPE="Debug";; 14 | esac 15 | # }}} 16 | # 计算版本号 17 | GIT_SHA1=`(git show-ref --head --hash=8 2> /dev/null || echo 00000000) | head -n1` 18 | GIT_DIRTY=`git diff --no-ext-diff 2> /dev/null | wc -l` 19 | BUILD_ID=`uname -n`"-"`date +%s` 20 | 21 | CMD="cmake -H. -B_build -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DGIT_SHA1=$GIT_SHA1 -DGIT_DIRTY=$GIT_DIRTY -DBUILD_ID=$BUILD_ID -DSTATICCOMPILE=ON " 22 | 23 | echo $CMD 24 | eval $CMD 25 | -------------------------------------------------------------------------------- /conf/aidt.yml: -------------------------------------------------------------------------------- 1 | # aidt config 2 | 3 | transfers: ~ 4 | # - 5 | # path: /tmp/test2 6 | # is_dir: true 7 | # is_binary: false 8 | # is_recursive: false 9 | # time_wait: 3 10 | # format: '%s' 11 | # topic_name: test 12 | -------------------------------------------------------------------------------- /conf/system.ini: -------------------------------------------------------------------------------- 1 | [system] 2 | appid=0 3 | macid=0 4 | daemon=no 5 | pidFile=./aidt.pid 6 | aidtConfig=../../conf/aidt.yml 7 | ; malloc trim 128kb 8 | mallocTrimPad=131072 9 | ; malloc trim interval 10 | ; 5s 11 | mallocTrimInterval=1000 12 | 13 | [logging] 14 | logsDir=logs/ 15 | ; 日志分卷大小 16 | logRollSize=52428800 17 | ; 1: LOG_TRACE 2: LOG_DEBUG 3: LOG_INFO 18 | logLevel=3 19 | isAsync=no 20 | 21 | [adserver] 22 | ; 是否启动 mc server 23 | mc=yes 24 | ; 是否启动 http server 25 | http=yes 26 | ; 是否启动 head server 27 | head=no 28 | 29 | [mc] 30 | host=0.0.0.0 31 | port=40011 32 | threadNum=4 33 | serverName=mc-server 34 | 35 | [http] 36 | host=0.0.0.0 37 | port=11013 38 | timeout=3 39 | threadNum=4 40 | serverName=adinf-adserver 41 | accessLogDir=logs/ 42 | accesslogRollSize=52428800 43 | defaultController=index 44 | defaultAction=index 45 | 46 | [head] 47 | host=0.0.0.0 48 | port=11012 49 | threadNum=4 50 | serverName=head-server 51 | 52 | [kafkap_in] 53 | brokerListIn=127.0.0.1:9192 54 | debugIn=none 55 | queueLengthIn=10000 56 | securityProtocol=plaintext 57 | saslMechanisms=PLAIN 58 | saslUsername= 59 | saslPassword= 60 | kerberosServiceName= 61 | kerberosPrincipal= 62 | kerberosCmd= 63 | kerberosKeytab= 64 | kerberosMinTime= 65 | 66 | [timer] 67 | ; 单位ms 68 | intervalSyncOffset = 2000 69 | 70 | [message] 71 | sendError=error 72 | queueLimit=10000 73 | 74 | [reader] 75 | maxLines=10000 76 | maxfd=2 77 | maxLineBytes=1048576 78 | offsetFile=./agent.offset 79 | -------------------------------------------------------------------------------- /docs/configure.md: -------------------------------------------------------------------------------- 1 | Aidt Configuration 2 | ================== 3 | 4 | ### system 5 | 6 | | Property | Default | Range | Desc | 7 | | -- | -- | -- | -- | 8 | | appid | 0 | 0-255 | 用来计算 baseid ，可忽略 | 9 | | macid | 0 | 0-255 | 用来计算 baseid ，可忽略 | 10 | | daemon | no | no/yes | 是否将程序以守护进程方式运行 | 11 | | pidFile | ./aidt.pid | - | 程序运行的 pid 文件路径 | 12 | | aidtConfig | ../conf/aidt.yml | - | aidt 文件传输配置文件路径 | 13 | 14 | ### logging 15 | 16 | | Property | Default | Range | Desc | 17 | | -- | -- | -- | -- | 18 | | logsDir | logs/ | - | 程序日志写入目录 | 19 | | logRollSize | 52428800 | - | 日志文件分卷大小 bytes | 20 | | logLevel | 1 | [1,2,3] | 1: LOG_TRACE 2:LOG_DEBUG 3:LOG_INFO| 21 | | isAsync | no | no/yes | 是否开启异步写日志的方式, 默认不开启会写入到 stdout 上 | 22 | 23 | ### adserver 24 | 25 | | Property | Default | Range | Desc | 26 | | -- | -- | -- | -- | 27 | | mc | yes | yes/no | 是否开启 memcache 代理服务 | 28 | | http | yes | yes | http 默认开启，请勿修改 | 29 | | head | no | no | head 在 aidt 中没有用到该协议，启动无效 | 30 | 31 | ### mc 32 | 33 | | Property | Default | Range | Desc | 34 | | -- | -- | -- | -- | 35 | | host | 0.0.0.0 | ip | 监听 mc server ip 地址 | 36 | | port | 40011 | port | 监听 mc server port | 37 | | threadNum | 4 | >0 | mc server work 线程数 | 38 | | serverName | adinf-adserver | - | mc server name | 39 | 40 | ### http 41 | 42 | | Property | Default | Range | Desc | 43 | | -- | -- | -- | -- | 44 | | host | 0.0.0.0 | ip | 监听 http server ip 地址 | 45 | | port | 40010 | port | 监听 http server port | 46 | | timeout | 3 | >0 | http server 读写超时，单位秒 | 47 | | threadNum | 4 | >0 | http server work 线程数 | 48 | | serverName | adinf-adserver | - | http server name | 49 | | accessLogDir | logs/ | - | http server 访问日志目录路劲 | 50 | | accesslogRollSize | 52428800 | >0 | http server 访问日志分卷大小 | 51 | | defaultController | index | - | 请勿修改 | 52 | | defaultAction | index | - | 请勿修改 | 53 | 54 | ### kafkap_in 55 | 56 | | Property | Default | Range | Desc | 57 | | -- | -- | -- | -- | 58 | | topicNameIn | test | - | 无效参数 | 59 | | brokerListIn | 127.0.0.1:9192 | - | kafka 集群地址, 注意此处不是 zookeeper 地址 | 60 | | debugIn | none | - | kafka 调试级别 | 61 | | queueLengthIn | 10000 | >0 | 代理缓存队列大小 | 62 | 63 | ### message 64 | 65 | | Property | Default | Range | Desc | 66 | | -- | -- | -- | -- | 67 | | sendError | error | - | 如果发送失败，消息落地到本地文件的路劲 | 68 | 69 | ### reader 70 | 71 | | Property | Default | Range | Desc | 72 | | -- | -- | -- | -- | 73 | | maxLines | 1000 | - | 单次事件读取最大行数 | 74 | | maxfd | 512 | - | 最大打开文件个数 | 75 | | maxLineBytes | 1048576 | - | 单行最大字节数，建议和kafka 最大消息保持一致 | 76 | | offsetFile | ./agent.offset | - | 保存读取文件位置的文件路劲 | 77 | -------------------------------------------------------------------------------- /docs/file_transfer.md: -------------------------------------------------------------------------------- 1 | # 文件目录数据传输 2 | 3 | aidt 采用 inotify 机制来监听文件目录数据的变更，将变更的数据实时的传输到 kafka, 目前该模块支持普通文件、目录、以及递归目录的方式传输。并且也支持二进制安全的方式传输文件内容 4 | 5 | ### 配置 6 | 7 | aidt 在文件目录传输配置上采用yaml 配置文件格式，在 `transfers` 段下配置一个或多个文件传输配置，配置项如下： 8 | 9 | ```yaml 10 | transfers 11 | - 12 | path: /tmp/test1 13 | is_dir: true 14 | is_binary: false 15 | is_recursive: false 16 | format: '%s' 17 | topic_name: test 18 | - 19 | path: /tmp/test2 20 | is_dir: true 21 | is_binary: false 22 | is_recursive: false 23 | format: '%s' 24 | topic_name: test 25 | ``` 26 | 27 | 如上例所示配置了传输 `/tmp/test1` 和 `/tmp/test2` 两个目录下所有文件内容到 topic 为 `test` 的kafka 集群 28 | 29 | #### 配置项格式 30 | 31 | | Property | Range | Default | Desc | 32 | | -- | -- | -- | -- | 33 | | path | string | None | 要监听传输的文件或目录 | 34 | | is_dir | true/false | false | 监听的路劲是否是目录 | 35 | | is_recursive | true/false | false | 是否递归目录方式读取数据，只有在监听路劲为目录时生效 | 36 | | is_binary | true/false | false | 传输的数据是否按照二进制方式传输，默认是按行传输 | 37 | | format | string | '%s' | 自定义格式化消息, 只有在传输普通行消息时生效 | 38 | | topic_name | string | None | 消息传输的 topic | 39 | 40 | ### 重新热加载配置 41 | 42 | aidt 支持热修改传输配置，通过发送 `-USR1` 信号进行配置 reload, 在重新加载时一定要检查配置文件语法， 检查语法的方式： 43 | 44 | ``` 45 | ./aidt -c ../config/system.ini -t 46 | ``` 47 | -------------------------------------------------------------------------------- /docs/image/aidt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weiboad/aidt/dabb0877675326f8bd80fdb450a4f3d7108ac5db/docs/image/aidt.png -------------------------------------------------------------------------------- /docs/upgrade.md: -------------------------------------------------------------------------------- 1 | ### 由 0.4.2.1 版本升级到 1.0.x 方法 2 | 3 | 首先确认旧节点所有传输的配置路劲 4 | 5 | cat agent-10023.offset|awk -F `echo -e '\x1a'` '{print $1}' |sort -nr |uniq 6 | 7 | 转化 offset 为升级的中间文本 8 | 9 | cat agent-10023.offset|awk -F `echo -e '\x1a'` '{print $3"\t"$1"\t"$2}' > tmp_agent_offset 10 | 11 | 如果修改传输路劲可以修改编辑 12 | 13 | :%s/\/data1\/www\//\/usr\/local\/adinf\/logs\//g 14 | :%s/\/data1\/run\//\/usr\/local\/adinf\/logs\//g 15 | 16 | 将新节点的 agent.offset 转化为中间升级文本 17 | 18 | ./aidt-cli -o dumpoffset -f agent.offset -d ./ 19 | 20 | 合并新旧的 offset 中间文本 21 | 22 | cat tmp_agent\_offset > dump\_offsets 23 | 24 | 通过中间文本生成最终的 agent.offset 25 | 26 | ./aidt-cli -o dynamicoffset -f dump_offsets -d ./ 27 | -------------------------------------------------------------------------------- /rpm/aidt.spec: -------------------------------------------------------------------------------- 1 | Summary: Weibo adinf data transfer 2 | Name: aidt 3 | Version:0.1.0 4 | Release:el6.5 5 | Group:Development/Tools 6 | License:BSD 7 | URL: https://github.com/weiboad/aidt 8 | Vendor: nmred 9 | Packager: nmred 10 | Distribution: Weibo adinf data transfer 11 | Source:%{name}-%{version}.tar 12 | Buildroot:%{_tmppath}/%{name}-%{version} 13 | Prefix:/usr/local/adinf/aidt 14 | Requires:sudo 15 | %define debug_packages %{nil} 16 | %define debug_package %{nil} 17 | %define _topdir /usr/home/zhongxiu/rpmbuild 18 | %description 19 | ------------------------------------- 20 | - Everything in order to facilitate - 21 | ------------------------------------- 22 | 23 | %prep 24 | %setup -q 25 | %build 26 | 27 | %pre 28 | if test -d %{prefix}; then 29 | echo "" 30 | echo "Install dir \'%{prefix}\' exists. You need resolve it manually before install aidt. Exit now." 31 | echo "" 32 | # return !0 for exit. 33 | test "" 34 | fi 35 | %install 36 | #rm -rf $RPM_BUILD_ROOT 37 | mkdir -p $RPM_BUILD_ROOT%{prefix} 38 | cp -r * $RPM_BUILD_ROOT%{prefix} 39 | 40 | %clean 41 | #rm -rf $RPM_BUILD_ROOT 42 | 43 | %post 44 | # build install 45 | echo "" 46 | echo "Build: START" 47 | echo -e "Build: DONE" 48 | %preun 49 | echo -n "unrpm ... " 50 | SW_TMP_BAK_PATH=%{prefix}-backup-$(date +%%Y%%m%%d-%%H%%M%%S) 51 | mv %{prefix} $SW_TMP_BAK_PATH 52 | echo "Old version '%{prefix}' backup to '$SW_TMP_BAK_PATH'" 53 | %postun 54 | %files 55 | %{prefix} 56 | 57 | %changelog 58 | -------------------------------------------------------------------------------- /rpm/aidt.spec.in: -------------------------------------------------------------------------------- 1 | Summary: Weibo adinf data transfer 2 | Name: aidt 3 | Version:${ADINFVERSION} 4 | Release:${OPVERSION} 5 | Group:Development/Tools 6 | License:BSD 7 | URL: https://github.com/weiboad/aidt 8 | Vendor: nmred 9 | Packager: nmred 10 | Distribution: Weibo adinf data transfer 11 | Source:%{name}-%{version}.tar 12 | Buildroot:%{_tmppath}/%{name}-%{version} 13 | Prefix:/usr/local/adinf/aidt 14 | Requires:sudo 15 | %define debug_packages %{nil} 16 | %define debug_package %{nil} 17 | %define _topdir /usr/home/zhongxiu/rpmbuild 18 | %description 19 | ------------------------------------- 20 | - Everything in order to facilitate - 21 | ------------------------------------- 22 | 23 | %prep 24 | %setup -q 25 | %build 26 | 27 | %pre 28 | if test -d %{prefix}; then 29 | echo "" 30 | echo "Install dir \'%{prefix}\' exists. You need resolve it manually before install aidt. Exit now." 31 | echo "" 32 | # return !0 for exit. 33 | test "" 34 | fi 35 | %install 36 | #rm -rf $RPM_BUILD_ROOT 37 | mkdir -p $RPM_BUILD_ROOT%{prefix} 38 | cp -r * $RPM_BUILD_ROOT%{prefix} 39 | 40 | %clean 41 | #rm -rf $RPM_BUILD_ROOT 42 | 43 | %post 44 | # build install 45 | echo "" 46 | echo "Build: START" 47 | echo -e "Build: DONE" 48 | %preun 49 | echo -n "unrpm ... " 50 | SW_TMP_BAK_PATH=%{prefix}-backup-$(date +%%Y%%m%%d-%%H%%M%%S) 51 | mv %{prefix} $SW_TMP_BAK_PATH 52 | echo "Old version '%{prefix}' backup to '$SW_TMP_BAK_PATH'" 53 | %postun 54 | %files 55 | %{prefix} 56 | 57 | %changelog -------------------------------------------------------------------------------- /rpm/build_rpm: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4 foldmethod=marker: # 3 | # adbase-common 出包工具 4 | 5 | ############################ 6 | # 相关配置项 ############# 7 | ############################ 8 | CUR_PWD=`pwd` 9 | 10 | # 编译后的临时目录 11 | PATH_DEV_SWAN=/usr/local/adinf/aidt 12 | 13 | # build rpm 包的目录 14 | BUILD_ROOT="/usr/home/zhongxiu/rpmbuild" 15 | SWANSOFT_NAME="aidt" 16 | SWANSOFT_VERSION="0.1.0" 17 | BUILD_TMP=/tmp 18 | BUILD_BALL=$BUILD_TMP/$SWANSOFT_NAME-$SWANSOFT_VERSION 19 | 20 | ############################ 21 | # 函数 ##### 22 | ############################ 23 | 24 | # {{{ check_return_exit() 25 | 26 | check_return_exit () 27 | { 28 | if ! test 0 = $?; then 29 | echo -e "\n!!!!!!!!!! Error, this script will exit !!!!!!!!!!\n" 30 | exit 1 31 | fi 32 | } 33 | 34 | # }}} 35 | # {{{ function sw_mkdir() 36 | 37 | ############################ 38 | # 创建目录 ################# 39 | ############################ 40 | function sw_mkdir() 41 | { 42 | echo "create $1 directory ..."; 43 | if [ -d $1 ] ;then 44 | echo "directory $1 already exists. " 45 | else 46 | mkdir -p $1 47 | check_return_exit 48 | fi 49 | } 50 | 51 | # }}} 52 | 53 | ############################### 54 | #copy opt , web code to /tmp # 55 | ############################### 56 | # {{{ function ad_copy_opt() 57 | 58 | function ad_copy_opt() 59 | { 60 | echo "copy opt, to $BUILD_BALL ..." 61 | echo "cp -r $PATH_DEV_SWAN/* $BUILD_BALL/ ..." 62 | sw_mkdir $BUILD_BALL/ 63 | cp -r $PATH_DEV_SWAN/* $BUILD_BALL/ 64 | check_return_exit 65 | } 66 | 67 | # }}} 68 | ############################### 69 | # 制作tar包 # 70 | ############################### 71 | # {{{ function ad_make_tar() 72 | 73 | function ad_make_tar() 74 | { 75 | echo "start make smeta tar $BUILD_BALL.tar ..." 76 | echo "cd $BUILD_TMP ..." 77 | cd $BUILD_TMP 78 | CMD="tar -zcf $BUILD_BALL.tar $SWANSOFT_NAME-$SWANSOFT_VERSION" 79 | echo $CMD 80 | eval $CMD 81 | check_return_exit 82 | 83 | echo "remove dir $BUILD_BALL ..." 84 | rm -rf $BUILD_BALL 85 | check_return_exit 86 | } 87 | 88 | # }}} 89 | ############################### 90 | # 建立打包环境 # 91 | ############################### 92 | # {{{ function ad_set_rpm_env() 93 | 94 | function ad_set_rpm_env() 95 | { 96 | echo "cd current $CUR_PWD ..." 97 | cd $CUR_PWD 98 | check_return_exit 99 | 100 | echo "mkdir rpmbuild ..." 101 | mkdir -p $BUILD_ROOT 102 | mkdir -p $BUILD_ROOT/{BUILD,RPMS,SOURCES,SPECS,SRPMS} 103 | check_return_exit 104 | 105 | echo "cd current $CUR_PWD ..." 106 | cd $CUR_PWD 107 | check_return_exit 108 | 109 | echo "mv spec to buildroot ..." 110 | cp $SWANSOFT_NAME.spec $BUILD_ROOT/SPECS 111 | check_return_exit 112 | 113 | echo "mv tar file to SOURCES ..." 114 | mv $BUILD_BALL.tar $BUILD_ROOT/SOURCES 115 | check_return_exit 116 | } 117 | 118 | # }}} 119 | ############################### 120 | # 执行打包 # 121 | ############################### 122 | # {{{ function ad_rpmbuild() 123 | 124 | function ad_rpmbuild() 125 | { 126 | echo "start rpmbuild ... $BUILD_ROOT" 127 | rpmbuild -bb $BUILD_ROOT/SPECS/$SWANSOFT_NAME.spec 128 | check_return_exit 129 | } 130 | 131 | # }}} 132 | ############################### 133 | # 预处理 # 134 | ############################### 135 | # {{{ function ad_pre_install() 136 | 137 | function ad_pre_install() 138 | { 139 | if [ -d $BUILD_ROOT ]; then 140 | echo "remove $BUILD_ROOT ..." 141 | rm -rf $BUILD_ROOT 142 | check_return_exit 143 | fi 144 | if [ -d $BUILD_TMP/$SWANSOFT_NAME-$SWANSOFT_VERSION ]; then 145 | echo "remove /tmp/$SWANSOFT_NAME-$SWANSOFT_VERSION ..." 146 | rm -rf $BUILD_TMP/$SWANSOFT_NAME-$SWANSOFT_VERSION 147 | check_return_exit 148 | fi 149 | } 150 | 151 | # }}} 152 | 153 | ad_pre_install 154 | 155 | ############################### 156 | # 打包全过程 # 157 | ############################### 158 | 159 | ad_copy_opt 160 | ad_make_tar 161 | ad_set_rpm_env 162 | ad_rpmbuild 163 | -------------------------------------------------------------------------------- /rpm/build_rpm.in: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4 foldmethod=marker: # 3 | # adbase-common 出包工具 4 | 5 | ############################ 6 | # 相关配置项 ############# 7 | ############################ 8 | CUR_PWD=`pwd` 9 | 10 | # 编译后的临时目录 11 | PATH_DEV_SWAN=/usr/local/adinf/aidt 12 | 13 | # build rpm 包的目录 14 | BUILD_ROOT="/usr/home/zhongxiu/rpmbuild" 15 | SWANSOFT_NAME="aidt" 16 | SWANSOFT_VERSION="@ADINFVERSION@" 17 | BUILD_TMP=/tmp 18 | BUILD_BALL=$BUILD_TMP/$SWANSOFT_NAME-$SWANSOFT_VERSION 19 | 20 | ############################ 21 | # 函数 ##### 22 | ############################ 23 | 24 | # {{{ check_return_exit() 25 | 26 | check_return_exit () 27 | { 28 | if ! test 0 = $?; then 29 | echo -e "\n!!!!!!!!!! Error, this script will exit !!!!!!!!!!\n" 30 | exit 1 31 | fi 32 | } 33 | 34 | # }}} 35 | # {{{ function sw_mkdir() 36 | 37 | ############################ 38 | # 创建目录 ################# 39 | ############################ 40 | function sw_mkdir() 41 | { 42 | echo "create $1 directory ..."; 43 | if [ -d $1 ] ;then 44 | echo "directory $1 already exists. " 45 | else 46 | mkdir -p $1 47 | check_return_exit 48 | fi 49 | } 50 | 51 | # }}} 52 | 53 | ############################### 54 | #copy opt , web code to /tmp # 55 | ############################### 56 | # {{{ function ad_copy_opt() 57 | 58 | function ad_copy_opt() 59 | { 60 | echo "copy opt, to $BUILD_BALL ..." 61 | echo "cp -r $PATH_DEV_SWAN/* $BUILD_BALL/ ..." 62 | sw_mkdir $BUILD_BALL/ 63 | cp -r $PATH_DEV_SWAN/* $BUILD_BALL/ 64 | check_return_exit 65 | } 66 | 67 | # }}} 68 | ############################### 69 | # 制作tar包 # 70 | ############################### 71 | # {{{ function ad_make_tar() 72 | 73 | function ad_make_tar() 74 | { 75 | echo "start make smeta tar $BUILD_BALL.tar ..." 76 | echo "cd $BUILD_TMP ..." 77 | cd $BUILD_TMP 78 | CMD="tar -zcf $BUILD_BALL.tar $SWANSOFT_NAME-$SWANSOFT_VERSION" 79 | echo $CMD 80 | eval $CMD 81 | check_return_exit 82 | 83 | echo "remove dir $BUILD_BALL ..." 84 | rm -rf $BUILD_BALL 85 | check_return_exit 86 | } 87 | 88 | # }}} 89 | ############################### 90 | # 建立打包环境 # 91 | ############################### 92 | # {{{ function ad_set_rpm_env() 93 | 94 | function ad_set_rpm_env() 95 | { 96 | echo "cd current $CUR_PWD ..." 97 | cd $CUR_PWD 98 | check_return_exit 99 | 100 | echo "mkdir rpmbuild ..." 101 | mkdir -p $BUILD_ROOT 102 | mkdir -p $BUILD_ROOT/{BUILD,RPMS,SOURCES,SPECS,SRPMS} 103 | check_return_exit 104 | 105 | echo "cd current $CUR_PWD ..." 106 | cd $CUR_PWD 107 | check_return_exit 108 | 109 | echo "mv spec to buildroot ..." 110 | cp $SWANSOFT_NAME.spec $BUILD_ROOT/SPECS 111 | check_return_exit 112 | 113 | echo "mv tar file to SOURCES ..." 114 | mv $BUILD_BALL.tar $BUILD_ROOT/SOURCES 115 | check_return_exit 116 | } 117 | 118 | # }}} 119 | ############################### 120 | # 执行打包 # 121 | ############################### 122 | # {{{ function ad_rpmbuild() 123 | 124 | function ad_rpmbuild() 125 | { 126 | echo "start rpmbuild ... $BUILD_ROOT" 127 | rpmbuild -bb $BUILD_ROOT/SPECS/$SWANSOFT_NAME.spec 128 | check_return_exit 129 | } 130 | 131 | # }}} 132 | ############################### 133 | # 预处理 # 134 | ############################### 135 | # {{{ function ad_pre_install() 136 | 137 | function ad_pre_install() 138 | { 139 | if [ -d $BUILD_ROOT ]; then 140 | echo "remove $BUILD_ROOT ..." 141 | rm -rf $BUILD_ROOT 142 | check_return_exit 143 | fi 144 | if [ -d $BUILD_TMP/$SWANSOFT_NAME-$SWANSOFT_VERSION ]; then 145 | echo "remove /tmp/$SWANSOFT_NAME-$SWANSOFT_VERSION ..." 146 | rm -rf $BUILD_TMP/$SWANSOFT_NAME-$SWANSOFT_VERSION 147 | check_return_exit 148 | fi 149 | } 150 | 151 | # }}} 152 | 153 | ad_pre_install 154 | 155 | ############################### 156 | # 打包全过程 # 157 | ############################### 158 | 159 | ad_copy_opt 160 | ad_make_tar 161 | ad_set_rpm_env 162 | ad_rpmbuild -------------------------------------------------------------------------------- /src/AdServer.cpp: -------------------------------------------------------------------------------- 1 | /// 该程序自动生成，禁止修改 2 | #include "AdServer.hpp" 3 | 4 | // {{{ AdServer::AdServer() 5 | 6 | AdServer::AdServer(AdServerContext* context) : 7 | _context(context) { 8 | _configure = _context->config; 9 | _base = _context->mainEventBase; 10 | } 11 | 12 | // }}} 13 | // {{{ AdServer::~AdServer() 14 | 15 | AdServer::~AdServer() { 16 | //stop(); 17 | } 18 | 19 | // }}} 20 | // {{{ void AdServer::run() 21 | 22 | void AdServer::run() { 23 | // 初始化 server 24 | init(); 25 | if (_configure->isStartHttp && _http != nullptr) { 26 | _http->start(_configure->httpThreadNum); 27 | } 28 | if (_configure->isStartMc) { 29 | _mcServer->start(_configure->mcThreadNum); 30 | } 31 | if (_configure->isStartHead) { 32 | _headServer->start(_configure->headThreadNum); 33 | } 34 | } 35 | 36 | // }}} 37 | // {{{ void AdServer::init() 38 | 39 | void AdServer::init() { 40 | if (_configure->isStartHttp) { 41 | initHttp(); 42 | } 43 | if (_configure->isStartMc) { 44 | initMc(); 45 | } 46 | if (_configure->isStartHead) { 47 | initHead(); 48 | } 49 | } 50 | 51 | // }}} 52 | // {{{ void AdServer::stop() 53 | 54 | void AdServer::stop() { 55 | if (_configure->isStartHttp && _http != nullptr) { 56 | LOG_TRACE << "Stop http server."; 57 | _http->stop(); 58 | delete _httpManager; 59 | } 60 | 61 | if (_configure->isStartMc && _mcServer != nullptr) { 62 | _mcServer->stop(); 63 | delete _mcServer; 64 | _mcServer = nullptr; 65 | delete _mcProcessor; 66 | delete _mcAddr; 67 | delete _mcInterface; 68 | delete _mcHandler; 69 | } 70 | 71 | if (_configure->isStartHead && _headServer != nullptr) { 72 | _headServer->stop(); 73 | delete _headServer; 74 | _headServer = nullptr; 75 | delete _headProcessor; 76 | delete _headAddr; 77 | delete _headInterface; 78 | delete _headHandler; 79 | } 80 | } 81 | 82 | // }}} 83 | // {{{ void AdServer::initHttp() 84 | 85 | void AdServer::initHttp() { 86 | adbase::TimeZone tz(8*3600, "CST"); 87 | LOG_TRACE << "Init http server, host:" 88 | << _configure->httpHost 89 | << " port:" 90 | << _configure->httpPort; 91 | adbase::http::Config config(_configure->httpHost, _configure->httpPort, _configure->httpTimeout); 92 | config.setTimeZone(tz); 93 | config.setServerName(_configure->httpServerName); 94 | config.setLogDir(_configure->httpAccessLogDir); 95 | config.setLogRollSize(_configure->httpAccesslogRollSize); 96 | _http = new adbase::http::Server(config); 97 | _httpManager = new Http(_context, _http); 98 | _httpManager->addController(); 99 | _http->setLocationFallback(std::bind(&AdServer::httpFallback, this, std::placeholders::_1)); 100 | } 101 | 102 | // }}} 103 | // {{{ std::string AdServer::httpFallback() 104 | 105 | std::string AdServer::httpFallback(const std::string& url) { 106 | std::string result = "/"; 107 | std::string trimUrl = adbase::trim(url, "/"); 108 | if (_httpManager != nullptr) { 109 | std::unordered_map rewrites = _httpManager->rewrite(); 110 | if (rewrites.find(trimUrl) != rewrites.end()) { 111 | return rewrites[trimUrl]; 112 | } 113 | } 114 | if (trimUrl.empty()) { 115 | return result + _configure->httpDefaultController + "/" + _configure->httpDefaultAction; 116 | } else if (trimUrl.find("/") == std::string::npos) { 117 | return result + trimUrl + "/" + _configure->httpDefaultAction; 118 | } else { 119 | return url; 120 | } 121 | } 122 | 123 | // }}} 124 | // {{{ void AdServer::initHead() 125 | 126 | void AdServer::initHead() { 127 | // head server 128 | _headProcessor = new HeadProcessor(_context); 129 | _headAddr = new adbase::InetAddress(_configure->headHost, static_cast(_configure->headPort)); 130 | _headServer = new adbase::TcpServer(_base, *_headAddr, _configure->headServerName); 131 | _headInterface = new adbase::head::Interface(); 132 | _headInterface->setReadHandler(std::bind(&HeadProcessor::readHandler, _headProcessor, std::placeholders::_1, 133 | std::placeholders::_2, std::placeholders::_3, 134 | std::placeholders::_4, std::placeholders::_5)); 135 | _headHandler = new adbase::head::Handler(_headInterface); 136 | _headServer->setConnectionCallback(std::bind(&adbase::head::Handler::onConnection, _headHandler, 137 | std::placeholders::_1)); 138 | _headServer->setCloseCallback(std::bind(&adbase::head::Handler::onClose, _headHandler, std::placeholders::_1)); 139 | _headServer->setMessageCallback(std::bind(&adbase::head::Handler::onMessage, _headHandler, std::placeholders::_1, 140 | std::placeholders::_2, std::placeholders::_3)); 141 | 142 | } 143 | 144 | // }}} 145 | // {{{ void AdServer::initMc() 146 | 147 | void AdServer::initMc() { 148 | // Mc server 149 | _mcProcessor = new McProcessor(_context); 150 | _mcAddr = new adbase::InetAddress(_configure->mcHost, static_cast(_configure->mcPort)); 151 | _mcServer = new adbase::TcpServer(_base, *_mcAddr, _configure->mcServerName); 152 | _mcInterface = new adbase::mc::Interface(); 153 | bindMcCallback(); 154 | _mcHandler = new adbase::mc::Handler(_mcInterface); 155 | _mcServer->setConnectionCallback(std::bind(&adbase::mc::Handler::onConnection, _mcHandler, 156 | std::placeholders::_1)); 157 | _mcServer->setCloseCallback(std::bind(&adbase::mc::Handler::onClose, _mcHandler, std::placeholders::_1)); 158 | _mcServer->setMessageCallback(std::bind(&adbase::mc::Handler::onMessage, _mcHandler, std::placeholders::_1, 159 | std::placeholders::_2, std::placeholders::_3)); 160 | 161 | } 162 | 163 | // }}} 164 | // {{{ void AdServer::bindMcCallback() 165 | 166 | void AdServer::bindMcCallback() { 167 | _mcInterface->setAddHandler(std::bind(&McProcessor::add, _mcProcessor, std::placeholders::_1, 168 | std::placeholders::_2, std::placeholders::_3, 169 | std::placeholders::_4, std::placeholders::_5, 170 | std::placeholders::_6, std::placeholders::_7)); 171 | _mcInterface->setAppendHandler(std::bind(&McProcessor::append, _mcProcessor, std::placeholders::_1, 172 | std::placeholders::_2, std::placeholders::_3, 173 | std::placeholders::_4, std::placeholders::_5, 174 | std::placeholders::_6)); 175 | _mcInterface->setDecrementHandler(std::bind(&McProcessor::decrement, _mcProcessor, std::placeholders::_1, 176 | std::placeholders::_2, std::placeholders::_3, 177 | std::placeholders::_4, std::placeholders::_5, 178 | std::placeholders::_6, std::placeholders::_7)); 179 | _mcInterface->setDeleteHandler(std::bind(&McProcessor::deleteOp, _mcProcessor, std::placeholders::_1, 180 | std::placeholders::_2, std::placeholders::_3)); 181 | _mcInterface->setFlushHandler(std::bind(&McProcessor::flush, _mcProcessor, std::placeholders::_1)); 182 | _mcInterface->setGetHandler(std::bind(&McProcessor::get, _mcProcessor, std::placeholders::_1, 183 | std::placeholders::_2, std::placeholders::_3)); 184 | _mcInterface->setIncrementHandler(std::bind(&McProcessor::increment, _mcProcessor, std::placeholders::_1, 185 | std::placeholders::_2, std::placeholders::_3, 186 | std::placeholders::_4, std::placeholders::_5, 187 | std::placeholders::_6, std::placeholders::_7)); 188 | _mcInterface->setPrependHandler(std::bind(&McProcessor::prepend, _mcProcessor, std::placeholders::_1, 189 | std::placeholders::_2, std::placeholders::_3, 190 | std::placeholders::_4, std::placeholders::_5, 191 | std::placeholders::_6)); 192 | _mcInterface->setNoopHandler(std::bind(&McProcessor::noop, _mcProcessor)); 193 | _mcInterface->setQuitHandler(std::bind(&McProcessor::quit, _mcProcessor)); 194 | _mcInterface->setReplaceHandler(std::bind(&McProcessor::replace, _mcProcessor, std::placeholders::_1, 195 | std::placeholders::_2, std::placeholders::_3, 196 | std::placeholders::_4, std::placeholders::_5, 197 | std::placeholders::_6, std::placeholders::_7, 198 | std::placeholders::_8)); 199 | _mcInterface->setSetHandler(std::bind(&McProcessor::set, _mcProcessor, std::placeholders::_1, 200 | std::placeholders::_2, std::placeholders::_3, 201 | std::placeholders::_4, std::placeholders::_5, 202 | std::placeholders::_6, std::placeholders::_7, 203 | std::placeholders::_8)); 204 | _mcInterface->setVersionHandler(std::bind(&McProcessor::version, _mcProcessor, std::placeholders::_1)); 205 | _mcInterface->setVerbosityHandler(std::bind(&McProcessor::verbosity, _mcProcessor, std::placeholders::_1)); 206 | _mcInterface->setStatHandler(std::bind(&McProcessor::stat, _mcProcessor, std::placeholders::_1, 207 | std::placeholders::_2, std::placeholders::_3)); 208 | _mcInterface->setPreExecute(std::bind(&McProcessor::preExecute, _mcProcessor)); 209 | _mcInterface->setPostExecute(std::bind(&McProcessor::postExecute, _mcProcessor)); 210 | _mcInterface->setUnknownExecute(std::bind(&McProcessor::unknownExecute, _mcProcessor)); 211 | } 212 | 213 | // }}} 214 | -------------------------------------------------------------------------------- /src/AdServer.hpp: -------------------------------------------------------------------------------- 1 | /// 该程序自动生成，禁止修改 2 | #ifndef AIDT_ADSERVER_HPP_ 3 | #define AIDT_ADSERVER_HPP_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "AdbaseConfig.hpp" 12 | #include "Http.hpp" 13 | #include "McProcessor.hpp" 14 | #include "HeadProcessor.hpp" 15 | 16 | class AdServer { 17 | public: 18 | AdServer(AdServerContext* context); 19 | ~AdServer(); 20 | void run(); 21 | void stop(); 22 | std::string httpFallback(const std::string& url); 23 | 24 | private: 25 | AdbaseConfig* _configure; 26 | /// 传输上下文指针 27 | AdServerContext* _context; 28 | adbase::EventBasePtr _base; 29 | 30 | /// http server 31 | adbase::http::Server* _http = nullptr; 32 | Http* _httpManager = nullptr; 33 | /// mc server 34 | McProcessor* _mcProcessor = nullptr; 35 | adbase::TcpServer* _mcServer = nullptr; 36 | adbase::InetAddress* _mcAddr = nullptr; 37 | adbase::mc::Interface* _mcInterface = nullptr; 38 | adbase::mc::Handler* _mcHandler = nullptr; 39 | /// head server 40 | HeadProcessor* _headProcessor = nullptr; 41 | adbase::TcpServer* _headServer = nullptr; 42 | adbase::InetAddress* _headAddr = nullptr; 43 | adbase::head::Interface* _headInterface = nullptr; 44 | adbase::head::Handler* _headHandler = nullptr; 45 | 46 | void init(); 47 | void initHttp(); 48 | void initHead(); 49 | void initMc(); 50 | void bindMcCallback(); 51 | }; 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /src/AdbaseConfig.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_ADBASE_CONFIG_HPP_ 2 | #define AIDT_ADBASE_CONFIG_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | // {{{ macros 9 | 10 | #ifndef DECLARE_KAFKA_CONSUMER_CONFIG 11 | #define DECLARE_KAFKA_CONSUMER_CONFIG(name) \ 12 | std::string topicNameConsumer##name;\ 13 | std::string groupId##name;\ 14 | std::string brokerListConsumer##name;\ 15 | std::string kafkaDebug##name;\ 16 | std::string offsetPath##name;\ 17 | std::string statInterval##name;\ 18 | bool isNewConsumer##name; 19 | #endif 20 | #ifndef DECLARE_KAFKA_PRODUCER_CONFIG 21 | #define DECLARE_KAFKA_PRODUCER_CONFIG(name) \ 22 | std::string brokerListProducer##name;\ 23 | std::string debug##name;\ 24 | int queueLength##name; \ 25 | std::string securityProtocol;\ 26 | std::string saslMechanisms;\ 27 | std::string kerberosServiceName;\ 28 | std::string kerberosPrincipal;\ 29 | std::string kerberosCmd;\ 30 | std::string kerberosKeytab;\ 31 | std::string kerberosMinTime;\ 32 | std::string saslUsername;\ 33 | std::string saslPassword; 34 | #endif 35 | #ifndef DECLARE_TIMER_CONFIG 36 | #define DECLARE_TIMER_CONFIG(name) \ 37 | int interval##name; 38 | #endif 39 | 40 | // }}} 41 | 42 | typedef struct adbaseConfig { 43 | bool daemon; 44 | std::string pidFile; 45 | int appid; 46 | int macid; 47 | int mallocTrimPad; 48 | int mallocTrimInterval; 49 | 50 | // logging config 51 | std::string logsDir; 52 | size_t logRollSize; 53 | int logLevel; 54 | bool isAsync; 55 | 56 | bool isStartMc; 57 | bool isStartHead; 58 | bool isStartHttp; 59 | 60 | std::string httpHost; 61 | int httpPort; 62 | int httpTimeout; 63 | int httpThreadNum; 64 | std::string httpDefaultController; 65 | std::string httpDefaultAction; 66 | std::string httpServerName; 67 | std::string httpAccessLogDir; 68 | int httpAccesslogRollSize; 69 | 70 | std::string headHost; 71 | int headPort; 72 | std::string headServerName; 73 | int headThreadNum; 74 | 75 | std::string mcHost; 76 | int mcPort; 77 | std::string mcServerName; 78 | int mcThreadNum; 79 | DECLARE_KAFKA_PRODUCER_CONFIG(In); 80 | DECLARE_TIMER_CONFIG(SyncOffset); 81 | 82 | std::string aidtConfig; 83 | 84 | // producer 85 | int messageQueueLimit; 86 | std::string messageSendError; 87 | 88 | // file reader 89 | int maxLines; 90 | int maxfd; 91 | int maxLineBytes; 92 | std::string offsetFile; 93 | } AdbaseConfig; 94 | 95 | class App; 96 | namespace app { 97 | class Message; 98 | class Reader; 99 | } 100 | typedef struct adserverContext { 101 | AdbaseConfig* config; 102 | adbase::EventBasePtr mainEventBase; 103 | App* app; 104 | adbase::metrics::Metrics* metrics; 105 | // 前后端交互数据指针添加到下面 106 | app::Message* message; 107 | } AdServerContext; 108 | 109 | typedef struct aimsContext { 110 | AdbaseConfig* config; 111 | App* app; 112 | // 消息队列交互上下文 113 | app::Message* message; 114 | } AimsContext; 115 | typedef struct timerContext { 116 | AdbaseConfig* config; 117 | adbase::EventBasePtr mainEventBase; 118 | App* app; 119 | // 定时器交互上下文 120 | app::Reader* reader; 121 | } TimerContext; 122 | 123 | #endif 124 | -------------------------------------------------------------------------------- /src/Aims.cpp: -------------------------------------------------------------------------------- 1 | #include "Aims.hpp" 2 | 3 | // {{{ Aims::Aims() 4 | 5 | Aims::Aims(AimsContext* context) : 6 | _context(context) { 7 | _configure = _context->config; 8 | } 9 | 10 | // }}} 11 | // {{{ Aims::~Aims() 12 | 13 | Aims::~Aims() { 14 | } 15 | 16 | // }}} 17 | // {{{ void Aims::run() 18 | 19 | void Aims::run() { 20 | // 初始化 server 21 | init(); 22 | _kafkaProducerIn->start(); 23 | } 24 | 25 | // }}} 26 | // {{{ void Aims::init() 27 | 28 | void Aims::init() { 29 | initKafkaProducer(); 30 | } 31 | 32 | // }}} 33 | // {{{ void Aims::stop() 34 | 35 | void Aims::stop() { 36 | if (_kafkaProducerIn != nullptr) { 37 | _kafkaProducerIn->stop(); 38 | } 39 | if (_kafkaProducerCallbackIn != nullptr) { 40 | delete _kafkaProducerCallbackIn; 41 | _kafkaProducerCallbackIn = nullptr; 42 | } 43 | } 44 | 45 | // }}} 46 | // {{{ adbase::kafka::Producer* Aims::getProducer() 47 | 48 | adbase::kafka::Producer* Aims::getProducer() { 49 | return _kafkaProducerIn; 50 | } 51 | 52 | // }}} 53 | // {{{ void Aims::initKafkaProducer() 54 | 55 | void Aims::initKafkaProducer() { 56 | _kafkaProducerCallbackIn = new aims::kafka::ProducerIn(_context); 57 | std::unordered_map configs; 58 | 59 | configs["metadata.broker.list"] = _configure->brokerListProducerIn; 60 | configs["debug"] = _configure->debugIn; 61 | configs["security.protocol"] = _configure->securityProtocol; 62 | configs["sasl.mechanisms"] = _configure->saslMechanisms; 63 | configs["sasl.kerberos.service.name"] = _configure->kerberosServiceName; 64 | configs["sasl.kerberos.principal"] = _configure->kerberosPrincipal; 65 | configs["sasl.kerberos.kinit.cmd"] = _configure->kerberosCmd; 66 | configs["sasl.kerberos.keytab"] = _configure->kerberosKeytab; 67 | configs["sasl.kerberos.min.time.before.relogin"] = _configure->kerberosMinTime; 68 | configs["sasl.username"] = _configure->saslUsername; 69 | configs["sasl.password"] = _configure->saslPassword; 70 | 71 | _kafkaProducerIn = new adbase::kafka::Producer(configs, _configure->queueLengthIn); 72 | _kafkaProducerIn->setSendHandler(std::bind(&aims::kafka::ProducerIn::send, 73 | _kafkaProducerCallbackIn, 74 | std::placeholders::_1, std::placeholders::_2, 75 | std::placeholders::_3)); 76 | _kafkaProducerIn->setErrorHandler(std::bind(&aims::kafka::ProducerIn::errorCallback, 77 | _kafkaProducerCallbackIn, 78 | std::placeholders::_1, std::placeholders::_2, 79 | std::placeholders::_3, std::placeholders::_4)); 80 | } 81 | 82 | // }}} 83 | -------------------------------------------------------------------------------- /src/Aims.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_AIMS_HPP_ 2 | #define AIDT_AIMS_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "AdbaseConfig.hpp" 8 | #include "Aims/Kafka/ProducerIn.hpp" 9 | 10 | #ifndef DECLARE_KAFKA_CONSUMER 11 | #define DECLARE_KAFKA_CONSUMER(name) \ 12 | adbase::kafka::Consumer* _kafkaConsumer##name = nullptr;\ 13 | aims::kafka::Consumer##name* _kafkaConsumerCallback##name = nullptr; 14 | #endif 15 | #ifndef DECLARE_KAFKA_PRODUCER 16 | #define DECLARE_KAFKA_PRODUCER(name) \ 17 | adbase::kafka::Producer* _kafkaProducer##name = nullptr;\ 18 | aims::kafka::Producer##name* _kafkaProducerCallback##name = nullptr; 19 | #endif 20 | 21 | class Aims { 22 | public: 23 | Aims(AimsContext* context); 24 | ~Aims(); 25 | void run(); 26 | void stop(); 27 | adbase::kafka::Producer* getProducer(); 28 | 29 | private: 30 | /// 传输上下文指针 31 | AimsContext* _context; 32 | AdbaseConfig* _configure; 33 | 34 | DECLARE_KAFKA_PRODUCER(In) 35 | void init(); 36 | void initKafkaProducer(); 37 | }; 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /src/Aims/Kafka/ProducerIn.cpp: -------------------------------------------------------------------------------- 1 | #include "ProducerIn.hpp" 2 | #include "App/Message.hpp" 3 | #include "App.hpp" 4 | 5 | namespace aims { 6 | namespace kafka { 7 | // {{{ ProducerIn::ProducerIn() 8 | 9 | ProducerIn::ProducerIn(AimsContext* context) : 10 | _context(context) { 11 | } 12 | 13 | // }}} 14 | // {{{ ProducerIn::~ProducerIn() 15 | 16 | ProducerIn::~ProducerIn() { 17 | } 18 | 19 | // }}} 20 | // {{{ bool ProducerIn::send() 21 | 22 | bool ProducerIn::send(std::string& topicName, int* partId, adbase::Buffer& data) { 23 | if (_context->message == nullptr) { 24 | return false; 25 | } 26 | if (_context->app == nullptr) { 27 | return false; 28 | } 29 | 30 | bool ret = _context->message->send(topicName, partId, data); 31 | if (!_context->app->checkValidTopicAndPartition(topicName, *partId)) { 32 | _context->message->errorCallback(topicName, *partId, data, "Topic or parttion is invalid."); 33 | return false; 34 | } 35 | return ret; 36 | } 37 | 38 | // }}} 39 | // {{{ void ProducerIn::errorCallback() 40 | 41 | void ProducerIn::errorCallback(const std::string& topicName, int partId, const adbase::Buffer& message, const std::string& error) { 42 | if (_context->message == nullptr) { 43 | return; 44 | } 45 | _context->message->errorCallback(topicName, partId, message, error); 46 | } 47 | 48 | // }}} 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Aims/Kafka/ProducerIn.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_AIMS_KAFKA_PRODUER_INHPP_ 2 | #define AIDT_AIMS_KAFKA_PRODUER_INHPP_ 3 | 4 | #include 5 | #include 6 | #include "AdbaseConfig.hpp" 7 | 8 | namespace aims { 9 | namespace kafka { 10 | class ProducerIn { 11 | public: 12 | ProducerIn(AimsContext* context); 13 | ~ProducerIn(); 14 | bool send(std::string& topicName, int* partId, adbase::Buffer& message); 15 | void errorCallback(const std::string& topicName, int partId, const adbase::Buffer& message, const std::string& error); 16 | private: 17 | AimsContext* _context; 18 | }; 19 | 20 | } 21 | } 22 | #endif 23 | -------------------------------------------------------------------------------- /src/App.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "App.hpp" 4 | #include "App/ConfigPaser.hpp" 5 | #include "App/Reader.hpp" 6 | #include "App/Watcher.hpp" 7 | 8 | //{{{ macros 9 | 10 | #define LOAD_KAFKA_CONSUMER_CONFIG(name, sectionName) do {\ 11 | _configure->isNewConsumer##name = config.getOptionBool("kafkac_"#sectionName, "isNewConsumer"#name);\ 12 | _configure->topicNameConsumer##name = config.getOption("kafkac_"#sectionName, "topicName"#name);\ 13 | _configure->groupId##name = config.getOption("kafkac_"#sectionName, "groupId"#name);\ 14 | _configure->brokerListConsumer##name = config.getOption("kafkac_"#sectionName, "brokerList"#name);\ 15 | _configure->kafkaDebug##name = config.getOption("kafkac_"#sectionName, "kafkaDebug"#name);\ 16 | _configure->offsetPath##name = config.getOption("kafkac_"#sectionName, "offsetPath"#name);\ 17 | _configure->statInterval##name = config.getOption("kafkac_"#sectionName, "statInterval"#name);\ 18 | } while(0) 19 | 20 | #define LOAD_KAFKA_PRODUCER_CONFIG(name, sectionName) do {\ 21 | _configure->brokerListProducer##name = config.getOption("kafkap_"#sectionName, "brokerList"#name);\ 22 | _configure->debug##name = config.getOption("kafkap_"#sectionName, "debug"#name);\ 23 | _configure->queueLength##name = config.getOptionUint32("kafkap_"#sectionName, "queueLength"#name);\ 24 | _configure->securityProtocol = config.getOption("kafkap_"#sectionName, "securityProtocol");\ 25 | _configure->saslMechanisms = config.getOption("kafkap_"#sectionName, "saslMechanisms");\ 26 | _configure->kerberosServiceName = config.getOption("kafkap_"#sectionName, "kerberosServiceName");\ 27 | _configure->kerberosPrincipal = config.getOption("kafkap_"#sectionName, "kerberosPrincipal");\ 28 | _configure->kerberosCmd = config.getOption("kafkap_"#sectionName, "kerberosCmd");\ 29 | _configure->kerberosKeytab = config.getOption("kafkap_"#sectionName, "kerberosKeytab");\ 30 | _configure->kerberosMinTime = config.getOption("kafkap_"#sectionName, "kerberosMinTime");\ 31 | _configure->saslUsername = config.getOption("kafkap_"#sectionName, "saslUsername");\ 32 | _configure->saslPassword = config.getOption("kafkap_"#sectionName, "saslPassword");\ 33 | } while(0) 34 | 35 | #define LOAD_TIMER_CONFIG(name) do {\ 36 | _configure->interval##name = config.getOptionUint32("timer", "interval"#name);\ 37 | } while(0) 38 | 39 | //}}} 40 | // {{{ App::App() 41 | 42 | App::App(AdbaseConfig* config) : 43 | _configure(config) { 44 | } 45 | 46 | // }}} 47 | // {{{ App::~App() 48 | 49 | App::~App() { 50 | } 51 | 52 | // }}} 53 | // {{{ void App::run() 54 | 55 | void App::run() { 56 | _config = std::shared_ptr(new app::Config()); 57 | _message = std::shared_ptr(new app::Message(_configure)); 58 | 59 | app::ConfigPaser::paser(_configure->aidtConfig, _config.get()); 60 | 61 | _reader = std::shared_ptr(new app::Reader(_config, this, _configure, _message)); 62 | _reader->load(); 63 | 64 | _watcher = std::shared_ptr(new app::Watcher(_configure, _config)); 65 | _watcher->setReadHandler(std::bind(&app::Reader::read, _reader, std::placeholders::_1, std::placeholders::_2)); 66 | _watcher->setClearHandler(std::bind(&app::Reader::clear, _reader, std::placeholders::_1)); 67 | _watcher->setMoveToHandler(std::bind(&app::Reader::moveto, _reader, std::placeholders::_1)); 68 | _watcher->setMoveSelfHandler(std::bind(&app::Reader::moveself, _reader, std::placeholders::_1)); 69 | 70 | _watcher->start(); 71 | _message->loadMessage(); 72 | } 73 | 74 | // }}} 75 | // {{{ void App::checkConfig() 76 | 77 | void App::checkConfig() { 78 | _config = std::shared_ptr(new app::Config()); 79 | app::ConfigPaser::paser(_configure->aidtConfig, _config.get()); 80 | } 81 | 82 | // }}} 83 | // {{{ void App::reload() 84 | 85 | void App::reload() { 86 | app::ConfigPaser::paser(_configure->aidtConfig, _config.get()); 87 | } 88 | 89 | // }}} 90 | // {{{ void App::resend() 91 | 92 | void App::resend() { 93 | if (_message) { 94 | _message->loadMessage(); 95 | } 96 | } 97 | 98 | // }}} 99 | // {{{ void App::stop() 100 | 101 | void App::stop() { 102 | if (_message) { 103 | _message->saveMessage(); 104 | } 105 | 106 | if (_watcher) { 107 | _watcher->stop(); 108 | if (_reader) { 109 | _reader->stop(); 110 | } 111 | } 112 | if (_reader) { 113 | _reader->save(); 114 | } 115 | } 116 | 117 | // }}} 118 | // {{{ void App::setAdServerContext() 119 | 120 | void App::setAdServerContext(AdServerContext* context) { 121 | context->app = this; 122 | context->message = _message.get(); 123 | } 124 | 125 | // }}} 126 | // {{{ void App::setAimsContext() 127 | 128 | void App::setAimsContext(AimsContext* context) { 129 | context->app = this; 130 | context->message = _message.get(); 131 | } 132 | 133 | // }}} 134 | // {{{ void App::setTimerContext() 135 | 136 | void App::setTimerContext(TimerContext* context) { 137 | context->app = this; 138 | context->reader = _reader.get(); 139 | } 140 | 141 | // }}} 142 | // {{{ void App::setAims() 143 | 144 | void App::setAims(std::shared_ptr& aims) { 145 | _aims = aims; 146 | } 147 | 148 | // }}} 149 | //{{{ void App::loadConfig() 150 | 151 | void App::loadConfig(adbase::IniConfig& config) { 152 | LOAD_KAFKA_PRODUCER_CONFIG(In, in); 153 | 154 | LOAD_TIMER_CONFIG(SyncOffset); 155 | 156 | _configure->aidtConfig = config.getOption("system", "aidtConfig"); 157 | _configure->mallocTrimInterval = config.getOptionUint32("system", "mallocTrimInterval"); 158 | _configure->mallocTrimPad = config.getOptionUint32("system", "mallocTrimPad"); 159 | _configure->messageQueueLimit = config.getOptionUint32("message", "queueLimit"); 160 | _configure->messageSendError = config.getOption("message", "sendError"); 161 | 162 | _configure->maxLines = config.getOptionUint32("reader", "maxLines"); 163 | _configure->maxfd = config.getOptionUint32("reader", "maxfd"); 164 | _configure->maxLineBytes = config.getOptionUint32("reader", "maxLineBytes"); 165 | _configure->offsetFile = config.getOption("reader", "offsetFile"); 166 | } 167 | 168 | //}}} 169 | //{{{ uint64_t App::getSeqId() 170 | 171 | uint64_t App::getSeqId() { 172 | std::lock_guard lk(_mut); 173 | adbase::Sequence seq; 174 | return seq.getSeqId(static_cast(_configure->macid), static_cast(_configure->appid)); 175 | } 176 | 177 | //}}} 178 | //{{{ bool App::checkValidTopicAndPartition() 179 | 180 | bool App::checkValidTopicAndPartition(const std::string& topicName, int partId) { 181 | if (topicName.empty()) { 182 | return false; 183 | } 184 | std::lock_guard lk(_mut); 185 | if (_topics.find(topicName) == _topics.end()) { 186 | _topics = _aims->getProducer()->getTopics(); 187 | } 188 | if (_topics.find(topicName) == _topics.end()) { 189 | return false; 190 | } 191 | 192 | if (partId == -1) { //不需要指定分区 193 | return true; 194 | } 195 | 196 | for (auto &t : _topics[topicName]) { 197 | if (t == static_cast(partId)) { 198 | return true; 199 | } 200 | } 201 | return false; 202 | } 203 | 204 | //}}} 205 | -------------------------------------------------------------------------------- /src/App.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_APP_HPP_ 2 | #define AIDT_APP_HPP_ 3 | 4 | #include 5 | #include "AdbaseConfig.hpp" 6 | #include "App/Config.hpp" 7 | #include "App/Message.hpp" 8 | #include "App/Watcher.hpp" 9 | #include "App/Reader.hpp" 10 | #include "Aims.hpp" 11 | 12 | class App { 13 | public: 14 | App(AdbaseConfig* config); 15 | ~App(); 16 | void reload(); 17 | void resend(); 18 | void run(); 19 | void stop(); 20 | void setAdServerContext(AdServerContext* context); 21 | void setTimerContext(TimerContext* context); 22 | void setAimsContext(AimsContext* context); 23 | void loadConfig(adbase::IniConfig& config); 24 | void checkConfig(); 25 | uint64_t getSeqId(); 26 | void setAims(std::shared_ptr& aims); 27 | bool checkValidTopicAndPartition(const std::string& topicName, int partId); 28 | 29 | private: 30 | AdbaseConfig* _configure; 31 | std::shared_ptr _config; 32 | std::shared_ptr _message; 33 | std::shared_ptr _reader; 34 | std::shared_ptr _watcher; 35 | std::shared_ptr _aims; 36 | mutable std::mutex _mut; 37 | std::unordered_map> _topics; 38 | }; 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /src/App/Config.cpp: -------------------------------------------------------------------------------- 1 | #include "Config.hpp" 2 | #include 3 | #include 4 | 5 | namespace app { 6 | // {{{ Config::Config() 7 | 8 | Config::Config() { 9 | adbase::metrics::Metrics::buildGauges("config", "watcher.mapsize", 1000, [this](){ 10 | std::lock_guard lk(_mut); 11 | return static_cast(_fileMap.size()); 12 | }); 13 | adbase::metrics::Metrics::buildGauges("config", "queue.size", 1000, [this](){ 14 | return _queue.getSize(); 15 | }); 16 | } 17 | 18 | // }}} 19 | // {{{ Config::~Config() 20 | 21 | Config::~Config() { 22 | } 23 | 24 | // }}} 25 | // {{{ void Config::updateFileInfo() 26 | 27 | void Config::updateFileInfo(const WatcherFileMap& infos) { 28 | std::lock_guard lk(_mut); 29 | std::vector deleteFileInfo; 30 | for (auto &t : _fileMap) { 31 | if (infos.find(t.first) == infos.end()) { 32 | deleteFileInfo.push_back(t.first); 33 | } 34 | } 35 | for (auto &t : deleteFileInfo) { 36 | _fileMap.erase(t); 37 | LOG_INFO << "Delete `" << t << "` config."; 38 | } 39 | 40 | for (auto &t : infos) { 41 | _queue.push(t.first); 42 | _fileMap[t.first] = t.second; 43 | 44 | } 45 | } 46 | 47 | // }}} 48 | // {{{ void Config::setMoveTime() 49 | 50 | void Config::setMoveTime(std::string &filePath, uint32_t time) { 51 | std::lock_guard lk(_mut); 52 | if (_fileMap.find(filePath) == _fileMap.end()) { 53 | return; 54 | } 55 | 56 | WatcherFileInfo info = _fileMap[filePath]; 57 | info.moveTime = time; 58 | _fileMap[filePath] = info; 59 | } 60 | 61 | // }}} 62 | // {{{ bool Config::findFile() 63 | 64 | bool Config::findFile(std::string &filePath, WatcherFileInfo& fileInfo) { 65 | std::lock_guard lk(_mut); 66 | if (_fileMap.find(filePath) == _fileMap.end()) { 67 | return false; 68 | } 69 | 70 | fileInfo = _fileMap[filePath]; 71 | return true; 72 | } 73 | 74 | // }}} 75 | // {{{ bool Config::patternPath() 76 | 77 | bool Config::patternPath(std::string origFile, std::string &pathFile) { 78 | std::lock_guard lk(_mut); 79 | for (auto &t : _fileMap) { 80 | if (t.second.deleteTime) { 81 | continue; 82 | } 83 | 84 | if (!t.second.isDir) { // 如果监听的是文件，需要全等 85 | if (origFile == t.first) { 86 | pathFile = t.first; 87 | return true; 88 | } 89 | } else { 90 | if (t.second.isRecursive) { 91 | if (origFile.find(t.first) != std::string::npos) { 92 | pathFile = t.first; 93 | return true; 94 | } 95 | } else { 96 | std::vector pathSplit = adbase::explode(origFile, '/', true); 97 | if (pathSplit.empty()) { 98 | return false; 99 | } 100 | std::string newOriginFile; 101 | for (uint32_t i = 0; i < static_cast(pathSplit.size()) - 1; i++) { 102 | newOriginFile.append("/"); 103 | newOriginFile.append(pathSplit[i]); 104 | } 105 | newOriginFile = adbase::trim(newOriginFile, "/"); 106 | 107 | if (newOriginFile == adbase::trim(t.first, "/")) { 108 | pathFile = t.first; 109 | return true; 110 | } 111 | } 112 | } 113 | } 114 | 115 | return false; 116 | } 117 | 118 | // }}} 119 | // {{{ bool Config::isHasWatcherFile() 120 | 121 | bool Config::isHasWatcherFile(std::string & pathFile) { 122 | std::vector pathSplit = adbase::explode(pathFile, '/', true); 123 | if (pathSplit.empty()) { 124 | return false; 125 | } 126 | std::string newPathFile; 127 | for (uint32_t i = 0; i < static_cast(pathSplit.size()) - 1; i++) { 128 | newPathFile.append("/"); 129 | newPathFile.append(pathSplit[i]); 130 | } 131 | std::lock_guard lk(_mut); 132 | for (auto &t : _fileMap) { 133 | if (t.second.isDir) { 134 | continue; 135 | } 136 | std::vector secSplits = adbase::explode(t.first, '/', true); 137 | if (secSplits.empty()) { 138 | continue; 139 | } 140 | std::string newSecFile; 141 | for (uint32_t i = 0; i < static_cast(secSplits.size()) - 1; i++) { 142 | newSecFile.append("/"); 143 | newSecFile.append(pathSplit[i]); 144 | } 145 | 146 | if (newSecFile == newPathFile) { 147 | return true; 148 | } 149 | } 150 | 151 | return false; 152 | } 153 | 154 | // }}} 155 | // {{{ bool Config::consumerConfig() 156 | 157 | bool Config::consumerConfig(WatcherFileInfo& info) { 158 | std::string pathFile; 159 | bool ret = _queue.tryPop(pathFile); 160 | if (!ret) { 161 | return false; 162 | } 163 | std::lock_guard lk(_mut); 164 | if (_fileMap.find(pathFile) == _fileMap.end()) { 165 | return false; 166 | } 167 | 168 | info = _fileMap[pathFile]; 169 | return true; 170 | } 171 | 172 | // }}} 173 | } 174 | -------------------------------------------------------------------------------- /src/App/Config.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_CONFIG_HPP_ 2 | #define AIDT_CONFIG_HPP_ 3 | 4 | #include 5 | 6 | namespace app { 7 | typedef struct watcherFileInfo { 8 | std::string pathFile; 9 | bool isDir; 10 | bool isBinary; 11 | bool isRecursive; 12 | int id; 13 | int timeWait; 14 | uint32_t deleteTime; 15 | uint32_t moveTime; 16 | std::string topicName; 17 | std::string format; 18 | std::vector excludes; 19 | } WatcherFileInfo; 20 | typedef std::unordered_map WatcherFileMap; 21 | 22 | class Config { 23 | public: 24 | Config(); 25 | ~Config(); 26 | void updateFileInfo(const WatcherFileMap& infos); 27 | bool findFile(std::string & filePath, WatcherFileInfo& fileInfo); 28 | bool patternPath(std::string origFile, std::string &pathFile); 29 | bool isHasWatcherFile(std::string & pathFile); 30 | bool consumerConfig(WatcherFileInfo& info); 31 | void setMoveTime(std::string &filePath, uint32_t time); 32 | 33 | private: 34 | WatcherFileMap _fileMap; 35 | adbase::Queue _queue; 36 | mutable std::mutex _mut; 37 | }; 38 | } 39 | #endif 40 | -------------------------------------------------------------------------------- /src/App/ConfigPaser.cpp: -------------------------------------------------------------------------------- 1 | #include "ConfigPaser.hpp" 2 | #include "App/Config.hpp" 3 | 4 | namespace app { 5 | // {{{ ConfigPaser::ConfigPaser() 6 | 7 | ConfigPaser::ConfigPaser() { 8 | } 9 | 10 | // }}} 11 | // {{{ ConfigPaser::~ConfigPaser() 12 | 13 | ConfigPaser::~ConfigPaser() { 14 | } 15 | 16 | // }}} 17 | // {{{ bool ConfigPaser::paser() 18 | 19 | void ConfigPaser::paser(std::string configFile, Config *configObj) { 20 | check(configFile); 21 | 22 | YAML::Node config; 23 | config = YAML::LoadFile(configFile); 24 | YAML::Node transfers = config["transfers"]; 25 | if (transfers.IsSequence()) { 26 | WatcherFileMap infos; 27 | for (std::size_t i = 0; i < transfers.size(); i++) { 28 | YAML::Node item = transfers[i]; 29 | std::string path = item["path"].as(); 30 | path = trimPath(path.c_str()); 31 | if (path.empty()) { 32 | continue; 33 | } 34 | std::vector excludes; 35 | if (item["excludes"]) { 36 | YAML::Node excludesYml = item["excludes"]; 37 | for (std::size_t j = 0; j < excludes.size(); j++) { 38 | excludes.push_back(excludesYml[i].as()); 39 | } 40 | } 41 | bool isDir = false; 42 | if (item["is_dir"]) { 43 | isDir = item["is_dir"].as(); 44 | } 45 | 46 | bool isBinary = false; 47 | if (item["is_binary"]) { 48 | isBinary = item["is_binary"].as(); 49 | } 50 | bool isRecursive = false; 51 | if (item["is_recursive"]) { 52 | isRecursive = item["is_recursive"].as(); 53 | } 54 | uint32_t timeWait = 3; 55 | if (item["time_wait"]) { 56 | timeWait = item["time_wait"].as(); 57 | } 58 | 59 | std::string format = "%s"; 60 | if (item["format"]) { 61 | format = item["format"].as(); 62 | } 63 | std::string topicName = item["topic_name"].as(); 64 | WatcherFileInfo info; 65 | info.isDir = isDir; 66 | info.pathFile = path; 67 | info.isBinary = isBinary; 68 | info.timeWait = timeWait; 69 | info.topicName = topicName; 70 | info.format = format; 71 | info.excludes = excludes; 72 | info.deleteTime = 0; 73 | info.moveTime = 0; 74 | info.isRecursive = isRecursive; 75 | infos[path] = info; 76 | } 77 | configObj->updateFileInfo(infos); 78 | } 79 | } 80 | 81 | // }}} 82 | // {{{ bool ConfigPaser::check() 83 | 84 | void ConfigPaser::check(std::string configFile) { 85 | YAML::Node config; 86 | try { 87 | config = YAML::LoadFile(configFile); 88 | } catch (std::exception& e) { 89 | LOG_SYSFATAL << "Paser yaml config :" << configFile << " fail, err:" << e.what(); 90 | } catch (...) { 91 | LOG_SYSFATAL << "Paser yaml config :" << configFile << " fail"; 92 | } 93 | 94 | if (config["transfers"]) { 95 | YAML::Node transfers = config["transfers"]; 96 | if (!transfers.IsSequence() && !transfers.IsNull()) { 97 | LOG_SYSFATAL << "`transfers` config item must is array or null"; 98 | } 99 | 100 | if (transfers.IsSequence()) { 101 | for (std::size_t i = 0; i < transfers.size(); i++) { 102 | YAML::Node item = transfers[i]; 103 | if (!item.IsMap()) { 104 | LOG_SYSFATAL << "`transfers->item` must is map"; 105 | } 106 | if (!item["path"]) { 107 | LOG_SYSFATAL << "`transfers->item[ " << i << " ]->path` must is define"; 108 | } 109 | item["path"].as(); 110 | 111 | if (item["excludes"]) { 112 | YAML::Node excludes = item["excludes"]; 113 | if (!excludes.IsSequence()) { 114 | LOG_SYSFATAL << "`transfers->item[ " << i << " ]->excludes` must is array"; 115 | } 116 | for (std::size_t j = 0; j < excludes.size(); j++) { 117 | excludes[i].as(); 118 | } 119 | } 120 | if (item["is_dir"]) { 121 | item["is_dir"].as(); 122 | } 123 | if (item["is_binary"]) { 124 | item["is_binary"].as(); 125 | } 126 | if (item["is_recursive"]) { 127 | item["is_recursive"].as(); 128 | } 129 | if (item["time_wait"]) { 130 | item["time_wait"].as(); 131 | } 132 | if (item["format"]) { 133 | item["format"].as(); 134 | } 135 | if (!item["topic_name"]) { 136 | LOG_SYSFATAL << "`transfers->item[ " << i << " ]->topic_name` must is define"; 137 | } 138 | item["topic_name"].as(); 139 | } 140 | } 141 | } else { 142 | LOG_SYSFATAL << "`transfers` config item must is define, if not file transfer you can set it `~`"; 143 | } 144 | } 145 | 146 | // }}} 147 | // {{{ std::string ConfigPaser::trimPath() 148 | 149 | std::string ConfigPaser::trimPath(const char *path) { 150 | std::string pathFile(path); 151 | std::string::size_type n = pathFile.find_last_not_of("/"); 152 | if (n == std::string::npos) { // 不遍历根目录 153 | return ""; 154 | } 155 | return pathFile.substr(0, n + 1); 156 | } 157 | 158 | // }}} 159 | } 160 | -------------------------------------------------------------------------------- /src/App/ConfigPaser.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_CONFIG_PASER_HPP_ 2 | #define AIDT_CONFIG_PASER_HPP_ 3 | #include 4 | #include "yaml-cpp/yaml.h" 5 | 6 | namespace app { 7 | class Config; 8 | class ConfigPaser { 9 | public: 10 | ConfigPaser(); 11 | ~ConfigPaser(); 12 | static void paser(std::string configStr, Config *config); 13 | static void check(std::string configStr); 14 | 15 | private: 16 | static std::string trimPath(const char *pathFile); 17 | }; 18 | } 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /src/App/DataStore.cpp: -------------------------------------------------------------------------------- 1 | #include "DataStore.hpp" 2 | #include 3 | 4 | namespace app { 5 | // {{{ DataStore::DataStore() 6 | 7 | DataStore::DataStore(MessageQueue* queue): _queue(queue) { 8 | } 9 | 10 | // }}} 11 | // {{{ DataStore::~DataStore() 12 | 13 | DataStore::~DataStore() { 14 | } 15 | 16 | // }}} 17 | // {{{ void DataStore::saveMessage() 18 | 19 | void DataStore::saveMessage(const std::string& messageFile) { 20 | std::ofstream ofs(messageFile.c_str(), std::ofstream::app | std::ofstream::binary); 21 | 22 | MessageItem item; 23 | uint32_t queueCount = 0; 24 | while (!_queue->empty()) { // 获取队列中的数据 25 | bool ret = _queue->tryPop(item); 26 | if (ret) { 27 | adbase::Buffer dumpBuffer; 28 | serialize(dumpBuffer, item); 29 | ofs.write(dumpBuffer.peek(), dumpBuffer.readableBytes()); 30 | queueCount++; 31 | } 32 | } 33 | 34 | LOG_INFO << "Save message, count " << queueCount; 35 | ofs.flush(); 36 | ofs.close(); 37 | } 38 | 39 | // }}} 40 | // {{{ void DataStore::saveMessage() 41 | 42 | void DataStore::saveMessage(const std::string& messageFile, MessageItem& item) { 43 | std::ofstream ofs(messageFile.c_str(), std::ofstream::app | std::ofstream::binary); 44 | adbase::Buffer dumpBuffer; 45 | if (ofs.is_open() && ofs.good()) { 46 | serialize(dumpBuffer, item); 47 | ofs.write(dumpBuffer.peek(), dumpBuffer.readableBytes()); 48 | } 49 | ofs.flush(); 50 | ofs.close(); 51 | } 52 | 53 | // }}} 54 | // {{{ void DataStore::loadMessage() 55 | 56 | void DataStore::loadMessage(const std::string& messageFile) { 57 | std::ifstream ifs(messageFile.c_str(), std::ios_base::in | std::ios_base::binary); 58 | if (!ifs.good() || !ifs.is_open()) { 59 | return; 60 | } 61 | 62 | while (true) { 63 | adbase::Buffer loadBuffer; 64 | uint32_t headerSize = 3 * static_cast(sizeof(uint32_t)); 65 | char lenBuf[headerSize]; 66 | memset(lenBuf, 0, headerSize); 67 | ifs.read(lenBuf, headerSize); 68 | if (!ifs.good() || ifs.gcount() != headerSize) { 69 | break; 70 | } 71 | loadBuffer.append(lenBuf, headerSize); 72 | 73 | uint32_t partId = loadBuffer.readInt32(); 74 | uint32_t tryNum = loadBuffer.readInt32(); 75 | uint32_t topicNameSize = loadBuffer.readInt32(); 76 | char topicNameBuf[topicNameSize]; 77 | memset(topicNameBuf, 0, topicNameSize); 78 | ifs.read(topicNameBuf, topicNameSize); 79 | if (!ifs.good() || ifs.gcount() != topicNameSize) { 80 | break; 81 | } 82 | std::string topicName(topicNameBuf, topicNameSize); 83 | 84 | char messageSizeBuf[sizeof(uint32_t)] = {0}; 85 | memset(messageSizeBuf, 0, sizeof(uint32_t)); 86 | ifs.read(messageSizeBuf, sizeof(uint32_t)); 87 | if (!ifs.good() || ifs.gcount() != sizeof(uint32_t)) { 88 | break; 89 | } 90 | uint32_t messageSize = 0; 91 | ::memcpy(&messageSize, messageSizeBuf, sizeof messageSize); 92 | messageSize = adbase::networkToHost32(messageSize); 93 | 94 | std::unique_ptr data(new char[messageSize]); 95 | ifs.read(data.get(), messageSize); 96 | if (!ifs.good() || ifs.gcount() != messageSize) { 97 | break; 98 | } 99 | adbase::Buffer message; 100 | message.append(data.get(), messageSize); 101 | 102 | MessageItem newItem; 103 | newItem.partId = partId; 104 | newItem.tryNum = tryNum; 105 | newItem.topicName = topicName; 106 | newItem.message = message; 107 | _queue->push(newItem); 108 | } 109 | 110 | ifs.close(); 111 | std::string bakPathName = messageFile + ".bak"; 112 | if (0 != rename(messageFile.c_str(), bakPathName.c_str())) { 113 | LOG_INFO << "Rename error file to bak fail."; 114 | } 115 | } 116 | 117 | // }}} 118 | // {{{ void DataStore::serialize() 119 | 120 | void DataStore::serialize(adbase::Buffer& buffer, MessageItem& item) { 121 | buffer.appendInt32(item.partId); 122 | buffer.appendInt32(item.tryNum); 123 | buffer.appendInt32(static_cast(item.topicName.size())); 124 | buffer.append(item.topicName); 125 | 126 | size_t messageSize = item.message.readableBytes(); 127 | buffer.appendInt32(static_cast(messageSize)); 128 | buffer.append(item.message.peek(), messageSize); 129 | } 130 | 131 | // }}} 132 | // {{{ bool DataStore::saveOffset() 133 | 134 | bool DataStore::saveOffset(std::ofstream* ofs, const FdItem& item) { 135 | struct stat fileStat; 136 | if (0 != stat(item.origfile.c_str(), &fileStat) || !S_ISREG(fileStat.st_mode)) { 137 | return false; 138 | } 139 | uint64_t inode = static_cast(fileStat.st_ino); 140 | uint64_t filesize = static_cast(fileStat.st_size); 141 | 142 | adbase::Buffer buffer; 143 | buffer.appendInt64(item.offset); 144 | buffer.appendInt64(inode); 145 | buffer.appendInt64(filesize); 146 | buffer.appendInt32(static_cast(item.pathfile.size())); 147 | buffer.appendInt32(static_cast(item.origfile.size())); 148 | buffer.append(item.pathfile); 149 | buffer.append(item.origfile); 150 | ofs->write(buffer.peek(), buffer.readableBytes()); 151 | 152 | return true; 153 | } 154 | 155 | // }}} 156 | // {{{ void DataStore::loadOffsets() 157 | 158 | void DataStore::loadOffsets(const std::string& offsetFile, FdMap& maps) { 159 | std::ifstream ifs(offsetFile.c_str(), std::ios_base::in | std::ios_base::binary); 160 | if (!ifs.good() || !ifs.is_open()) { 161 | return; 162 | } 163 | while (true) { 164 | adbase::Buffer loadBuffer; 165 | uint32_t headerSize = 8 * static_cast(sizeof(uint32_t)); 166 | char lenBuf[headerSize]; 167 | memset(lenBuf, 0, headerSize); 168 | ifs.read(lenBuf, headerSize); 169 | if (!ifs.good() || ifs.gcount() != headerSize) { 170 | break; 171 | } 172 | loadBuffer.append(lenBuf, headerSize); 173 | 174 | uint64_t offset = loadBuffer.readInt64(); 175 | loadBuffer.readInt64(); // inode 176 | loadBuffer.readInt64(); // filesize 177 | uint32_t pathFileSize = loadBuffer.readInt32(); 178 | uint32_t origFileSize = loadBuffer.readInt32(); 179 | char pathFileBuf[pathFileSize]; 180 | char origFileBuf[origFileSize]; 181 | memset(pathFileBuf, 0, pathFileSize); 182 | memset(origFileBuf, 0, origFileSize); 183 | ifs.read(pathFileBuf, pathFileSize); 184 | if (!ifs.good() || ifs.gcount() != pathFileSize) { 185 | break; 186 | } 187 | std::string pathFile(pathFileBuf, pathFileSize); 188 | ifs.read(origFileBuf, origFileSize); 189 | if (!ifs.good() || ifs.gcount() != origFileSize) { 190 | break; 191 | } 192 | std::string origFile(origFileBuf, origFileSize); 193 | FdItem item; 194 | item.pathfile = pathFile; 195 | item.origfile = origFile; 196 | item.offset = offset; 197 | item.fs = nullptr; 198 | maps[origFile] = item; 199 | LOG_INFO << "Load origin file offset info, originFile: " << item.origfile 200 | << ", pathFile: " << item.pathfile 201 | << ", offset: " << item.offset; 202 | } 203 | ifs.close(); 204 | } 205 | 206 | // }}} 207 | // {{{ void DataStore::saveOffsets() 208 | 209 | void DataStore::saveOffsets(const std::string& offsetFile, const FdMap& maps) { 210 | std::ofstream ofs(offsetFile.c_str(), std::ofstream::trunc | std::ofstream::binary); 211 | for (auto &t : maps) { 212 | saveOffset(&ofs, t.second); 213 | } 214 | ofs.flush(); 215 | ofs.close(); 216 | } 217 | 218 | // }}} 219 | 220 | } 221 | -------------------------------------------------------------------------------- /src/App/DataStore.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_DATASTORE_HPP_ 2 | #define AIDT_DATASTORE_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "AdbaseConfig.hpp" 12 | 13 | namespace app { 14 | // message queue 15 | typedef struct MessageItem { 16 | int partId; 17 | int tryNum; 18 | std::string topicName; 19 | adbase::Buffer message; 20 | } MessageItem; 21 | typedef adbase::Queue MessageQueue; 22 | 23 | typedef struct FdItem { 24 | std::string pathfile; 25 | std::string origfile; 26 | uint64_t offset; 27 | int lastTime; 28 | bool moveProtected; // 是否处于保护期，如果处于文件保护状态在gc过程中保留 29 | std::fstream *fs; 30 | } FdItem; 31 | typedef std::unordered_map FdMap; 32 | 33 | class DataStore { 34 | public: 35 | DataStore(MessageQueue* queue); 36 | ~DataStore(); 37 | void saveMessage(const std::string& messageFile); 38 | void loadMessage(const std::string& messageFile); 39 | void saveMessage(const std::string& messageFile, MessageItem& item); 40 | bool saveOffset(std::ofstream* ofs, const FdItem& item); 41 | void saveOffsets(const std::string& offsetFile, const FdMap& maps); 42 | void loadOffsets(const std::string& offsetFile, FdMap& maps); 43 | private: 44 | MessageQueue* _queue; 45 | void serialize(adbase::Buffer& buffer, MessageItem& item); 46 | }; 47 | } 48 | #endif 49 | -------------------------------------------------------------------------------- /src/App/Message.cpp: -------------------------------------------------------------------------------- 1 | #include "Message.hpp" 2 | #include 3 | 4 | namespace app { 5 | // {{{ Message::Message() 6 | 7 | Message::Message(AdbaseConfig* configure): 8 | _configure(configure) { 9 | adbase::metrics::Metrics::buildGauges("message", "queue.size", 1, [this](){ 10 | return _queue.getSize(); 11 | }); 12 | _setRate = adbase::metrics::Metrics::buildMeters("message", "set.rate"); 13 | _dataStore = std::shared_ptr(new DataStore(&_queue)); 14 | } 15 | 16 | // }}} 17 | // {{{ Message::~Message() 18 | 19 | Message::~Message() { 20 | } 21 | 22 | // }}} 23 | // {{{ bool Message::send() 24 | 25 | bool Message::send(std::string& topicName, int* partId, adbase::Buffer& message) { 26 | MessageItem item; 27 | bool ret = _queue.tryPop(item); 28 | if (!ret) { 29 | return false; 30 | } 31 | 32 | *partId = item.partId; 33 | message = item.message; 34 | topicName = item.topicName; 35 | return true; 36 | } 37 | 38 | // }}} 39 | // {{{ bool Message::setMessage() 40 | 41 | bool Message::setMessage(const std::string& topicName, int partId, adbase::Buffer& message) { 42 | // 防止队列堆积 43 | if (static_cast(_queue.getSize()) > _configure->messageQueueLimit) { 44 | return false; 45 | } 46 | MessageItem item; 47 | item.partId = partId; 48 | item.message = message; 49 | item.topicName = topicName; 50 | item.tryNum = 0; 51 | _queue.push(item); 52 | if (_setRate != nullptr) { 53 | _setRate->mark(); 54 | } 55 | return true; 56 | } 57 | 58 | // }}} 59 | // {{{ void Message::errorCallback() 60 | 61 | void Message::errorCallback(const std::string& topicName, int partId, const adbase::Buffer& message, const std::string&) { 62 | std::lock_guard lk(_mut); 63 | if (topicName.empty() || message.readableBytes() == 0) { 64 | return; 65 | } 66 | 67 | LOG_ERROR << "Send message fail, message size:" << message.readableBytes() << " topic name:" << topicName << " partId:" << partId; 68 | MessageItem item; 69 | item.topicName = topicName; 70 | item.partId = partId; 71 | item.message = message; 72 | _dataStore->saveMessage(_configure->messageSendError, item); 73 | } 74 | 75 | // }}} 76 | // {{{ void Message::saveMessage() 77 | 78 | void Message::saveMessage() { 79 | std::lock_guard lk(_mut); 80 | _dataStore->saveMessage(_configure->messageSendError); 81 | } 82 | 83 | // }}} 84 | // {{{ void Message::loadMessage() 85 | 86 | void Message::loadMessage() { 87 | std::lock_guard lk(_mut); 88 | _dataStore->loadMessage(_configure->messageSendError); 89 | } 90 | 91 | // }}} 92 | } 93 | -------------------------------------------------------------------------------- /src/App/Message.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_MESSAGE_HPP_ 2 | #define AIDT_MESSAGE_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "AdbaseConfig.hpp" 12 | #include "DataStore.hpp" 13 | 14 | namespace app { 15 | class Message { 16 | public: 17 | Message(AdbaseConfig* configure); 18 | ~Message(); 19 | bool send(std::string& topicName, int* partId, adbase::Buffer& message); 20 | void errorCallback(const std::string& topicName, int partId, const adbase::Buffer& message, const std::string& error); 21 | bool setMessage(const std::string& topicName, int partId, adbase::Buffer& message); 22 | void saveMessage(); 23 | void loadMessage(); 24 | 25 | private: 26 | AdbaseConfig *_configure; 27 | MessageQueue _queue; 28 | adbase::metrics::Meters* _setRate = nullptr; 29 | mutable std::mutex _mut; 30 | std::shared_ptr _dataStore; 31 | }; 32 | } 33 | #endif 34 | -------------------------------------------------------------------------------- /src/App/Reader.cpp: -------------------------------------------------------------------------------- 1 | #include "Reader.hpp" 2 | #include "Watcher.hpp" 3 | #include "Message.hpp" 4 | #include 5 | #include "App.hpp" 6 | 7 | namespace app { 8 | // {{{ Reader::Reader() 9 | 10 | Reader::Reader(std::shared_ptr& watcherConfig, App* app, AdbaseConfig *configure, std::shared_ptr& message) : 11 | _configure(configure), 12 | _watcherConfig(watcherConfig), 13 | _app(app), 14 | _message(message), 15 | _running(true) { 16 | _currentFdNum = 0; 17 | _dataStore = std::shared_ptr(new DataStore(nullptr)); 18 | } 19 | 20 | // }}} 21 | // {{{ Reader::~Reader() 22 | 23 | Reader::~Reader() { 24 | } 25 | 26 | // }}} 27 | 28 | // {{{ void Reader::read() 29 | 30 | void Reader::read(const InotifyEvent& event, bool& isContinue) { 31 | WatcherFileInfo info; 32 | bool ret = _watcherConfig->findFile(const_cast(event.pathfile), info); 33 | if (!ret) { 34 | LOG_INFO << "Can't find pathFile " << event.pathfile << " in config manager"; 35 | return; 36 | } 37 | 38 | if (info.isBinary) { // binary safe 39 | readBinary(event, info, isContinue); 40 | } else { 41 | readString(event, info, isContinue); 42 | } 43 | } 44 | 45 | // }}} 46 | // {{{ void Reader::clear() 47 | 48 | void Reader::clear(const InotifyEvent& event) { 49 | std::lock_guard lk(_mut); 50 | LOG_INFO << "Delete file `" << event.origfile << "`, start."; 51 | for (auto &t : _fdMap) { 52 | LOG_INFO << "Map key: " << t.first << ", find: " << event.origfile; 53 | } 54 | 55 | if (_fdMap.find(event.origfile) != _fdMap.end()) { 56 | FdItem item = _fdMap[event.origfile]; 57 | closeStream(item.fs); 58 | if (item.fs != nullptr) { 59 | delete item.fs; 60 | } 61 | _fdMap.erase(event.origfile); 62 | LOG_INFO << "Delete file `" << event.origfile << "`, offset map clear."; 63 | } 64 | 65 | // 删除 alias 数据 66 | std::vector keys; 67 | for (auto &t : _aliasMap) { 68 | if (t.second == event.origfile) { 69 | keys.push_back(t.first); 70 | } 71 | } 72 | for (auto &t : keys) { 73 | _aliasMap.erase(t); 74 | } 75 | } 76 | 77 | // }}} 78 | // {{{ void Reader::moveself() 79 | 80 | void Reader::moveself(const InotifyEvent& event) { 81 | _watcherConfig->setMoveTime(const_cast(event.pathfile), static_cast(time(nullptr))); 82 | { 83 | std::lock_guard lk(_mut); 84 | if (_fdMap.find(event.origfile) != _fdMap.end()) { 85 | FdItem& item = _fdMap[event.origfile]; 86 | item.moveProtected = true; 87 | } 88 | } 89 | LOG_INFO << "Set file move time:" << static_cast(time(nullptr)); 90 | } 91 | 92 | // }}} 93 | // {{{ void Reader::moveto() 94 | 95 | void Reader::moveto(const InotifyEvent& event) { 96 | std::lock_guard lk(_mut); 97 | if (_fdMap.find(event.aliasfile) != _fdMap.end()) { 98 | _fdMap[event.origfile] = _fdMap[event.aliasfile]; 99 | _fdMap[event.origfile].origfile = event.origfile; 100 | // 用来更正在 move 操作以前生成的队列数据 101 | _aliasMap[event.aliasfile] = event.origfile; 102 | _fdMap.erase(event.aliasfile); 103 | } 104 | } 105 | 106 | // }}} 107 | // {{{ void Reader::save() 108 | 109 | void Reader::save() { 110 | std::lock_guard lk(_mut); 111 | std::string pathName = _configure->offsetFile; 112 | pathName.append(".swp"); 113 | std::string bakPathName = _configure->offsetFile; 114 | bakPathName.append(".bak"); 115 | std::ofstream ofs(pathName.c_str(), std::ofstream::trunc | std::ofstream::binary); 116 | for (auto &t : _fdMap) { 117 | WatcherFileInfo info; 118 | bool ret = _watcherConfig->findFile(const_cast(t.second.pathfile), info); 119 | if (!ret) { 120 | LOG_INFO << "Can't find pathFile " << t.second.pathfile << " in config manager"; 121 | continue; 122 | } 123 | 124 | _dataStore->saveOffset(&ofs, t.second); 125 | } 126 | ofs.flush(); 127 | ofs.close(); 128 | if (0 != rename(_configure->offsetFile.c_str(), bakPathName.c_str())) { 129 | LOG_INFO << "Rename offset file to bak fail."; 130 | } 131 | if (0 != rename(pathName.c_str(), _configure->offsetFile.c_str())) { 132 | LOG_INFO << "Rename swp offset file to offset file fail."; 133 | } 134 | LOG_TRACE << "Save file offset file success."; 135 | } 136 | 137 | // }}} 138 | // {{{ void Reader::load() 139 | 140 | void Reader::load() { 141 | std::lock_guard lk(_mut); 142 | _dataStore->loadOffsets(_configure->offsetFile, _fdMap); 143 | } 144 | 145 | // }}} 146 | // {{{ void Reader::stop() 147 | 148 | void Reader::stop() { 149 | _running = false; 150 | } 151 | 152 | // }}} 153 | 154 | // {{{ void Reader::readString() 155 | 156 | void Reader::readString(const InotifyEvent& event, const WatcherFileInfo& info, bool& isContinue) { 157 | isContinue = false; 158 | FdItem item; 159 | LOG_TRACE << "file " << event.origfile ; 160 | if (!initStream(event, item, info, isContinue)) { 161 | return; 162 | } 163 | 164 | std::filebuf* pbuf = item.fs->rdbuf(); 165 | pbuf->pubseekpos(item.offset); 166 | LOG_TRACE << "file " << event.origfile << ", seek:" << item.offset; 167 | 168 | size_t readSize = 0; 169 | int lines = 0; 170 | char nextchar; 171 | adbase::Buffer buffer; 172 | while((nextchar = pbuf->sbumpc()) != EOF && _running) { 173 | readSize++; 174 | if (nextchar != '\n') { 175 | buffer.append(&nextchar, 1); 176 | if (static_cast(buffer.readableBytes()) > _configure->maxLineBytes) { 177 | LOG_INFO << "File " << event.origfile << " single line > " << _configure->maxLineBytes; 178 | isContinue = true; 179 | break; 180 | } 181 | continue; 182 | } 183 | 184 | if (buffer.readableBytes()) { 185 | // 更新文件读取的位置 186 | if (!sendMessage(buffer, info)) { 187 | isContinue = true; 188 | break; 189 | } 190 | //LOG_INFO << "FILE:" << event.origfile << " size:" << readSize << "mess:" << std::string(buffer.peek(), buffer.readableBytes()); 191 | setFileOffset(const_cast(event.origfile), readSize); 192 | readSize = 0; 193 | lines++; 194 | 195 | buffer.retrieveAll(); 196 | if (lines > _configure->maxLines) { // 超过单次事件的行数 197 | isContinue = true; 198 | break; 199 | } 200 | } 201 | } 202 | } 203 | 204 | // }}} 205 | // {{{ void Reader::readBinary() 206 | 207 | void Reader::readBinary(const InotifyEvent& event, const WatcherFileInfo& info, bool& isContinue) { 208 | isContinue = false; 209 | FdItem item; 210 | LOG_INFO << "file " << event.origfile ; 211 | if (!initStream(event, item, info, isContinue)) { 212 | return; 213 | } 214 | 215 | int lines = 0; 216 | item.fs->seekg(item.offset, std::ios::beg); 217 | LOG_INFO << "file " << event.origfile << ", seek:" << item.offset; 218 | while (_running) { 219 | char lenBuf[sizeof(uint32_t)] = {0}; 220 | uint32_t len = 0; 221 | item.fs->read(lenBuf, sizeof(uint32_t)); 222 | if (item.fs->good() && item.fs->gcount() == sizeof(uint32_t)) { 223 | ::memcpy(&len, lenBuf, sizeof len); 224 | len = adbase::networkToHost32(len); 225 | } else { 226 | break; 227 | } 228 | 229 | std::unique_ptr data(new char[len]); 230 | item.fs->read(data.get(), len); 231 | if (item.fs->good() && item.fs->gcount() == len) { 232 | adbase::Buffer buffer; 233 | buffer.append(data.get(), len); 234 | if (!sendMessage(buffer, info)) { 235 | isContinue = true; 236 | break; 237 | } 238 | item.offset += len + sizeof(uint32_t); 239 | setFileOffset(const_cast(event.origfile), len + sizeof(uint32_t)); 240 | lines++; 241 | } else { 242 | break; 243 | } 244 | item.fs->seekg(item.offset, std::ios::beg); 245 | if (lines > _configure->maxLines) { // 超过单次事件的行数 246 | isContinue = true; 247 | break; 248 | } 249 | } 250 | } 251 | 252 | // }}} 253 | // {{{ void Reader::sendMessage() 254 | 255 | bool Reader::sendMessage(adbase::Buffer &msg, const WatcherFileInfo& info) { 256 | uint32_t partId = -1; 257 | adbase::Buffer message; 258 | if (info.isBinary) { 259 | LOG_TRACE << "Message: binary(" << msg.readableBytes() << "), topic:" << info.topicName 260 | << ", partId: " << partId; 261 | message = msg; 262 | } else { 263 | //message = msg; 264 | uint32_t formatBufferSize = static_cast(msg.readableBytes()) 265 | + static_cast(info.format.size()) - 1; 266 | std::unique_ptr buffer(new char[formatBufferSize]); 267 | snprintf(buffer.get(), formatBufferSize, info.format.c_str(), msg.peek()); 268 | // 去除尾部 snprintf 自动添加的 \0 字符 269 | message.append(buffer.get(), formatBufferSize - 1); 270 | } 271 | 272 | return _message->setMessage(info.topicName, partId, message); 273 | } 274 | 275 | // }}} 276 | // {{{ bool Reader::initStream() 277 | 278 | bool Reader::initStream(const InotifyEvent& event, FdItem& item, const WatcherFileInfo& info, bool& isContinue) { 279 | int currentTime = static_cast(time(nullptr)); 280 | isContinue = false; 281 | std::lock_guard lk(_mut); 282 | 283 | InotifyEvent tmpEvent = event; 284 | if (_currentFdNum > _configure->maxfd) { // 如果已经到打开文件最大数，主动gc 285 | LOG_TRACE << "Current not has enough fd, gc fd, max fd set is " << _configure->maxfd; 286 | gcFstreamNonLock(1); 287 | } 288 | if (_currentFdNum > _configure->maxfd) { 289 | LOG_TRACE << "open file max limit, pathfile: " << tmpEvent.origfile; 290 | isContinue = true; 291 | return false; 292 | } 293 | 294 | 295 | if (_aliasMap.find(tmpEvent.origfile) != _aliasMap.end()) { 296 | tmpEvent.origfile = _aliasMap[tmpEvent.origfile]; 297 | } 298 | if (_fdMap.find(tmpEvent.origfile) == _fdMap.end()) { 299 | LOG_INFO << "Can't find `" << tmpEvent.origfile << "` fd item, create new it"; 300 | item.pathfile = tmpEvent.pathfile; 301 | item.origfile = tmpEvent.origfile; 302 | item.offset = 0; 303 | item.fs = nullptr; 304 | } else { 305 | item = _fdMap[tmpEvent.origfile]; 306 | if (info.moveTime != 0 && (currentTime - info.moveTime) > info.timeWait) { // 需要重新重置 offset 307 | LOG_INFO << "File move timeout, need reset origfile info, timewait:" << info.timeWait; 308 | item.offset = 0; 309 | closeStream(item.fs); 310 | item.moveProtected = false; 311 | _watcherConfig->setMoveTime(const_cast(tmpEvent.origfile), 0); 312 | } 313 | } 314 | 315 | item.fs = openStream(tmpEvent.origfile, info, item.fs); 316 | item.lastTime = currentTime; 317 | _fdMap[tmpEvent.origfile] = item; 318 | if (item.fs != nullptr) { 319 | return true; 320 | } 321 | LOG_INFO << "open file fail, pathfile: " << tmpEvent.origfile; 322 | return false; 323 | } 324 | 325 | // }}} 326 | // {{{ void Reader::setFileOffset() 327 | 328 | void Reader::setFileOffset(std::string &pathFile, uint64_t offset) { 329 | std::lock_guard lk(_mut); 330 | FdItem item; 331 | if (_fdMap.find(pathFile) == _fdMap.end()) { 332 | return; 333 | } else { 334 | item = _fdMap[pathFile]; 335 | item.offset += offset; 336 | } 337 | 338 | _fdMap[pathFile] = item; 339 | } 340 | 341 | // }}} 342 | 343 | // {{{ void Reader::gcFstreamNonLock() 344 | 345 | void Reader::gcFstreamNonLock(int lifeTime) { 346 | int currentTime = static_cast(time(nullptr)); 347 | int count = 0; 348 | for (auto &t : _fdMap) { 349 | if (t.second.fs != nullptr && t.second.fs->is_open()) { 350 | count++; 351 | } 352 | if (!t.second.lastTime || (currentTime - t.second.lastTime) < lifeTime || t.second.moveProtected) { 353 | continue; 354 | } 355 | closeStream(t.second.fs); 356 | } 357 | _currentFdNum = count; 358 | } 359 | 360 | // }}} 361 | // {{{ void Reader::closeStream() 362 | 363 | void Reader::closeStream(std::fstream* fs) { 364 | if (fs != nullptr && fs->is_open()) { 365 | fs->close(); 366 | } 367 | } 368 | 369 | // }}} 370 | // {{{ std::fstream* Reader::openStream() 371 | 372 | std::fstream* Reader::openStream(const std::string& file, const WatcherFileInfo& info, std::fstream* stream) { 373 | if (stream != nullptr && stream->is_open() && stream->good()) { 374 | return stream; 375 | } 376 | 377 | if (stream != nullptr && !stream->good() && stream->is_open()) { 378 | closeStream(stream); 379 | } 380 | 381 | if (stream == nullptr) { 382 | if (info.isBinary) { 383 | stream = new std::fstream(file, std::fstream::in | std::fstream::binary); 384 | } else { 385 | stream = new std::fstream(file, std::fstream::in); 386 | } 387 | LOG_INFO << "New open file:" << file; 388 | } else { 389 | if (info.isBinary) { 390 | stream->open(file, std::fstream::in | std::fstream::binary); 391 | } else { 392 | stream->open(file, std::fstream::in); 393 | } 394 | LOG_INFO << "Re open file:" << file; 395 | } 396 | _currentFdNum++; 397 | 398 | if (!stream->good()) { 399 | stream->close(); 400 | } 401 | if (!stream->is_open()) { 402 | if (stream != nullptr) { 403 | delete stream; 404 | } 405 | stream = nullptr; 406 | } 407 | return stream; 408 | } 409 | 410 | // }}} 411 | 412 | } 413 | -------------------------------------------------------------------------------- /src/App/Reader.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_READER_HPP_ 2 | #define AIDT_READER_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "AdbaseConfig.hpp" 8 | #include "App/Config.hpp" 9 | #include "App/DataStore.hpp" 10 | 11 | class App; 12 | namespace app { 13 | class Message; 14 | class InotifyEvent; 15 | class Reader { 16 | public: 17 | Reader(std::shared_ptr& watcherConfig, App* app, AdbaseConfig *configure, std::shared_ptr& message); 18 | ~Reader(); 19 | void read(const InotifyEvent& event, bool& isContinue); 20 | void clear(const InotifyEvent& event); 21 | void moveself(const InotifyEvent& event); 22 | void moveto(const InotifyEvent& event); 23 | void save(); 24 | void load(); 25 | void stop(); 26 | 27 | private: 28 | AdbaseConfig *_configure; 29 | std::shared_ptr& _watcherConfig; 30 | App *_app; 31 | std::shared_ptr& _message; 32 | std::shared_ptr _dataStore; 33 | mutable std::mutex _mut; 34 | FdMap _fdMap; 35 | std::unordered_map _aliasMap; 36 | int _currentFdNum; 37 | bool _running; 38 | 39 | void readString(const InotifyEvent& event, const WatcherFileInfo& info, bool& isContinue); 40 | void readBinary(const InotifyEvent& event, const WatcherFileInfo& info, bool& isContinue); 41 | bool sendMessage(adbase::Buffer &msg, const WatcherFileInfo& info); 42 | bool initStream(const InotifyEvent& event, FdItem& item, const WatcherFileInfo& info, bool& isContinue); 43 | void gcFstreamNonLock(int lifeTime); 44 | void setFileOffset(std::string &pathFile, uint64_t offset); 45 | void closeStream(std::fstream* fs); 46 | std::fstream* openStream(const std::string& file, const WatcherFileInfo& info, std::fstream* stream); 47 | 48 | }; 49 | } 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /src/App/Watcher.cpp: -------------------------------------------------------------------------------- 1 | #include "Watcher.hpp" 2 | #include 3 | 4 | namespace app { 5 | // {{{ Watcher::Watcher() 6 | 7 | Watcher::Watcher(AdbaseConfig* config, std::shared_ptr& watcherConfig) : 8 | _configure(config), 9 | _watcherConfig(watcherConfig), 10 | _running(true), 11 | _watcherNum(0), 12 | _eventThreadNumber(0) { 13 | } 14 | 15 | // }}} 16 | // {{{ void Watcher::start() 17 | 18 | void Watcher::start() { 19 | // 初始化 inotify 实例 20 | if (!inotifytools_initialize()) { 21 | LOG_ERROR << strerror(inotifytools_error()); 22 | } 23 | 24 | adbase::metrics::Metrics::buildGauges("watcher", "inotify.size", 1000, [this](){ 25 | std::lock_guard lk(_mut); 26 | return _watcherNum; 27 | }); 28 | adbase::metrics::Metrics::buildGauges("watcher", "inotify.event.size", 1000, [this](){ 29 | return _eventQueue.getSize(); 30 | }); 31 | 32 | ThreadPtr watcherThread(new std::thread(std::bind(&Watcher::watcherThread, this, std::placeholders::_1), nullptr), &Watcher::deleteThread); 33 | LOG_DEBUG << "Create inotify watch thread success"; 34 | Threads.push_back(std::move(watcherThread)); 35 | ThreadPtr eventThread(new std::thread(std::bind(&Watcher::eventThread, this, std::placeholders::_1), nullptr), &Watcher::deleteThread); 36 | Threads.push_back(std::move(eventThread)); 37 | } 38 | 39 | // }}} 40 | // {{{ void Watcher::stop() 41 | 42 | void Watcher::stop() { 43 | _running = false; 44 | InotifyEvent eventItem; 45 | eventItem.event = STOP; 46 | _eventQueue.push(eventItem); 47 | LOG_DEBUG << "Wacher event stopping"; 48 | std::unique_lock lk(_mut); 49 | _dataCond.wait(lk, [this]{return (_eventThreadNumber == 0);}); 50 | } 51 | 52 | // }}} 53 | // {{{ void Watcher::watcherThread() 54 | 55 | void Watcher::watcherThread(void *data) { 56 | (void)data; 57 | 58 | struct inotify_event* event; 59 | while (_running) { 60 | consumerConfigQueue(); 61 | event = inotifytools_next_event(1); 62 | if (!event) { 63 | continue; 64 | } 65 | 66 | paserEvent(event); 67 | } 68 | LOG_DEBUG << "Watcher watcher thread stop."; 69 | } 70 | 71 | // }}} 72 | // {{{ void Watcher::eventThread() 73 | 74 | void Watcher::eventThread(void *) { 75 | _eventThreadNumber++; 76 | while (true) { 77 | InotifyEvent item; 78 | _eventQueue.waitPop(item); 79 | if (item.event == DELETE) { // 删除文件 80 | _clearHandler(item); 81 | LOG_DEBUG << "pop event delete, pathfile: " << item.pathfile << ", origfile: " << item.origfile; 82 | } else if (item.event == MOVE_SELF) { 83 | item.event = MODIFY; 84 | _eventQueue.push(item); 85 | _moveSelfHandler(item); 86 | LOG_TRACE << "pop event move, pathfile: " << item.pathfile 87 | << ", origfile: " << item.origfile; 88 | } else if (item.event == MODIFY) { 89 | bool isContinue = false; 90 | LOG_TRACE << "pop event modify, pathfile: " << item.pathfile 91 | << ", origfile: " << item.origfile; 92 | _readHandler(item, isContinue); 93 | if (isContinue) { 94 | _eventQueue.push(item); 95 | } 96 | } else if (item.event == MOVE_TO) { 97 | LOG_TRACE << "pop event move to, pathfile: " << item.pathfile 98 | << ", origfile: " << item.origfile; 99 | item.event = MODIFY; 100 | _eventQueue.push(item); 101 | _moveToHandler(item); 102 | } else { 103 | LOG_ERROR << "Stop watcher event thread"; 104 | break; 105 | } 106 | } 107 | std::unique_lock lk(_mut); 108 | _eventThreadNumber--; 109 | _dataCond.notify_all(); 110 | LOG_DEBUG << "Watcher event thread stop."; 111 | } 112 | 113 | // }}} 114 | // {{{ void Watcher::consumerConfigQueue() 115 | 116 | void Watcher::consumerConfigQueue() { 117 | WatcherFileInfo info; 118 | bool ret = _watcherConfig->consumerConfig(info); 119 | if (!ret) { 120 | return; 121 | } 122 | 123 | LOG_INFO << "Start process pathfile `" << info.pathFile << "` config"; 124 | createWatcher(info.pathFile, info.excludes, info.isDir, info.isRecursive); 125 | } 126 | 127 | // }}} 128 | // {{{ void Watcher::createWatcher() 129 | 130 | void Watcher::createWatcher(std::string pathFile, std::vector excludes, bool isDir, bool isRecursive) { 131 | if (!isDir) { // 监测单个文件 132 | struct stat fileStat; 133 | if (stat(pathFile.c_str(), &fileStat) != 0 || !S_ISREG(fileStat.st_mode)) { 134 | LOG_ERROR << "Create Watcher fail, pathfile `" << pathFile << "` is not file or not exists."; 135 | return; 136 | } 137 | std::vector pathSplit = adbase::explode(pathFile, '/', true); 138 | if (pathSplit.empty()) { 139 | LOG_ERROR << "Create Watcher fail, pathFile is invalid."; 140 | return; 141 | } 142 | std::string newPathFile; 143 | for (int i = 0; i < static_cast(pathSplit.size()) - 1; i++) { 144 | newPathFile.append("/"); 145 | newPathFile.append(pathSplit[i]); 146 | } 147 | if(!inotifytools_watch_file(newPathFile.c_str(), IN_ALL_EVENTS)) { 148 | LOG_ERROR << "Create Watcher fail, pathFile: " << pathFile 149 | << ", dirPath: " << newPathFile 150 | << ", " << strerror(inotifytools_error()); 151 | } 152 | LOG_INFO << "Create Watcher file success, path: " << newPathFile << " watches:" << inotifytools_get_num_watches(); 153 | paserWatcherFile(pathFile); 154 | { 155 | std::lock_guard lk(_mut); 156 | _watcherNum = inotifytools_get_num_watches(); 157 | } 158 | return; 159 | } 160 | 161 | struct stat fileStat; 162 | if (stat(pathFile.c_str(), &fileStat) != 0 || !S_ISDIR(fileStat.st_mode)) { 163 | LOG_WARN << "pathfile `" << pathFile << "` is not dir or not exists, creating.."; 164 | if (!adbase::mkdirRecursive(pathFile, 0777)) { 165 | LOG_ERROR << "pathfile `" << pathFile << "` create dir fail"; 166 | return; 167 | } 168 | } 169 | 170 | if (isRecursive) { 171 | if (excludes.empty()) { 172 | if(!inotifytools_watch_recursively(pathFile.c_str(), IN_ALL_EVENTS)) { 173 | LOG_ERROR << "Create Watcher fail, " << strerror(inotifytools_error()); 174 | } 175 | LOG_INFO << "Create Watcher recursive dir success, path: " << pathFile << " watches:" << inotifytools_get_num_watches(); 176 | } else { 177 | uint32_t excludesSize = static_cast(excludes.size()); 178 | char *p[excludesSize + 1]; 179 | for (uint32_t i = 0; i < static_cast(excludes.size()); i++) { 180 | uint32_t excludeLen = static_cast(excludes[i].size()); 181 | *(p + i) = new char[excludeLen + 1]; 182 | memcpy(*(p + i), excludes[i].c_str(), excludeLen); 183 | *(*(p + i) + excludeLen) = '\0'; 184 | LOG_INFO << "Excludes: " << excludes[i]; 185 | } 186 | *(p + excludesSize) = nullptr; 187 | 188 | if(!inotifytools_watch_recursively_with_exclude(pathFile.c_str(), IN_ALL_EVENTS, const_cast(p))) { 189 | LOG_ERROR << "Create Watcher fail, " << strerror(inotifytools_error()); 190 | } 191 | 192 | for (uint32_t i = 0; i < static_cast(excludesSize); i++) { 193 | delete [] *(p + i); 194 | } 195 | LOG_INFO << "Create Watcher recursive dir and excluses success, path: " << pathFile << " watches:" << inotifytools_get_num_watches(); 196 | } 197 | } else { 198 | if(!inotifytools_watch_file(pathFile.c_str(), IN_ALL_EVENTS)) { 199 | LOG_ERROR << "Create Watcher fail, " << strerror(inotifytools_error()); 200 | } 201 | LOG_INFO << "Create Watcher dir not recursive success, path: " << pathFile << " watches:" << inotifytools_get_num_watches(); 202 | } 203 | paserWatcherFile(pathFile); 204 | { 205 | std::lock_guard lk(_mut); 206 | _watcherNum = inotifytools_get_num_watches(); 207 | } 208 | } 209 | 210 | // }}} 211 | // {{{ void Watcher::deleteWatcher() 212 | 213 | void Watcher::deleteWatcher(std::string pathFile, bool isDir) { 214 | if (isDir) { // 监测单个文件 215 | return; 216 | } 217 | struct stat fileStat; 218 | if (stat(pathFile.c_str(), &fileStat) != 0 || !S_ISREG(fileStat.st_mode)) { 219 | LOG_ERROR << "Delete Watcher fail, pathfile `" << pathFile << "` is not file or not exists."; 220 | return; 221 | } 222 | 223 | if (_watcherConfig->isHasWatcherFile(pathFile)) { 224 | LOG_INFO << "Have an other file watcher in this dir, not need remove watcher. pathFile: " << pathFile; 225 | } else { 226 | std::vector pathSplit = adbase::explode(pathFile, '/', true); 227 | if (pathSplit.empty()) { 228 | LOG_ERROR << "Remove Watcher fail, pathFile is invalid."; 229 | return; 230 | } 231 | std::string newPathFile; 232 | for (uint32_t i = 0; i < static_cast(pathSplit.size()) - 1; i++) { 233 | newPathFile.append("/"); 234 | newPathFile.append(pathSplit[i]); 235 | } 236 | if(!inotifytools_remove_watch_by_filename(newPathFile.c_str())) { 237 | LOG_ERROR << "Delete Watcher fail, pathFile: " << pathFile << ", dirPath: " 238 | << newPathFile << ", " << strerror(inotifytools_error()); 239 | } 240 | } 241 | { 242 | std::lock_guard lk(_mut); 243 | _watcherNum = inotifytools_get_num_watches(); 244 | } 245 | LOG_INFO << "Remove Watcher file success, path: " << pathFile; 246 | return; 247 | } 248 | 249 | // }}} 250 | // {{{ bool Watcher::patternPathFile() 251 | 252 | bool Watcher::patternPathFile(std::string &originFile, WatcherFileInfo &info) { 253 | std::string pathFile; 254 | // 先查看缓存 255 | if (static_cast(_cache.size()) > 1024) { 256 | int currentTime = static_cast(time(nullptr)); 257 | std::vector keys; 258 | for (auto &t : _cache) { 259 | if (currentTime - t.second.lastTime > 1) { 260 | keys.push_back(t.first); 261 | } 262 | } 263 | for (auto &t : keys) { 264 | _cache.erase(t); 265 | } 266 | } 267 | 268 | if (_cache.find(originFile) == _cache.end()) { 269 | bool ret = _watcherConfig->patternPath(originFile, pathFile); 270 | if (!ret) { 271 | LOG_DEBUG << "pattern originFile: " << originFile << " pathFile: " << pathFile << " fail."; 272 | } 273 | InotifyPaserCacheItem item; 274 | item.pathfile = pathFile; 275 | item.lastTime = static_cast(time(nullptr)); 276 | _cache[originFile] = item; 277 | } else { 278 | pathFile = _cache[originFile].pathfile; 279 | _cache[originFile].lastTime = static_cast(time(nullptr)); 280 | } 281 | 282 | bool ret = _watcherConfig->findFile(pathFile, info); 283 | return ret; 284 | } 285 | 286 | // }}} 287 | // {{{ void Watcher::paserEvent() 288 | 289 | void Watcher::paserEvent(struct inotify_event *event) { 290 | std::string origFile(inotifytools_filename_from_wd(event->wd)); 291 | 292 | std::string::size_type n = origFile.find_last_not_of("/"); 293 | if (n == std::string::npos) { // 不遍历根目录 294 | origFile = ""; 295 | } 296 | origFile = origFile.substr(0, n + 1); 297 | origFile.append("/"); 298 | origFile.append(event->name); 299 | LOG_TRACE << "Paser event origin file: " << origFile; 300 | WatcherFileInfo info; 301 | if (!patternPathFile(origFile, info)) { 302 | return; 303 | } 304 | 305 | // 新增监控节点 306 | if (event->mask & IN_CREATE) { 307 | if (!info.isDir || !info.isRecursive) { 308 | return; 309 | } 310 | struct stat fileStat; 311 | if (stat(origFile.c_str(), &fileStat) != 0 || !S_ISDIR(fileStat.st_mode)) { 312 | return; 313 | } 314 | std::vector excludes; 315 | createWatcher(origFile, excludes, true, true); 316 | LOG_INFO << "Push Event DIR CREATE `" << origFile << "` pathFile: " << info.pathFile; 317 | } 318 | 319 | if (event->mask & IN_DELETE || event->mask & IN_DELETE_SELF) { // 删除事件，用来清除打开的文件 320 | InotifyEvent eventItem; 321 | eventItem.pathfile = info.pathFile; 322 | eventItem.origfile = origFile; 323 | eventItem.aliasfile = ""; 324 | eventItem.event = DELETE; 325 | _eventQueue.push(eventItem); 326 | LOG_INFO << "Push Event DELETE `" << origFile << "` pathFile: " 327 | << eventItem.pathfile; 328 | return; 329 | } 330 | 331 | if (event->mask & IN_MODIFY) { // 文件修改事件，通知读取 332 | InotifyEvent eventItem; 333 | eventItem.pathfile = info.pathFile; 334 | eventItem.origfile = origFile; 335 | eventItem.aliasfile = ""; 336 | eventItem.event = MODIFY; 337 | _eventQueue.push(eventItem); 338 | LOG_TRACE << "Push Event MODIFY `" << origFile << "` pathFile: " << eventItem.pathfile; 339 | return; 340 | } 341 | 342 | if (event->mask & IN_MOVE_SELF || event->mask & IN_MOVED_FROM) { // 文件类型的监控生效 343 | if (info.isDir) { 344 | LOG_INFO << "Dir type file Event MOVE_SLEF `" << origFile << "` pathFile: " << info.pathFile; 345 | _dirTypeMoveFrom = origFile; 346 | return; 347 | } 348 | 349 | // 处理文件 350 | InotifyEvent eventItem; 351 | eventItem.pathfile = info.pathFile; 352 | eventItem.origfile = origFile; 353 | eventItem.aliasfile = ""; 354 | eventItem.event = MOVE_SELF; 355 | _eventQueue.push(eventItem); 356 | LOG_INFO << "Push Event MOVE_SLEF `" << origFile << "` pathFile: " << eventItem.pathfile; 357 | return; 358 | } 359 | 360 | if (event->mask & IN_MOVED_TO) { 361 | if (info.isDir) { 362 | LOG_INFO << "Dir type file Event MOVE_TO `" << origFile << "` pathFile: " << info.pathFile; 363 | if (_dirTypeMoveFrom != "") { 364 | InotifyEvent eventItem; 365 | eventItem.pathfile = info.pathFile; 366 | eventItem.origfile = origFile; 367 | eventItem.aliasfile = _dirTypeMoveFrom; 368 | eventItem.event = MOVE_TO; 369 | _eventQueue.push(eventItem); 370 | LOG_INFO << "Push dir type Event MOVE_TO `" << origFile << "` alias `" 371 | << _dirTypeMoveFrom << "` pathFile: " << eventItem.pathfile; 372 | } 373 | _dirTypeMoveFrom = ""; 374 | } 375 | } 376 | } 377 | 378 | // }}} 379 | // {{{ void Watcher::paserWatcherFile() 380 | 381 | void Watcher::paserWatcherFile(std::string pathFile) { 382 | WatcherFileInfo info; 383 | bool ret = _watcherConfig->findFile(pathFile, info); 384 | if (!ret) { 385 | return; 386 | } 387 | 388 | if (!info.isDir) { 389 | addModifyEvent(pathFile, pathFile); 390 | } else { 391 | std::vector originFiles; 392 | if (!info.isRecursive) { 393 | adbase::recursiveDir(pathFile, false, info.excludes, originFiles); 394 | } else { 395 | adbase::recursiveDir(pathFile, true, info.excludes, originFiles); 396 | } 397 | 398 | for (auto &t : originFiles) { 399 | addModifyEvent(pathFile, t); 400 | } 401 | } 402 | } 403 | 404 | // }}} 405 | // {{{ void Watcher::addModifyEvent() 406 | 407 | void Watcher::addModifyEvent(std::string pathFile, std::string originFile) { 408 | struct stat fileStat; 409 | if (0 != stat(originFile.c_str(), &fileStat) || !S_ISREG(fileStat.st_mode)) { 410 | return; 411 | } 412 | InotifyEvent eventItem; 413 | eventItem.pathfile = pathFile; 414 | eventItem.origfile = originFile; 415 | eventItem.event = MODIFY; 416 | _eventQueue.push(eventItem); 417 | LOG_INFO << "Restart Event MODIFY `" << originFile << "` pathFile: " << eventItem.pathfile; 418 | } 419 | 420 | // }}}} 421 | // {{{ void Watcher::deleteThread() 422 | 423 | void Watcher::deleteThread(std::thread *t) { 424 | t->join(); 425 | delete t; 426 | } 427 | 428 | // }}} 429 | // {{{ Watcher::~Watcher() 430 | 431 | Watcher::~Watcher() { 432 | } 433 | 434 | // }}} 435 | } 436 | -------------------------------------------------------------------------------- /src/App/Watcher.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_WATCHER_HPP_ 2 | #define AIDT_WATCHER_HPP_ 3 | 4 | #include 5 | #pragma GCC diagnostic ignored "-Wconversion" 6 | #pragma GCC diagnostic ignored "-Wold-style-cast" 7 | #include "inotify/inotifytools.h" 8 | #include "inotify/inotify.h" 9 | #pragma GCC diagnostic warning "-Wconversion" 10 | #pragma GCC diagnostic warning "-Wold-style-cast" 11 | #include "AdbaseConfig.hpp" 12 | #include "App/Config.hpp" 13 | 14 | namespace app { 15 | typedef enum EventType { 16 | DELETE=1, 17 | MOVE_SELF=2, 18 | MODIFY=3, 19 | MOVE_TO=4, 20 | STOP = 999, 21 | } EventType; 22 | 23 | typedef struct InotifyEvent { 24 | std::string pathfile; // 配置监听的文件或目录 25 | std::string origfile; // 真正的事件文件 26 | std::string aliasfile; // 移动前的名称 27 | EventType event; 28 | } InotifyEvent; 29 | // InotifyPaser Cache 管理 30 | typedef struct InotifyPaserCacheItem { 31 | std::string pathfile; 32 | int lastTime; 33 | } InotifyPaserCacheItem; 34 | typedef std::unordered_map InotifyPaserCache; 35 | 36 | typedef std::function ReaderCallbackRead; 37 | typedef std::function ReaderCallbackClear; 38 | typedef std::function ReaderCallbackMoveSelf; 39 | typedef std::function ReaderCallbackMoveTo; 40 | 41 | class Watcher { 42 | public: 43 | Watcher(AdbaseConfig* configure, std::shared_ptr& watcherConfig); 44 | ~Watcher(); 45 | void start(); 46 | void stop(); 47 | void watcherThread(void *data); 48 | void eventThread(void *data); 49 | void createWatcher(std::string pathFile, std::vector excludes, bool isDir, bool isRecursive); 50 | void deleteWatcher(std::string pathFile, bool isDir); 51 | void setReadHandler(const ReaderCallbackRead& readHandler) { 52 | _readHandler = readHandler; 53 | } 54 | void setClearHandler(const ReaderCallbackClear& clearHandler) { 55 | _clearHandler = clearHandler; 56 | } 57 | void setMoveSelfHandler(const ReaderCallbackMoveSelf& moveSelfHandler) { 58 | _moveSelfHandler = moveSelfHandler; 59 | } 60 | void setMoveToHandler(const ReaderCallbackMoveTo& moveToHandler) { 61 | _moveToHandler = moveToHandler; 62 | } 63 | static void deleteThread(std::thread *t); 64 | private: 65 | typedef std::unique_ptr ThreadPtr; 66 | typedef std::vector ThreadPool; 67 | ThreadPool Threads; 68 | 69 | AdbaseConfig *_configure; 70 | std::shared_ptr& _watcherConfig; 71 | bool _running; 72 | InotifyPaserCache _cache; 73 | adbase::Queue _eventQueue; 74 | std::string _dirTypeMoveFrom; 75 | int _watcherNum; 76 | mutable std::mutex _mut; 77 | std::condition_variable _dataCond; 78 | int _eventThreadNumber; 79 | ReaderCallbackRead _readHandler; 80 | ReaderCallbackClear _clearHandler; 81 | ReaderCallbackMoveSelf _moveSelfHandler; 82 | ReaderCallbackMoveTo _moveToHandler; 83 | 84 | void consumerConfigQueue(); 85 | void paserEvent(struct inotify_event *event); 86 | bool patternPathFile(std::string &origfile, WatcherFileInfo& info); 87 | void paserWatcherFile(std::string pathFile); 88 | void addModifyEvent(std::string pathFile, std::string originFile); 89 | }; 90 | 91 | } 92 | 93 | #endif 94 | -------------------------------------------------------------------------------- /src/BootStrap.cpp: -------------------------------------------------------------------------------- 1 | /// 该程序自动生成，禁止修改 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "BootStrap.hpp" 9 | #include "App.hpp" 10 | #include "Version.hpp" 11 | #include "AdServer.hpp" 12 | #include "Aims.hpp" 13 | #include "Timer.hpp" 14 | #include "App/ConfigPaser.hpp" 15 | 16 | 17 | // {{{ BootStrap::BootStrap() 18 | 19 | BootStrap::BootStrap() { 20 | } 21 | 22 | // }}} 23 | // {{{ BootStrap::~BootStrap() 24 | 25 | BootStrap::~BootStrap() { 26 | } 27 | 28 | // }}} 29 | // {{{ void BootStrap::init() 30 | 31 | void BootStrap::init(int argc, char **argv) { 32 | _configure = std::shared_ptr(new AdbaseConfig()); 33 | std::unique_ptr tmpApp(new App(_configure.get())); 34 | _app = std::move(tmpApp); 35 | 36 | // 解析指定的参数 37 | parseOption(argc, argv); 38 | loadConfig(); 39 | // 时区设置 40 | adbase::TimeZone beijing(8*3600, "CST"); 41 | // 初始化 daemon 程序 42 | daemonInit(); 43 | loggerInit(beijing); 44 | _loop = std::shared_ptr(new adbase::EventLoop()); 45 | } 46 | 47 | // }}} 48 | // {{{ void BootStrap::run() 49 | 50 | void BootStrap::run() { 51 | // 创建定时器 52 | TimerContext timerContext; 53 | timerContext.config = _configure.get(); 54 | timerContext.mainEventBase = _loop->getBase(); 55 | _timer = std::shared_ptr(new Timer(&timerContext)); 56 | _timer->init(); 57 | 58 | // 初始化 metric 信息 59 | adbase::metrics::Metrics* metrics = adbase::metrics::Metrics::init(_timer->getTimer()); 60 | 61 | _app->run(); 62 | _app->setTimerContext(&timerContext); 63 | 64 | // 构建 adserver 上下文 65 | // @todo 66 | AdServerContext context; 67 | context.config = _configure.get(); 68 | context.metrics = metrics; 69 | context.mainEventBase = _loop->getBase(); 70 | _app->setAdServerContext(&context); 71 | _adServer = std::shared_ptr(new AdServer(&context)); 72 | if (_adServer) { 73 | _adServer->run(); 74 | } 75 | 76 | AimsContext aimsContext; 77 | aimsContext.config = _configure.get(); 78 | _app->setAimsContext(&aimsContext); 79 | _aims = std::shared_ptr(new Aims(&aimsContext)); 80 | _app->setAims(_aims); 81 | if (_aims) { 82 | _aims->run(); 83 | } 84 | 85 | _loop->start(); 86 | } 87 | 88 | // }}} 89 | // {{{ void BootStrap::reload() 90 | 91 | void BootStrap::reload() { 92 | try { 93 | loadConfig(); 94 | } catch (...) { 95 | LOG_ERROR << "Reload config: " << _configFile << " is invalid, please check it."; 96 | return; 97 | } 98 | setLoggerLevel(); 99 | _app->reload(); 100 | } 101 | 102 | // }}} 103 | // {{{ void BootStrap::resend() 104 | 105 | void BootStrap::resend() { 106 | _app->resend(); 107 | } 108 | 109 | // }}} 110 | // {{{ void BootStrap::stop() 111 | 112 | void BootStrap::stop(const int sig) { 113 | LOG_ERROR << "Stop callback sig:" << sig; 114 | remove(_configure->pidFile.c_str()); 115 | 116 | if (_aims) { 117 | _aims->stop(); 118 | } 119 | 120 | if (_adServer) { 121 | _adServer->stop(); 122 | } 123 | 124 | if (_app) { 125 | _app->stop(); 126 | } 127 | 128 | if (_loop) { 129 | _loop->stop(); 130 | } 131 | } 132 | 133 | // }}} 134 | //{{{ AdbaseConfig* BootStrap::getConfig() 135 | 136 | AdbaseConfig* BootStrap::getConfig() { 137 | std::lock_guard lk(_mut); 138 | return _configure.get(); 139 | } 140 | 141 | //}}} 142 | // {{{ void BootStrap::daemonInit() 143 | 144 | void BootStrap::daemonInit() { 145 | if (_configure->daemon) { 146 | pid_t pid; 147 | pid = fork(); 148 | if (pid < 0) { 149 | LOG_SYSFATAL << "Fork parent process fail."; 150 | } 151 | if (pid > 0) { 152 | exit(EXIT_SUCCESS); 153 | } 154 | setsid(); /* create a new session */ 155 | } 156 | 157 | FILE *fpPidfile; 158 | fpPidfile = fopen(_configure->pidFile.c_str(), "w"); 159 | fprintf(fpPidfile, "%d\n", getpid()); 160 | fclose(fpPidfile); 161 | } 162 | 163 | // }}} 164 | // {{{ void BootStrap::loggerInit() 165 | 166 | void BootStrap::loggerInit(const adbase::TimeZone& tz) { 167 | adbase::Logger::setTimeZone(tz); 168 | if (_configure->isAsync) { // 只有是守护进程开启的时候日志开启异步写入 169 | adbase::Logger::setOutput(std::bind(&BootStrap::asyncLogger, this, 170 | std::placeholders::_1, std::placeholders::_2)); 171 | // 启动异步日志落地 172 | std::string basename = _configure->logsDir + std::string(::basename("aidt")); 173 | _asnclog = std::shared_ptr(new adbase::AsyncLogging(basename, _configure->logRollSize)); 174 | _asnclog->start(); 175 | } 176 | setLoggerLevel(); 177 | } 178 | 179 | // }}} 180 | // {{{ void BootStrap::asyncLogger() 181 | 182 | void BootStrap::asyncLogger(const char* msg, int len) { 183 | if (_asnclog) { 184 | _asnclog->append(msg, static_cast(len)); 185 | } 186 | } 187 | 188 | // }}} 189 | // {{{ void BootStrap::setLoggerLevel() 190 | 191 | void BootStrap::setLoggerLevel() { 192 | switch (_configure->logLevel) { 193 | case 1: 194 | adbase::Logger::setLogLevel(adbase::Logger::TRACE); 195 | break; 196 | case 2: 197 | adbase::Logger::setLogLevel(adbase::Logger::DEBUG); 198 | break; 199 | case 3: 200 | default: 201 | adbase::Logger::setLogLevel(adbase::Logger::INFO); 202 | } 203 | } 204 | 205 | // }}} 206 | //{{{ void BootStrap::loadConfig() 207 | 208 | void BootStrap::loadConfig() { 209 | std::lock_guard lk(_mut); 210 | adbase::IniConfig config = adbase::IniParse::loadFile(_configFile); 211 | 212 | _configure->daemon = config.getOptionBool("system", "daemon"); 213 | _configure->pidFile = config.getOption("system", "pidFile"); 214 | _configure->appid = config.getOptionUint32("system", "appid"); 215 | _configure->macid = config.getOptionUint32("system", "macid"); 216 | 217 | _configure->logsDir = config.getOption("logging", "logsDir"); 218 | _configure->logRollSize = config.getOptionUint32("logging", "logRollSize"); 219 | _configure->logLevel = config.getOptionUint32("logging", "logLevel"); 220 | _configure->isAsync = config.getOptionBool("logging", "isAsync"); 221 | 222 | _configure->isStartMc = config.getOptionBool("adserver", "mc"); 223 | _configure->isStartHead = config.getOptionBool("adserver", "head"); 224 | _configure->isStartHttp = config.getOptionBool("adserver", "http"); 225 | 226 | _configure->httpHost = config.getOption("http", "host"); 227 | _configure->httpPort = config.getOptionUint32("http", "port"); 228 | _configure->httpTimeout = config.getOptionUint32("http", "timeout"); 229 | _configure->httpThreadNum = config.getOptionUint32("http", "threadNum"); 230 | _configure->httpServerName = config.getOption("http", "serverName"); 231 | _configure->httpDefaultController = config.getOption("http", "defaultController"); 232 | _configure->httpDefaultAction = config.getOption("http", "defaultAction"); 233 | _configure->httpAccessLogDir = config.getOption("http", "accessLogDir"); 234 | _configure->httpAccesslogRollSize = config.getOptionUint32("http", "accesslogRollSize"); 235 | 236 | _configure->mcHost = config.getOption("mc", "host"); 237 | _configure->mcPort = config.getOptionUint32("mc", "port"); 238 | _configure->mcServerName = config.getOption("mc", "serverName"); 239 | _configure->mcThreadNum = config.getOptionUint32("mc", "threadNum"); 240 | 241 | _configure->headHost = config.getOption("head", "host"); 242 | _configure->headPort = config.getOptionUint32("head", "port"); 243 | _configure->headServerName = config.getOption("head", "serverName"); 244 | _configure->headThreadNum = config.getOptionUint32("head", "threadNum"); 245 | 246 | if (_app) { 247 | _app->loadConfig(config); 248 | } 249 | } 250 | 251 | //}}} 252 | // {{{ void BootStrap::usage() 253 | 254 | void BootStrap::usage() { 255 | std::cout << "Usage: aidt [options...] " << std::endl; 256 | std::cout << "\t-c: 主配置文件路径" << std::endl; 257 | std::cout << "\t-h: 帮助" << std::endl; 258 | exit(0); 259 | } 260 | 261 | // }}} 262 | // {{{ void BootStrap::printVersion() 263 | 264 | void BootStrap::printVersion() { 265 | std::cout << "VERSION : " << VERSION << std::endl; 266 | std::cout << "GIT SHA1 : " << GIT_SHA1 << std::endl; 267 | std::cout << "GIT DIRTY: " << GIT_DIRTY << std::endl; 268 | std::cout << "BUILD ID : " << BUILD_ID << std::endl; 269 | exit(0); 270 | } 271 | 272 | // }}} 273 | // {{{ void BootStrap::parseOption() 274 | 275 | void BootStrap::parseOption(int argc, char **argv) { 276 | int ch; 277 | while((ch = getopt(argc, argv, "c:hvt")) != -1) { 278 | if (ch == 'c') { 279 | _configFile = optarg; 280 | } else if (ch == 'h') { 281 | usage(); 282 | } else if (ch == 't') { 283 | checkConfig(); 284 | } else if (ch == 'v') { 285 | printVersion(); 286 | } 287 | } 288 | } 289 | 290 | // }}} 291 | // {{{ void BootStrap::checkConfig() 292 | 293 | void BootStrap::checkConfig() { 294 | loadConfig(); 295 | _app->checkConfig(); 296 | LOG_INFO << "Config OK+"; 297 | exit(0); 298 | } 299 | 300 | // }}} 301 | -------------------------------------------------------------------------------- /src/BootStrap.hpp: -------------------------------------------------------------------------------- 1 | /// 该程序自动生成，禁止修改 2 | #ifndef AIDT_BOOTSTRAP_HPP_ 3 | #define AIDT_BOOTSTRAP_HPP_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "AdbaseConfig.hpp" 11 | 12 | class App; 13 | class Timer; 14 | class AdServer; 15 | class Aims; 16 | class BootStrap { 17 | public: 18 | BootStrap(); 19 | ~BootStrap(); 20 | void init(int argc, char **argv); 21 | void run(); 22 | void reload(); 23 | void resend(); 24 | void stop(const int sig); 25 | AdbaseConfig* getConfig(); 26 | 27 | private: 28 | std::shared_ptr _configure; 29 | mutable std::mutex _mut; 30 | std::shared_ptr _asnclog; 31 | std::string _configFile; 32 | std::shared_ptr _adServer; 33 | std::shared_ptr _loop; 34 | std::shared_ptr _aims; 35 | std::shared_ptr _timer; 36 | std::unique_ptr _app; 37 | 38 | void daemonInit(); 39 | void asyncLogger(const char* msg, int len); 40 | void loggerInit(const adbase::TimeZone& tz); 41 | void setLoggerLevel(); 42 | void loadConfig(); 43 | void parseOption(int argc, char **argv); 44 | void usage(); 45 | void printVersion(); 46 | void checkConfig(); 47 | }; 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /src/BootStrapCli.cpp: -------------------------------------------------------------------------------- 1 | /// 该程序自动生成，禁止修改 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "BootStrapCli.hpp" 7 | #include "Version.hpp" 8 | 9 | // {{{ BootStrapCli::BootStrapCli() 10 | 11 | BootStrapCli::BootStrapCli() { 12 | } 13 | 14 | // }}} 15 | // {{{ BootStrapCli::~BootStrapCli() 16 | 17 | BootStrapCli::~BootStrapCli() { 18 | } 19 | 20 | // }}} 21 | // {{{ void BootStrapCli::init() 22 | 23 | void BootStrapCli::init(int argc, char **argv) { 24 | // 解析指定的参数 25 | parseOption(argc, argv); 26 | } 27 | 28 | // }}} 29 | // {{{ void BootStrapCli::run() 30 | 31 | void BootStrapCli::run() { 32 | int ret = 0; 33 | if (_opt == "dumpmessage") { 34 | ret = dumpMessage(); 35 | } else if (_opt == "dumpoffset") { 36 | ret = dumpOffset(); 37 | } else if (_opt == "dynamicoffset") { 38 | ret = dynamicOffset(); 39 | } 40 | 41 | exit(ret); 42 | } 43 | 44 | // }}} 45 | // {{{ void BootStrapCli::stop() 46 | 47 | void BootStrapCli::stop(const int sig) { 48 | } 49 | 50 | // }}} 51 | // {{{ void BootStrapCli::usage() 52 | 53 | void BootStrapCli::usage() { 54 | std::cout << "Usage: aidt -o [opt] [options...] " << std::endl; 55 | std::cout << "\t-o: 操作命令" << std::endl; 56 | std::cout << "\t\tdumpmessage: 导出消息数据到文件" << std::endl; 57 | std::cout << "\t-f: 操作的文件" << std::endl; 58 | std::cout << "\t-h: 帮助" << std::endl; 59 | exit(0); 60 | } 61 | 62 | // }}} 63 | // {{{ void BootStrapCli::printVersion() 64 | 65 | void BootStrapCli::printVersion() { 66 | std::cout << "VERSION : " << VERSION << std::endl; 67 | std::cout << "GIT SHA1 : " << GIT_SHA1 << std::endl; 68 | std::cout << "GIT DIRTY: " << GIT_DIRTY << std::endl; 69 | std::cout << "BUILD ID : " << BUILD_ID << std::endl; 70 | exit(0); 71 | } 72 | 73 | // }}} 74 | // {{{ void BootStrapCli::parseOption() 75 | 76 | void BootStrapCli::parseOption(int argc, char **argv) { 77 | int ch; 78 | while((ch = getopt(argc, argv, "o:f:d:hv")) != -1) { 79 | if (ch == 'o') { 80 | _opt = optarg; 81 | if (_opt != "dumpmessage" && _opt != "dumpoffset" && _opt != "dynamicoffset") { 82 | usage(); 83 | } 84 | } else if (ch == 'f') { 85 | _optFileName = optarg; 86 | } else if (ch == 'd') { 87 | _dumpDir = optarg; 88 | } else if (ch == 'h') { 89 | usage(); 90 | } else if (ch == 'v') { 91 | printVersion(); 92 | } 93 | } 94 | } 95 | 96 | // }}} 97 | // {{{ int BootStrapCli::dumpMessage() 98 | 99 | int BootStrapCli::dumpMessage() { 100 | struct stat fileStat; 101 | if (stat(_optFileName.c_str(), &fileStat) != 0 || !S_ISREG(fileStat.st_mode)) { 102 | LOG_ERROR << "Opt file `" << _optFileName << "` is not file or not exists."; 103 | return 1; 104 | } 105 | if (stat(_dumpDir.c_str(), &fileStat) != 0 || !S_ISDIR(fileStat.st_mode)) { 106 | LOG_ERROR << "Dump directory `" << _dumpDir << "` is not dir or not exists."; 107 | return 2; 108 | } 109 | 110 | app::MessageQueue queue; 111 | app::DataStore data(&queue); 112 | data.loadMessage(_optFileName); 113 | if (!queue.empty()) { 114 | LOG_INFO << "Load message size:" << queue.getSize(); 115 | } 116 | while (!queue.empty()) { // 获取队列中的数据 117 | app::MessageItem item; 118 | bool ret = queue.tryPop(item); 119 | if (!ret) { 120 | break; 121 | } 122 | dumpMessageFile(item); 123 | } 124 | return 0; 125 | } 126 | 127 | // }}} 128 | // {{{ void BootStrapCli::dumpMessageFile() 129 | 130 | void BootStrapCli::dumpMessageFile(app::MessageItem& item) { 131 | std::string topicName = item.topicName; 132 | std::string fileName = _dumpDir; 133 | std::string::size_type n = fileName.find_last_not_of("/"); 134 | if (n != std::string::npos) { 135 | fileName.append("/"); 136 | } 137 | fileName.append(topicName); 138 | std::ofstream ofs(fileName.c_str(), std::ofstream::app); 139 | ofs << item.message.retrieveAllAsString() << '\n'; 140 | ofs.flush(); 141 | ofs.close(); 142 | } 143 | 144 | // }}} 145 | // {{{ int BootStrapCli::dumpOffset() 146 | 147 | int BootStrapCli::dumpOffset() { 148 | struct stat fileStat; 149 | if (stat(_optFileName.c_str(), &fileStat) != 0 || !S_ISREG(fileStat.st_mode)) { 150 | LOG_ERROR << "Opt file `" << _optFileName << "` is not file or not exists."; 151 | return 1; 152 | } 153 | if (stat(_dumpDir.c_str(), &fileStat) != 0 || !S_ISDIR(fileStat.st_mode)) { 154 | LOG_ERROR << "Dump directory `" << _dumpDir << "` is not dir or not exists."; 155 | return 2; 156 | } 157 | 158 | app::FdMap maps; 159 | app::DataStore data(nullptr); 160 | data.loadOffsets(_optFileName, maps); 161 | 162 | if (!maps.empty()) { 163 | LOG_INFO << "Load offset size:" << maps.size(); 164 | } 165 | 166 | std::string fileName = _dumpDir; 167 | std::string::size_type n = fileName.find_last_not_of("/"); 168 | if (n != std::string::npos) { 169 | fileName.append("/"); 170 | } 171 | fileName.append("dump_offsets"); 172 | 173 | std::ofstream ofs(fileName.c_str(), std::ofstream::app); 174 | 175 | for (auto &t : maps) { 176 | ofs << t.second.offset << '\t' << t.second.pathfile << '\t' << t.second.origfile << '\n'; 177 | } 178 | ofs.flush(); 179 | ofs.close(); 180 | return 0; 181 | } 182 | 183 | // }}} 184 | // {{{ int BootStrapCli::dynamicOffset() 185 | 186 | int BootStrapCli::dynamicOffset() { 187 | struct stat fileStat; 188 | if (stat(_optFileName.c_str(), &fileStat) != 0 || !S_ISREG(fileStat.st_mode)) { 189 | LOG_ERROR << "Opt file `" << _optFileName << "` is not file or not exists."; 190 | return 1; 191 | } 192 | if (stat(_dumpDir.c_str(), &fileStat) != 0 || !S_ISDIR(fileStat.st_mode)) { 193 | LOG_ERROR << "Dump directory `" << _dumpDir << "` is not dir or not exists."; 194 | return 2; 195 | } 196 | 197 | app::FdMap maps; 198 | app::DataStore data(nullptr); 199 | 200 | std::ifstream ifs(_optFileName.c_str(), std::ios_base::in | std::ios_base::binary); 201 | if (!ifs.good() || !ifs.is_open()) { 202 | return 1; 203 | } 204 | 205 | std::string line; 206 | while(std::getline(ifs, line)) { 207 | std::vector infos = adbase::explode(line, '\t', true); 208 | if (static_cast(infos.size()) != 3) { 209 | LOG_ERROR << "Offset format is invalid, offset item:" << line; 210 | continue; 211 | } 212 | 213 | errno = 0; 214 | uint64_t offset = static_cast(strtoull(infos[0].c_str(), nullptr, 10)); 215 | if (errno != 0) { 216 | continue; 217 | } 218 | app::FdItem item; 219 | item.pathfile = infos[1]; 220 | item.origfile = infos[2]; 221 | item.offset = offset; 222 | maps[item.origfile] = item; 223 | } 224 | 225 | if (!maps.empty()) { 226 | LOG_INFO << "Load offset size:" << maps.size(); 227 | } 228 | 229 | std::string fileName = _dumpDir; 230 | std::string::size_type n = fileName.find_last_not_of("/"); 231 | if (n != std::string::npos) { 232 | fileName.append("/"); 233 | } 234 | fileName.append("dynamic_offsets"); 235 | data.saveOffsets(fileName, maps); 236 | return 0; 237 | } 238 | 239 | // }}} 240 | -------------------------------------------------------------------------------- /src/BootStrapCli.hpp: -------------------------------------------------------------------------------- 1 | /// 该程序自动生成，禁止修改 2 | #ifndef AIDT_BOOTSTRAPCLI_HPP_ 3 | #define AIDT_BOOTSTRAPCLI_HPP_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "AdbaseConfig.hpp" 11 | #include "App/DataStore.hpp" 12 | 13 | class App; 14 | class Timer; 15 | class AdServer; 16 | class Aims; 17 | class BootStrapCli { 18 | public: 19 | BootStrapCli(); 20 | ~BootStrapCli(); 21 | void init(int argc, char **argv); 22 | void run(); 23 | void stop(const int sig); 24 | 25 | private: 26 | std::string _optFileName; 27 | std::string _opt; 28 | std::string _dumpDir; 29 | void parseOption(int argc, char **argv); 30 | void usage(); 31 | void printVersion(); 32 | int dumpMessage(); 33 | void dumpMessageFile(app::MessageItem& item); 34 | int dumpOffset(); 35 | int dynamicOffset(); 36 | }; 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.6) 2 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/src) 3 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/src/thridpart) 4 | LINK_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) 5 | 6 | SET(CMAKE_SOURCE_DIR .) 7 | SET(CMAKE_MODULE_PATH ${CMAKE_ROOT}/Modules ${CMAKE_SOURCE_DIR}/cmake/Modules 8 | ${PROJECT_SOURCE_DIR}/Modules) 9 | 10 | configure_file ( 11 | "${PROJECT_SOURCE_DIR}/src/Version.hpp.in" 12 | "${PROJECT_SOURCE_DIR}/src/Version.hpp" 13 | ) 14 | 15 | SET(AIDT_SRC aidt.cpp 16 | BootStrap.cpp 17 | App.cpp 18 | AdServer.cpp 19 | Http.cpp 20 | Http/HttpInterface.cpp 21 | Http/Server.cpp 22 | Http/Index.cpp 23 | Http/Config.cpp 24 | Http/Topic.cpp 25 | Http/Message.cpp 26 | McProcessor.cpp 27 | HeadProcessor.cpp 28 | Aims.cpp 29 | Aims/Kafka/ProducerIn.cpp 30 | Timer.cpp 31 | App/Config.cpp 32 | App/ConfigPaser.cpp 33 | App/Message.cpp 34 | App/DataStore.cpp 35 | App/Watcher.cpp 36 | App/Reader.cpp 37 | thridpart/inotify/inotifytools.c 38 | thridpart/inotify/redblack.c 39 | ) 40 | ADD_EXECUTABLE(aidt ${AIDT_SRC}) 41 | 42 | SET(AIDT_CLI_SRC aidtCli.cpp 43 | BootStrapCli.cpp 44 | App/DataStore.cpp 45 | ) 46 | ADD_EXECUTABLE(aidt-cli ${AIDT_CLI_SRC}) 47 | 48 | # adbase 49 | FIND_PACKAGE( libadbase REQUIRED) 50 | MARK_AS_ADVANCED( 51 | LIBADBASE_INCLUDE_DIR 52 | LIBADBASE_LIBRARIES 53 | ) 54 | IF (LIBADBASE_INCLUDE_DIR AND LIBADBASE_LIBRARIES) 55 | MESSAGE(STATUS "Found libadbase libraries") 56 | INCLUDE_DIRECTORIES(${LIBADBASE_INCLUDE_DIR}) 57 | MESSAGE( ${LIBADBASE_LIBRARIES} ) 58 | TARGET_LINK_LIBRARIES(aidt ${LIBADBASE_LIBRARIES} ) 59 | TARGET_LINK_LIBRARIES(aidt-cli ${LIBADBASE_LIBRARIES} ) 60 | ELSE (LIBADBASE_INCLUDE_DIR AND LIBADBASE_LIBRARIES) 61 | MESSAGE(FATAL_ERROR "Failed to find libadbase libraries") 62 | ENDIF (LIBADBASE_INCLUDE_DIR AND LIBADBASE_LIBRARIES) 63 | 64 | # adbase_kafka 65 | FIND_PACKAGE( libadbase_kafka REQUIRED) 66 | MARK_AS_ADVANCED( 67 | LIBADBASE_KAFKA_INCLUDE_DIR 68 | LIBADBASE_KAFKA_LIBRARIES 69 | ) 70 | IF (LIBADBASE_KAFKA_INCLUDE_DIR AND LIBADBASE_KAFKA_LIBRARIES) 71 | MESSAGE(STATUS "Found libadbase_kafka libraries") 72 | INCLUDE_DIRECTORIES(${LIBADBASE_KAFKA_INCLUDE_DIR}) 73 | MESSAGE( ${LIBADBASE_KAFKA_LIBRARIES} ) 74 | TARGET_LINK_LIBRARIES(aidt ${LIBADBASE_KAFKA_LIBRARIES} ) 75 | ELSE (LIBADBASE_KAFKA_INCLUDE_DIR AND LIBADBASE_KAFKA_LIBRARIES) 76 | MESSAGE(FATAL_ERROR "Failed to find libadbase_kafka libraries") 77 | ENDIF (LIBADBASE_KAFKA_INCLUDE_DIR AND LIBADBASE_KAFKA_LIBRARIES) 78 | 79 | # event 80 | #FIND_PACKAGE( libevent REQUIRED) 81 | #MARK_AS_ADVANCED( 82 | # LIBEVENT_INCLUDE_DIR 83 | # LIBEVENT_LIBRARIES 84 | #) 85 | #IF (LIBEVENT_INCLUDE_DIR AND LIBEVENT_LIBRARIES) 86 | # MESSAGE(STATUS "Found libevent libraries") 87 | # INCLUDE_DIRECTORIES(${LIBEVENT_INCLUDE_DIR}) 88 | # MESSAGE( ${LIBEVENT_LIBRARIES} ) 89 | # MESSAGE( ${LIBEVENT_STATIC_LIBRARIES} ) 90 | # TARGET_LINK_LIBRARIES(aidt ${LIBEVENT_STATIC_LIBRARIES} ) 91 | #ELSE (LIBEVENT_INCLUDE_DIR AND LIBEVENT_LIBRARIES) 92 | # MESSAGE(FATAL_ERROR "Failed to find libevent libraries") 93 | #ENDIF (LIBEVENT_INCLUDE_DIR AND LIBEVENT_LIBRARIES) 94 | # 95 | # rdkafka++ 96 | FIND_PACKAGE( librdkafka REQUIRED) 97 | MARK_AS_ADVANCED( 98 | LIBRDKAFKA_INCLUDE_DIR 99 | LIBRDKAFKA_LIBRARIES 100 | ) 101 | IF (LIBRDKAFKA_INCLUDE_DIR AND LIBRDKAFKA_LIBRARIES) 102 | MESSAGE(STATUS "Found librdkafka libraries") 103 | INCLUDE_DIRECTORIES(${LIBRDKAFKA_INCLUDE_DIR}) 104 | MESSAGE( ${LIBRDKAFKA_LIBRARIES} ) 105 | TARGET_LINK_LIBRARIES(aidt ${LIBRDKAFKA_CPP_LIBRARIES} ) 106 | TARGET_LINK_LIBRARIES(aidt ${LIBRDKAFKA_LIBRARIES} ) 107 | ELSE (LIBRDKAFKA_INCLUDE_DIR AND LIBRDKAFKA_LIBRARIES) 108 | MESSAGE(FATAL_ERROR "Failed to find librdkafka libraries") 109 | ENDIF (LIBRDKAFKA_INCLUDE_DIR AND LIBRDKAFKA_LIBRARIES) 110 | 111 | # yaml-cpp 112 | FIND_PACKAGE( libyaml-cpp REQUIRED) 113 | MARK_AS_ADVANCED( 114 | LIBYAMLCPP_INCLUDE_DIR 115 | LIBYAMLCPP_LIBRARIES 116 | ) 117 | IF (LIBYAMLCPP_INCLUDE_DIR AND LIBYAMLCPP_LIBRARIES) 118 | MESSAGE(STATUS "Found libyaml-cpp libraries") 119 | INCLUDE_DIRECTORIES(${LIBYAMLCPP_INCLUDE_DIR}) 120 | MESSAGE( ${LIBYAMLCPP_LIBRARIES} ) 121 | TARGET_LINK_LIBRARIES(aidt ${LIBYAMLCPP_LIBRARIES} ) 122 | ELSE (LIBYAMLCPP_INCLUDE_DIR AND LIBYAMLCPP_LIBRARIES) 123 | MESSAGE(FATAL_ERROR "Failed to find libyaml-cpp libraries") 124 | ENDIF (LIBYAMLCPP_INCLUDE_DIR AND LIBYAMLCPP_LIBRARIES) 125 | 126 | TARGET_LINK_LIBRARIES(aidt libevent.a rt libz.a ssl sasl2 dl) 127 | TARGET_LINK_LIBRARIES(aidt-cli rt dl) 128 | # pthread 129 | FIND_PACKAGE( libpthread REQUIRED) 130 | MARK_AS_ADVANCED( 131 | LIBPTHREAD_INCLUDE_DIR 132 | LIBPTHREAD_LIBRARIES 133 | ) 134 | IF (LIBPTHREAD_INCLUDE_DIR AND LIBPTHREAD_LIBRARIES) 135 | MESSAGE(STATUS "Found libpthread libraries") 136 | INCLUDE_DIRECTORIES(${LIBPTHREAD_INCLUDE_DIR}) 137 | MESSAGE( ${LIBPTHREAD_LIBRARIES} ) 138 | TARGET_LINK_LIBRARIES(aidt ${LIBPTHREAD_LIBRARIES} ) 139 | ELSE (LIBPTHREAD_INCLUDE_DIR AND LIBPTHREAD_LIBRARIES) 140 | MESSAGE(FATAL_ERROR "Failed to find libpthread libraries") 141 | ENDIF (LIBPTHREAD_INCLUDE_DIR AND LIBPTHREAD_LIBRARIES) 142 | 143 | 144 | INSTALL(TARGETS aidt RUNTIME DESTINATION bin) 145 | INSTALL(TARGETS aidt-cli RUNTIME DESTINATION bin) 146 | -------------------------------------------------------------------------------- /src/HeadProcessor.cpp: -------------------------------------------------------------------------------- 1 | #include "HeadProcessor.hpp" 2 | #include "Version.hpp" 3 | 4 | // {{{ HeadProcessor::HeadProcessor() 5 | 6 | HeadProcessor::HeadProcessor(AdServerContext* context) : 7 | _context(context) { 8 | _configure = _context->config; 9 | } 10 | 11 | // }}} 12 | // {{{ HeadProcessor::~HeadProcessor() 13 | 14 | HeadProcessor::~HeadProcessor() { 15 | } 16 | 17 | // }}} 18 | // {{{ adbase::head::ProtocolBinaryResponseStatus HeadProcessor::readHandler() 19 | 20 | adbase::head::ProtocolBinaryResponseStatus HeadProcessor::readHandler(adbase::head::ProtocolBinaryDataType datatype, 21 | const void *data, 22 | ssize_t datalen, 23 | adbase::head::ProtocolBinaryDataType* resDataType, 24 | adbase::Buffer* responseData) { 25 | (void)data; 26 | (void)datatype; 27 | (void)datalen; 28 | *resDataType = adbase::head::PROTOCOL_BINARY_TYPE_JSON; 29 | // 如下 json 手动拼接为了减少对 json 库的依赖，在实际项目中推荐用 rapidjson 30 | std::unordered_map procs = adbase::procStats(); 31 | procs["version"] = VERSION; 32 | procs["git_sha1"] = GIT_SHA1; 33 | procs["git_dirty"] = GIT_DIRTY; 34 | procs["build_id"] = BUILD_ID; 35 | procs["build_type"] = BUILD_TYPE; 36 | std::string result = "{"; 37 | for (auto &t : procs) { 38 | result +="\"" + t.first + "\":\"" + t.second + "\","; 39 | } 40 | result = adbase::rightTrim(result, ","); 41 | result += "}"; 42 | responseData->append(result); 43 | return adbase::head::PROTOCOL_BINARY_RESPONSE_SUCCESS; 44 | } 45 | 46 | // }}} -------------------------------------------------------------------------------- /src/HeadProcessor.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_HEAD_PROCESSOR_HPP_ 2 | #define AIDT_HEAD_PROCESSOR_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "AdbaseConfig.hpp" 8 | 9 | class HeadProcessor { 10 | public: 11 | HeadProcessor(AdServerContext* context); 12 | ~HeadProcessor(); 13 | adbase::head::ProtocolBinaryResponseStatus readHandler(adbase::head::ProtocolBinaryDataType datatype, 14 | const void *data, 15 | ssize_t datalen, 16 | adbase::head::ProtocolBinaryDataType* resDataType, 17 | adbase::Buffer* responseData); 18 | 19 | private: 20 | /// 传输上下文指针 21 | AdServerContext* _context; 22 | AdbaseConfig* _configure; 23 | }; 24 | 25 | #endif -------------------------------------------------------------------------------- /src/Http.cpp: -------------------------------------------------------------------------------- 1 | #include "Http.hpp" 2 | 3 | // {{{ Http::Http() 4 | 5 | Http::Http(AdServerContext* context, adbase::http::Server* http) : 6 | _context(context), 7 | _http(http) { 8 | } 9 | 10 | // }}} 11 | // {{{ Http::~Http() 12 | 13 | Http::~Http() { 14 | ADSERVER_HTTP_STOP(Server); 15 | ADSERVER_HTTP_STOP(Index); 16 | ADSERVER_HTTP_STOP(Config); 17 | ADSERVER_HTTP_STOP(Topic); 18 | ADSERVER_HTTP_STOP(Message); 19 | } 20 | 21 | // }}} 22 | // {{{ void Http::addController() 23 | 24 | void Http::addController() { 25 | ADSERVER_HTTP_ADD_CONTROLLER(Server); 26 | ADSERVER_HTTP_ADD_CONTROLLER(Index); 27 | ADSERVER_HTTP_ADD_CONTROLLER(Config); 28 | ADSERVER_HTTP_ADD_CONTROLLER(Topic); 29 | ADSERVER_HTTP_ADD_CONTROLLER(Message); 30 | } 31 | 32 | // }}} 33 | // {{{ std::unordered_map Http::rewrite() 34 | 35 | std::unordered_map Http::rewrite() { 36 | std::unordered_map urls; 37 | return urls; 38 | } 39 | 40 | // }}} -------------------------------------------------------------------------------- /src/Http.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_HTTP_HPP_ 2 | #define AIDT_HTTP_HPP_ 3 | 4 | #include "AdbaseConfig.hpp" 5 | #include "Http/Index.hpp" 6 | #include "Http/Config.hpp" 7 | #include "Http/Topic.hpp" 8 | #include "Http/Message.hpp" 9 | #include "Http/Server.hpp" 10 | 11 | // {{{ macros 12 | 13 | #ifndef DECLARE_HTTP_CONTROLLER 14 | #define DECLARE_HTTP_CONTROLLER(name) \ 15 | adserver::http:: name* _http##name = nullptr; 16 | #endif 17 | #ifndef ADSERVER_HTTP_ADD_CONTROLLER 18 | #define ADSERVER_HTTP_ADD_CONTROLLER(name) do {\ 19 | _http##name = new adserver::http:: name(_context);\ 20 | _http##name->registerLocation(_http);\ 21 | } while(0) 22 | #endif 23 | #ifndef ADSERVER_HTTP_STOP 24 | #define ADSERVER_HTTP_STOP(name) \ 25 | delete _http##name; 26 | #endif 27 | 28 | // }}} 29 | class Http { 30 | public: 31 | Http(AdServerContext* context, adbase::http::Server* http); 32 | ~Http(); 33 | void addController(); 34 | std::unordered_map rewrite(); 35 | private: 36 | AdServerContext* _context; 37 | adbase::http::Server* _http; 38 | DECLARE_HTTP_CONTROLLER(Server); 39 | DECLARE_HTTP_CONTROLLER(Index); 40 | DECLARE_HTTP_CONTROLLER(Config); 41 | DECLARE_HTTP_CONTROLLER(Topic); 42 | DECLARE_HTTP_CONTROLLER(Message); 43 | 44 | }; 45 | 46 | #endif -------------------------------------------------------------------------------- /src/Http/Config.cpp: -------------------------------------------------------------------------------- 1 | #include "Config.hpp" 2 | 3 | namespace adserver { 4 | namespace http { 5 | // {{{ Config::Config() 6 | 7 | Config::Config(AdServerContext* context) : 8 | HttpInterface(context) { 9 | } 10 | 11 | // }}} 12 | // {{{ void Config::registerLocation() 13 | 14 | void Config::registerLocation(adbase::http::Server* http) { 15 | ADSERVER_HTTP_ADD_API(http, Config, index) 16 | } 17 | 18 | // }}} 19 | // {{{ void Config::index() 20 | 21 | void Config::index(adbase::http::Request* request, adbase::http::Response* response, void*) { 22 | (void)request; 23 | responseJson(response, "{\"msg\": \"hello adinf\"}", 0, ""); 24 | } 25 | 26 | // }}} 27 | } 28 | } -------------------------------------------------------------------------------- /src/Http/Config.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_HTTP_CONFIG_HPP_ 2 | #define AIDT_HTTP_CONFIG_HPP_ 3 | 4 | #include "HttpInterface.hpp" 5 | 6 | namespace adserver { 7 | namespace http { 8 | class Config : HttpInterface { 9 | public: 10 | Config(AdServerContext* context); 11 | void registerLocation(adbase::http::Server* http); 12 | void index(adbase::http::Request* request, adbase::http::Response* response, void*); 13 | }; 14 | } 15 | } 16 | 17 | #endif -------------------------------------------------------------------------------- /src/Http/HttpInterface.cpp: -------------------------------------------------------------------------------- 1 | #include "HttpInterface.hpp" 2 | #include "App.hpp" 3 | 4 | namespace adserver { 5 | namespace http { 6 | // {{{ HttpInterface::HttpInterface() 7 | 8 | HttpInterface::HttpInterface(AdServerContext* context) : 9 | _context(context) { 10 | } 11 | 12 | // }}} 13 | // {{{ HttpInterface::~HttpInterface() 14 | 15 | HttpInterface::~HttpInterface() { 16 | } 17 | 18 | // }}} 19 | // {{{ void HttpInterface::registerLocation() 20 | 21 | void HttpInterface::registerLocation(adbase::http::Server* http) { 22 | (void)http; 23 | } 24 | 25 | // }}} 26 | // {{{ void HttpInterface::responseHeader() 27 | 28 | void HttpInterface::responseHeader(adbase::http::Response* response) { 29 | response->addHeader("Content-Type", "application/json; charset=utf-8"); 30 | response->setHeader("Server", _context->config->httpServerName, true); 31 | response->setHeader("Connection", "keep-alive"); 32 | } 33 | 34 | // }}} 35 | // {{{ void HttpInterface::responseJson() 36 | 37 | void HttpInterface::responseJson(adbase::http::Response* response, const std::string& data, uint32_t ret, const std::string& msg, bool isRawString) { 38 | std::string result = "{\"code\":" + std::to_string(ret) + ","; 39 | result += "\"baseid\":" + std::to_string(_context->app->getSeqId()) + ","; 40 | std::string rawData = data; 41 | if (isRawString) { 42 | rawData = "\"" + rawData + "\""; 43 | } 44 | result += "\"data\":" + rawData + ","; 45 | std::string rawMsg = "\"" + msg + "\"}"; 46 | result += "\"msg\":" + rawMsg; 47 | responseHeader(response); 48 | response->setContent(result); 49 | response->sendReply(); 50 | } 51 | 52 | // }}} 53 | } 54 | } -------------------------------------------------------------------------------- /src/Http/HttpInterface.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_HTTPINTERFACE_HPP_ 2 | #define AIDT_HTTPINTERFACE_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "AdbaseConfig.hpp" 8 | 9 | namespace adserver { 10 | namespace http { 11 | // {{{ ADSERVER_HTTP_ADD_API 12 | class SourceFile { 13 | public: 14 | template 15 | inline SourceFile(const char (&arr)[N]) : 16 | _data(arr), 17 | _size(N - 1) { 18 | const char* slash = strrchr(_data, '/'); 19 | if (slash) { 20 | _data = slash + 1; 21 | _size -= static_cast(_data - arr) + 4; // 后缀 .cpp 22 | } 23 | } 24 | 25 | const char* _data; 26 | int _size; 27 | }; 28 | #ifndef ADSERVER_HTTP_ADD_API 29 | #define ADSERVER_HTTP_ADD_API(http, class, api) do { \ 30 | SourceFile file##api(__FILE__);\ 31 | std::string module##api(file##api._data, file##api._size);\ 32 | transform(module##api.begin(), module##api.end(), module##api.begin(), ::tolower);\ 33 | std::string apiPath##api = "/" + module##api + "/"#api;\ 34 | http->registerLocation(apiPath##api, std::bind(& class:: api, this, \ 35 | std::placeholders::_1, \ 36 | std::placeholders::_2, \ 37 | std::placeholders::_3), nullptr); } while (0); 38 | #endif 39 | // }}} 40 | class HttpInterface { 41 | public: 42 | HttpInterface(AdServerContext* context); 43 | virtual ~HttpInterface(); 44 | virtual void registerLocation(adbase::http::Server* http); 45 | 46 | protected: 47 | /// 传输上下文指针 48 | AdServerContext* _context; 49 | void responseHeader(adbase::http::Response* response); 50 | void responseJson(adbase::http::Response* response, const std::string& data, uint32_t ret, const std::string& msg, bool isRawString = false); 51 | }; 52 | } 53 | } 54 | #endif -------------------------------------------------------------------------------- /src/Http/Index.cpp: -------------------------------------------------------------------------------- 1 | #include "Index.hpp" 2 | 3 | namespace adserver { 4 | namespace http { 5 | // {{{ Index::Index() 6 | 7 | Index::Index(AdServerContext* context) : 8 | HttpInterface(context) { 9 | } 10 | 11 | // }}} 12 | // {{{ void Index::registerLocation() 13 | 14 | void Index::registerLocation(adbase::http::Server* http) { 15 | ADSERVER_HTTP_ADD_API(http, Index, index) 16 | } 17 | 18 | // }}} 19 | // {{{ void Index::index() 20 | 21 | void Index::index(adbase::http::Request* request, adbase::http::Response* response, void*) { 22 | (void)request; 23 | responseJson(response, "{\"msg\": \"hello adinf\"}", 0, ""); 24 | } 25 | 26 | // }}} 27 | } 28 | } -------------------------------------------------------------------------------- /src/Http/Index.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_HTTP_INDEX_HPP_ 2 | #define AIDT_HTTP_INDEX_HPP_ 3 | 4 | #include "HttpInterface.hpp" 5 | 6 | namespace adserver { 7 | namespace http { 8 | class Index : HttpInterface { 9 | public: 10 | Index(AdServerContext* context); 11 | void registerLocation(adbase::http::Server* http); 12 | void index(adbase::http::Request* request, adbase::http::Response* response, void*); 13 | }; 14 | } 15 | } 16 | 17 | #endif -------------------------------------------------------------------------------- /src/Http/Message.cpp: -------------------------------------------------------------------------------- 1 | #include "Message.hpp" 2 | #include "App.hpp" 3 | 4 | namespace adserver { 5 | namespace http { 6 | // {{{ Message::Message() 7 | 8 | Message::Message(AdServerContext* context) : 9 | HttpInterface(context) { 10 | } 11 | 12 | // }}} 13 | // {{{ void Message::registerLocation() 14 | 15 | void Message::registerLocation(adbase::http::Server* http) { 16 | ADSERVER_HTTP_ADD_API(http, Message, index) 17 | } 18 | 19 | // }}} 20 | // {{{ void Message::index() 21 | 22 | void Message::index(adbase::http::Request* request, adbase::http::Response* response, void*) { 23 | if (_context->app == nullptr || _context->message == nullptr) { 24 | responseJson(response, "", 1000, "System is starting.", true); 25 | return; 26 | } 27 | 28 | std::string topicName = request->getQuery("topic_name"); 29 | if (topicName == "") { 30 | responseJson(response, "", 1000, "Must defined `topic_name`.", true); 31 | return; 32 | } 33 | std::string partIdStr = request->getQuery("part_id"); 34 | errno = 0; 35 | uint32_t partId = static_cast(strtoul(partIdStr.c_str(), nullptr, 10)); 36 | if (errno != 0 || !partId) { 37 | partId = -1; 38 | } 39 | 40 | std::string data = request->getPost("data"); 41 | if (data == "") { 42 | responseJson(response, "", 1000, "Must defined `data`.", true); 43 | return; 44 | } 45 | 46 | if (!_context->app->checkValidTopicAndPartition(topicName, partId)) { 47 | responseJson(response, "", 1000, "Topic or partId is invalid.", true); 48 | return; 49 | } 50 | 51 | adbase::Buffer buffer; 52 | buffer.append(data); 53 | bool ret = _context->message->setMessage(topicName, partId, buffer); 54 | if (!ret) { 55 | responseJson(response, "", 1000, "Message queue fully.", true); 56 | return; 57 | } 58 | responseJson(response, "", 0, "Message send success", true); 59 | } 60 | 61 | // }}} 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/Http/Message.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_HTTP_MESSAGE_HPP_ 2 | #define AIDT_HTTP_MESSAGE_HPP_ 3 | 4 | #include "HttpInterface.hpp" 5 | 6 | namespace adserver { 7 | namespace http { 8 | class Message : HttpInterface { 9 | public: 10 | Message(AdServerContext* context); 11 | void registerLocation(adbase::http::Server* http); 12 | void index(adbase::http::Request* request, adbase::http::Response* response, void*); 13 | }; 14 | } 15 | } 16 | 17 | #endif -------------------------------------------------------------------------------- /src/Http/Server.cpp: -------------------------------------------------------------------------------- 1 | #include "Server.hpp" 2 | #include "Version.hpp" 3 | 4 | namespace adserver { 5 | namespace http { 6 | // {{{ Server::Server() 7 | 8 | Server::Server(AdServerContext* context) : 9 | HttpInterface(context) { 10 | } 11 | 12 | // }}} 13 | // {{{ void Server::registerLocation() 14 | 15 | void Server::registerLocation(adbase::http::Server* http) { 16 | ADSERVER_HTTP_ADD_API(http, Server, status) 17 | ADSERVER_HTTP_ADD_API(http, Server, metrics) 18 | } 19 | 20 | // }}} 21 | // {{{ void Server::status() 22 | 23 | void Server::status(adbase::http::Request* request, adbase::http::Response* response, void*) { 24 | (void)request; 25 | std::string result; 26 | // 如下 json 手动拼接为了减少对 json 库的依赖，在实际项目中推荐用 rapidjson 27 | std::unordered_map procs = adbase::procStats(); 28 | procs["version"] = VERSION; 29 | procs["git_sha1"] = GIT_SHA1; 30 | procs["git_dirty"] = GIT_DIRTY; 31 | procs["build_id"] = BUILD_ID; 32 | procs["build_type"] = BUILD_TYPE; 33 | std::string system = "{"; 34 | for (auto &t : procs) { 35 | system +="\"" + t.first + "\":\"" + t.second + "\","; 36 | } 37 | system = adbase::rightTrim(system, ","); 38 | 39 | // Metrics 40 | std::unordered_map gauges; 41 | std::unordered_map counters; 42 | std::unordered_map meters; 43 | std::unordered_map histograms; 44 | std::unordered_map timers; 45 | if (_context->metrics != nullptr) { 46 | gauges = _context->metrics->getGauges(); 47 | counters = _context->metrics->getCounter(); 48 | histograms = _context->metrics->getHistograms(); 49 | meters = _context->metrics->getMeters(); 50 | timers = _context->metrics->getTimers(); 51 | } 52 | 53 | std::unordered_map metricItems; 54 | for (auto &t : gauges) { 55 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 56 | std::unordered_map metricTags = name.tags; 57 | if (name.moduleName == "self") { 58 | continue; 59 | } 60 | 61 | metricTags["value"] = std::to_string(t.second); 62 | std::string item = "{"; 63 | for (auto &tag : metricTags) { 64 | item += "\"" + tag.first + "\":\"" + tag.second + "\","; 65 | } 66 | item = adbase::rightTrim(item, ","); 67 | item += "}"; 68 | metricItems[getKey(name.moduleName, name.metricName)] += item + ","; 69 | } 70 | 71 | 72 | for (auto &t : counters) { 73 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 74 | std::unordered_map metricTags = name.tags; 75 | if (name.moduleName == "self") { 76 | continue; 77 | } 78 | 79 | metricTags["value"] = std::to_string(t.second); 80 | std::string item = "{"; 81 | for (auto &tag : metricTags) { 82 | item += "\"" + tag.first + "\":\"" + tag.second + "\","; 83 | } 84 | item = adbase::rightTrim(item, ","); 85 | item += "}"; 86 | 87 | metricItems[getKey(name.moduleName, name.metricName)] += item + ","; 88 | } 89 | 90 | for (auto &t : meters) { 91 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 92 | std::unordered_map metricTags = name.tags; 93 | if (name.moduleName == "self") { 94 | continue; 95 | } 96 | 97 | metricTags["count"] = std::to_string(t.second.count); 98 | metricTags["mean"] = std::to_string(t.second.meanRate); 99 | metricTags["min1"] = std::to_string(t.second.min1Rate); 100 | metricTags["min5"] = std::to_string(t.second.min5Rate); 101 | metricTags["min15"] = std::to_string(t.second.min15Rate); 102 | std::string item = "{"; 103 | for (auto &tag : metricTags) { 104 | item += "\"" + tag.first + "\":\"" + tag.second + "\","; 105 | } 106 | item = adbase::rightTrim(item, ","); 107 | item += "}"; 108 | 109 | metricItems[getKey(name.moduleName, name.metricName)] += item + ","; 110 | } 111 | for (auto &t : histograms) { 112 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 113 | std::unordered_map metricTags = name.tags; 114 | if (name.moduleName == "self") { 115 | continue; 116 | } 117 | 118 | metricTags["min"] = std::to_string(t.second.min); 119 | metricTags["max"] = std::to_string(t.second.max); 120 | metricTags["stddev"] = std::to_string(t.second.stddev); 121 | metricTags["median"] = std::to_string(t.second.median); 122 | metricTags["percent75"] = std::to_string(t.second.percent75); 123 | metricTags["percent95"] = std::to_string(t.second.percent95); 124 | metricTags["percent98"] = std::to_string(t.second.percent98); 125 | metricTags["percent99"] = std::to_string(t.second.percent99); 126 | metricTags["percent999"] = std::to_string(t.second.percent999); 127 | std::string item = "{"; 128 | for (auto &tag : metricTags) { 129 | item += "\"" + tag.first + "\":" + tag.second + ","; 130 | } 131 | item = adbase::rightTrim(item, ","); 132 | item += "}"; 133 | 134 | metricItems[getKey(name.moduleName, name.metricName)] += item + ","; 135 | } 136 | for (auto &t : timers) { 137 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 138 | std::unordered_map metricTags = name.tags; 139 | 140 | metricTags["count"] = std::to_string(t.second.meter.count); 141 | metricTags["mean"] = std::to_string(t.second.meter.meanRate); 142 | metricTags["min1"] = std::to_string(t.second.meter.min1Rate); 143 | metricTags["min5"] = std::to_string(t.second.meter.min5Rate); 144 | metricTags["min15"] = std::to_string(t.second.meter.min15Rate); 145 | metricTags["min"] = std::to_string(t.second.histogram.min); 146 | metricTags["max"] = std::to_string(t.second.histogram.max); 147 | metricTags["stddev"] = std::to_string(t.second.histogram.stddev); 148 | metricTags["median"] = std::to_string(t.second.histogram.median); 149 | metricTags["percent75"] = std::to_string(t.second.histogram.percent75); 150 | metricTags["percent95"] = std::to_string(t.second.histogram.percent95); 151 | metricTags["percent98"] = std::to_string(t.second.histogram.percent98); 152 | metricTags["percent99"] = std::to_string(t.second.histogram.percent99); 153 | metricTags["percent999"] = std::to_string(t.second.histogram.percent999); 154 | std::string item = "{"; 155 | for (auto &tag : metricTags) { 156 | item += "\"" + tag.first + "\":\"" + tag.second + "\","; 157 | } 158 | item = adbase::rightTrim(item, ","); 159 | item += "}"; 160 | 161 | metricItems[getKey(name.moduleName, name.metricName)] += item + ","; 162 | } 163 | 164 | std::unordered_map modulesItems; 165 | modulesItems["system"] = system; 166 | for (auto &t : metricItems) { 167 | std::vector result = adbase::explode(t.first, 26); 168 | if (modulesItems.find(result[0]) == modulesItems.end()) { 169 | modulesItems[result[0]] = "{"; 170 | } 171 | 172 | modulesItems[result[0]] += "\"" + result[1] + "\":[" + adbase::rightTrim(t.second, ",") + "],"; 173 | } 174 | 175 | result = "{"; 176 | for (auto &t : modulesItems) { 177 | result += "\"" + t.first + "\":" + adbase::rightTrim(t.second, ",") + "},"; 178 | } 179 | result = adbase::rightTrim(result, ","); 180 | result += "}"; 181 | 182 | responseJson(response, result, 0, ""); 183 | } 184 | 185 | // }}} 186 | // {{{ void Server::metrics() 187 | 188 | void Server::metrics(adbase::http::Request* request, adbase::http::Response* response, void*) { 189 | (void)request; 190 | std::string result; 191 | std::string tagValue = request->getQuery("tags"); 192 | std::unordered_map tags; 193 | if (!tagValue.empty()) { 194 | std::vector tagStrs = adbase::explode(tagValue, '|', true); 195 | for (auto &t: tagStrs) { 196 | std::vector tagKV = adbase::explode(t, ':', true); 197 | if (tagKV.size() == 2) { 198 | tags[tagKV[0]] = tagKV[1]; 199 | } 200 | } 201 | } 202 | 203 | int count = 0; 204 | std::unordered_map procs = adbase::procStats(); 205 | tags["name"] = adbase::trim(procs["name"], "()"); 206 | 207 | 208 | tags["service"] = request->getServerAddress(); 209 | tags["version"] = VERSION; 210 | 211 | for (auto &t : procs) { 212 | std::string key = adbase::replace(".", "_", t.first, count); 213 | tags["metric_type"] = "gauges"; 214 | result += formatMetric(key, toUint64(t.second), tags); 215 | } 216 | 217 | // Metrics 218 | std::unordered_map gauges; 219 | std::unordered_map counters; 220 | std::unordered_map meters; 221 | std::unordered_map histograms; 222 | std::unordered_map timers; 223 | if (_context->metrics != nullptr) { 224 | gauges = _context->metrics->getGauges(); 225 | counters = _context->metrics->getCounter(); 226 | histograms = _context->metrics->getHistograms(); 227 | meters = _context->metrics->getMeters(); 228 | timers = _context->metrics->getTimers(); 229 | } 230 | 231 | tags["metric_type"] = "gauges"; 232 | for (auto &t : gauges) { 233 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 234 | std::unordered_map metricTags = name.tags; 235 | if (name.moduleName == "self") { 236 | continue; 237 | } 238 | 239 | for (auto &t : tags) { 240 | metricTags[t.first] = t.second; 241 | } 242 | std::string key = name.moduleName + "_" + name.metricName; 243 | key = adbase::replace(".", "_", key, count); 244 | result += formatMetric(key, t.second, metricTags); 245 | } 246 | tags["metric_type"] = "counters"; 247 | for (auto &t : counters) { 248 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 249 | std::unordered_map metricTags = name.tags; 250 | if (name.moduleName == "self") { 251 | continue; 252 | } 253 | std::string key = name.moduleName + "_" + name.metricName; 254 | key = adbase::replace(".", "_", key, count); 255 | for (auto &t : tags) { 256 | metricTags[t.first] = t.second; 257 | } 258 | result += formatMetric(key, t.second, metricTags); 259 | } 260 | 261 | tags["metric_type"] = "meters"; 262 | for (auto &t : meters) { 263 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 264 | std::unordered_map metricTags = name.tags; 265 | if (name.moduleName == "self") { 266 | continue; 267 | } 268 | std::string key = name.moduleName + "_" + name.metricName; 269 | key = adbase::replace(".", "_", key, count); 270 | for (auto &t : tags) { 271 | metricTags[t.first] = t.second; 272 | } 273 | metricTags["meter_type"] = "count"; 274 | result += formatMetric(key, t.second.count, metricTags); 275 | metricTags["meter_type"] = "mean"; 276 | result += formatMetric(key, toUint64(t.second.meanRate), metricTags); 277 | metricTags["meter_type"] = "min1"; 278 | result += formatMetric(key, toUint64(t.second.min1Rate), metricTags); 279 | metricTags["meter_type"] = "min5"; 280 | result += formatMetric(key, toUint64(t.second.min5Rate), metricTags); 281 | metricTags["meter_type"] = "min15"; 282 | result += formatMetric(key, toUint64(t.second.min15Rate), metricTags); 283 | } 284 | 285 | tags["metric_type"] = "histograms"; 286 | for (auto &t : histograms) { 287 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 288 | std::unordered_map metricTags = name.tags; 289 | if (name.moduleName == "self") { 290 | continue; 291 | } 292 | 293 | std::string key = name.moduleName + "_" + name.metricName; 294 | key = adbase::replace(".", "_", key, count); 295 | for (auto &t : tags) { 296 | metricTags[t.first] = t.second; 297 | } 298 | metricTags["histograms_type"] = "min"; 299 | result += formatMetric(key, toUint64(t.second.min), metricTags); 300 | metricTags["histograms_type"] = "max"; 301 | result += formatMetric(key, toUint64(t.second.max), metricTags); 302 | metricTags["histograms_type"] = "mean"; 303 | result += formatMetric(key, toUint64(t.second.mean), metricTags); 304 | metricTags["histograms_type"] = "stddev"; 305 | result += formatMetric(key, toUint64(t.second.stddev), metricTags); 306 | metricTags["histograms_type"] = "median"; 307 | result += formatMetric(key, toUint64(t.second.median), metricTags); 308 | metricTags["histograms_type"] = "percent75"; 309 | result += formatMetric(key, toUint64(t.second.percent75), metricTags); 310 | metricTags["histograms_type"] = "percent95"; 311 | result += formatMetric(key, toUint64(t.second.percent95), metricTags); 312 | metricTags["histograms_type"] = "percent98"; 313 | result += formatMetric(key, toUint64(t.second.percent98), metricTags); 314 | metricTags["histograms_type"] = "percent99"; 315 | result += formatMetric(key, toUint64(t.second.percent99), metricTags); 316 | metricTags["histograms_type"] = "percent999"; 317 | result += formatMetric(key, toUint64(t.second.percent999), metricTags); 318 | } 319 | 320 | tags["metric_type"] = "timers"; 321 | for (auto &t : timers) { 322 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 323 | std::unordered_map metricTags = name.tags; 324 | 325 | std::string key = name.moduleName + "_" + name.metricName; 326 | key = adbase::replace(".", "_", key, count); 327 | for (auto &t : tags) { 328 | metricTags[t.first] = t.second; 329 | } 330 | metricTags["timers_type"] = "count"; 331 | result += formatMetric(key, t.second.meter.count, metricTags); 332 | metricTags["timers_type"] = "mean"; 333 | result += formatMetric(key, toUint64(t.second.meter.meanRate), metricTags); 334 | metricTags["timers_type"] = "min1"; 335 | result += formatMetric(key, toUint64(t.second.meter.min1Rate), metricTags); 336 | metricTags["timers_type"] = "min5"; 337 | result += formatMetric(key, toUint64(t.second.meter.min5Rate), metricTags); 338 | metricTags["timers_type"] = "min15"; 339 | result += formatMetric(key, toUint64(t.second.meter.min15Rate), metricTags); 340 | metricTags["timers_type"] = "min"; 341 | result += formatMetric(key, toUint64(t.second.histogram.min), metricTags); 342 | metricTags["timers_type"] = "max"; 343 | result += formatMetric(key, toUint64(t.second.histogram.max), metricTags); 344 | metricTags["timers_type"] = "mean"; 345 | result += formatMetric(key, toUint64(t.second.histogram.mean), metricTags); 346 | metricTags["timers_type"] = "stddev"; 347 | result += formatMetric(key, toUint64(t.second.histogram.stddev), metricTags); 348 | metricTags["timers_type"] = "median"; 349 | result += formatMetric(key, toUint64(t.second.histogram.median), metricTags); 350 | metricTags["timers_type"] = "percent75"; 351 | result += formatMetric(key, toUint64(t.second.histogram.percent75), metricTags); 352 | metricTags["timers_type"] = "percent95"; 353 | result += formatMetric(key, toUint64(t.second.histogram.percent95), metricTags); 354 | metricTags["timers_type"] = "percent98"; 355 | result += formatMetric(key, toUint64(t.second.histogram.percent98), metricTags); 356 | metricTags["timers_type"] = "percent99"; 357 | result += formatMetric(key, toUint64(t.second.histogram.percent99), metricTags); 358 | metricTags["timers_type"] = "percent999"; 359 | result += formatMetric(key, toUint64(t.second.histogram.percent999), metricTags); 360 | } 361 | 362 | responseHeader(response); 363 | response->setContent(result); 364 | response->sendReply(); 365 | } 366 | 367 | // }}} 368 | // {{{ const std::string Server::formatMetric() 369 | 370 | const std::string Server::formatMetric(std::string key, uint64_t value, std::unordered_map tags) { 371 | std::string result = "adbase_"; 372 | 373 | int count = 0; 374 | std::string keyFormat = adbase::replace(".", "_", key, count); 375 | result += keyFormat; 376 | if (!tags.empty()) { 377 | result += "{"; 378 | for (auto &t : tags) { 379 | result += t.first; 380 | result += "=\""; 381 | result += t.second; 382 | result += "\","; 383 | } 384 | result = adbase::rightTrim(result, ","); 385 | result += "} "; 386 | } 387 | 388 | result += " "; 389 | result += std::to_string(value) + "\n"; 390 | return result; 391 | } 392 | 393 | // }}} 394 | // {{{ uint64_t Server::toUint64() 395 | 396 | uint64_t Server::toUint64(std::string value) { 397 | errno = 0; 398 | uint64_t result = static_cast(strtoull(value.c_str(), nullptr, 10)); 399 | if (errno != 0) { 400 | return 0; 401 | } 402 | return result; 403 | } 404 | 405 | // }}} 406 | // {{{ uint64_t Server::toUint64() 407 | 408 | uint64_t Server::toUint64(double value) { 409 | return static_cast(value * 10000); 410 | } 411 | 412 | // }}} 413 | // {{{ const std::string Server::getKey() 414 | 415 | const std::string Server::getKey(const std::string& moduleName, const std::string& metricName) { 416 | std::string result = moduleName; 417 | result.append(1, 26); 418 | result.append(metricName); 419 | return result; 420 | } 421 | 422 | // }}} 423 | } 424 | } 425 | -------------------------------------------------------------------------------- /src/Http/Server.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_HTTP_SERVER_HPP_ 2 | #define AIDT_HTTP_SERVER_HPP_ 3 | 4 | #include "HttpInterface.hpp" 5 | 6 | namespace adserver { 7 | namespace http { 8 | class Server : HttpInterface { 9 | public: 10 | Server(AdServerContext* context); 11 | void registerLocation(adbase::http::Server* http); 12 | void status(adbase::http::Request* request, adbase::http::Response* response, void*); 13 | void metrics(adbase::http::Request* request, adbase::http::Response* response, void*); 14 | 15 | private: 16 | const std::string formatMetric(std::string key, uint64_t value, std::unordered_map tags); 17 | const std::string getKey(const std::string& moduleName, const std::string& metricName); 18 | uint64_t toUint64(std::string value); 19 | uint64_t toUint64(double value); 20 | }; 21 | } 22 | } 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /src/Http/Topic.cpp: -------------------------------------------------------------------------------- 1 | #include "Topic.hpp" 2 | 3 | namespace adserver { 4 | namespace http { 5 | // {{{ Topic::Topic() 6 | 7 | Topic::Topic(AdServerContext* context) : 8 | HttpInterface(context) { 9 | } 10 | 11 | // }}} 12 | // {{{ void Topic::registerLocation() 13 | 14 | void Topic::registerLocation(adbase::http::Server* http) { 15 | ADSERVER_HTTP_ADD_API(http, Topic, index) 16 | } 17 | 18 | // }}} 19 | // {{{ void Topic::index() 20 | 21 | void Topic::index(adbase::http::Request* request, adbase::http::Response* response, void*) { 22 | (void)request; 23 | responseJson(response, "{\"msg\": \"hello adinf\"}", 0, ""); 24 | } 25 | 26 | // }}} 27 | } 28 | } -------------------------------------------------------------------------------- /src/Http/Topic.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_HTTP_TOPIC_HPP_ 2 | #define AIDT_HTTP_TOPIC_HPP_ 3 | 4 | #include "HttpInterface.hpp" 5 | 6 | namespace adserver { 7 | namespace http { 8 | class Topic : HttpInterface { 9 | public: 10 | Topic(AdServerContext* context); 11 | void registerLocation(adbase::http::Server* http); 12 | void index(adbase::http::Request* request, adbase::http::Response* response, void*); 13 | }; 14 | } 15 | } 16 | 17 | #endif -------------------------------------------------------------------------------- /src/McProcessor.cpp: -------------------------------------------------------------------------------- 1 | #include "McProcessor.hpp" 2 | #include "App.hpp" 3 | #include "Version.hpp" 4 | 5 | // {{{ McProcessor::McProcessor() 6 | 7 | McProcessor::McProcessor(AdServerContext* context) : 8 | _context(context) { 9 | _configure = _context->config; 10 | } 11 | 12 | // }}} 13 | // {{{ McProcessor::~McProcessor() 14 | 15 | McProcessor::~McProcessor() { 16 | } 17 | 18 | // }}} 19 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::add() 20 | 21 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::add(const void* key, 22 | uint16_t keylen, 23 | const void *data, 24 | uint32_t datalen, 25 | uint32_t flags, 26 | uint32_t exptime, 27 | uint64_t* cas) { 28 | (void)flags; 29 | (void)exptime; 30 | (void)cas; 31 | std::string keyData(static_cast(key), static_cast(keylen)); 32 | adbase::Buffer bufferBody; 33 | bufferBody.append(static_cast(data), static_cast(datalen)); 34 | LOG_INFO << "Mc ADD key:" << keyData; 35 | LOG_INFO << "Mc ADD data size:" << bufferBody.readableBytes(); 36 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 37 | } 38 | 39 | // }}} 40 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::append() 41 | 42 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::append(const void* key, 43 | uint16_t keylen, 44 | const void *data, 45 | uint32_t datalen, 46 | uint64_t cas, 47 | uint64_t* resultCas) { 48 | (void)cas; 49 | (void)resultCas; 50 | std::string keyData(static_cast(key), static_cast(keylen)); 51 | adbase::Buffer bufferBody; 52 | bufferBody.append(static_cast(data), static_cast(datalen)); 53 | LOG_INFO << "Mc APPEND key:" << keyData; 54 | LOG_INFO << "Mc APPEND data size:" << bufferBody.readableBytes(); 55 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 56 | } 57 | 58 | // }}} 59 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::decrement() 60 | 61 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::decrement(const void* key, 62 | uint16_t keylen, 63 | uint64_t delta, 64 | uint64_t initial, 65 | uint32_t expiration, 66 | uint64_t* result, 67 | uint64_t* resultCas) { 68 | (void)delta; 69 | (void)initial; 70 | (void)expiration; 71 | (void)result; 72 | (void)resultCas; 73 | std::string keyData(static_cast(key), static_cast(keylen)); 74 | LOG_INFO << "Mc DECREMENT key:" << keyData; 75 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 76 | } 77 | 78 | // }}} 79 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::deleteOp() 80 | 81 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::deleteOp(const void* key, 82 | uint16_t keylen, 83 | uint64_t cas) { 84 | (void)cas; 85 | std::string keyData(static_cast(key), static_cast(keylen)); 86 | LOG_INFO << "Mc DELETE key:" << keyData; 87 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 88 | } 89 | 90 | // }}} 91 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::flush() 92 | 93 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::flush(uint32_t when) { 94 | LOG_INFO << "Mc FLUSH when:" << when; 95 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 96 | } 97 | 98 | // }}} 99 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::get() 100 | 101 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::get(const void* key, 102 | uint16_t keylen, 103 | adbase::Buffer *data) { 104 | std::string keyData(static_cast(key), static_cast(keylen)); 105 | data->append(static_cast(key), static_cast(keylen)); 106 | LOG_INFO << "Mc GET key:" << keyData; 107 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 108 | } 109 | 110 | // }}} 111 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::increment() 112 | 113 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::increment(const void* key, 114 | uint16_t keylen, 115 | uint64_t delta, 116 | uint64_t initial, 117 | uint32_t expiration, 118 | uint64_t* result, 119 | uint64_t* resultCas) { 120 | (void)delta; 121 | (void)initial; 122 | (void)expiration; 123 | (void)result; 124 | (void)resultCas; 125 | std::string keyData(static_cast(key), static_cast(keylen)); 126 | LOG_INFO << "Mc INCREMENT key:" << keyData; 127 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 128 | } 129 | 130 | // }}} 131 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::prepend() 132 | 133 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::prepend(const void* key, 134 | uint16_t keylen, 135 | const void *data, 136 | uint32_t datalen, 137 | uint64_t cas, 138 | uint64_t* resultCas) { 139 | (void)cas; 140 | (void)resultCas; 141 | std::string keyData(static_cast(key), static_cast(keylen)); 142 | adbase::Buffer bufferBody; 143 | bufferBody.append(static_cast(data), static_cast(datalen)); 144 | LOG_INFO << "Mc PREPEND key:" << keyData; 145 | LOG_INFO << "Mc PREPEND data size:" << bufferBody.readableBytes(); 146 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 147 | } 148 | 149 | // }}} 150 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::noop() 151 | 152 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::noop() { 153 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 154 | } 155 | 156 | // }}} 157 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::quit() 158 | 159 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::quit() { 160 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 161 | } 162 | 163 | // }}} 164 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::replace() 165 | 166 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::replace(const void* key, 167 | uint16_t keylen, 168 | const void *data, 169 | uint32_t datalen, 170 | uint32_t flags, 171 | uint32_t exptime, 172 | uint64_t cas, 173 | uint64_t* resultCas) { 174 | (void)flags; 175 | (void)exptime; 176 | (void)cas; 177 | (void)resultCas; 178 | std::string keyData(static_cast(key), static_cast(keylen)); 179 | adbase::Buffer bufferBody; 180 | bufferBody.append(static_cast(data), static_cast(datalen)); 181 | LOG_INFO << "Mc REPLACE key:" << keyData; 182 | LOG_INFO << "Mc REPLACE data size:" << bufferBody.readableBytes(); 183 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 184 | } 185 | 186 | // }}} 187 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::set() 188 | 189 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::set(const void* key, 190 | uint16_t keylen, 191 | const void *data, 192 | uint32_t datalen, 193 | uint32_t flags, 194 | uint32_t exptime, 195 | uint64_t cas, 196 | uint64_t* resultCas) { 197 | (void)flags; 198 | (void)exptime; 199 | (void)cas; 200 | (void)resultCas; 201 | 202 | if (_context->app == nullptr || _context->message == nullptr) { 203 | LOG_ERROR << "System is starting."; 204 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_NOT_STORED; 205 | } 206 | 207 | std::string keyData(static_cast(key), static_cast(keylen)); 208 | adbase::Buffer bufferBody; 209 | bufferBody.append(static_cast(data), static_cast(datalen)); 210 | 211 | std::vector keys = adbase::explode(keyData, '#', true); 212 | if (keys.size() != 2 && keys.size() != 1) { 213 | LOG_TRACE << "Key: `" << keyData << "` is invalid."; 214 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_AUTH_ERROR; 215 | } 216 | 217 | int partId = 0; 218 | if (keys.size() == 2) { 219 | errno = 0; 220 | partId = static_cast(strtoul(keys[1].c_str(), nullptr, 10)); 221 | if (errno != 0) { 222 | LOG_TRACE << "Key partId: `" << keys[1] << "` is invalid."; 223 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_AUTH_ERROR; 224 | } 225 | } else { 226 | partId = -1; 227 | } 228 | 229 | if (!_context->app->checkValidTopicAndPartition(keys[0], partId)) { 230 | LOG_TRACE << "Key topic or partId: `" << keyData << "` is invalid."; 231 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_AUTH_ERROR; 232 | } 233 | 234 | bool ret = _context->message->setMessage(keys[0], partId, bufferBody); 235 | if (!ret) { 236 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_NOT_STORED; 237 | } 238 | 239 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 240 | } 241 | 242 | // }}} 243 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::stat() 244 | 245 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::stat(const void* key, 246 | uint16_t keylen, 247 | adbase::Buffer *data) { 248 | std::string keyData(static_cast(key), static_cast(keylen)); 249 | data->append(static_cast(key), static_cast(keylen)); 250 | LOG_INFO << "Mc STAT key:" << keyData; 251 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 252 | } 253 | 254 | // }}} 255 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::version() 256 | 257 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::version(adbase::Buffer *data) { 258 | data->append(VERSION); 259 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 260 | } 261 | 262 | // }}} 263 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::verbosity() 264 | 265 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::verbosity(uint32_t verbose) { 266 | LOG_INFO << "Mc FLUSH verbose:" << verbose; 267 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 268 | } 269 | 270 | // }}} 271 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::preExecute() 272 | 273 | void McProcessor::preExecute() { 274 | LOG_TRACE << "CMD exec pre."; 275 | } 276 | 277 | // }}} 278 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::postExecute() 279 | 280 | void McProcessor::postExecute() { 281 | LOG_TRACE << "CMD exec post."; 282 | } 283 | 284 | // }}} 285 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::unknownExecute() 286 | 287 | void McProcessor::unknownExecute() { 288 | LOG_TRACE << "CMD exec unknown."; 289 | } 290 | 291 | // }}} 292 | -------------------------------------------------------------------------------- /src/McProcessor.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_MC_PROCESSOR_HPP_ 2 | #define AIDT_MC_PROCESSOR_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "AdbaseConfig.hpp" 8 | 9 | class McProcessor { 10 | public: 11 | McProcessor(AdServerContext* context); 12 | ~McProcessor(); 13 | adbase::mc::ProtocolBinaryResponseStatus add(const void* key, 14 | uint16_t keylen, 15 | const void *data, 16 | uint32_t datalen, 17 | uint32_t flags, 18 | uint32_t exptime, 19 | uint64_t* cas); 20 | adbase::mc::ProtocolBinaryResponseStatus append(const void* key, 21 | uint16_t keylen, 22 | const void *data, 23 | uint32_t datalen, 24 | uint64_t cas, 25 | uint64_t* resultCas); 26 | adbase::mc::ProtocolBinaryResponseStatus decrement(const void* key, 27 | uint16_t keylen, 28 | uint64_t delta, 29 | uint64_t initial, 30 | uint32_t expiration, 31 | uint64_t* result, 32 | uint64_t* resultCas); 33 | adbase::mc::ProtocolBinaryResponseStatus deleteOp(const void* key, 34 | uint16_t keylen, 35 | uint64_t cas); 36 | adbase::mc::ProtocolBinaryResponseStatus flush(uint32_t when); 37 | adbase::mc::ProtocolBinaryResponseStatus get(const void* key, 38 | uint16_t keylen, 39 | adbase::Buffer *data); 40 | adbase::mc::ProtocolBinaryResponseStatus increment(const void* key, 41 | uint16_t keylen, 42 | uint64_t delta, 43 | uint64_t initial, 44 | uint32_t expiration, 45 | uint64_t* result, 46 | uint64_t* resultCas); 47 | adbase::mc::ProtocolBinaryResponseStatus prepend(const void* key, 48 | uint16_t keylen, 49 | const void *data, 50 | uint32_t datalen, 51 | uint64_t cas, 52 | uint64_t* resultCas); 53 | adbase::mc::ProtocolBinaryResponseStatus noop(); 54 | adbase::mc::ProtocolBinaryResponseStatus quit(); 55 | adbase::mc::ProtocolBinaryResponseStatus replace(const void* key, 56 | uint16_t keylen, 57 | const void *data, 58 | uint32_t datalen, 59 | uint32_t flags, 60 | uint32_t exptime, 61 | uint64_t cas, 62 | uint64_t* resultCas); 63 | adbase::mc::ProtocolBinaryResponseStatus set(const void* key, 64 | uint16_t keylen, 65 | const void *data, 66 | uint32_t datalen, 67 | uint32_t flags, 68 | uint32_t exptime, 69 | uint64_t cas, 70 | uint64_t* resultCas); 71 | adbase::mc::ProtocolBinaryResponseStatus stat(const void* key, 72 | uint16_t keylen, 73 | adbase::Buffer *data); 74 | adbase::mc::ProtocolBinaryResponseStatus version(adbase::Buffer *data); 75 | adbase::mc::ProtocolBinaryResponseStatus verbosity(uint32_t); 76 | void preExecute(); 77 | void postExecute(); 78 | void unknownExecute(); 79 | 80 | private: 81 | /// 传输上下文指针 82 | AdServerContext* _context; 83 | AdbaseConfig* _configure; 84 | }; 85 | 86 | #endif -------------------------------------------------------------------------------- /src/Timer.cpp: -------------------------------------------------------------------------------- 1 | #include "Timer.hpp" 2 | #include "App.hpp" 3 | 4 | // {{{ macros 5 | 6 | #define ADD_AFTER_TIMER(interval, name) do { \ 7 | uint64_t timerId##name = _timer->runAfter((interval),\ 8 | std::bind(&Timer:: name, this, std::placeholders::_1), nullptr);\ 9 | _timerIds.push_back(timerId##name);\ 10 | } while (0); 11 | #define ADD_EVERY_TIMER(interval, name) do { \ 12 | uint64_t timerId##name = _timer->runEvery((interval),\ 13 | std::bind(&Timer:: name, this, std::placeholders::_1), nullptr);\ 14 | _timerIds.push_back(timerId##name);\ 15 | } while (0); 16 | 17 | // }}} 18 | // {{{ Timer::Timer() 19 | 20 | Timer::Timer(TimerContext* context) : 21 | _context(context) { 22 | _configure = _context->config; 23 | _timer = new adbase::Timer(_context->mainEventBase); 24 | } 25 | 26 | // }}} 27 | // {{{ adbase::Timer* Timer::getTimer() 28 | 29 | adbase::Timer* Timer::getTimer() { 30 | return _timer; 31 | } 32 | 33 | // }}} 34 | // {{{ Timer::~Timer() 35 | 36 | Timer::~Timer() { 37 | for (auto &t : _timerIds) { 38 | _timer->cancel(t); 39 | } 40 | delete _timer; 41 | } 42 | 43 | // }}} 44 | // {{{ void Timer::init() 45 | 46 | void Timer::init() { 47 | /// 间隔一段时间执行一次 48 | //ADD_EVERY_TIMER(1 * 1000, one); 49 | /// 一段时间后仅执行一次 50 | //ADD_AFTER_TIMER(1 * 1000, one); 51 | ADD_EVERY_TIMER(_configure->intervalSyncOffset, syncOffset); 52 | ADD_EVERY_TIMER(_configure->mallocTrimInterval, mallocTrim); 53 | } 54 | 55 | // }}} 56 | // {{{ void Timer::syncOffset() 57 | 58 | void Timer::syncOffset(void*) { 59 | LOG_TRACE << "Timer " << "syncOffset"; 60 | if (_context->reader != nullptr) { 61 | _context->reader->save(); 62 | } 63 | } 64 | 65 | // }}} 66 | // {{{ void Timer::mallocTrim() 67 | 68 | void Timer::mallocTrim(void*) { 69 | adbase::mallocTrim(_configure->mallocTrimPad); 70 | } 71 | 72 | // }}} 73 | -------------------------------------------------------------------------------- /src/Timer.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDT_TIMER_HPP_ 2 | #define AIDT_TIMER_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "AdbaseConfig.hpp" 8 | 9 | class Timer { 10 | public: 11 | Timer(TimerContext* context); 12 | ~Timer(); 13 | void init(); 14 | adbase::Timer* getTimer(); 15 | void syncOffset(void* args); 16 | void mallocTrim(void* args); 17 | 18 | private: 19 | /// 传输上下文指针 20 | TimerContext* _context; 21 | AdbaseConfig* _configure; 22 | adbase::Timer* _timer; 23 | std::vector _timerIds; 24 | }; 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /src/Version.hpp.in: -------------------------------------------------------------------------------- 1 | /// 该程序自动生成，禁止修改 2 | #ifndef AIDT_VERSION_HPP_ 3 | #define AIDT_VERSION_HPP_ 4 | #define VERSION "@PROJECT_VERSION_MAJOR@.@PROJECT_VERSION_MINOR@.@PROJECT_VERSION_PATCH@" 5 | #define SOVERSION "${PROJECT_VERSION_MAJOR}" 6 | #define GIT_DIRTY "${GIT_DIRTY}" 7 | #define GIT_SHA1 "${GIT_SHA1}" 8 | #define BUILD_ID "${BUILD_ID}" 9 | #define BUILD_TYPE "${CMAKE_BUILD_TYPE}" 10 | #endif 11 | -------------------------------------------------------------------------------- /src/aidt.cpp: -------------------------------------------------------------------------------- 1 | /// 该程序自动生成，禁止修改 2 | #include "BootStrap.hpp" 3 | // {{{ global 4 | 5 | // 引导程序 6 | BootStrap* bootMain; 7 | 8 | // }}} 9 | // {{{ static void killSignal() 10 | 11 | static void killSignal(const int sig) { 12 | if (bootMain != nullptr) { 13 | bootMain->stop(sig); 14 | delete bootMain; 15 | } 16 | } 17 | 18 | // }}} 19 | // {{{ static void reloadConf() 20 | 21 | static void reloadConf(const int sig) { 22 | (void)sig; 23 | LOG_ERROR << "Start reload config..."; 24 | bootMain->reload(); 25 | } 26 | 27 | // }}} 28 | // {{{ static void resend() 29 | 30 | static void resend(const int sig) { 31 | (void)sig; 32 | LOG_ERROR << "Start resend message..."; 33 | bootMain->resend(); 34 | } 35 | 36 | // }}} 37 | // {{{ static void registerSignal() 38 | 39 | static void registerSignal() { 40 | /* 忽略Broken Pipe信号 */ 41 | signal(SIGPIPE, SIG_IGN); 42 | /* 处理kill信号 */ 43 | signal(SIGINT, killSignal); 44 | signal(SIGKILL, killSignal); 45 | signal(SIGQUIT, killSignal); 46 | signal(SIGTERM, killSignal); 47 | signal(SIGHUP, killSignal); 48 | signal(SIGSEGV, killSignal); 49 | signal(SIGUSR1, reloadConf); 50 | signal(SIGUSR2, resend); 51 | } 52 | 53 | // }}} 54 | // {{{ int main() 55 | 56 | int main(int argc, char **argv) { 57 | try { 58 | // 初始化 BootStrap 59 | bootMain = new BootStrap(); 60 | bootMain->init(argc, argv); 61 | // 注册信号处理 62 | registerSignal(); 63 | // 启动运行 64 | bootMain->run(); 65 | } catch (std::exception &e) { 66 | LOG_SYSFATAL << "Main exception: " << e.what(); 67 | } catch (...) { 68 | LOG_SYSFATAL << "System exception."; 69 | } 70 | 71 | return 0; 72 | } 73 | 74 | // }}} 75 | -------------------------------------------------------------------------------- /src/aidtCli.cpp: -------------------------------------------------------------------------------- 1 | /// 该程序自动生成，禁止修改 2 | #include "BootStrapCli.hpp" 3 | // {{{ global 4 | 5 | // 引导程序 6 | BootStrapCli* bootMain; 7 | 8 | // }}} 9 | // {{{ static void killSignal() 10 | 11 | static void killSignal(const int sig) { 12 | if (bootMain != nullptr) { 13 | bootMain->stop(sig); 14 | delete bootMain; 15 | } 16 | } 17 | 18 | // }}} 19 | // {{{ static void registerSignal() 20 | 21 | static void registerSignal() { 22 | /* 忽略Broken Pipe信号 */ 23 | signal(SIGPIPE, SIG_IGN); 24 | /* 处理kill信号 */ 25 | signal(SIGINT, killSignal); 26 | signal(SIGKILL, killSignal); 27 | signal(SIGQUIT, killSignal); 28 | signal(SIGTERM, killSignal); 29 | signal(SIGHUP, killSignal); 30 | signal(SIGSEGV, killSignal); 31 | } 32 | 33 | // }}} 34 | // {{{ int main() 35 | 36 | int main(int argc, char **argv) { 37 | try { 38 | // 初始化 BootStrap 39 | bootMain = new BootStrapCli(); 40 | bootMain->init(argc, argv); 41 | // 注册信号处理 42 | registerSignal(); 43 | // 启动运行 44 | bootMain->run(); 45 | } catch (std::exception &e) { 46 | LOG_SYSFATAL << "Main exception: " << e.what(); 47 | } catch (...) { 48 | LOG_SYSFATAL << "System exception."; 49 | } 50 | 51 | return 0; 52 | } 53 | 54 | // }}} 55 | -------------------------------------------------------------------------------- /src/thridpart/inotify/config.h: -------------------------------------------------------------------------------- 1 | /* config.h.in. Generated from configure.ac by autoheader. */ 2 | 3 | /* Define to 1 if you have the header file. */ 4 | #undef HAVE_INTTYPES_H 5 | 6 | /* Define to 1 if you have the header file. */ 7 | #undef HAVE_MCHECK_H 8 | 9 | /* Define to 1 if you have the header file. */ 10 | #undef HAVE_MEMORY_H 11 | 12 | /* Define to 1 if you have the header file. */ 13 | #undef HAVE_STDINT_H 14 | 15 | /* Define to 1 if you have the header file. */ 16 | #undef HAVE_STDLIB_H 17 | 18 | /* Define to 1 if you have the header file. */ 19 | #undef HAVE_STRINGS_H 20 | 21 | /* Define to 1 if you have the header file. */ 22 | #undef HAVE_STRING_H 23 | 24 | /* Define to 1 if you have the header file. */ 25 | #undef HAVE_SYS_INOTIFY_H 26 | 27 | /* Define to 1 if you have the header file. */ 28 | #undef HAVE_SYS_STAT_H 29 | 30 | /* Define to 1 if you have the header file. */ 31 | #undef HAVE_SYS_TYPES_H 32 | 33 | /* Define to 1 if you have the header file. */ 34 | #undef HAVE_UNISTD_H 35 | 36 | /* Name of package */ 37 | #undef PACKAGE 38 | 39 | /* Define to the address where bug reports for this package should be sent. */ 40 | #undef PACKAGE_BUGREPORT 41 | 42 | /* Define to the full name of this package. */ 43 | #undef PACKAGE_NAME 44 | 45 | /* Define to the full name and version of this package. */ 46 | #undef PACKAGE_STRING 47 | 48 | /* Define to the one symbol short name of this package. */ 49 | #undef PACKAGE_TARNAME 50 | 51 | /* Define to the version of this package. */ 52 | #undef PACKAGE_VERSION 53 | 54 | /* Define to 1 if you have the ANSI C header files. */ 55 | #undef STDC_HEADERS 56 | 57 | /* sys/inotify.h exists and works correctly */ 58 | #undef SYS_INOTIFY_H_EXISTS_AND_WORKS 59 | 60 | /* Version number of package */ 61 | #undef VERSION 62 | 63 | /* For a few GNU-specific functions */ 64 | #undef _GNU_SOURCE 65 | 66 | /* Define to empty if `const' does not conform to ANSI C. */ 67 | #undef const 68 | 69 | /* Define to `__inline__' or `__inline' if that's what the C compiler 70 | calls it, or to nothing if 'inline' is not supported under any name. */ 71 | #ifndef __cplusplus 72 | #undef inline 73 | #endif 74 | -------------------------------------------------------------------------------- /src/thridpart/inotify/inotify-nosys.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This header is used if cannot be found. 3 | * 4 | * Inode based directory notification for Linux 5 | * 6 | * Copyright (C) 2005 John McCutchan 7 | */ 8 | 9 | #ifndef _LINUX_INOTIFY_H 10 | #define _LINUX_INOTIFY_H 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | /* 17 | * struct inotify_event - structure read from the inotify device for each event 18 | * 19 | * When you are watching a directory, you will receive the filename for events 20 | * such as IN_CREATE, IN_DELETE, IN_OPEN, IN_CLOSE, ..., relative to the wd. 21 | */ 22 | struct inotify_event { 23 | int wd; /* watch descriptor */ 24 | uint32_t mask; /* watch mask */ 25 | uint32_t cookie; /* cookie to synchronize two events */ 26 | uint32_t len; /* length (including nulls) of name */ 27 | char name __flexarr; /* stub for possible name */ 28 | }; 29 | 30 | /* the following are legal, implemented events that user-space can watch for */ 31 | #define IN_ACCESS 0x00000001 /* File was accessed */ 32 | #define IN_MODIFY 0x00000002 /* File was modified */ 33 | #define IN_ATTRIB 0x00000004 /* Metadata changed */ 34 | #define IN_CLOSE_WRITE 0x00000008 /* Writtable file was closed */ 35 | #define IN_CLOSE_NOWRITE 0x00000010 /* Unwrittable file closed */ 36 | #define IN_OPEN 0x00000020 /* File was opened */ 37 | #define IN_MOVED_FROM 0x00000040 /* File was moved from X */ 38 | #define IN_MOVED_TO 0x00000080 /* File was moved to Y */ 39 | #define IN_CREATE 0x00000100 /* Subfile was created */ 40 | #define IN_DELETE 0x00000200 /* Subfile was deleted */ 41 | #define IN_DELETE_SELF 0x00000400 /* Self was deleted */ 42 | #define IN_MOVE_SELF 0x00000800 /* Self was moved */ 43 | 44 | /* the following are legal events. they are sent as needed to any watch */ 45 | #define IN_UNMOUNT 0x00002000 /* Backing fs was unmounted */ 46 | #define IN_Q_OVERFLOW 0x00004000 /* Event queued overflowed */ 47 | #define IN_IGNORED 0x00008000 /* File was ignored */ 48 | 49 | /* helper events */ 50 | #define IN_CLOSE (IN_CLOSE_WRITE | IN_CLOSE_NOWRITE) /* close */ 51 | #define IN_MOVE (IN_MOVED_FROM | IN_MOVED_TO) /* moves */ 52 | 53 | /* special flags */ 54 | #define IN_ONLYDIR 0x01000000 /* only watch the path if it is a directory */ 55 | #define IN_DONT_FOLLOW 0x02000000 /* don't follow a sym link */ 56 | #define IN_MASK_ADD 0x20000000 /* add to the mask of an already existing watch */ 57 | #define IN_ISDIR 0x40000000 /* event occurred against dir */ 58 | #define IN_ONESHOT 0x80000000 /* only send event once */ 59 | 60 | /* 61 | * All of the events - we build the list by hand so that we can add flags in 62 | * the future and not break backward compatibility. Apps will get only the 63 | * events that they originally wanted. Be sure to add new events here! 64 | */ 65 | #define IN_ALL_EVENTS (IN_ACCESS | IN_MODIFY | IN_ATTRIB | IN_CLOSE_WRITE | \ 66 | IN_CLOSE_NOWRITE | IN_OPEN | IN_MOVED_FROM | \ 67 | IN_MOVED_TO | IN_DELETE | IN_CREATE | IN_DELETE_SELF | \ 68 | IN_MOVE_SELF) 69 | 70 | #if defined (__alpha__) 71 | # define __NR_inotify_init 444 72 | # define __NR_inotify_add_watch 445 73 | # define __NR_inotify_rm_watch 446 74 | 75 | #elif defined (__arm__) 76 | # define __NR_inotify_init (__NR_SYSCALL_BASE+316) 77 | # define __NR_inotify_add_watch (__NR_SYSCALL_BASE+317) 78 | # define __NR_inotify_rm_watch (__NR_SYSCALL_BASE+318) 79 | 80 | #elif defined (__frv__) 81 | # define __NR_inotify_init 291 82 | # define __NR_inotify_add_watch 292 83 | # define __NR_inotify_rm_watch 293 84 | 85 | #elif defined(__i386__) 86 | # define __NR_inotify_init 291 87 | # define __NR_inotify_add_watch 292 88 | # define __NR_inotify_rm_watch 293 89 | 90 | #elif defined (__ia64__) 91 | # define __NR_inotify_init 1277 92 | # define __NR_inotify_add_watch 1278 93 | # define __NR_inotify_rm_watch 1279 94 | 95 | #elif defined (__mips__) 96 | # if _MIPS_SIM == _MIPS_SIM_ABI32 97 | # define __NR_inotify_init (__NR_Linux + 284) 98 | # define __NR_inotify_add_watch (__NR_Linux + 285) 99 | # define __NR_inotify_rm_watch (__NR_Linux + 286) 100 | # endif 101 | # if _MIPS_SIM == _MIPS_SIM_ABI64 102 | # define __NR_inotify_init (__NR_Linux + 243) 103 | # define __NR_inotify_add_watch (__NR_Linux + 243) 104 | # define __NR_inotify_rm_watch (__NR_Linux + 243) 105 | # endif 106 | # if _MIPS_SIM == _MIPS_SIM_NABI32 107 | # define __NR_inotify_init (__NR_Linux + 247) 108 | # define __NR_inotify_add_watch (__NR_Linux + 248) 109 | # define __NR_inotify_rm_watch (__NR_Linux + 249) 110 | # endif 111 | 112 | #elif defined(__parisc__) 113 | # define __NR_inotify_init (__NR_Linux + 269) 114 | # define __NR_inotify_add_watch (__NR_Linux + 270) 115 | # define __NR_inotify_rm_watch (__NR_Linux + 271) 116 | 117 | #elif defined(__powerpc__) || defined(__powerpc64__) 118 | # define __NR_inotify_init 275 119 | # define __NR_inotify_add_watch 276 120 | # define __NR_inotify_rm_watch 277 121 | 122 | #elif defined (__s390__) 123 | # define __NR_inotify_init 284 124 | # define __NR_inotify_add_watch 285 125 | # define __NR_inotify_rm_watch 286 126 | 127 | #elif defined (__sh__) 128 | # define __NR_inotify_init 290 129 | # define __NR_inotify_add_watch 291 130 | # define __NR_inotify_rm_watch 292 131 | 132 | #elif defined (__sh64__) 133 | # define __NR_inotify_init 318 134 | # define __NR_inotify_add_watch 319 135 | # define __NR_inotify_rm_watch 320 136 | 137 | #elif defined (__sparc__) || defined (__sparc64__) 138 | # define __NR_inotify_init 151 139 | # define __NR_inotify_add_watch 152 140 | # define __NR_inotify_rm_watch 156 141 | 142 | #elif defined(__x86_64__) 143 | # define __NR_inotify_init 253 144 | # define __NR_inotify_add_watch 254 145 | # define __NR_inotify_rm_watch 255 146 | 147 | #else 148 | # error "Unsupported architecture!" 149 | #endif 150 | 151 | static inline int inotify_init (void) 152 | { 153 | return syscall (__NR_inotify_init); 154 | } 155 | 156 | static inline int inotify_add_watch (int fd, const char *name, uint32_t mask) 157 | { 158 | return syscall (__NR_inotify_add_watch, fd, name, mask); 159 | } 160 | 161 | static inline int inotify_rm_watch (int fd, uint32_t wd) 162 | { 163 | return syscall (__NR_inotify_rm_watch, fd, wd); 164 | } 165 | 166 | 167 | #endif /* _LINUX_INOTIFY_H */ 168 | -------------------------------------------------------------------------------- /src/thridpart/inotify/inotify.h: -------------------------------------------------------------------------------- 1 | #ifndef _INOTIFYTOOLS_INOTIFY_H 2 | #define _INOTIFYTOOLS_INOTIFY_H 1 3 | 4 | /* this is defined to 1 if works. */ 5 | #undef SYS_INOTIFY_H_EXISTS_AND_WORKS 6 | 7 | 8 | #ifdef SYS_INOTIFY_H_EXISTS_AND_WORKS 9 | #include 10 | #else 11 | #include "inotify-nosys.h" 12 | #endif // SYS_INOTIFY_H_EXISTS_AND_WORKS 13 | 14 | 15 | #endif 16 | 17 | -------------------------------------------------------------------------------- /src/thridpart/inotify/inotifytools.h: -------------------------------------------------------------------------------- 1 | #ifndef _inotifytools_H 2 | #define _inotifytools_H 3 | 4 | #ifdef __cplusplus 5 | extern "C" 6 | { 7 | #endif 8 | 9 | #include 10 | 11 | int inotifytools_str_to_event(char const * event); 12 | int inotifytools_str_to_event_sep(char const * event, char sep); 13 | char * inotifytools_event_to_str(int events); 14 | char * inotifytools_event_to_str_sep(int events, char sep); 15 | void inotifytools_set_filename_by_wd( int wd, char const * filename ); 16 | void inotifytools_set_filename_by_filename( char const * oldname, 17 | char const * newname ); 18 | void inotifytools_replace_filename( char const * oldname, 19 | char const * newname ); 20 | char * inotifytools_filename_from_wd( int wd ); 21 | int inotifytools_wd_from_filename( char const * filename ); 22 | int inotifytools_remove_watch_by_filename( char const * filename ); 23 | int inotifytools_remove_watch_by_wd( int wd ); 24 | int inotifytools_watch_file( char const * filename, int events ); 25 | int inotifytools_watch_files( char const * filenames[], int events ); 26 | int inotifytools_watch_recursively( char const * path, int events ); 27 | int inotifytools_watch_recursively_with_exclude( char const * path, 28 | int events, 29 | char const ** exclude_list ); 30 | // [UH] 31 | int inotifytools_ignore_events_by_regex( char const *pattern, int flags ); 32 | int inotifytools_ignore_events_by_inverted_regex( char const *pattern, int flags ); 33 | struct inotify_event * inotifytools_next_event( long int timeout ); 34 | struct inotify_event * inotifytools_next_events( long int timeout, int num_events ); 35 | int inotifytools_error(); 36 | int inotifytools_get_stat_by_wd( int wd, int event ); 37 | int inotifytools_get_stat_total( int event ); 38 | int inotifytools_get_stat_by_filename( char const * filename, 39 | int event ); 40 | void inotifytools_initialize_stats(); 41 | int inotifytools_initialize(); 42 | void inotifytools_cleanup(); 43 | int inotifytools_get_num_watches(); 44 | 45 | int inotifytools_printf( struct inotify_event* event, char* fmt ); 46 | int inotifytools_fprintf( FILE* file, struct inotify_event* event, char* fmt ); 47 | int inotifytools_sprintf( char * out, struct inotify_event* event, char* fmt ); 48 | int inotifytools_snprintf( char * out, int size, struct inotify_event* event, 49 | char* fmt ); 50 | void inotifytools_set_printf_timefmt( char * fmt ); 51 | 52 | int inotifytools_get_max_user_watches(); 53 | int inotifytools_get_max_user_instances(); 54 | int inotifytools_get_max_queued_events(); 55 | 56 | #ifdef __cplusplus 57 | } 58 | #endif 59 | 60 | #endif // _inotifytools_H 61 | -------------------------------------------------------------------------------- /src/thridpart/inotify/inotifytools_p.h: -------------------------------------------------------------------------------- 1 | #ifndef INOTIFYTOOLS_P_H 2 | #define INOTIFYTOOLS_P_H 3 | 4 | #include "redblack.h" 5 | 6 | struct rbtree *inotifytools_wd_sorted_by_event(int sort_event); 7 | 8 | typedef struct watch { 9 | char *filename; 10 | int wd; 11 | unsigned hit_access; 12 | unsigned hit_modify; 13 | unsigned hit_attrib; 14 | unsigned hit_close_write; 15 | unsigned hit_close_nowrite; 16 | unsigned hit_open; 17 | unsigned hit_moved_from; 18 | unsigned hit_moved_to; 19 | unsigned hit_create; 20 | unsigned hit_delete; 21 | unsigned hit_delete_self; 22 | unsigned hit_unmount; 23 | unsigned hit_move_self; 24 | unsigned hit_total; 25 | } watch; 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /src/thridpart/inotify/redblack.h: -------------------------------------------------------------------------------- 1 | /* 2 | * RCS $Id: redblack.h,v 1.9 2003/10/24 01:31:21 damo Exp $ 3 | */ 4 | 5 | /* 6 | Redblack balanced tree algorithm 7 | Copyright (C) Damian Ivereigh 2000 8 | 9 | This program is free software; you can redistribute it and/or modify 10 | it under the terms of the GNU Lesser General Public License as published by 11 | the Free Software Foundation; either version 2.1 of the License, or 12 | (at your option) any later version. See the file COPYING for details. 13 | 14 | This program is distributed in the hope that it will be useful, 15 | but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | GNU General Public License for more details. 18 | 19 | You should have received a copy of the GNU Lesser General Public License 20 | along with this program; if not, write to the Free Software 21 | Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 22 | */ 23 | 24 | /* Header file for redblack.c, should be included by any code that 25 | ** uses redblack.c since it defines the functions 26 | */ 27 | 28 | /* Stop multiple includes */ 29 | #ifndef _REDBLACK_H 30 | 31 | #ifndef RB_CUSTOMIZE 32 | /* 33 | * Without customization, the data member in the tree nodes is a void 34 | * pointer, and you need to pass in a comparison function to be 35 | * applied at runtime. With customization, you specify the data type 36 | * as the macro RB_ENTRY(data_t) (has to be a macro because compilers 37 | * gag on typdef void) and the name of the compare function as the 38 | * value of the macro RB_CMP. Because the comparison function is 39 | * compiled in, RB_CMP only needs to take two arguments. If your 40 | * content type is not a pointer, define INLINE to get direct access. 41 | */ 42 | #define rbdata_t void 43 | #define RB_CMP(s, t, e) (*rbinfo->rb_cmp)(s, t, e) 44 | #undef RB_INLINE 45 | #define RB_ENTRY(name) rb##name 46 | #endif /* RB_CUSTOMIZE */ 47 | 48 | #ifndef RB_STATIC 49 | #define RB_STATIC 50 | #endif 51 | 52 | /* Modes for rblookup */ 53 | #define RB_NONE -1 /* None of those below */ 54 | #define RB_LUEQUAL 0 /* Only exact match */ 55 | #define RB_LUGTEQ 1 /* Exact match or greater */ 56 | #define RB_LULTEQ 2 /* Exact match or less */ 57 | #define RB_LULESS 3 /* Less than key (not equal to) */ 58 | #define RB_LUGREAT 4 /* Greater than key (not equal to) */ 59 | #define RB_LUNEXT 5 /* Next key after current */ 60 | #define RB_LUPREV 6 /* Prev key before current */ 61 | #define RB_LUFIRST 7 /* First key in index */ 62 | #define RB_LULAST 8 /* Last key in index */ 63 | 64 | /* For rbwalk - pinched from search.h */ 65 | typedef enum 66 | { 67 | preorder, 68 | postorder, 69 | endorder, 70 | leaf 71 | } 72 | VISIT; 73 | 74 | struct RB_ENTRY(lists) { 75 | const struct RB_ENTRY(node) *rootp; 76 | const struct RB_ENTRY(node) *nextp; 77 | }; 78 | 79 | #define RBLIST struct RB_ENTRY(lists) 80 | 81 | struct RB_ENTRY(tree) { 82 | #ifndef RB_CUSTOMIZE 83 | /* comparison routine */ 84 | int (*rb_cmp)(const void *, const void *, const void *); 85 | /* config data to be passed to rb_cmp */ 86 | const void *rb_config; 87 | /* root of tree */ 88 | #endif /* RB_CUSTOMIZE */ 89 | struct RB_ENTRY(node) *rb_root; 90 | }; 91 | 92 | #ifndef RB_CUSTOMIZE 93 | RB_STATIC struct RB_ENTRY(tree) *rbinit(int (*)(const void *, const void *, const void *), 94 | const void *); 95 | #else 96 | RB_STATIC struct RB_ENTRY(tree) *RB_ENTRY(init)(void); 97 | #endif /* RB_CUSTOMIZE */ 98 | 99 | #ifndef no_delete 100 | RB_STATIC const RB_ENTRY(data_t) *RB_ENTRY(delete)(const RB_ENTRY(data_t) *, struct RB_ENTRY(tree) *); 101 | #endif 102 | 103 | #ifndef no_find 104 | RB_STATIC const RB_ENTRY(data_t) *RB_ENTRY(find)(const RB_ENTRY(data_t) *, struct RB_ENTRY(tree) *); 105 | #endif 106 | 107 | #ifndef no_lookup 108 | RB_STATIC const RB_ENTRY(data_t) *RB_ENTRY(lookup)(int, const RB_ENTRY(data_t) *, struct RB_ENTRY(tree) *); 109 | #endif 110 | 111 | #ifndef no_search 112 | RB_STATIC const RB_ENTRY(data_t) *RB_ENTRY(search)(const RB_ENTRY(data_t) *, struct RB_ENTRY(tree) *); 113 | #endif 114 | 115 | #ifndef no_destroy 116 | RB_STATIC void RB_ENTRY(destroy)(struct RB_ENTRY(tree) *); 117 | #endif 118 | 119 | #ifndef no_walk 120 | RB_STATIC void RB_ENTRY(walk)(const struct RB_ENTRY(tree) *, 121 | void (*)(const RB_ENTRY(data_t) *, const VISIT, const int, void *), 122 | void *); 123 | #endif 124 | 125 | #ifndef no_readlist 126 | RB_STATIC RBLIST *RB_ENTRY(openlist)(const struct RB_ENTRY(tree) *); 127 | RB_STATIC const RB_ENTRY(data_t) *RB_ENTRY(readlist)(RBLIST *); 128 | RB_STATIC void RB_ENTRY(closelist)(RBLIST *); 129 | #endif 130 | 131 | /* Some useful macros */ 132 | #define rbmin(rbinfo) RB_ENTRY(lookup)(RB_LUFIRST, NULL, (rbinfo)) 133 | #define rbmax(rbinfo) RB_ENTRY(lookup)(RB_LULAST, NULL, (rbinfo)) 134 | 135 | #define _REDBLACK_H 136 | #endif /* _REDBLACK_H */ 137 | 138 | /* 139 | * 140 | * $Log: redblack.h,v $ 141 | * Revision 1.9 2003/10/24 01:31:21 damo 142 | * Patches from Eric Raymond: %prefix is implemented.  Various other small 143 | * changes avoid stepping on global namespaces and improve the documentation. 144 | * 145 | * Revision 1.8 2003/10/23 04:18:47 damo 146 | * Fixed up the rbgen stuff ready for the 1.3 release 147 | * 148 | * Revision 1.7 2002/08/26 03:11:40 damo 149 | * Fixed up a bunch of compiler warnings when compiling example4 150 | * 151 | * Tidies up the Makefile.am & Specfile. 152 | * 153 | * Renamed redblack to rbgen 154 | * 155 | * Revision 1.6 2002/08/26 01:03:35 damo 156 | * Patch from Eric Raymond to change the way the library is used:- 157 | * 158 | * Eric's idea is to convert libredblack into a piece of in-line code 159 | * generated by another program. This should be faster, smaller and easier 160 | * to use. 161 | * 162 | * This is the first check-in of his code before I start futzing with it! 163 | * 164 | * Revision 1.5 2002/01/30 07:54:53 damo 165 | * Fixed up the libtool versioning stuff (finally) 166 | * Fixed bug 500600 (not detecting a NULL return from malloc) 167 | * Fixed bug 509485 (no longer needs search.h) 168 | * Cleaned up debugging section 169 | * Allow multiple inclusions of redblack.h 170 | * Thanks to Matthias Andree for reporting (and fixing) these 171 | * 172 | * Revision 1.4 2000/06/06 14:43:43 damo 173 | * Added all the rbwalk & rbopenlist stuff. Fixed up malloc instead of sbrk. 174 | * Added two new examples 175 | * 176 | * Revision 1.3 2000/05/24 06:45:27 damo 177 | * Converted everything over to using const 178 | * Added a new example1.c file to demonstrate the worst case scenario 179 | * Minor fixups of the spec file 180 | * 181 | * Revision 1.2 2000/05/24 06:17:10 damo 182 | * Fixed up the License (now the LGPL) 183 | * 184 | * Revision 1.1 2000/05/24 04:15:53 damo 185 | * Initial import of files. Versions are now all over the place. Oh well 186 | * 187 | */ 188 | 189 | --------------------------------------------------------------------------------