├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── Modules ├── Findlibadbase.cmake ├── Findlibadbase_kafka.cmake ├── Findlibadbase_lua.cmake ├── Findlibevent.cmake ├── Findliblua.cmake ├── Findlibpthread.cmake └── Findlibrdkafka.cmake ├── README.md ├── README_CN.md ├── adbase.ini ├── cmake.sh ├── conf └── system.ini ├── docs └── images │ ├── aidp_struct.png │ ├── aidp_struct_pro.png │ └── empty ├── dusty.lua ├── example ├── dusty.lua └── http.lua └── src ├── AdServer.cpp ├── AdServer.hpp ├── AdbaseConfig.hpp ├── Aims.cpp ├── Aims.hpp ├── Aims └── Kafka │ ├── ConsumerOut.cpp │ └── ConsumerOut.hpp ├── App.cpp ├── App.hpp ├── App ├── Message.cpp ├── Message.hpp ├── Metrics.cpp ├── Metrics.hpp ├── Storage.cpp └── Storage.hpp ├── BootStrap.cpp ├── BootStrap.hpp ├── CMakeLists.txt ├── HeadProcessor.cpp ├── HeadProcessor.hpp ├── Http.cpp ├── Http.hpp ├── Http ├── HttpInterface.cpp ├── HttpInterface.hpp ├── Index.cpp ├── Index.hpp ├── Server.cpp └── Server.hpp ├── McProcessor.cpp ├── McProcessor.hpp ├── Timer.cpp ├── Timer.hpp ├── Version.hpp ├── Version.hpp.in └── aidp.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | build/* 2 | adbase/Version.hpp 3 | doxygen.tag 4 | _build/* 5 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | PROJECT(aidp) 3 | 4 | SET (CMAKE_VERBOSE_MAKEFILE on) 5 | SET (PROJECT_VERSION_MAJOR 1) 6 | SET (PROJECT_VERSION_MINOR 0) 7 | SET (PROJECT_VERSION_PATCH 1) 8 | 9 | SET(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules) 10 | SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wl,-Bstatic -static-libstdc++ -static-libgcc") 11 | SET(LINK_FLAGS "${LINK_FLAGS}") 12 | 13 | ADD_SUBDIRECTORY(src bin) 14 | 15 | IF (NOT DEFINED CMAKE_BINARY_DIR) 16 | SET(CMAKE_BINARY_DIR ${PROJECT_SOURCE_DIR}/_build) 17 | ENDIF (NOT DEFINED CMAKE_BINARY_DIR) 18 | 19 | if (NOT DEFINED CMAKE_INSTALL_PREFIX) 20 | set(CMAKE_INSTALL_PREFIX "/usr/local/aidp") 21 | endif (NOT DEFINED CMAKE_INSTALL_PREFIX) 22 | 23 | INCLUDE (InstallRequiredSystemLibraries) 24 | SET (CPACK_RESOURCE_FILE_LICENSE 25 | "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE") 26 | IF(NOT DEFINED CPACK_SYSTEM_NAME) 27 | SET(CPACK_SYSTEM_NAME "${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}") 28 | ENDIF(NOT DEFINED CPACK_SYSTEM_NAME) 29 | 30 | SET (CPACK_PACKAGE_VERSION_MAJOR "${PROJECT_VERSION_MAJOR}") 31 | SET (CPACK_PACKAGE_VERSION_MINOR "${PROJECT_VERSION_MINOR}") 32 | SET (CPACK_PACKAGE_VERSION_PATCH "${PROJECT_VERSION_PATCH}") 33 | SET(CPACK_SOURCE_GENERATOR "TGZ;TBZ2;TZ") 34 | INCLUDE (CPack) 35 | 36 | INSTALL(DIRECTORY conf DESTINATION . PATTERN ".git" EXCLUDE) 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Modules/Findlibadbase.cmake: -------------------------------------------------------------------------------- 1 | MESSAGE(STATUS "Using bundled Findlibadbase.cmake...") 2 | FIND_PATH( 3 | LIBADBASE_INCLUDE_DIR 4 | Version.hpp 5 | /usr/local/include/adbase 6 | /usr/include/ 7 | ) 8 | 9 | FIND_LIBRARY( 10 | LIBADBASE_LIBRARIES NAMES libadbase.a 11 | PATHS /usr/lib/ /usr/local/lib/ /usr/lib64 /usr/local/lib64 12 | ) 13 | -------------------------------------------------------------------------------- /Modules/Findlibadbase_kafka.cmake: -------------------------------------------------------------------------------- 1 | MESSAGE(STATUS "Using bundled Findlibadbase_kafka.cmake...") 2 | FIND_PATH( 3 | LIBADBASE_KAFKA_INCLUDE_DIR 4 | Kafka.hpp 5 | /usr/local/include/adbase 6 | /usr/include/ 7 | ) 8 | 9 | FIND_LIBRARY( 10 | LIBADBASE_KAFKA_LIBRARIES NAMES libadbase_kafka.a 11 | PATHS /usr/lib/ /usr/local/lib/ /usr/lib64 /usr/local/lib64 12 | ) 13 | -------------------------------------------------------------------------------- /Modules/Findlibadbase_lua.cmake: -------------------------------------------------------------------------------- 1 | MESSAGE(STATUS "Using bundled Findlibadbase_lua.cmake...") 2 | FIND_PATH( 3 | LIBADBASE_LUA_INCLUDE_DIR 4 | Lua.hpp 5 | /usr/local/include/adbase 6 | /usr/include/ 7 | ) 8 | 9 | FIND_LIBRARY( 10 | LIBADBASE_LUA_LIBRARIES NAMES libadbase_lua.a 11 | PATHS /usr/lib/ /usr/local/lib/ /usr/lib64 /usr/local/lib64 12 | ) 13 | -------------------------------------------------------------------------------- /Modules/Findlibevent.cmake: -------------------------------------------------------------------------------- 1 | MESSAGE(STATUS "Using bundled Findlibevent.cmake...") 2 | FIND_PATH( 3 | LIBEVENT_INCLUDE_DIR 4 | event.h 5 | /usr/include/ 6 | /usr/local/include/event2 7 | ) 8 | 9 | FIND_LIBRARY( 10 | LIBEVENT_LIBRARIES NAMES event 11 | PATHS /usr/lib/ /usr/local/lib/ /usr/lib64 /usr/local/lib64 12 | ) 13 | FIND_LIBRARY( 14 | LIBEVENT_STATIC_LIBRARIES NAMES libevent.a 15 | PATHS /usr/lib/ /usr/local/lib/ /usr/lib64 /usr/local/lib64 16 | ) 17 | -------------------------------------------------------------------------------- /Modules/Findliblua.cmake: -------------------------------------------------------------------------------- 1 | MESSAGE(STATUS "Using bundled Findliblua.cmake...") 2 | FIND_PATH( 3 | LIBLUA_INCLUDE_DIR 4 | lua.h 5 | /usr/include/ 6 | /usr/local/include/ 7 | ) 8 | 9 | FIND_LIBRARY( 10 | LIBLUA_LIBRARIES NAMES lua 11 | PATHS /usr/lib/ /usr/local/lib/ /usr/lib64 /usr/local/lib64 12 | ) 13 | FIND_LIBRARY( 14 | LIBLUA_STATIC_LIBRARIES NAMES liblua.a 15 | PATHS /usr/lib/ /usr/local/lib/ /usr/lib64 /usr/local/lib64 16 | ) 17 | -------------------------------------------------------------------------------- /Modules/Findlibpthread.cmake: -------------------------------------------------------------------------------- 1 | MESSAGE(STATUS "Using Findlibpthread.cmake...") 2 | FIND_PATH( 3 | LIBPTHREAD_INCLUDE_DIR 4 | pthread.h 5 | /usr/include/ 6 | /usr/local/include/ 7 | ) 8 | 9 | FIND_LIBRARY( 10 | LIBPTHREAD_LIBRARIES NAMES pthread 11 | PATHS /usr/lib/ /usr/local/lib/ /usr/lib64 /usr/local/lib64 12 | ) 13 | -------------------------------------------------------------------------------- /Modules/Findlibrdkafka.cmake: -------------------------------------------------------------------------------- 1 | MESSAGE(STATUS "Using bundled Findlibrdkafka.cmake...") 2 | FIND_PATH( 3 | LIBRDKAFKA_INCLUDE_DIR 4 | librdkafka/rdkafka.h 5 | /usr/include/ 6 | /usr/local/include/ 7 | ) 8 | 9 | FIND_LIBRARY( 10 | LIBRDKAFKA_LIBRARIES NAMES librdkafka.a 11 | PATHS /usr/lib/ /usr/local/lib/ 12 | ) 13 | FIND_LIBRARY( 14 | LIBRDKAFKA_CPP_LIBRARIES NAMES librdkafka++.a 15 | PATHS /usr/lib/ /usr/local/lib/ 16 | ) 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AIDP (Data Process) 2 | 3 | [中文文档](README_CN.md) 4 | 5 | AIDP is responsible for a series of messages such as message queue consumption. After getting the message, the specific processing is handled by Lua script. The developer can customize the processing of the consumption queue, and the framework provides http server service. You can use the Lua script to customize the http API Interface, and then consumption message logic and http server between the framework provides a simple Key-Val storage structure for data sharing. The figure is the structure of the AIDP data processing framework: 6 | 7 | ![structure](docs/images/aidp_struct.png) 8 | 9 | AIDP invoke flow： 10 | 11 | ![data process flow](docs/images/aidp_struct_pro.png) 12 | 13 | ### Why use lua script customize ? 14 | 15 | Most of the application scenarios for the message queue are back-end with a consumer module for data processing, the final data stored in a DB, and finally through the RPC interface for the user use. Some application scenarios data process logic may be particularly simple, but most of the language is very complicated for Kafka's consumption (Kafka consumption logic) When the specific data process logic to increase the development costs, AIDP through C++ this general development model abstract package, in order to increase the custom through the embedded Lua scripting language. To complete the custom part of the logic. AIDP is a compromise between development difficulty, cost and flexibility. Can meet a large number of data processing demand 16 | 17 | ### Quick Start 18 | 19 | #### RPM install 20 | 21 | RPM currently support centos7.x version, other versions will be uploaded, there are other system version of the demand can add the issue 22 | 23 | #### Complie install 24 | 25 | First, install AIDP depend package adbase, Installation method refer : https://nmred.gitbooks.io/adbase/content/en/basic/install.html 26 | 27 | Second, complie and install aidp 28 | 29 | ``` 30 | git clone git@github.com:weiboad/aidp.git 31 | cd aidp 32 | ./cmake.sh 33 | cd build 34 | make 35 | make install 36 | ``` 37 | 38 | #### Example 39 | 40 | For example, using aidp get kafka cluster topic `test` data write to the local disk, and statistics the number of messages, the final number of messages through the http api interface can be obtained. In the installed aidp directory modify the configuration `conf/system.ini`, configure kafka consumption calls lua script path, by modifying `consumer` section `scriptName` field. Set up on the lua script can be configured to handle the script as follows: 41 | 42 | ```lua 43 | -- 44 | -- 45 | -- This sctipt process message write to the local disk and statistics the number of messages store in local storage 46 | -- 47 | -- 48 | local obj = aidp.message.get() 49 | local filename = os.date('%Y-%m-%d-%H') .. '.log' 50 | local files = {}; 51 | -- get storage object 52 | local storage = aidp.storage() 53 | for k,v in pairs(obj) do 54 | if #v == 3 then 55 | if files[v[2]] == nil then 56 | files[v[2]] = io.open(v[2]..filename, 'a+'); 57 | files[v[2]]:setvbuf('full') 58 | end 59 | files[v[2]]:write(v[3]..'\n') 60 | -- store number of message in local storage 61 | storage:incr(v[2] .. ':message_size', 1) 62 | end 63 | end 64 | for k,v in pairs(files) do 65 | v:flush() 66 | v:close() 67 | end 68 | ``` 69 | 70 | Modify the configuration file `http` section `scriptName` field set http interface call lua script path, lua script processing logic as follows: 71 | 72 | ```lua 73 | -- get a topic number of has process message 74 | local request = aidp.http.request(); 75 | local topic_name = request:get_query('t') 76 | local response = aidp.http.response(); 77 | local storage = aidp.storage() 78 | response:set_content(topic_name .. ':message_size ' .. storage:get(topic_name .. ':message_size')); 79 | ``` 80 | 81 | Start aidp service: 82 | 83 | ``` 84 | cd /usr/local/adinf/aidp/bin 85 | ./aidp -c ../conf/system.ini 86 | ``` 87 | 88 | ### Message Consumer 89 | 90 | The message consumption module is responsible for getting the message from Kafka and invoking the Lua consumption script after getting the message. The Lua script can invoke the `aidp.message.get()` function to get the message 91 | 92 | #### aidp.message.get 93 | 94 | Get kafka message 95 | 96 | Params：None 97 | 98 | Scope：Only the message consumption script valid 99 | 100 | Return：Data structure is follow: 101 | 102 | ```lua 103 | -- { 104 | -- { 105 | -- id, 106 | -- topicName, 107 | -- messageBody 108 | -- }, 109 | -- { 110 | -- id, 111 | -- topicName, 112 | -- messageBody 113 | -- }, 114 | -- { 115 | -- id, 116 | -- topicName, 117 | -- messageBody 118 | -- } 119 | -- } 120 | ``` 121 | 122 | ```lua 123 | --This sctipt process message write to the local disk 124 | -- 125 | -- get message 126 | local obj = aidp.message.get() 127 | 128 | local filename = os.date('%Y-%m-%d-%H') .. '.log' 129 | local files = {}; 130 | local storage = aidp.storage() 131 | for k,v in pairs(obj) do 132 | if #v == 3 then 133 | if files[v[2]] == nil then 134 | files[v[2]] = io.open(v[2]..filename, 'a+'); 135 | files[v[2]]:setvbuf('full') 136 | end 137 | files[v[2]]:write(v[3]..'\n') 138 | storage:incr(v[2] .. ':message_size', 1) 139 | end 140 | end 141 | for k,v in pairs(files) do 142 | v:flush() 143 | v:close() 144 | end 145 | 146 | ``` 147 | 148 | ### Http Server 149 | 150 | Provide HTTP interface Lua custom development, for the Lua side to provide Request, Response object, through the Request to obtain the request data, and through the Lua script. The final response data will be customized through the Response interface, Request and Response object methods only in the Http Server processing script is valid for the Message consumer processing script is invalid 151 | 152 | #### Request 153 | 154 | | Method | Method Type | Description | 155 | |-----|-----|-----| 156 | | aidp.http.request() | Static | Construct request object | 157 | | request:get_uri() | Object Method | Get request URI | 158 | | request:get_remote_address() | Object Method | Get remote address | 159 | | request:get_post_data() | Object Method | Get origin post data | 160 | | request:get_post([key]) | Object Method | Get POST Form-Data type data | 161 | | request:get_query(key) | Object Method | Get GET method data | 162 | | request:get_header(key) | Object Method | Get Header val | 163 | | request:get_location() | Object Method | Get location | 164 | | request:get_method() | Object Method | Get request method, will return enum type METHOD_GET、METHOD_POST、METHOD_OTHER | 165 | 166 | #### Response 167 | 168 | | Method | Method Type | Description | 169 | |-----|-----|-----| 170 | | aidp.http.response() | Static | Construct response object | 171 | | response:set_header(key, val) | Object Method | Set response header, if already exists will cover header | 172 | | response:add_header(key, val) | Object Method | Add response header | 173 | | response:set_content(body) | Object Method | Set response body | 174 | | response:append_content(body) | Object Method | Append response body | 175 | | response:get_code() | Object Method | Get response http code | 176 | | response:set_body_size(size) | Object Method | Set response body size | 177 | 178 | #### Sample 179 | 180 | ```lua 181 | local request = aidp.http.request(); 182 | local topic_name = request:get_query('t') 183 | local response = aidp.http.response(); 184 | local storage = aidp.storage() 185 | response:set_content(topic_name .. ':message_size ' .. storage:get(topic_name .. ':message_size')); 186 | ``` 187 | 188 | ### Simple Storage 189 | 190 | Storage provides a simple stand-alone storage similar to Redis's key-val structure, helping to implement some data cache storage logic. 191 | 192 | The following Lua Api is common in the Http lua script and the Message Consumer valid. 193 | 194 | | Method | Method Type | Description | 195 | |-----|-----|-----| 196 | | aidp.storage() | Static | Construct storage object | 197 | | storage:set(key, val, [ttl]) | Object Method | Set the data in the storage, if set ttl is expired after the key will be recovered, ttl units is second | 198 | | storage:incr(key, step) | Object Method | Increment val step | 199 | | storage:decr(key, step) | Object Method | Decrement val step | 200 | | storage:get(key) | Object Method | Get value | 201 | | storage:exists(key) | Object Method | Get whether a key exists | 202 | | storage:del(key) | Object Method | Delete a key | 203 | -------------------------------------------------------------------------------- /README_CN.md: -------------------------------------------------------------------------------- 1 | # AIDP (Data Process) 2 | 3 | AIDP 负责消息队列消费等一系列逻辑，在获取到消息后具体的处理交由 Lua 脚本处理，开发者可以定制消费队列的处理，并且本框架提供了 Http Server 服务，可以使用 Lua 脚本定制 Http API 接口，再消费消息逻辑和 Http Server 间本框架提供了简单的 Key-Val 存储结构做数据共享。如图是该数据处理框架的结构： 4 | 5 | ![结构图](docs/images/aidp_struct.png) 6 | 7 | AIDP 处理调用流程： 8 | 9 | ![数据处理流程](docs/images/aidp_struct_pro.png) 10 | 11 | ### Why? 12 | 13 | 对于消息队列的大部分应用场景是后端有一个消费模块进行数据处理，将最终的数据存储到一个地方，最终通过 RPC 接口供使用方使用。有的应用场景业务逻辑可能特别的简单，但是大部分语言对于 Kafka 的消费实现都很复杂（Kafka 消费逻辑决定）使得在开发具体业务逻辑的时候增加开发成本，AIDP 通过 C++ 将这一通用开发模型进行抽象封装，为了增加定制性通过嵌入 Lua 脚本语言来完成定制化部分的逻辑。AIDP 是在开发难度、成本与灵活性之间权衡的折中方案。可以满足大量的数据处理需求 14 | 15 | ### Quick Start 16 | 17 | #### RPM 安装 18 | 19 | RPM 目前支持 centos7.x 版本，其他版本陆续会上传, 有其他系统版本的需求可以提 issue 20 | 21 | #### 编译安装 22 | 23 | 1. 安装 adbase, 参见：https://nmred.gitbooks.io/adbase/content/zh/basic/install.html 24 | 25 | 2. 安装 aidp 26 | 27 | ``` 28 | - git clone git@github.com:weiboad/aidp.git 29 | - cd aidp 30 | - ./cmake.sh 31 | - cd build 32 | - make 33 | - make install 34 | ``` 35 | 36 | #### Example 37 | 38 | 例如使用 aidp 将 kafka 集群中 topic test 落地到本地，并且统计消息的个数，最终消息个数通过 http api 接口可以获取 39 | 40 | 在安装好的 aidp 中修改配置 `conf/system.ini` ，配置 kafka 消费调用的 lua 脚本，通过修改 [consumer] -> scriptName 配置配置上 lua 脚本即可, 处理脚本如下： 41 | 42 | ```lua 43 | -- 44 | -- 45 | -- 该脚本实现将 kafka 中的数据落地到文件中，并且统计落地消息数 46 | -- 47 | -- 48 | local obj = aidp.message.get() 49 | local filename = os.date('%Y-%m-%d-%H') .. '.log' 50 | local files = {}; 51 | -- 实例化存储 52 | local storage = aidp.storage() 53 | for k,v in pairs(obj) do 54 | if #v == 3 then 55 | if files[v[2]] == nil then 56 | files[v[2]] = io.open(v[2]..filename, 'a+'); 57 | files[v[2]]:setvbuf('full') 58 | end 59 | files[v[2]]:write(v[3]..'\n') 60 | -- 存储计数 61 | storage:incr(v[2] .. ':message_size', 1) 62 | end 63 | end 64 | for k,v in pairs(files) do 65 | v:flush() 66 | v:close() 67 | end 68 | ``` 69 | 70 | 修改配置文件 [http]->scriptName 设置 http 接口调用的 lua 脚本，lua脚本处理内容如下： 71 | 72 | ```lua 73 | -- 获取某个 topic 当前消费的数量 74 | local request = aidp.http.request(); 75 | local topic_name = request:get_query('t') 76 | local response = aidp.http.response(); 77 | local storage = aidp.storage() 78 | response:set_content(topic_name .. ':message_size ' .. storage:get(topic_name .. ':message_size')); 79 | ``` 80 | 81 | 启动 aidp 82 | 83 | ``` 84 | cd /usr/local/adinf/aidp/bin 85 | ./aidp -c ../conf/system.ini 86 | ``` 87 | 88 | ### Message Consumer 89 | 90 | 消息消费模块负责从 Kafka 中获取消息，在获取到消息后会调用 Lua 消费脚本，Lua 脚本中可以调用 `aidp.message.get()` 这个函数获取消息，做相应的处理, 例如： 91 | 92 | #### aidp.message.get 93 | 94 | Lua 脚本获取 kafka 消息的接口 95 | 96 | 参数：无 97 | 98 | 作用范围：仅 Message Consumer 脚本有效 99 | 100 | 返回： 101 | 102 | ```lua 103 | -- 返回多条消息： 104 | -- { 105 | -- { 106 | -- id, 107 | -- topicName, 108 | -- messageBody 109 | -- }, 110 | -- { 111 | -- id, 112 | -- topicName, 113 | -- messageBody 114 | -- }, 115 | -- { 116 | -- id, 117 | -- topicName, 118 | -- messageBody 119 | -- } 120 | -- } 121 | ``` 122 | 123 | ```lua 124 | -- 将 Kafka 消息写入到本地文件中 125 | 126 | -- 获取消息 127 | local obj = aidp.message.get() 128 | 129 | local filename = os.date('%Y-%m-%d-%H') .. '.log' 130 | local files = {}; 131 | local storage = aidp.storage() 132 | for k,v in pairs(obj) do 133 | if #v == 3 then 134 | if files[v[2]] == nil then 135 | files[v[2]] = io.open(v[2]..filename, 'a+'); 136 | files[v[2]]:setvbuf('full') 137 | end 138 | files[v[2]]:write(v[3]..'\n') 139 | storage:incr(v[2] .. ':message_size', 1) 140 | end 141 | end 142 | for k,v in pairs(files) do 143 | v:flush() 144 | v:close() 145 | end 146 | 147 | ``` 148 | 149 | ### Http Server 150 | 151 | 提供 HTTP 接口 Lua 定制开发，对于 Lua 端提供 Request、Response 对象，通过 Request 获取请求数据，并且通过 Lua 脚本处理将最终的响应数据通过 Response 接口定制，Request 和 Response 相关的接口仅仅在 Http Server 处理脚本上有效，对于 Message consumer 处理脚本无效 152 | 153 | #### Request 154 | 155 | | 方法 | 类型 | 描述 | 156 | |-----|-----|-----| 157 | | aidp.http.request() | 静态构造函数 | 构造 request 对象 | 158 | | request:get_uri() | 对象方法 | 获取请求 URI | 159 | | request:get_remote_address() | 对象方法 | 获取请求远程的 IP 地址 | 160 | | request:get_post_data() | 对象方法 | 获取 POST 请求的原始数据 | 161 | | request:get_post([key]) | 对象方法 | 获取 POST Form-Data 类型的数据 | 162 | | request:get_query(key) | 对象方法 | 获取 GET 数据 | 163 | | request:get_header(key) | 对象方法 | 获取 Header 数据 | 164 | | request:get_location() | 对象方法 | 获取 Location 数据 | 165 | | request:get_method() | 对象方法 | 获取请求类型, 返回枚举 METHOD_GET、METHOD_POST、METHOD_OTHER | 166 | 167 | #### Response 168 | 169 | | 方法 | 类型 | 描述 | 170 | |-----|-----|-----| 171 | | aidp.http.response() | 静态构造函数 | 构造 response 对象 | 172 | | response:set_header(key, val) | 对象方法 | 设置响应 header, 如果存在则覆盖原有的 header | 173 | | response:(key, val) | 对象方法 | 设置响应 header, 如果存在则覆盖原有的 header | 174 | | response:add_header(key, val) | 对象方法 | 添加响应 header | 175 | | response:add_header(key, val) | 对象方法 | 添加响应 header | 176 | | response:set_content(body) | 对象方法 | 设置响应 body | 177 | | response:append_content(body) | 对象方法 | 追加响应 body | 178 | | response:get_code() | 对象方法 | 获取响应的 HTTP 状态码 | 179 | | response:set_body_size(size) | 对象方法 | 设置 body 体大小 | 180 | 181 | #### Sample 182 | 183 | ```lua 184 | local request = aidp.http.request(); 185 | local topic_name = request:get_query('t') 186 | local response = aidp.http.response(); 187 | local storage = aidp.storage() 188 | response:set_content(topic_name .. ':message_size ' .. storage:get(topic_name .. ':message_size')); 189 | ``` 190 | 191 | ### Simple Storage 192 | 193 | 存储提供了类似于 Redis 的 key-val 结构的简单的单机存储，辅助实现一些数据缓存存储逻辑 194 | 195 | 如下 Lua Api 在 Http lua 脚本和 Message Consumer 中通用 196 | 197 | | 方法 | 类型 | 描述 | 198 | |-----|-----|-----| 199 | | aidp.storage() | 静态构造函数 | 构造 storage 对象 | 200 | | storage:set(key, val, [ttl]) | 对象方法 | 在存储中设置数据, 如果设置 ttl 则到期后该key 会被回收掉，ttl 单位为秒 | 201 | | storage:incr(key, step) | 对象方法 | 对某个值累加 step | 202 | | storage:decr(key, step) | 对象方法 | 对某个值累减 step | 203 | | storage:get(key) | 对象方法 | 获取数据 | 204 | | storage:exists(key) | 对象方法 | 判断某个 key 是否存在 | 205 | | storage:del(key) | 对象方法 | 删除某个 key | 206 | -------------------------------------------------------------------------------- /adbase.ini: -------------------------------------------------------------------------------- 1 | [project] 2 | ADINF_PROJECT_NAME=aidp 3 | ADINF_PROJECT_SUMMARY=Weibo adinf kafka consumer using lua script 4 | ADINF_PROJECT_URL=http://adinf.weiboad.org 5 | ADINF_PROJECT_VENDOR=nmred 6 | ADINF_PROJECT_PACKAGER=nmred 7 | 8 | [module] 9 | adserver=yes 10 | timer=yes 11 | kafkac=yes 12 | kafkap=no 13 | logging=yes 14 | 15 | [params] 16 | timers=default 17 | http_controllers=Index 18 | kafka_consumers=Out 19 | kafka_consumers_topics=test 20 | kafka_consumers_groups=test_group_id 21 | kafka_producers=In 22 | kafka_producers_topics=in 23 | 24 | [files] 25 | src/main.cpp=src/@ADINF_PROJECT_NAME@.cpp 26 | rpm/main.spec.in=rpm/@ADINF_PROJECT_NAME@.spec.in 27 | 28 | [execs] 29 | cmake.sh=1 30 | build_rpm.in=1 31 | -------------------------------------------------------------------------------- /cmake.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #{{{ options 3 | 4 | echo "Start cmake configure....." 5 | echo -e "\033[31m\033[05mCompile debug level [Debug(D)|Release(R)]: \033[0m\c" 6 | read makelevel 7 | case $makelevel in 8 | Debug | D ) 9 | CMAKE_BUILD_TYPE="Debug";; 10 | Release | R ) 11 | CMAKE_BUILD_TYPE="Release";; 12 | *) 13 | CMAKE_BUILD_TYPE="Debug";; 14 | esac 15 | # }}} 16 | # 计算版本号 17 | GIT_SHA1=`(git show-ref --head --hash=8 2> /dev/null || echo 00000000) | head -n1` 18 | GIT_DIRTY=`git diff --no-ext-diff 2> /dev/null | wc -l` 19 | BUILD_ID=`uname -n`"-"`date +%s` 20 | 21 | CMD="cmake -H. -B_build -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DGIT_SHA1=$GIT_SHA1 -DGIT_DIRTY=$GIT_DIRTY -DBUILD_ID=$BUILD_ID -DSTATICCOMPILE=ON " 22 | 23 | echo $CMD 24 | eval $CMD 25 | -------------------------------------------------------------------------------- /conf/system.ini: -------------------------------------------------------------------------------- 1 | [system] 2 | appid=0 3 | macid=0 4 | daemon=no 5 | pidFile=./aidp.pid 6 | ; malloc trim 128kb 7 | mallocTrimPad=131072 8 | ; malloc trim interval 9 | ; 5s 10 | mallocTrimInterval=1000 11 | 12 | [logging] 13 | logsDir=logs/ 14 | ; 日志分卷大小 15 | logRollSize=52428800 16 | ; 1: LOG_TRACE 2: LOG_DEBUG 3: LOG_INFO 17 | logLevel=3 18 | isAsync=no 19 | 20 | [adserver] 21 | ; 是否启动 mc server 22 | mc=no 23 | ; 是否启动 http server 24 | http=yes 25 | ; 是否启动 head server 26 | head=no 27 | 28 | [mc] 29 | host=0.0.0.0 30 | port=10011 31 | threadNum=4 32 | serverName=mc-server 33 | 34 | [http] 35 | host=0.0.0.0 36 | port=10053 37 | timeout=3 38 | threadNum=1 39 | serverName=adinf-adserver 40 | accessLogDir=logs/ 41 | accesslogRollSize=52428800 42 | defaultController=index 43 | defaultAction=index 44 | scriptName=http.lua 45 | 46 | [head] 47 | host=0.0.0.0 48 | port=10012 49 | threadNum=4 50 | serverName=head-server 51 | 52 | [kafkac_out] 53 | ; 支持同时消费多个 topic, 多个用逗号分隔 54 | topicNameOut=test 55 | groupIdOut=test_group_id 56 | brokerListOut=127.0.0.1:9192 57 | kafkaDebugOut=none 58 | statIntervalOut=60000 59 | 60 | [timer] 61 | ; 单位ms 62 | intervalClearStorage=1000 63 | 64 | [lua] 65 | ; 如果是 debug 模式脚本执行将每次重新加载 66 | debug=no 67 | scriptPath=/usr/home/zhongxiu/code/lua 68 | 69 | [consumer] 70 | scriptName=../../example/dusty.lua 71 | batchNumber=1000 72 | messageDir=./message/ 73 | messageRollSize=52428800 74 | threadNumber=8 75 | tryNumber=10 76 | maxNumber=10000 77 | -------------------------------------------------------------------------------- /docs/images/aidp_struct.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weiboad/aidp/8bf7d0cbd580d892922e3f772690b39fd2b0203b/docs/images/aidp_struct.png -------------------------------------------------------------------------------- /docs/images/aidp_struct_pro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weiboad/aidp/8bf7d0cbd580d892922e3f772690b39fd2b0203b/docs/images/aidp_struct_pro.png -------------------------------------------------------------------------------- /docs/images/empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weiboad/aidp/8bf7d0cbd580d892922e3f772690b39fd2b0203b/docs/images/empty -------------------------------------------------------------------------------- /dusty.lua: -------------------------------------------------------------------------------- 1 | local obj = aidp.message.get() 2 | local filename = 'test' .. os.date('%Y-%m-%d-%H') .. '.log' 3 | local file = io.open(filename, 'a+') 4 | file:setvbuf('full') 5 | for k,v in pairs(obj) do 6 | if #v == 2 then 7 | print(v[2]) 8 | file:write(v[2]..'\n') 9 | end 10 | end 11 | file:flush() 12 | file:close() 13 | -------------------------------------------------------------------------------- /example/dusty.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- 3 | -- 该脚本实现将 kafka 中的数据落地到文件中，并且统计落地消息数 4 | -- 5 | -- 6 | local obj = aidp.message.get() 7 | local filename = os.date('%Y-%m-%d-%H') .. '.log' 8 | local files = {}; 9 | local storage = aidp.storage() 10 | for k,v in pairs(obj) do 11 | if #v == 3 then 12 | if files[v[2]] == nil then 13 | files[v[2]] = io.open(v[2]..filename, 'a+'); 14 | files[v[2]]:setvbuf('full') 15 | end 16 | files[v[2]]:write(v[3]..'\n') 17 | storage:incr(v[2] .. ':message_size', 1) 18 | end 19 | end 20 | for k,v in pairs(files) do 21 | v:flush() 22 | v:close() 23 | end 24 | -------------------------------------------------------------------------------- /example/http.lua: -------------------------------------------------------------------------------- 1 | local request = aidp.http.request(); 2 | local topic_name = request:get_query('t') 3 | local response = aidp.http.response(); 4 | local storage = aidp.storage() 5 | response:set_content(topic_name .. ':message_size ' .. storage:get(topic_name .. ':message_size')); 6 | -------------------------------------------------------------------------------- /src/AdServer.cpp: -------------------------------------------------------------------------------- 1 | /// 该程序自动生成，禁止修改 2 | #include "AdServer.hpp" 3 | 4 | // {{{ AdServer::AdServer() 5 | 6 | AdServer::AdServer(AdServerContext* context) : 7 | _context(context) { 8 | _configure = _context->config; 9 | _base = _context->mainEventBase; 10 | } 11 | 12 | // }}} 13 | // {{{ AdServer::~AdServer() 14 | 15 | AdServer::~AdServer() { 16 | stop(); 17 | } 18 | 19 | // }}} 20 | // {{{ void AdServer::run() 21 | 22 | void AdServer::run() { 23 | // 初始化 server 24 | init(); 25 | if (_configure->isStartHttp && _http != nullptr) { 26 | _http->start(_configure->httpThreadNum); 27 | } 28 | if (_configure->isStartMc) { 29 | _mcServer->start(_configure->mcThreadNum); 30 | } 31 | if (_configure->isStartHead) { 32 | _headServer->start(_configure->headThreadNum); 33 | } 34 | } 35 | 36 | // }}} 37 | // {{{ void AdServer::init() 38 | 39 | void AdServer::init() { 40 | if (_configure->isStartHttp) { 41 | initHttp(); 42 | } 43 | if (_configure->isStartMc) { 44 | initMc(); 45 | } 46 | if (_configure->isStartHead) { 47 | initHead(); 48 | } 49 | } 50 | 51 | // }}} 52 | // {{{ void AdServer::stop() 53 | 54 | void AdServer::stop() { 55 | if (_configure->isStartHttp && _http != nullptr) { 56 | LOG_TRACE << "Stop http server."; 57 | _http->stop(); 58 | delete _httpManager; 59 | } 60 | 61 | if (_configure->isStartMc && _mcServer != nullptr) { 62 | _mcServer->stop(); 63 | delete _mcServer; 64 | _mcServer = nullptr; 65 | delete _mcProcessor; 66 | delete _mcAddr; 67 | delete _mcInterface; 68 | delete _mcHandler; 69 | } 70 | 71 | if (_configure->isStartHead && _mcServer != nullptr) { 72 | _headServer->stop(); 73 | delete _headServer; 74 | _headServer = nullptr; 75 | delete _headProcessor; 76 | delete _headAddr; 77 | delete _headInterface; 78 | delete _headHandler; 79 | } 80 | } 81 | 82 | // }}} 83 | // {{{ void AdServer::initHttp() 84 | 85 | void AdServer::initHttp() { 86 | adbase::TimeZone tz(8*3600, "CST"); 87 | LOG_TRACE << "Init http server, host:" 88 | << _configure->httpHost 89 | << " port:" 90 | << _configure->httpPort; 91 | adbase::http::Config config(_configure->httpHost, _configure->httpPort, _configure->httpTimeout); 92 | config.setTimeZone(tz); 93 | config.setServerName(_configure->httpServerName); 94 | config.setLogDir(_configure->httpAccessLogDir); 95 | config.setLogRollSize(_configure->httpAccesslogRollSize); 96 | _http = new adbase::http::Server(config); 97 | _httpManager = new Http(_context, _http); 98 | _httpManager->addController(); 99 | _http->setLocationFallback(std::bind(&AdServer::httpFallback, this, std::placeholders::_1)); 100 | } 101 | 102 | // }}} 103 | // {{{ std::string AdServer::httpFallback() 104 | 105 | std::string AdServer::httpFallback(const std::string&) { 106 | return "/index/index"; 107 | //std::string result = "/"; 108 | //std::string trimUrl = adbase::trim(url, "/"); 109 | //if (_httpManager != nullptr) { 110 | // std::unordered_map rewrites = _httpManager->rewrite(); 111 | // if (rewrites.find(trimUrl) != rewrites.end()) { 112 | // return rewrites[trimUrl]; 113 | // } 114 | //} 115 | //if (trimUrl.empty()) { 116 | // return result + _configure->httpDefaultController + "/" + _configure->httpDefaultAction; 117 | //} else if (trimUrl.find("/") == std::string::npos) { 118 | // return result + trimUrl + "/" + _configure->httpDefaultAction; 119 | //} else { 120 | // return url; 121 | //} 122 | } 123 | 124 | // }}} 125 | // {{{ void AdServer::initHead() 126 | 127 | void AdServer::initHead() { 128 | // head server 129 | _headProcessor = new HeadProcessor(_context); 130 | _headAddr = new adbase::InetAddress(_configure->headHost, static_cast(_configure->headPort)); 131 | _headServer = new adbase::TcpServer(_base, *_headAddr, _configure->headServerName); 132 | _headInterface = new adbase::head::Interface(); 133 | _headInterface->setReadHandler(std::bind(&HeadProcessor::readHandler, _headProcessor, std::placeholders::_1, 134 | std::placeholders::_2, std::placeholders::_3, 135 | std::placeholders::_4, std::placeholders::_5)); 136 | _headHandler = new adbase::head::Handler(_headInterface); 137 | _headServer->setConnectionCallback(std::bind(&adbase::head::Handler::onConnection, _headHandler, 138 | std::placeholders::_1)); 139 | _headServer->setCloseCallback(std::bind(&adbase::head::Handler::onClose, _headHandler, std::placeholders::_1)); 140 | _headServer->setMessageCallback(std::bind(&adbase::head::Handler::onMessage, _headHandler, std::placeholders::_1, 141 | std::placeholders::_2, std::placeholders::_3)); 142 | 143 | } 144 | 145 | // }}} 146 | // {{{ void AdServer::initMc() 147 | 148 | void AdServer::initMc() { 149 | // Mc server 150 | _mcProcessor = new McProcessor(_context); 151 | _mcAddr = new adbase::InetAddress(_configure->mcHost, static_cast(_configure->mcPort)); 152 | _mcServer = new adbase::TcpServer(_base, *_mcAddr, _configure->mcServerName); 153 | _mcInterface = new adbase::mc::Interface(); 154 | bindMcCallback(); 155 | _mcHandler = new adbase::mc::Handler(_mcInterface); 156 | _mcServer->setConnectionCallback(std::bind(&adbase::mc::Handler::onConnection, _mcHandler, 157 | std::placeholders::_1)); 158 | _mcServer->setCloseCallback(std::bind(&adbase::mc::Handler::onClose, _mcHandler, std::placeholders::_1)); 159 | _mcServer->setMessageCallback(std::bind(&adbase::mc::Handler::onMessage, _mcHandler, std::placeholders::_1, 160 | std::placeholders::_2, std::placeholders::_3)); 161 | 162 | } 163 | 164 | // }}} 165 | // {{{ void AdServer::bindMcCallback() 166 | 167 | void AdServer::bindMcCallback() { 168 | _mcInterface->setAddHandler(std::bind(&McProcessor::add, _mcProcessor, std::placeholders::_1, 169 | std::placeholders::_2, std::placeholders::_3, 170 | std::placeholders::_4, std::placeholders::_5, 171 | std::placeholders::_6, std::placeholders::_7)); 172 | _mcInterface->setAppendHandler(std::bind(&McProcessor::append, _mcProcessor, std::placeholders::_1, 173 | std::placeholders::_2, std::placeholders::_3, 174 | std::placeholders::_4, std::placeholders::_5, 175 | std::placeholders::_6)); 176 | _mcInterface->setDecrementHandler(std::bind(&McProcessor::decrement, _mcProcessor, std::placeholders::_1, 177 | std::placeholders::_2, std::placeholders::_3, 178 | std::placeholders::_4, std::placeholders::_5, 179 | std::placeholders::_6, std::placeholders::_7)); 180 | _mcInterface->setDeleteHandler(std::bind(&McProcessor::deleteOp, _mcProcessor, std::placeholders::_1, 181 | std::placeholders::_2, std::placeholders::_3)); 182 | _mcInterface->setFlushHandler(std::bind(&McProcessor::flush, _mcProcessor, std::placeholders::_1)); 183 | _mcInterface->setGetHandler(std::bind(&McProcessor::get, _mcProcessor, std::placeholders::_1, 184 | std::placeholders::_2, std::placeholders::_3)); 185 | _mcInterface->setIncrementHandler(std::bind(&McProcessor::increment, _mcProcessor, std::placeholders::_1, 186 | std::placeholders::_2, std::placeholders::_3, 187 | std::placeholders::_4, std::placeholders::_5, 188 | std::placeholders::_6, std::placeholders::_7)); 189 | _mcInterface->setPrependHandler(std::bind(&McProcessor::prepend, _mcProcessor, std::placeholders::_1, 190 | std::placeholders::_2, std::placeholders::_3, 191 | std::placeholders::_4, std::placeholders::_5, 192 | std::placeholders::_6)); 193 | _mcInterface->setNoopHandler(std::bind(&McProcessor::noop, _mcProcessor)); 194 | _mcInterface->setQuitHandler(std::bind(&McProcessor::quit, _mcProcessor)); 195 | _mcInterface->setReplaceHandler(std::bind(&McProcessor::replace, _mcProcessor, std::placeholders::_1, 196 | std::placeholders::_2, std::placeholders::_3, 197 | std::placeholders::_4, std::placeholders::_5, 198 | std::placeholders::_6, std::placeholders::_7, 199 | std::placeholders::_8)); 200 | _mcInterface->setSetHandler(std::bind(&McProcessor::set, _mcProcessor, std::placeholders::_1, 201 | std::placeholders::_2, std::placeholders::_3, 202 | std::placeholders::_4, std::placeholders::_5, 203 | std::placeholders::_6, std::placeholders::_7, 204 | std::placeholders::_8)); 205 | _mcInterface->setVersionHandler(std::bind(&McProcessor::version, _mcProcessor, std::placeholders::_1)); 206 | _mcInterface->setVerbosityHandler(std::bind(&McProcessor::verbosity, _mcProcessor, std::placeholders::_1)); 207 | _mcInterface->setStatHandler(std::bind(&McProcessor::stat, _mcProcessor, std::placeholders::_1, 208 | std::placeholders::_2, std::placeholders::_3)); 209 | _mcInterface->setPreExecute(std::bind(&McProcessor::preExecute, _mcProcessor)); 210 | _mcInterface->setPostExecute(std::bind(&McProcessor::postExecute, _mcProcessor)); 211 | _mcInterface->setUnknownExecute(std::bind(&McProcessor::unknownExecute, _mcProcessor)); 212 | } 213 | 214 | // }}} 215 | -------------------------------------------------------------------------------- /src/AdServer.hpp: -------------------------------------------------------------------------------- 1 | /// 该程序自动生成，禁止修改 2 | #ifndef AIDP_ADSERVER_HPP_ 3 | #define AIDP_ADSERVER_HPP_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "AdbaseConfig.hpp" 12 | #include "Http.hpp" 13 | #include "McProcessor.hpp" 14 | #include "HeadProcessor.hpp" 15 | 16 | class AdServer { 17 | public: 18 | AdServer(AdServerContext* context); 19 | ~AdServer(); 20 | void run(); 21 | std::string httpFallback(const std::string& url); 22 | 23 | private: 24 | AdbaseConfig* _configure; 25 | /// 传输上下文指针 26 | AdServerContext* _context; 27 | adbase::EventBasePtr _base; 28 | 29 | /// http server 30 | adbase::http::Server* _http = nullptr; 31 | Http* _httpManager = nullptr; 32 | /// mc server 33 | McProcessor* _mcProcessor = nullptr; 34 | adbase::TcpServer* _mcServer = nullptr; 35 | adbase::InetAddress* _mcAddr = nullptr; 36 | adbase::mc::Interface* _mcInterface = nullptr; 37 | adbase::mc::Handler* _mcHandler = nullptr; 38 | /// head server 39 | HeadProcessor* _headProcessor = nullptr; 40 | adbase::TcpServer* _headServer = nullptr; 41 | adbase::InetAddress* _headAddr = nullptr; 42 | adbase::head::Interface* _headInterface = nullptr; 43 | adbase::head::Handler* _headHandler = nullptr; 44 | 45 | void init(); 46 | void stop(); 47 | void initHttp(); 48 | void initHead(); 49 | void initMc(); 50 | void bindMcCallback(); 51 | }; 52 | 53 | #endif -------------------------------------------------------------------------------- /src/AdbaseConfig.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDP_ADBASE_CONFIG_HPP_ 2 | #define AIDP_ADBASE_CONFIG_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | // {{{ macros 10 | 11 | #ifndef DECLARE_KAFKA_CONSUMER_CONFIG 12 | #define DECLARE_KAFKA_CONSUMER_CONFIG(name) \ 13 | std::string topicNameConsumer##name;\ 14 | std::string groupId##name;\ 15 | std::string brokerListConsumer##name;\ 16 | std::string kafkaDebug##name;\ 17 | std::string statInterval##name; 18 | #endif 19 | #ifndef DECLARE_KAFKA_PRODUCER_CONFIG 20 | #define DECLARE_KAFKA_PRODUCER_CONFIG(name) \ 21 | std::string topicNameProducer##name;\ 22 | std::string brokerListProducer##name;\ 23 | std::string debug##name;\ 24 | int queueLength##name; 25 | #endif 26 | #ifndef DECLARE_TIMER_CONFIG 27 | #define DECLARE_TIMER_CONFIG(name) \ 28 | int interval##name; 29 | #endif 30 | 31 | // }}} 32 | 33 | typedef struct adbaseConfig { 34 | bool daemon; 35 | std::string pidFile; 36 | int appid; 37 | int macid; 38 | int mallocTrimPad; 39 | int mallocTrimInterval; 40 | 41 | // logging config 42 | std::string logsDir; 43 | size_t logRollSize; 44 | int logLevel; 45 | bool isAsync; 46 | 47 | bool isStartMc; 48 | bool isStartHead; 49 | bool isStartHttp; 50 | 51 | std::string httpHost; 52 | int httpPort; 53 | int httpTimeout; 54 | int httpThreadNum; 55 | std::string httpDefaultController; 56 | std::string httpDefaultAction; 57 | std::string httpServerName; 58 | std::string httpAccessLogDir; 59 | int httpAccesslogRollSize; 60 | std::string httpScriptName; 61 | 62 | std::string headHost; 63 | int headPort; 64 | std::string headServerName; 65 | int headThreadNum; 66 | 67 | std::string mcHost; 68 | int mcPort; 69 | std::string mcServerName; 70 | int mcThreadNum; 71 | DECLARE_KAFKA_CONSUMER_CONFIG(Out); 72 | DECLARE_TIMER_CONFIG(ClearStorage); 73 | 74 | bool luaDebug; 75 | std::string luaScriptPath; 76 | 77 | std::string consumerScriptName; 78 | int consumerBatchNumber; 79 | int consumerThreadNumber; 80 | int consumerMaxNumber; 81 | int consumerTryNumber; 82 | int messageRollSize; 83 | std::string messageDir; 84 | } AdbaseConfig; 85 | 86 | class App; 87 | namespace app { 88 | class Message; 89 | class Storage; 90 | class Metrics; 91 | } 92 | typedef struct adserverContext { 93 | AdbaseConfig* config; 94 | adbase::EventBasePtr mainEventBase; 95 | App* app; 96 | adbase::metrics::Metrics* metrics; 97 | // 前后端交互数据指针添加到下面 98 | app::Storage* storage; 99 | app::Metrics* appMetrics; 100 | } AdServerContext; 101 | 102 | typedef struct aimsContext { 103 | AdbaseConfig* config; 104 | App* app; 105 | // 消息队列交互上下文 106 | app::Message* message; 107 | app::Storage* storage; 108 | app::Metrics* appMetrics; 109 | } AimsContext; 110 | typedef struct timerContext { 111 | AdbaseConfig* config; 112 | adbase::EventBasePtr mainEventBase; 113 | App* app; 114 | // 定时器交互上下文 115 | app::Storage* storage; 116 | app::Metrics* appMetrics; 117 | } TimerContext; 118 | 119 | #endif 120 | -------------------------------------------------------------------------------- /src/Aims.cpp: -------------------------------------------------------------------------------- 1 | #include "Aims.hpp" 2 | 3 | // {{{ Aims::Aims() 4 | 5 | Aims::Aims(AimsContext* context) : 6 | _context(context) { 7 | _configure = _context->config; 8 | } 9 | 10 | // }}} 11 | // {{{ Aims::~Aims() 12 | 13 | Aims::~Aims() { 14 | } 15 | 16 | // }}} 17 | // {{{ void Aims::run() 18 | 19 | void Aims::run() { 20 | // 初始化 server 21 | init(); 22 | _kafka->start(); 23 | _state = RUNNING; 24 | } 25 | 26 | // }}} 27 | // {{{ void Aims::reload() 28 | 29 | void Aims::reload() { 30 | std::vector topicNames = adbase::explode(_configure->topicNameConsumerOut, ',', true); 31 | _kafka->setTopics(topicNames); 32 | } 33 | 34 | // }}} 35 | // {{{ void Aims::init() 36 | 37 | void Aims::init() { 38 | initKafkaConsumer(); 39 | } 40 | 41 | // }}} 42 | // {{{ void Aims::stop() 43 | 44 | void Aims::stop() { 45 | if (_kafka != nullptr) { 46 | _kafka->stop(); 47 | delete _kafka; 48 | } 49 | if (_kafkaConsumerCallbackOut != nullptr) { 50 | delete _kafkaConsumerCallbackOut; 51 | _kafkaConsumerCallbackOut = nullptr; 52 | } 53 | } 54 | 55 | // }}} 56 | // {{{ void Aims::pause() 57 | 58 | void Aims::pause() { 59 | if (_state == RUNNING) { 60 | _kafka->pause(); 61 | _state = PAUSE; 62 | } 63 | } 64 | 65 | // }}} 66 | // {{{ void Aims::resume() 67 | 68 | void Aims::resume() { 69 | if (_state == PAUSE) { 70 | _kafka->resume(); 71 | _state = RUNNING; 72 | } 73 | } 74 | 75 | // }}} 76 | // {{{ void Aims::initKafkaConsumer() 77 | 78 | void Aims::initKafkaConsumer() { 79 | _kafkaConsumerCallbackOut = new aims::kafka::ConsumerOut(_context); 80 | std::vector topicNames = adbase::explode(_configure->topicNameConsumerOut, ',', true); 81 | LOG_INFO << "Topic list:" << _configure->topicNameConsumerOut; 82 | 83 | _kafka = new adbase::kafka::ConsumerBatch(topicNames, _configure->groupIdOut, 84 | _configure->brokerListConsumerOut); 85 | _kafka->setMessageHandler(std::bind(&aims::kafka::ConsumerOut::pull, 86 | _kafkaConsumerCallbackOut, 87 | std::placeholders::_1, std::placeholders::_2, 88 | std::placeholders::_3, std::placeholders::_4)); 89 | _kafka->setStatCallback(std::bind(&aims::kafka::ConsumerOut::stat, 90 | _kafkaConsumerCallbackOut, 91 | std::placeholders::_1, std::placeholders::_2)); 92 | _kafka->setKafkaDebug(_configure->kafkaDebugOut); 93 | _kafka->setKafkaStatInterval(_configure->statIntervalOut); 94 | } 95 | 96 | // }}} 97 | -------------------------------------------------------------------------------- /src/Aims.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDP_AIMS_HPP_ 2 | #define AIDP_AIMS_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "AdbaseConfig.hpp" 8 | #include "Aims/Kafka/ConsumerOut.hpp" 9 | 10 | class Aims { 11 | public: 12 | Aims(AimsContext* context); 13 | ~Aims(); 14 | void run(); 15 | void stop(); 16 | void reload(); 17 | void pause(); 18 | void resume(); 19 | 20 | private: 21 | /// 传输上下文指针 22 | AimsContext* _context; 23 | AdbaseConfig* _configure; 24 | bool _isResume = true; 25 | bool _isPause = true; 26 | 27 | void init(); 28 | void initKafkaConsumer(); 29 | aims::kafka::ConsumerOut* _kafkaConsumerCallbackOut = nullptr; 30 | adbase::kafka::ConsumerBatch* _kafka; 31 | enum runState { 32 | STOP = 0, 33 | RUNNING = 1, 34 | PAUSE = 2, 35 | }; 36 | runState _state = STOP; 37 | }; 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /src/Aims/Kafka/ConsumerOut.cpp: -------------------------------------------------------------------------------- 1 | #include "ConsumerOut.hpp" 2 | #include "App/Message.hpp" 3 | 4 | namespace aims { 5 | namespace kafka { 6 | // {{{ ConsumerOut::ConsumerOut() 7 | 8 | ConsumerOut::ConsumerOut(AimsContext* context) : 9 | _context(context) { 10 | } 11 | 12 | // }}} 13 | // {{{ ConsumerOut::~ConsumerOut() 14 | 15 | ConsumerOut::~ConsumerOut() { 16 | } 17 | 18 | // }}} 19 | // {{{ void ConsumerOut::pull() 20 | 21 | bool ConsumerOut::pull(const std::string& topicName, int partId, uint64_t offset, const adbase::Buffer& data) { 22 | app::MessageItem item; 23 | item.partId = partId; 24 | item.offset = offset; 25 | item.message = data; 26 | item.topicName = topicName; 27 | item.tryNum = 0; 28 | if (_context->message != nullptr) { 29 | return _context->message->push(item); 30 | } 31 | return false; 32 | } 33 | 34 | // }}} 35 | // {{{ void ConsumerOut::stat() 36 | 37 | void ConsumerOut::stat(adbase::kafka::ConsumerBatch*, const std::string& stats) { 38 | LOG_INFO << "Stats:" << stats.substr(0, 1024); 39 | } 40 | 41 | // }}} 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/Aims/Kafka/ConsumerOut.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDP_AIMS_KAFKA_CONSUMER_OUTHPP_ 2 | #define AIDP_AIMS_KAFKA_CONSUMER_OUTHPP_ 3 | 4 | #include 5 | #include 6 | #include "AdbaseConfig.hpp" 7 | 8 | namespace aims { 9 | namespace kafka { 10 | class ConsumerOut { 11 | public: 12 | ConsumerOut(AimsContext* context); 13 | ~ConsumerOut(); 14 | bool pull(const std::string& topicName, int partId, uint64_t offset, const adbase::Buffer& data); 15 | void stat(adbase::kafka::ConsumerBatch* consumer, const std::string& stats); 16 | private: 17 | AimsContext* _context; 18 | }; 19 | 20 | } 21 | } 22 | #endif 23 | -------------------------------------------------------------------------------- /src/App.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "App.hpp" 5 | 6 | //{{{ macros 7 | 8 | #define LOAD_TIMER_CONFIG(name) do {\ 9 | _configure->interval##name = config.getOptionUint32("timer", "interval"#name);\ 10 | } while(0) 11 | 12 | #define LOAD_KAFKA_CONSUMER_CONFIG(name, sectionName) do {\ 13 | _configure->topicNameConsumer##name = config.getOption("kafkac_"#sectionName, "topicName"#name);\ 14 | _configure->groupId##name = config.getOption("kafkac_"#sectionName, "groupId"#name);\ 15 | _configure->brokerListConsumer##name = config.getOption("kafkac_"#sectionName, "brokerList"#name);\ 16 | _configure->kafkaDebug##name = config.getOption("kafkac_"#sectionName, "kafkaDebug"#name);\ 17 | _configure->statInterval##name = config.getOption("kafkac_"#sectionName, "statInterval"#name);\ 18 | } while(0) 19 | 20 | //}}} 21 | // {{{ App::App() 22 | 23 | App::App(AdbaseConfig* config) : 24 | _configure(config) { 25 | } 26 | 27 | // }}} 28 | // {{{ App::~App() 29 | 30 | App::~App() { 31 | } 32 | 33 | // }}} 34 | // {{{ void App::run() 35 | 36 | void App::run() { 37 | _storage = std::shared_ptr(new app::Storage(_configure)); 38 | _storage->init(); 39 | 40 | _metrics = std::shared_ptr(new app::Metrics()); 41 | 42 | _message = std::shared_ptr(new app::Message(_configure, _storage.get(), _metrics.get())); 43 | _message->start(); 44 | } 45 | 46 | // }}} 47 | // {{{ void App::reload() 48 | 49 | void App::reload() { 50 | // 重新加载 message 51 | _message->reload(); 52 | } 53 | 54 | // }}} 55 | // {{{ void App::stop() 56 | 57 | void App::stop() { 58 | _message->stop(); 59 | } 60 | 61 | // }}} 62 | // {{{ void App::checkQueue() 63 | 64 | void App::checkQueue() { 65 | if (!_message->queueCheck()) { 66 | if (_aims) { 67 | _aims->pause(); 68 | } 69 | } else { 70 | if (_aims) { 71 | _aims->resume(); 72 | } 73 | } 74 | } 75 | 76 | // }}} 77 | // {{{ void App::setAdServerContext() 78 | 79 | void App::setAdServerContext(AdServerContext* context) { 80 | context->app = this; 81 | context->storage = _storage.get(); 82 | context->appMetrics = _metrics.get(); 83 | } 84 | 85 | // }}} 86 | // {{{ void App::setAimsContext() 87 | 88 | void App::setAimsContext(AimsContext* context) { 89 | context->app = this; 90 | context->message = _message.get(); 91 | context->storage = _storage.get(); 92 | context->appMetrics = _metrics.get(); 93 | } 94 | 95 | // }}} 96 | // {{{ void App::setTimerContext() 97 | 98 | void App::setTimerContext(TimerContext* context) { 99 | context->app = this; 100 | context->storage = _storage.get(); 101 | context->appMetrics = _metrics.get(); 102 | } 103 | 104 | // }}} 105 | // {{{ void App::setAims() 106 | 107 | void App::setAims(std::shared_ptr& aims) { 108 | _aims = aims; 109 | } 110 | 111 | // }}} 112 | //{{{ void App::loadConfig() 113 | 114 | void App::loadConfig(adbase::IniConfig& config) { 115 | _configure->mallocTrimInterval = config.getOptionUint32("system", "mallocTrimInterval"); 116 | _configure->mallocTrimPad = config.getOptionUint32("system", "mallocTrimPad"); 117 | _configure->luaDebug = config.getOptionBool("lua", "debug"); 118 | _configure->luaScriptPath = config.getOption("lua", "scriptPath"); 119 | _configure->consumerScriptName = config.getOption("consumer", "scriptName"); 120 | _configure->consumerBatchNumber = config.getOptionUint32("consumer", "batchNumber"); 121 | _configure->consumerThreadNumber = config.getOptionUint32("consumer", "threadNumber"); 122 | _configure->consumerMaxNumber = config.getOptionUint32("consumer", "maxNumber"); 123 | _configure->consumerTryNumber = config.getOptionUint32("consumer", "tryNumber"); 124 | _configure->messageDir = config.getOption("consumer", "messageDir"); 125 | _configure->messageRollSize = config.getOptionUint32("consumer", "messageRollSize"); 126 | _configure->httpScriptName = config.getOption("http", "scriptName"); 127 | 128 | LOAD_KAFKA_CONSUMER_CONFIG(Out, out); 129 | LOAD_TIMER_CONFIG(ClearStorage); 130 | } 131 | 132 | //}}} 133 | //{{{ uint64_t App::getSeqId() 134 | 135 | uint64_t App::getSeqId() { 136 | std::lock_guard lk(_mut); 137 | adbase::Sequence seq; 138 | return seq.getSeqId(static_cast(_configure->macid), static_cast(_configure->appid)); 139 | } 140 | 141 | //}}} 142 | -------------------------------------------------------------------------------- /src/App.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDP_APP_HPP_ 2 | #define AIDP_APP_HPP_ 3 | 4 | #include 5 | #include 6 | #include "AdbaseConfig.hpp" 7 | #include "Aims.hpp" 8 | #include "App/Message.hpp" 9 | #include "App/Storage.hpp" 10 | #include "App/Metrics.hpp" 11 | 12 | class App { 13 | public: 14 | App(AdbaseConfig* config); 15 | ~App(); 16 | void reload(); 17 | void run(); 18 | void stop(); 19 | void checkQueue(); 20 | void setAdServerContext(AdServerContext* context); 21 | void setTimerContext(TimerContext* context); 22 | void setAimsContext(AimsContext* context); 23 | void loadConfig(adbase::IniConfig& config); 24 | uint64_t getSeqId(); 25 | void setAims(std::shared_ptr& aims); 26 | 27 | private: 28 | AdbaseConfig* _configure; 29 | std::shared_ptr _storage; 30 | std::shared_ptr _message; 31 | std::shared_ptr _metrics; 32 | std::shared_ptr _aims; 33 | mutable std::mutex _mut; 34 | void bindLuaMessage(); 35 | }; 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /src/App/Message.cpp: -------------------------------------------------------------------------------- 1 | #include "Message.hpp" 2 | #include "Storage.hpp" 3 | #include "Metrics.hpp" 4 | #include 5 | #include 6 | 7 | namespace app { 8 | thread_local std::unique_ptr messageLuaEngine; 9 | thread_local std::unordered_map messageLuaMessages; 10 | thread_local size_t messageLuaMessagesSize; 11 | 12 | // {{{ Message::Message() 13 | 14 | Message::Message(AdbaseConfig* configure, Storage* storage, Metrics* metrics): _configure(configure), 15 | _storage(storage), 16 | _metrics(metrics), 17 | _isRunning(false) { 18 | } 19 | 20 | // }}} 21 | // {{{ Message::~Message() 22 | 23 | Message::~Message() { 24 | for(auto &t : _queues) { 25 | if (t.second != nullptr) { 26 | delete t.second; 27 | t.second = nullptr; 28 | } 29 | } 30 | } 31 | 32 | // }}} 33 | // {{{ void Message::start() 34 | 35 | void Message::start() { 36 | _threadNumber = _configure->consumerThreadNumber; 37 | _isRunning = true; 38 | _luaProcessTimer = adbase::metrics::Metrics::buildTimers("message", "process", 60000); 39 | for (int i = 0; i < _configure->consumerThreadNumber; i++) { 40 | _queues[i] = new MessageQueue; 41 | std::unordered_map tags; 42 | tags["queue_id"] = std::to_string(i); 43 | adbase::metrics::Metrics::buildGaugesWithTag("message", "queue.size", tags, 1000, [this, i](){ 44 | return _queues[i]->getSize(); 45 | }); 46 | ThreadPtr callThread(new std::thread(std::bind(&Message::call, this, std::placeholders::_1), i), &Message::deleteThread); 47 | LOG_DEBUG << "Create call lua thread success"; 48 | Threads.push_back(std::move(callThread)); 49 | } 50 | std::string messageName = _configure->messageDir + "/message_error"; 51 | _logger = std::shared_ptr(new adbase::AsyncLogging(messageName, _configure->messageRollSize)); 52 | _logger->start(); 53 | 54 | } 55 | 56 | // }}} 57 | // {{{ void Message::stop() 58 | 59 | void Message::stop() { 60 | for (auto &t : _queues) { 61 | MessageItem newItem; 62 | newItem.mask = 0x01; 63 | newItem.partId = 0; 64 | newItem.offset = 0; 65 | newItem.tryNum = 0; 66 | t.second->push(newItem); 67 | } 68 | _isRunning = false; 69 | std::unique_lock lk(_mut); 70 | _dataCond.wait(lk, [this]{return (_threadNumber == 0);}); 71 | } 72 | 73 | // }}} 74 | // {{{ void Message::reload() 75 | 76 | void Message::reload() { 77 | } 78 | 79 | // }}} 80 | // {{{ void Message::call() 81 | 82 | void Message::call(int i) { 83 | // 初始化 lua 引擎 84 | initLua(); 85 | std::unordered_map tags; 86 | tags["map_id"] = std::to_string(i); 87 | adbase::metrics::Metrics::buildGaugesWithTag("message", "lua.message.map", tags, 1000, [this, i](){ 88 | return messageLuaMessagesSize; 89 | }); 90 | 91 | int batchNum = _configure->consumerBatchNumber; 92 | while(_isRunning) { 93 | MessageItem item; 94 | _queues[i]->waitPop(item); 95 | if (item.mask == 0x01) { 96 | break; 97 | } 98 | 99 | addLuaMessage(item); 100 | 101 | if (static_cast(_queues[i]->getSize()) > batchNum) { 102 | bool ret = false; 103 | bool exit = false; 104 | do { 105 | MessageItem itemBatch; 106 | ret = _queues[i]->tryPop(itemBatch); 107 | if (item.mask == 0x01) { 108 | exit = true; 109 | continue; 110 | } 111 | if (!ret) { 112 | break; 113 | } 114 | addLuaMessage(itemBatch); 115 | batchNum--; 116 | } while(batchNum > 0); 117 | 118 | if (exit) { 119 | break; 120 | } 121 | } 122 | callMessage(); 123 | } 124 | 125 | for (auto &t : messageLuaMessages) { 126 | saveMessage(t.second); 127 | } 128 | 129 | while(_queues[i]->getSize()) { 130 | MessageItem item; 131 | bool ret = _queues[i]->tryPop(item); 132 | if (!ret) { 133 | break; 134 | } 135 | saveMessage(item); 136 | } 137 | 138 | std::unique_lock lk(_mut); 139 | _threadNumber--; 140 | _dataCond.notify_all(); 141 | } 142 | 143 | // }}} 144 | // {{{ void Message::initLua() 145 | 146 | void Message::initLua() { 147 | std::unique_ptr engine(new adbase::lua::Engine()); 148 | messageLuaEngine = std::move(engine); 149 | messageLuaEngine->init(); 150 | messageLuaEngine->addSearchPath(_configure->luaScriptPath, true); 151 | 152 | adbase::lua::BindingClass clazz("message", "aidp", messageLuaEngine->getLuaState()); 153 | typedef std::function GetMessageFn; 154 | GetMessageFn getMessageFn = std::bind(&app::Message::getMessage, this); 155 | clazz.addMethod("get", getMessageFn); 156 | 157 | typedef std::function)> RollBackMessageFn; 158 | RollBackMessageFn rollbackFn = std::bind(&app::Message::rollback, this, std::placeholders::_1); 159 | clazz.addMethod("rollback", rollbackFn); 160 | 161 | _storage->bindClass(messageLuaEngine.get()); 162 | _metrics->bindClass(messageLuaEngine.get()); 163 | } 164 | 165 | // }}} 166 | // {{{ void Message::callMessage() 167 | 168 | void Message::callMessage() { 169 | adbase::metrics::Timer timer; 170 | timer.start(); 171 | bool isCall = messageLuaEngine->runFile(_configure->consumerScriptName.c_str()); 172 | if (!isCall) { 173 | for (auto &t : messageLuaMessages) { 174 | saveMessage(t.second); 175 | } 176 | } 177 | messageLuaMessages.clear(); 178 | messageLuaMessagesSize = messageLuaMessages.size(); 179 | double time = timer.stop(); 180 | if (_luaProcessTimer != nullptr) { 181 | _luaProcessTimer->setTimer(time); 182 | } 183 | } 184 | 185 | // }}} 186 | // {{{ MessageToLua Message::getMessage() 187 | 188 | MessageToLua Message::getMessage() { 189 | MessageToLua ret; 190 | for (auto &t : messageLuaMessages) { 191 | std::list item; 192 | std::string id = convertKey(t.second); 193 | adbase::Buffer tmp = t.second.message; 194 | std::string message = tmp.retrieveAllAsString(); 195 | item.push_back(id); 196 | item.push_back(t.second.topicName); 197 | item.push_back(message); 198 | ret.push_back(item); 199 | } 200 | return ret; 201 | } 202 | 203 | // }}} 204 | // {{{ int Message::rollback() 205 | 206 | int Message::rollback(std::list ids) { 207 | int count = 0; 208 | for (auto &t : ids) { 209 | if (messageLuaMessages.find(t) != messageLuaMessages.end()) { 210 | MessageItem item = messageLuaMessages[t]; 211 | saveMessage(item); 212 | count++; 213 | } 214 | } 215 | messageLuaMessagesSize = messageLuaMessages.size(); 216 | return count; 217 | } 218 | 219 | // }}} 220 | // {{{ bool Message::push() 221 | 222 | bool Message::push(MessageItem& item) { 223 | int processQueueNum = item.partId % _configure->consumerThreadNumber; 224 | if (item.message.readableBytes()) { 225 | _queues[processQueueNum]->push(item); 226 | } 227 | 228 | return true; 229 | } 230 | 231 | // }}} 232 | // {{{ bool Message::queueCheck() 233 | 234 | bool Message::queueCheck() { 235 | for(auto &t : _queues) { 236 | if (static_cast(t.second->getSize()) > _configure->consumerMaxNumber) { 237 | return false; 238 | } 239 | } 240 | 241 | return true; 242 | } 243 | 244 | // }}} 245 | // {{{ void Message::deleteThread() 246 | 247 | void Message::deleteThread(std::thread *t) { 248 | t->join(); 249 | delete t; 250 | } 251 | 252 | // }}} 253 | // {{{ void Message::addLuaMessage() 254 | 255 | void Message::addLuaMessage(MessageItem& item) { 256 | messageLuaMessages[convertKey(item)] = item; 257 | messageLuaMessagesSize = messageLuaMessages.size(); 258 | } 259 | 260 | // }}} 261 | // {{{ const std::string Message::convertKey() 262 | 263 | const std::string Message::convertKey(MessageItem& item) { 264 | std::string key = std::to_string(item.partId) + "_" + std::to_string(item.offset) + "_" + item.topicName; 265 | return key; 266 | } 267 | 268 | // }}} 269 | // {{{ void Message::serialize() 270 | 271 | void Message::serialize(adbase::Buffer& buffer, MessageItem& item) { 272 | buffer.appendInt32(item.partId); 273 | buffer.appendInt32(item.tryNum); 274 | buffer.appendInt32(static_cast(item.topicName.size())); 275 | buffer.append(item.topicName); 276 | 277 | size_t messageSize = item.message.readableBytes(); 278 | buffer.appendInt32(static_cast(messageSize)); 279 | buffer.append(item.message.peek(), messageSize); 280 | } 281 | 282 | // }}} 283 | // {{{ void Message::saveMessage() 284 | 285 | void Message::saveMessage(MessageItem& item) { 286 | if (item.tryNum < _configure->consumerTryNumber) { 287 | int processQueueNum = item.partId % _configure->consumerThreadNumber; 288 | item.tryNum = item.tryNum + 1; 289 | _queues[processQueueNum]->push(item); 290 | } else { 291 | adbase::Buffer buffer; 292 | serialize(buffer, item); 293 | if (_logger) { 294 | _logger->append(buffer.peek(), static_cast(buffer.readableBytes())); 295 | } 296 | if (_errorMessageMeter.find(item.topicName) == _errorMessageMeter.end()) { 297 | std::unordered_map tags; 298 | tags["topic_name"] = item.topicName; 299 | _errorMessageMeter[item.topicName] = adbase::metrics::Metrics::buildMetersWithTag("message", "error", tags); 300 | } 301 | 302 | if (_errorMessageMeter[item.topicName] != nullptr) { 303 | _errorMessageMeter[item.topicName]->mark(); 304 | } 305 | } 306 | } 307 | 308 | // }}} 309 | } 310 | -------------------------------------------------------------------------------- /src/App/Message.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDP_APP_MESSAGE_HPP_ 2 | #define AIDP_APP_MESSAGE_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "AdbaseConfig.hpp" 15 | 16 | namespace app { 17 | class Storage; 18 | class Metrics; 19 | // message queue 20 | typedef struct MessageItem { 21 | int partId; 22 | int mask; // 0x00 normal, 0x01 stop 23 | uint64_t offset; 24 | int tryNum; 25 | std::string topicName; 26 | adbase::Buffer message; 27 | } MessageItem; 28 | 29 | typedef adbase::Queue MessageQueue; 30 | typedef std::list> MessageToLua; 31 | 32 | 33 | class Message { 34 | public: 35 | Message(AdbaseConfig* configure, Storage* storage, Metrics* metrics); 36 | ~Message(); 37 | void start(); 38 | void stop(); 39 | void reload(); 40 | void call(int i); 41 | MessageToLua getMessage(); 42 | int rollback(std::list ids); 43 | bool push(MessageItem& item); 44 | bool queueCheck(); 45 | static void deleteThread(std::thread *t); 46 | 47 | private: 48 | mutable std::mutex _mut; 49 | std::condition_variable _dataCond; 50 | typedef std::unique_ptr ThreadPtr; 51 | typedef std::vector ThreadPool; 52 | ThreadPool Threads; 53 | int _threadNumber; 54 | 55 | AdbaseConfig *_configure; 56 | Storage* _storage; 57 | Metrics* _metrics; 58 | bool _isRunning; 59 | std::unordered_map _queues; 60 | std::shared_ptr _logger; 61 | std::unordered_map _errorMessageMeter; 62 | adbase::metrics::Timers* _luaProcessTimer; 63 | 64 | void callMessage(); 65 | void initLua(); 66 | void addLuaMessage(MessageItem& item); 67 | const std::string convertKey(MessageItem& item); 68 | void serialize(adbase::Buffer& buffer, MessageItem& item); 69 | void saveMessage(MessageItem& item); 70 | }; 71 | } 72 | #endif 73 | -------------------------------------------------------------------------------- /src/App/Metrics.cpp: -------------------------------------------------------------------------------- 1 | #include "Metrics.hpp" 2 | 3 | namespace app { 4 | thread_local std::unordered_map> metricsCounters; 5 | thread_local std::unordered_map> metricsMeters; 6 | thread_local std::unordered_map> metricsHistograms; 7 | thread_local std::unordered_map> metricsTimers; 8 | thread_local std::shared_ptr metricsTimer; 9 | // {{{ Metrics::Metrics() 10 | 11 | Metrics::Metrics() { 12 | } 13 | 14 | // }}} 15 | // {{{ Metrics::~Metrics() 16 | 17 | Metrics::~Metrics() { 18 | } 19 | 20 | // }}} 21 | // {{{ void Metrics::bindClass() 22 | 23 | void Metrics::bindClass(adbase::lua::Engine* engine) { 24 | // metrics 25 | adbase::lua::BindingClass metrics("metrics", "aidp", engine->getLuaState()); 26 | adbase::lua::BindingNamespace metricsCs = metrics.getOwnerNamespace(); 27 | typedef std::function()> GetMetrics; 28 | GetMetrics metricsFn = std::bind(&Metrics::getMetrics, this); 29 | metricsCs.addMethod("metrics", metricsFn); 30 | metrics.addMethod("counter", &Metrics::buildCounter); 31 | metrics.addMethod("meters", &Metrics::buildMeters); 32 | metrics.addMethod("histograms", &Metrics::buildHistograms); 33 | metrics.addMethod("timers", &Metrics::buildTimers); 34 | metrics.addMethod("timer", &Metrics::getTimer); 35 | 36 | // counter 37 | adbase::lua::BindingClass counter("counter", "aidp", engine->getLuaState()); 38 | counter.addMethod("module_name", &adbase::metrics::Counter::getModuleName); 39 | counter.addMethod("metric_name", &adbase::metrics::Counter::getMetricName); 40 | counter.addMethod("add", &adbase::metrics::Counter::add); 41 | counter.addMethod("dec", &adbase::metrics::Counter::dec); 42 | 43 | // meters 44 | adbase::lua::BindingClass meters("meters", "aidp", engine->getLuaState()); 45 | meters.addMethod("module_name", &adbase::metrics::Meters::getModuleName); 46 | meters.addMethod("metric_name", &adbase::metrics::Meters::getMetricName); 47 | meters.addMethod("mark", &adbase::metrics::Meters::mark); 48 | meters.addMethod("get_count", &adbase::metrics::Meters::getCounter); 49 | 50 | LOG_INFO << "TOP:" << lua_gettop(engine->getLuaState()); 51 | 52 | // histograms 53 | adbase::lua::BindingClass histograms("histograms", "aidp", engine->getLuaState()); 54 | histograms.addMethod("module_name", &adbase::metrics::Histograms::getModuleName); 55 | histograms.addMethod("metric_name", &adbase::metrics::Histograms::getMetricName); 56 | histograms.addMethod("update", &adbase::metrics::Histograms::update); 57 | 58 | // timers 59 | adbase::lua::BindingClass timers("timers", "aidp", engine->getLuaState()); 60 | timers.addMethod("module_name", &adbase::metrics::Timers::getModuleName); 61 | timers.addMethod("metric_name", &adbase::metrics::Timers::getMetricName); 62 | timers.addMethod("set_timer", &adbase::metrics::Timers::setTimer); 63 | 64 | // timer 65 | adbase::lua::BindingClass timer("timer", "aidp", engine->getLuaState()); 66 | timer.addMethod("start", &adbase::metrics::Timer::start); 67 | timer.addMethod("stop", &adbase::metrics::Timer::stop); 68 | } 69 | 70 | // }}} 71 | // {{{ std::weak_ptr Metrics::getMetrics() 72 | 73 | std::weak_ptr Metrics::getMetrics() { 74 | return shared_from_this(); 75 | } 76 | 77 | // }}} 78 | // {{{ std::weak_ptr Metrics::getTimer() 79 | 80 | std::weak_ptr Metrics::getTimer() { 81 | if (metricsTimer == false) { 82 | metricsTimer = std::shared_ptr(new adbase::metrics::Timer()); 83 | } 84 | return metricsTimer; 85 | } 86 | 87 | // }}} 88 | // {{{ std::weak_ptr Metrics::buildCounter() 89 | 90 | std::weak_ptr Metrics::buildCounter(const std::string& moduleName, const std::string& metricName) { 91 | adbase::metrics::Counter* counter = adbase::metrics::Metrics::buildCounter(moduleName, metricName); 92 | auto sharedCounter = std::shared_ptr(counter, [](adbase::metrics::Counter*) { LOG_INFO << "delete Counter";}); 93 | std::string key = getKey(moduleName, metricName); 94 | metricsCounters[key] = sharedCounter; 95 | return metricsCounters[key]; 96 | } 97 | 98 | // }}} 99 | // {{{ std::weak_ptr Metrics::buildMeters() 100 | 101 | std::weak_ptr Metrics::buildMeters(const std::string& moduleName, const std::string& metricName) { 102 | adbase::metrics::Meters* meters = adbase::metrics::Metrics::buildMeters(moduleName, metricName); 103 | auto sharedMeters = std::shared_ptr(meters, [](adbase::metrics::Meters*) { LOG_INFO << "delete meters";}); 104 | std::string key = getKey(moduleName, metricName); 105 | metricsMeters[key] = sharedMeters; 106 | return metricsMeters[key]; 107 | } 108 | 109 | // }}} 110 | // {{{ std::weak_ptr Metrics::buildHistograms() 111 | 112 | std::weak_ptr Metrics::buildHistograms(const std::string& moduleName, const std::string& metricName, uint32_t interval) { 113 | adbase::metrics::Histograms* histograms = adbase::metrics::Metrics::buildHistograms(moduleName, metricName, interval); 114 | auto sharedHistograms = std::shared_ptr(histograms, [](adbase::metrics::Histograms*) { LOG_INFO << "delete histograms";}); 115 | std::string key = getKey(moduleName, metricName); 116 | metricsHistograms[key] = sharedHistograms; 117 | return metricsHistograms[key]; 118 | } 119 | 120 | // }}} 121 | // {{{ std::weak_ptr Metrics::buildHistograms() 122 | 123 | std::weak_ptr Metrics::buildTimers(const std::string& moduleName, const std::string& metricName, uint32_t interval) { 124 | adbase::metrics::Timers* timers = adbase::metrics::Metrics::buildTimers(moduleName, metricName, interval); 125 | auto sharedTimers = std::shared_ptr(timers, [](adbase::metrics::Timers*) { LOG_INFO << "delete timers";}); 126 | std::string key = getKey(moduleName, metricName); 127 | metricsTimers[key] = sharedTimers; 128 | return metricsTimers[key]; 129 | } 130 | 131 | // }}} 132 | // {{{ const std::string Metrics::getKey() 133 | 134 | const std::string Metrics::getKey(const std::string& moduleName, const std::string& metricName) { 135 | std::string result = moduleName; 136 | result.append(1, 26); 137 | result.append(metricName); 138 | return result; 139 | } 140 | 141 | // }}} 142 | } 143 | -------------------------------------------------------------------------------- /src/App/Metrics.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDP_APP_METRICS_HPP_ 2 | #define AIDP_APP_METRICS_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "AdbaseConfig.hpp" 10 | 11 | namespace app { 12 | class Metrics : public std::enable_shared_from_this { 13 | public: 14 | Metrics(); 15 | ~Metrics(); 16 | void bindClass(adbase::lua::Engine* engine); 17 | std::weak_ptr getMetrics(); 18 | std::weak_ptr buildCounter(const std::string& moduleName, const std::string& metricName); 19 | std::weak_ptr buildMeters(const std::string& moduleName, const std::string& metricName); 20 | std::weak_ptr buildHistograms(const std::string& moduleName, const std::string& metricName, uint32_t interval); 21 | std::weak_ptr buildTimers(const std::string& moduleName, const std::string& metricName, uint32_t interval); 22 | std::weak_ptr getTimer(); 23 | private: 24 | const std::string getKey(const std::string& moduleName, const std::string& metricName); 25 | }; 26 | } 27 | #endif 28 | -------------------------------------------------------------------------------- /src/App/Storage.cpp: -------------------------------------------------------------------------------- 1 | #include "Storage.hpp" 2 | 3 | namespace app { 4 | // {{{ Storage::Storage() 5 | 6 | Storage::Storage(AdbaseConfig* configure): _configure(configure) { 7 | } 8 | 9 | // }}} 10 | // {{{ Storage::~Storage() 11 | 12 | Storage::~Storage() { 13 | } 14 | 15 | // }}} 16 | // {{{ void Storage::init() 17 | 18 | void Storage::init() { 19 | adbase::metrics::Metrics::buildGauges("storage", "incrmap.size", 1000, [this](){ 20 | std::lock_guard lk(_mut); 21 | return _incrMap.size(); 22 | }); 23 | adbase::metrics::Metrics::buildGauges("storage", "keyvalmap.size", 1000, [this](){ 24 | std::lock_guard lk(_mut); 25 | return _keyValMap.size(); 26 | }); 27 | adbase::metrics::Metrics::buildGauges("storage", "ttlmap.size", 1000, [this](){ 28 | std::lock_guard lk(_mut); 29 | return _ttlMap.size(); 30 | }); 31 | } 32 | 33 | // }}} 34 | // {{{ void Storage::clear() 35 | 36 | void Storage::clear() { 37 | std::lock_guard lk(_mut); 38 | adbase::Timestamp current = adbase::Timestamp::now(); 39 | std::list deleteKeys; 40 | for (auto &t : _ttlMap) { 41 | if (t.second < current) { 42 | continue; 43 | } 44 | deleteKeys.push_back(t.first); 45 | } 46 | for (auto &t : deleteKeys) { 47 | if (_keyValMap.find(t) != _keyValMap.end()) { 48 | _keyValMap.erase(t); 49 | } 50 | if (_ttlMap.find(t) != _ttlMap.end()) { 51 | _ttlMap.erase(t); 52 | } 53 | } 54 | } 55 | 56 | // }}} 57 | // {{{ void Storage::bindClass() 58 | 59 | void Storage::bindClass(adbase::lua::Engine* engine) { 60 | adbase::lua::BindingClass storage("storage", "aidp", engine->getLuaState()); 61 | adbase::lua::BindingNamespace storageCs = storage.getOwnerNamespace(); 62 | typedef std::function()> GetStorage; 63 | GetStorage storageFn = std::bind(&Storage::getStorage, this); 64 | storageCs.addMethod("storage", storageFn); 65 | 66 | storage.addMethod("set", &Storage::set); 67 | storage.addMethod("set", &Storage::set); 68 | storage.addMethod("incr", &Storage::incr); 69 | storage.addMethod("decr", &Storage::decr); 70 | storage.addMethod("get", &Storage::get); 71 | storage.addMethod("exists", &Storage::exists); 72 | storage.addMethod("del", &Storage::deleteKey); 73 | } 74 | 75 | // }}} 76 | // {{{ std::weak_ptr Storage::getStorage() 77 | 78 | std::weak_ptr Storage::getStorage() { 79 | return shared_from_this(); 80 | } 81 | 82 | // }}} 83 | // {{{ bool Storage::set() 84 | 85 | bool Storage::set(const std::string key, const std::string value) { 86 | std::lock_guard lk(_mut); 87 | _keyValMap[key] = value; 88 | return true; 89 | } 90 | 91 | // }}} 92 | // {{{ bool Storage::set() 93 | 94 | bool Storage::set(const std::string key, const std::string value, int ttl) { 95 | std::lock_guard lk(_mut); 96 | _keyValMap[key] = value; 97 | _ttlMap[key] = adbase::addTime(adbase::Timestamp::now(), ttl); 98 | return true; 99 | } 100 | 101 | // }}} 102 | // {{{ int64_t Storage::incr() 103 | 104 | int64_t Storage::incr(const std::string key, int step) { 105 | std::lock_guard lk(_mut); 106 | if (_incrMap.find(key) == _incrMap.end()) { 107 | _incrMap[key] = step; 108 | } else { 109 | _incrMap[key] += step; 110 | } 111 | return _incrMap[key]; 112 | } 113 | 114 | // }}} 115 | // {{{ int64_t Storage::decr() 116 | 117 | int64_t Storage::decr(const std::string key, int step) { 118 | std::lock_guard lk(_mut); 119 | if (_incrMap.find(key) == _incrMap.end()) { 120 | _incrMap[key] = -step; 121 | } else { 122 | _incrMap[key] -= step; 123 | } 124 | return _incrMap[key]; 125 | } 126 | 127 | // }}} 128 | // {{{ const std::string Storage::get() 129 | 130 | const std::string Storage::get(const std::string key) { 131 | std::lock_guard lk(_mut); 132 | if (_keyValMap.find(key) != _keyValMap.end()) { 133 | return _keyValMap[key]; 134 | } 135 | if (_incrMap.find(key) != _incrMap.end()) { 136 | return std::to_string(_incrMap[key]); 137 | } 138 | return ""; 139 | } 140 | 141 | // }}} 142 | // {{{ bool Storage::exists() 143 | 144 | bool Storage::exists(const std::string key) { 145 | std::lock_guard lk(_mut); 146 | if (_keyValMap.find(key) != _keyValMap.end()) { 147 | return true; 148 | } 149 | if (_incrMap.find(key) != _incrMap.end()) { 150 | return true; 151 | } 152 | return false; 153 | } 154 | 155 | // }}} 156 | // {{{ bool Storage::deleteKey() 157 | 158 | bool Storage::deleteKey(const std::string key) { 159 | std::lock_guard lk(_mut); 160 | if (_keyValMap.find(key) != _keyValMap.end()) { 161 | _keyValMap.erase(key); 162 | } 163 | if (_ttlMap.find(key) != _ttlMap.end()) { 164 | _ttlMap.erase(key); 165 | } 166 | if (_incrMap.find(key) != _incrMap.end()) { 167 | _incrMap.erase(key); 168 | } 169 | return true; 170 | } 171 | 172 | // }}} 173 | } 174 | -------------------------------------------------------------------------------- /src/App/Storage.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDP_APP_STORAGE_HPP_ 2 | #define AIDP_APP_STORAGE_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "AdbaseConfig.hpp" 11 | 12 | namespace app { 13 | class Storage : public std::enable_shared_from_this { 14 | public: 15 | Storage(AdbaseConfig* configure); 16 | ~Storage(); 17 | void init(); 18 | 19 | // key - value 20 | bool set(const std::string key, const std::string value); 21 | bool set(const std::string key, const std::string value, int ttl); 22 | int64_t incr(const std::string key, int step); 23 | int64_t decr(const std::string key, int step); 24 | const std::string get(const std::string key); 25 | bool deleteKey(const std::string key); 26 | bool exists(const std::string key); 27 | 28 | void clear(); 29 | void bindClass(adbase::lua::Engine* engine); 30 | std::weak_ptr getStorage(); 31 | 32 | private: 33 | AdbaseConfig *_configure; 34 | mutable std::mutex _mut; 35 | 36 | std::unordered_map _incrMap; 37 | std::unordered_map _keyValMap; 38 | std::unordered_map _ttlMap; 39 | }; 40 | } 41 | #endif 42 | -------------------------------------------------------------------------------- /src/BootStrap.cpp: -------------------------------------------------------------------------------- 1 | /// 该程序自动生成，禁止修改 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "BootStrap.hpp" 9 | #include "App.hpp" 10 | #include "Version.hpp" 11 | #include "AdServer.hpp" 12 | #include "Aims.hpp" 13 | #include "Timer.hpp" 14 | 15 | // {{{ BootStrap::BootStrap() 16 | 17 | BootStrap::BootStrap() { 18 | } 19 | 20 | // }}} 21 | // {{{ BootStrap::~BootStrap() 22 | 23 | BootStrap::~BootStrap() { 24 | } 25 | 26 | // }}} 27 | // {{{ void BootStrap::init() 28 | 29 | void BootStrap::init(int argc, char **argv) { 30 | _configure = std::shared_ptr(new AdbaseConfig()); 31 | std::unique_ptr tmpApp(new App(_configure.get())); 32 | _app = std::move(tmpApp); 33 | 34 | // 解析指定的参数 35 | parseOption(argc, argv); 36 | loadConfig(); 37 | // 时区设置 38 | adbase::TimeZone beijing(8*3600, "CST"); 39 | // 初始化 daemon 程序 40 | daemonInit(); 41 | loggerInit(beijing); 42 | _loop = std::shared_ptr(new adbase::EventLoop()); 43 | } 44 | 45 | // }}} 46 | // {{{ void BootStrap::run() 47 | 48 | void BootStrap::run() { 49 | // 创建定时器 50 | TimerContext timerContext; 51 | timerContext.config = _configure.get(); 52 | timerContext.mainEventBase = _loop->getBase(); 53 | _timer = std::shared_ptr(new Timer(&timerContext)); 54 | _timer->init(); 55 | 56 | // 初始化 metric 信息 57 | adbase::metrics::Metrics* metrics = adbase::metrics::Metrics::init(_timer->getTimer()); 58 | 59 | _app->run(); 60 | _app->setTimerContext(&timerContext); 61 | 62 | // 构建 adserver 上下文 63 | // @todo 64 | AdServerContext context; 65 | context.config = _configure.get(); 66 | context.metrics = metrics; 67 | context.mainEventBase = _loop->getBase(); 68 | _app->setAdServerContext(&context); 69 | 70 | _adServer = std::shared_ptr(new AdServer(&context)); 71 | if (_adServer) { 72 | _adServer->run(); 73 | } 74 | 75 | AimsContext aimsContext; 76 | aimsContext.config = _configure.get(); 77 | _app->setAimsContext(&aimsContext); 78 | _aims = std::shared_ptr(new Aims(&aimsContext)); 79 | _app->setAims(_aims); 80 | if (_aims) { 81 | _aims->run(); 82 | } 83 | 84 | _loop->start(); 85 | } 86 | 87 | // }}} 88 | // {{{ void BootStrap::reload() 89 | 90 | void BootStrap::reload() { 91 | try { 92 | loadConfig(); 93 | } catch (...) { 94 | LOG_ERROR << "Reload config: " << _configFile << " is invalid, please check it."; 95 | return; 96 | } 97 | setLoggerLevel(); 98 | _app->reload(); 99 | if (_aims) { 100 | _aims->reload(); 101 | } 102 | } 103 | 104 | // }}} 105 | // {{{ void BootStrap::stop() 106 | 107 | void BootStrap::stop(const int sig) { 108 | LOG_ERROR << "Stop callback sig:" << sig; 109 | remove(_configure->pidFile.c_str()); 110 | if (_aims) { 111 | _aims->stop(); 112 | } 113 | 114 | if (_app) { 115 | _app->stop(); 116 | } 117 | 118 | if (_loop) { 119 | _loop->stop(); 120 | } 121 | } 122 | 123 | // }}} 124 | //{{{ AdbaseConfig* BootStrap::getConfig() 125 | 126 | AdbaseConfig* BootStrap::getConfig() { 127 | std::lock_guard lk(_mut); 128 | return _configure.get(); 129 | } 130 | 131 | //}}} 132 | // {{{ void BootStrap::daemonInit() 133 | 134 | void BootStrap::daemonInit() { 135 | if (_configure->daemon) { 136 | pid_t pid; 137 | pid = fork(); 138 | if (pid < 0) { 139 | LOG_SYSFATAL << "Fork parent process fail."; 140 | } 141 | if (pid > 0) { 142 | exit(EXIT_SUCCESS); 143 | } 144 | setsid(); /* create a new session */ 145 | } 146 | 147 | FILE *fpPidfile; 148 | fpPidfile = fopen(_configure->pidFile.c_str(), "w"); 149 | fprintf(fpPidfile, "%d\n", getpid()); 150 | fclose(fpPidfile); 151 | } 152 | 153 | // }}} 154 | // {{{ void BootStrap::loggerInit() 155 | 156 | void BootStrap::loggerInit(const adbase::TimeZone& tz) { 157 | adbase::Logger::setTimeZone(tz); 158 | if (_configure->isAsync) { // 只有是守护进程开启的时候日志开启异步写入 159 | adbase::Logger::setOutput(std::bind(&BootStrap::asyncLogger, this, 160 | std::placeholders::_1, std::placeholders::_2)); 161 | // 启动异步日志落地 162 | std::string basename = _configure->logsDir + std::string(::basename("aidp")); 163 | _asnclog = std::shared_ptr(new adbase::AsyncLogging(basename, _configure->logRollSize)); 164 | _asnclog->start(); 165 | } 166 | setLoggerLevel(); 167 | } 168 | 169 | // }}} 170 | // {{{ void BootStrap::asyncLogger() 171 | 172 | void BootStrap::asyncLogger(const char* msg, int len) { 173 | if (_asnclog) { 174 | _asnclog->append(msg, static_cast(len)); 175 | } 176 | } 177 | 178 | // }}} 179 | // {{{ void BootStrap::setLoggerLevel() 180 | 181 | void BootStrap::setLoggerLevel() { 182 | switch (_configure->logLevel) { 183 | case 1: 184 | adbase::Logger::setLogLevel(adbase::Logger::TRACE); 185 | break; 186 | case 2: 187 | adbase::Logger::setLogLevel(adbase::Logger::DEBUG); 188 | break; 189 | case 3: 190 | default: 191 | adbase::Logger::setLogLevel(adbase::Logger::INFO); 192 | } 193 | } 194 | 195 | // }}} 196 | //{{{ void BootStrap::loadConfig() 197 | 198 | void BootStrap::loadConfig() { 199 | std::lock_guard lk(_mut); 200 | adbase::IniConfig config = adbase::IniParse::loadFile(_configFile); 201 | 202 | _configure->daemon = config.getOptionBool("system", "daemon"); 203 | _configure->pidFile = config.getOption("system", "pidFile"); 204 | _configure->appid = config.getOptionUint32("system", "appid"); 205 | _configure->macid = config.getOptionUint32("system", "macid"); 206 | 207 | _configure->logsDir = config.getOption("logging", "logsDir"); 208 | _configure->logRollSize = config.getOptionUint32("logging", "logRollSize"); 209 | _configure->logLevel = config.getOptionUint32("logging", "logLevel"); 210 | _configure->isAsync = config.getOptionBool("logging", "isAsync"); 211 | 212 | _configure->isStartMc = config.getOptionBool("adserver", "mc"); 213 | _configure->isStartHead = config.getOptionBool("adserver", "head"); 214 | _configure->isStartHttp = config.getOptionBool("adserver", "http"); 215 | 216 | _configure->httpHost = config.getOption("http", "host"); 217 | _configure->httpPort = config.getOptionUint32("http", "port"); 218 | _configure->httpTimeout = config.getOptionUint32("http", "timeout"); 219 | _configure->httpThreadNum = config.getOptionUint32("http", "threadNum"); 220 | _configure->httpServerName = config.getOption("http", "serverName"); 221 | _configure->httpDefaultController = config.getOption("http", "defaultController"); 222 | _configure->httpDefaultAction = config.getOption("http", "defaultAction"); 223 | _configure->httpAccessLogDir = config.getOption("http", "accessLogDir"); 224 | _configure->httpAccesslogRollSize = config.getOptionUint32("http", "accesslogRollSize"); 225 | 226 | _configure->mcHost = config.getOption("mc", "host"); 227 | _configure->mcPort = config.getOptionUint32("mc", "port"); 228 | _configure->mcServerName = config.getOption("mc", "serverName"); 229 | _configure->mcThreadNum = config.getOptionUint32("mc", "threadNum"); 230 | 231 | _configure->headHost = config.getOption("head", "host"); 232 | _configure->headPort = config.getOptionUint32("head", "port"); 233 | _configure->headServerName = config.getOption("head", "serverName"); 234 | _configure->headThreadNum = config.getOptionUint32("head", "threadNum"); 235 | 236 | _app->loadConfig(config); 237 | } 238 | 239 | //}}} 240 | // {{{ void BootStrap::usage() 241 | 242 | void BootStrap::usage() { 243 | std::cout << "Usage: aidp [options...] " << std::endl; 244 | std::cout << "\t-c: 主配置文件路径" << std::endl; 245 | std::cout << "\t-h: 帮助" << std::endl; 246 | exit(0); 247 | } 248 | 249 | // }}} 250 | // {{{ void BootStrap::printVersion() 251 | 252 | void BootStrap::printVersion() { 253 | std::cout << "VERSION : " << VERSION << std::endl; 254 | std::cout << "GIT SHA1 : " << GIT_SHA1 << std::endl; 255 | std::cout << "GIT DIRTY: " << GIT_DIRTY << std::endl; 256 | std::cout << "BUILD ID : " << BUILD_ID << std::endl; 257 | exit(0); 258 | } 259 | 260 | // }}} 261 | // {{{ void BootStrap::parseOption() 262 | 263 | void BootStrap::parseOption(int argc, char **argv) { 264 | int ch; 265 | while((ch = getopt(argc, argv, "c:hv")) != -1) { 266 | if (ch == 'c') { 267 | _configFile = optarg; 268 | } else if (ch == 'h') { 269 | usage(); 270 | } else if (ch == 'v') { 271 | printVersion(); 272 | } 273 | } 274 | } 275 | 276 | // }}} 277 | -------------------------------------------------------------------------------- /src/BootStrap.hpp: -------------------------------------------------------------------------------- 1 | /// 该程序自动生成，禁止修改 2 | #ifndef AIDP_BOOTSTRAP_HPP_ 3 | #define AIDP_BOOTSTRAP_HPP_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "AdbaseConfig.hpp" 11 | 12 | class App; 13 | class Timer; 14 | class AdServer; 15 | class Aims; 16 | class BootStrap { 17 | public: 18 | BootStrap(); 19 | ~BootStrap(); 20 | void init(int argc, char **argv); 21 | void run(); 22 | void reload(); 23 | void stop(const int sig); 24 | AdbaseConfig* getConfig(); 25 | 26 | private: 27 | std::shared_ptr _configure; 28 | mutable std::mutex _mut; 29 | std::shared_ptr _asnclog; 30 | std::string _configFile; 31 | std::shared_ptr _adServer; 32 | std::shared_ptr _loop; 33 | std::shared_ptr _aims; 34 | std::shared_ptr _timer; 35 | std::unique_ptr _app; 36 | 37 | void daemonInit(); 38 | void asyncLogger(const char* msg, int len); 39 | void loggerInit(const adbase::TimeZone& tz); 40 | void setLoggerLevel(); 41 | void loadConfig(); 42 | void parseOption(int argc, char **argv); 43 | void usage(); 44 | void printVersion(); 45 | }; 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.6) 2 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/src) 3 | INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/src/thridpart) 4 | LINK_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) 5 | 6 | SET(CMAKE_SOURCE_DIR .) 7 | SET(CMAKE_MODULE_PATH ${CMAKE_ROOT}/Modules ${CMAKE_SOURCE_DIR}/cmake/Modules 8 | ${PROJECT_SOURCE_DIR}/Modules) 9 | 10 | configure_file ( 11 | "${PROJECT_SOURCE_DIR}/src/Version.hpp.in" 12 | "${PROJECT_SOURCE_DIR}/src/Version.hpp" 13 | ) 14 | 15 | SET(AIDP_SRC aidp.cpp 16 | BootStrap.cpp 17 | App.cpp 18 | AdServer.cpp 19 | Http.cpp 20 | Http/HttpInterface.cpp 21 | Http/Server.cpp 22 | Http/Index.cpp 23 | McProcessor.cpp 24 | HeadProcessor.cpp 25 | Aims.cpp 26 | Aims/Kafka/ConsumerOut.cpp 27 | Timer.cpp 28 | App/Message.cpp 29 | App/Storage.cpp 30 | App/Metrics.cpp 31 | ) 32 | ADD_EXECUTABLE(aidp ${AIDP_SRC}) 33 | 34 | # adbase 35 | FIND_PACKAGE( libadbase REQUIRED) 36 | MARK_AS_ADVANCED( 37 | LIBADBASE_INCLUDE_DIR 38 | LIBADBASE_LIBRARIES 39 | ) 40 | IF (LIBADBASE_INCLUDE_DIR AND LIBADBASE_LIBRARIES) 41 | MESSAGE(STATUS "Found libadbase libraries") 42 | INCLUDE_DIRECTORIES(${LIBADBASE_INCLUDE_DIR}) 43 | MESSAGE( ${LIBADBASE_LIBRARIES} ) 44 | TARGET_LINK_LIBRARIES(aidp ${LIBADBASE_LIBRARIES} ) 45 | ELSE (LIBADBASE_INCLUDE_DIR AND LIBADBASE_LIBRARIES) 46 | MESSAGE(FATAL_ERROR "Failed to find libadbase libraries") 47 | ENDIF (LIBADBASE_INCLUDE_DIR AND LIBADBASE_LIBRARIES) 48 | 49 | # adbase_kafka 50 | FIND_PACKAGE( libadbase_kafka REQUIRED) 51 | MARK_AS_ADVANCED( 52 | LIBADBASE_KAFKA_INCLUDE_DIR 53 | LIBADBASE_KAFKA_LIBRARIES 54 | ) 55 | IF (LIBADBASE_KAFKA_INCLUDE_DIR AND LIBADBASE_KAFKA_LIBRARIES) 56 | MESSAGE(STATUS "Found libadbase_kafka libraries") 57 | INCLUDE_DIRECTORIES(${LIBADBASE_KAFKA_INCLUDE_DIR}) 58 | MESSAGE( ${LIBADBASE_KAFKA_LIBRARIES} ) 59 | TARGET_LINK_LIBRARIES(aidp ${LIBADBASE_KAFKA_LIBRARIES} ) 60 | ELSE (LIBADBASE_KAFKA_INCLUDE_DIR AND LIBADBASE_KAFKA_LIBRARIES) 61 | MESSAGE(FATAL_ERROR "Failed to find libadbase_kafka libraries") 62 | ENDIF (LIBADBASE_KAFKA_INCLUDE_DIR AND LIBADBASE_KAFKA_LIBRARIES) 63 | 64 | # rdkafka++ 65 | FIND_PACKAGE( librdkafka REQUIRED) 66 | MARK_AS_ADVANCED( 67 | LIBRDKAFKA_INCLUDE_DIR 68 | LIBRDKAFKA_LIBRARIES 69 | ) 70 | IF (LIBRDKAFKA_INCLUDE_DIR AND LIBRDKAFKA_LIBRARIES) 71 | MESSAGE(STATUS "Found librdkafka libraries") 72 | INCLUDE_DIRECTORIES(${LIBRDKAFKA_INCLUDE_DIR}) 73 | MESSAGE( ${LIBRDKAFKA_LIBRARIES} ) 74 | TARGET_LINK_LIBRARIES(aidp ${LIBRDKAFKA_CPP_LIBRARIES} ) 75 | TARGET_LINK_LIBRARIES(aidp ${LIBRDKAFKA_LIBRARIES} ) 76 | ELSE (LIBRDKAFKA_INCLUDE_DIR AND LIBRDKAFKA_LIBRARIES) 77 | MESSAGE(FATAL_ERROR "Failed to find librdkafka libraries") 78 | ENDIF (LIBRDKAFKA_INCLUDE_DIR AND LIBRDKAFKA_LIBRARIES) 79 | 80 | # adbase_lua 81 | FIND_PACKAGE( libadbase_lua REQUIRED) 82 | MARK_AS_ADVANCED( 83 | LIBADBASE_LUA_INCLUDE_DIR 84 | LIBADBASE_LUA_LIBRARIES 85 | ) 86 | IF (LIBADBASE_LUA_INCLUDE_DIR AND LIBADBASE_LUA_LIBRARIES) 87 | MESSAGE(STATUS "Found libadbase_kafka libraries") 88 | INCLUDE_DIRECTORIES(${LIBADBASE_LUA_INCLUDE_DIR}) 89 | MESSAGE( ${LIBADBASE_LUA_LIBRARIES} ) 90 | TARGET_LINK_LIBRARIES(aidp ${LIBADBASE_LUA_LIBRARIES} ) 91 | ELSE (LIBADBASE_LUA_INCLUDE_DIR AND LIBADBASE_LUA_LIBRARIES) 92 | MESSAGE(FATAL_ERROR "Failed to find libadbase_lua libraries") 93 | ENDIF (LIBADBASE_LUA_INCLUDE_DIR AND LIBADBASE_LUA_LIBRARIES) 94 | 95 | # liblua 96 | FIND_PACKAGE( liblua REQUIRED) 97 | MARK_AS_ADVANCED( 98 | LIBLUA_INCLUDE_DIR 99 | LIBLUA_LIBRARIES 100 | ) 101 | IF (LIBLUA_INCLUDE_DIR AND LIBLUA_LIBRARIES) 102 | MESSAGE(STATUS "Found liblua libraries") 103 | INCLUDE_DIRECTORIES(${LIBLUA_INCLUDE_DIR}) 104 | MESSAGE( ${LIBLUA_LIBRARIES} ) 105 | TARGET_LINK_LIBRARIES(aidp ${LIBLUA_LIBRARIES} ) 106 | ELSE (LIBLUA_INCLUDE_DIR AND LIBLUA_LIBRARIES) 107 | MESSAGE(FATAL_ERROR "Failed to find liblua libraries") 108 | ENDIF (LIBLUA_INCLUDE_DIR AND LIBLUA_LIBRARIES) 109 | 110 | # pthread 111 | #FIND_PACKAGE( libpthread REQUIRED) 112 | #MARK_AS_ADVANCED( 113 | #LIBPTHREAD_INCLUDE_DIR 114 | #LIBPTHREAD_LIBRARIES 115 | #) 116 | #IF (LIBPTHREAD_INCLUDE_DIR AND LIBPTHREAD_LIBRARIES) 117 | # MESSAGE(STATUS "Found libpthread libraries") 118 | # INCLUDE_DIRECTORIES(${LIBPTHREAD_INCLUDE_DIR}) 119 | # MESSAGE( ${LIBPTHREAD_LIBRARIES} ) 120 | # TARGET_LINK_LIBRARIES(aidp ${LIBPTHREAD_LIBRARIES} ) 121 | #ELSE (LIBPTHREAD_INCLUDE_DIR AND LIBPTHREAD_LIBRARIES) 122 | # MESSAGE(FATAL_ERROR "Failed to find libpthread libraries") 123 | #ENDIF (LIBPTHREAD_INCLUDE_DIR AND LIBPTHREAD_LIBRARIES) 124 | 125 | TARGET_LINK_LIBRARIES(aidp libevent.a pthread rt dl libz.a) 126 | 127 | INSTALL(TARGETS aidp RUNTIME DESTINATION bin) 128 | -------------------------------------------------------------------------------- /src/HeadProcessor.cpp: -------------------------------------------------------------------------------- 1 | #include "HeadProcessor.hpp" 2 | #include "Version.hpp" 3 | 4 | // {{{ HeadProcessor::HeadProcessor() 5 | 6 | HeadProcessor::HeadProcessor(AdServerContext* context) : 7 | _context(context) { 8 | _configure = _context->config; 9 | } 10 | 11 | // }}} 12 | // {{{ HeadProcessor::~HeadProcessor() 13 | 14 | HeadProcessor::~HeadProcessor() { 15 | } 16 | 17 | // }}} 18 | // {{{ adbase::head::ProtocolBinaryResponseStatus HeadProcessor::readHandler() 19 | 20 | adbase::head::ProtocolBinaryResponseStatus HeadProcessor::readHandler(adbase::head::ProtocolBinaryDataType datatype, 21 | const void *data, 22 | ssize_t datalen, 23 | adbase::head::ProtocolBinaryDataType* resDataType, 24 | adbase::Buffer* responseData) { 25 | (void)data; 26 | (void)datatype; 27 | (void)datalen; 28 | *resDataType = adbase::head::PROTOCOL_BINARY_TYPE_JSON; 29 | // 如下 json 手动拼接为了减少对 json 库的依赖，在实际项目中推荐用 rapidjson 30 | std::unordered_map procs = adbase::procStats(); 31 | procs["version"] = VERSION; 32 | procs["git_sha1"] = GIT_SHA1; 33 | procs["git_dirty"] = GIT_DIRTY; 34 | procs["build_id"] = BUILD_ID; 35 | procs["build_type"] = BUILD_TYPE; 36 | std::string result = "{"; 37 | for (auto &t : procs) { 38 | result +="\"" + t.first + "\":\"" + t.second + "\","; 39 | } 40 | result = adbase::rightTrim(result, ","); 41 | result += "}"; 42 | responseData->append(result); 43 | return adbase::head::PROTOCOL_BINARY_RESPONSE_SUCCESS; 44 | } 45 | 46 | // }}} -------------------------------------------------------------------------------- /src/HeadProcessor.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDP_HEAD_PROCESSOR_HPP_ 2 | #define AIDP_HEAD_PROCESSOR_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "AdbaseConfig.hpp" 8 | 9 | class HeadProcessor { 10 | public: 11 | HeadProcessor(AdServerContext* context); 12 | ~HeadProcessor(); 13 | adbase::head::ProtocolBinaryResponseStatus readHandler(adbase::head::ProtocolBinaryDataType datatype, 14 | const void *data, 15 | ssize_t datalen, 16 | adbase::head::ProtocolBinaryDataType* resDataType, 17 | adbase::Buffer* responseData); 18 | 19 | private: 20 | /// 传输上下文指针 21 | AdServerContext* _context; 22 | AdbaseConfig* _configure; 23 | }; 24 | 25 | #endif -------------------------------------------------------------------------------- /src/Http.cpp: -------------------------------------------------------------------------------- 1 | #include "Http.hpp" 2 | 3 | // {{{ Http::Http() 4 | 5 | Http::Http(AdServerContext* context, adbase::http::Server* http) : 6 | _context(context), 7 | _http(http) { 8 | } 9 | 10 | // }}} 11 | // {{{ Http::~Http() 12 | 13 | Http::~Http() { 14 | ADSERVER_HTTP_STOP(Server); 15 | ADSERVER_HTTP_STOP(Index); 16 | } 17 | 18 | // }}} 19 | // {{{ void Http::addController() 20 | 21 | void Http::addController() { 22 | ADSERVER_HTTP_ADD_CONTROLLER(Server); 23 | ADSERVER_HTTP_ADD_CONTROLLER(Index); 24 | } 25 | 26 | // }}} 27 | // {{{ std::unordered_map Http::rewrite() 28 | 29 | std::unordered_map Http::rewrite() { 30 | std::unordered_map urls; 31 | return urls; 32 | } 33 | 34 | // }}} 35 | -------------------------------------------------------------------------------- /src/Http.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDP_HTTP_HPP_ 2 | #define AIDP_HTTP_HPP_ 3 | 4 | #include "AdbaseConfig.hpp" 5 | #include "Http/Index.hpp" 6 | #include "Http/Server.hpp" 7 | 8 | // {{{ macros 9 | 10 | #ifndef DECLARE_HTTP_CONTROLLER 11 | #define DECLARE_HTTP_CONTROLLER(name) \ 12 | adserver::http:: name* _http##name = nullptr; 13 | #endif 14 | #ifndef ADSERVER_HTTP_ADD_CONTROLLER 15 | #define ADSERVER_HTTP_ADD_CONTROLLER(name) do {\ 16 | _http##name = new adserver::http:: name(_context);\ 17 | _http##name->registerLocation(_http);\ 18 | } while(0) 19 | #endif 20 | #ifndef ADSERVER_HTTP_STOP 21 | #define ADSERVER_HTTP_STOP(name) \ 22 | delete _http##name; 23 | #endif 24 | 25 | // }}} 26 | 27 | class Http { 28 | public: 29 | Http(AdServerContext* context, adbase::http::Server* http); 30 | ~Http(); 31 | void addController(); 32 | std::unordered_map rewrite(); 33 | private: 34 | AdServerContext* _context; 35 | adbase::http::Server* _http; 36 | DECLARE_HTTP_CONTROLLER(Server); 37 | DECLARE_HTTP_CONTROLLER(Index); 38 | 39 | }; 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /src/Http/HttpInterface.cpp: -------------------------------------------------------------------------------- 1 | #include "HttpInterface.hpp" 2 | #include "App.hpp" 3 | 4 | namespace adserver { 5 | namespace http { 6 | // {{{ HttpInterface::HttpInterface() 7 | 8 | HttpInterface::HttpInterface(AdServerContext* context) : 9 | _context(context) { 10 | } 11 | 12 | // }}} 13 | // {{{ HttpInterface::~HttpInterface() 14 | 15 | HttpInterface::~HttpInterface() { 16 | } 17 | 18 | // }}} 19 | // {{{ void HttpInterface::registerLocation() 20 | 21 | void HttpInterface::registerLocation(adbase::http::Server* http) { 22 | (void)http; 23 | } 24 | 25 | // }}} 26 | // {{{ void HttpInterface::responseHeader() 27 | 28 | void HttpInterface::responseHeader(adbase::http::Response* response) { 29 | response->addHeader("Content-Type", "application/json; charset=utf-8"); 30 | response->setHeader("Server", _context->config->httpServerName, true); 31 | response->setHeader("Connection", "keep-alive"); 32 | } 33 | 34 | // }}} 35 | // {{{ void HttpInterface::responseJson() 36 | 37 | void HttpInterface::responseJson(adbase::http::Response* response, const std::string& data, uint32_t ret, const std::string& msg, bool isRawString) { 38 | std::string result = "{\"code\":" + std::to_string(ret) + ","; 39 | result += "\"baseid\":" + std::to_string(_context->app->getSeqId()) + ","; 40 | std::string rawData = data; 41 | if (isRawString) { 42 | rawData = "\"" + rawData + "\""; 43 | } 44 | result += "\"data\":" + rawData + ","; 45 | std::string rawMsg = "\"" + msg + "\"}"; 46 | result += "\"msg\":" + rawMsg; 47 | responseHeader(response); 48 | response->setContent(result); 49 | response->sendReply(); 50 | } 51 | 52 | // }}} 53 | } 54 | } -------------------------------------------------------------------------------- /src/Http/HttpInterface.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDP_HTTPINTERFACE_HPP_ 2 | #define AIDP_HTTPINTERFACE_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "AdbaseConfig.hpp" 8 | 9 | namespace adserver { 10 | namespace http { 11 | // {{{ ADSERVER_HTTP_ADD_API 12 | class SourceFile { 13 | public: 14 | template 15 | inline SourceFile(const char (&arr)[N]) : 16 | _data(arr), 17 | _size(N - 1) { 18 | const char* slash = strrchr(_data, '/'); 19 | if (slash) { 20 | _data = slash + 1; 21 | _size -= static_cast(_data - arr) + 4; // 后缀 .cpp 22 | } 23 | } 24 | 25 | const char* _data; 26 | int _size; 27 | }; 28 | #ifndef ADSERVER_HTTP_ADD_API 29 | #define ADSERVER_HTTP_ADD_API(http, class, api) do { \ 30 | SourceFile file##api(__FILE__);\ 31 | std::string module##api(file##api._data, file##api._size);\ 32 | transform(module##api.begin(), module##api.end(), module##api.begin(), ::tolower);\ 33 | std::string apiPath##api = "/" + module##api + "/"#api;\ 34 | http->registerLocation(apiPath##api, std::bind(& class:: api, this, \ 35 | std::placeholders::_1, \ 36 | std::placeholders::_2, \ 37 | std::placeholders::_3), nullptr); } while (0); 38 | #endif 39 | // }}} 40 | class HttpInterface { 41 | public: 42 | HttpInterface(AdServerContext* context); 43 | virtual ~HttpInterface(); 44 | virtual void registerLocation(adbase::http::Server* http); 45 | 46 | protected: 47 | /// 传输上下文指针 48 | AdServerContext* _context; 49 | void responseHeader(adbase::http::Response* response); 50 | void responseJson(adbase::http::Response* response, const std::string& data, uint32_t ret, const std::string& msg, bool isRawString = false); 51 | }; 52 | } 53 | } 54 | #endif -------------------------------------------------------------------------------- /src/Http/Index.cpp: -------------------------------------------------------------------------------- 1 | #include "Index.hpp" 2 | #include "App/Storage.hpp" 3 | #include "App/Metrics.hpp" 4 | #include 5 | 6 | namespace adserver { 7 | namespace http { 8 | 9 | thread_local std::weak_ptr tmpRequest; 10 | thread_local std::weak_ptr tmpResponse; 11 | 12 | // {{{ Index::Index() 13 | 14 | Index::Index(AdServerContext* context) : 15 | HttpInterface(context) { 16 | } 17 | 18 | // }}} 19 | // {{{ void Index::registerLocation() 20 | 21 | void Index::registerLocation(adbase::http::Server* http) { 22 | ADSERVER_HTTP_ADD_API(http, Index, index) 23 | } 24 | 25 | // }}} 26 | // {{{ void Index::index() 27 | 28 | void Index::index(adbase::http::Request* request, adbase::http::Response* response, void*) { 29 | thread_local adbase::lua::Engine* httpLuaEngine = nullptr; 30 | if (httpLuaEngine == nullptr) { 31 | httpLuaEngine = new adbase::lua::Engine(); 32 | httpLuaEngine->init(); 33 | httpLuaEngine->addSearchPath(_context->config->luaScriptPath, true); 34 | bindLuaClass(httpLuaEngine); 35 | LOG_INFO << "Http Lua engine init...."; 36 | } 37 | 38 | auto sharedRequest = std::shared_ptr(request, [](adbase::http::Request*) { LOG_INFO << "delete Request";}); 39 | auto sharedResponse = std::shared_ptr(response, [](adbase::http::Response*) { LOG_INFO << "delete Response";}); 40 | tmpRequest = sharedRequest; 41 | tmpResponse = sharedResponse; 42 | responseHeader(response); 43 | response->setContent(""); 44 | httpLuaEngine->runFile(_context->config->httpScriptName.c_str()); 45 | //responseJson(response, "{\"msg\": \"hello xxxx adinf\"}", 0, ""); 46 | sharedResponse.reset(); 47 | sharedRequest.reset(); 48 | response->sendReply(); 49 | } 50 | 51 | // }}} 52 | // {{{ void Index::getRequest() 53 | 54 | std::weak_ptr& Index::getRequest() { 55 | LOG_INFO << "return request...."; 56 | return tmpRequest; 57 | } 58 | 59 | // }}} 60 | // {{{ void Index::getResponse() 61 | 62 | std::weak_ptr& Index::getResponse() { 63 | return tmpResponse; 64 | } 65 | 66 | // }}} 67 | // {{{ void Index::bindLuaClass() 68 | 69 | void Index::bindLuaClass(adbase::lua::Engine* engine) { 70 | adbase::lua::BindingClass request("request", "aidp.http", engine->getLuaState()); 71 | adbase::lua::BindingNamespace requestCs = request.getOwnerNamespace(); 72 | typedef std::function()> GetRequest; 73 | GetRequest requestFn = std::bind(&adserver::http::Index::getRequest, this); 74 | requestCs.addMethod("request", requestFn); 75 | 76 | request.addMethod("get_uri", &adbase::http::Request::getUri); 77 | request.addMethod("get_remote_address", &adbase::http::Request::getRemoteAddress); 78 | request.addMethod("get_post_data", &adbase::http::Request::getPostData); 79 | request.addMethod("get_post", &adbase::http::Request::getPost); 80 | request.addMethod("get_query", &adbase::http::Request::getQuery); 81 | request.addMethod("get_header", &adbase::http::Request::getHeader); 82 | request.addMethod("get_location", &adbase::http::Request::getLocation); 83 | request.addMethod("get_method", &adbase::http::Request::getMethod); 84 | request.addConst("METHOD_GET", adbase::http::Request::httpMethod::METHOD_GET); 85 | request.addConst("METHOD_POST", adbase::http::Request::httpMethod::METHOD_POST); 86 | request.addConst("METHOD_OTHER", adbase::http::Request::httpMethod::METHOD_OTHER); 87 | 88 | // bind response 89 | adbase::lua::BindingClass response("response", "aidp.http", engine->getLuaState()); 90 | adbase::lua::BindingNamespace responseCs = response.getOwnerNamespace(); 91 | typedef std::function()> GetResponse; 92 | GetResponse responseFn = std::bind(&adserver::http::Index::getResponse, this); 93 | responseCs.addMethod("response", responseFn); // 构造函数 94 | 95 | response.addMethod("set_header", &adbase::http::Response::setHeader); 96 | response.addMethod("add_header", &adbase::http::Response::addHeader); 97 | response.addMethod("set_content", &adbase::http::Response::setContent); 98 | response.addMethod("set_content", &adbase::http::Response::setContent); 99 | response.addMethod("append_content", &adbase::http::Response::appendContent); 100 | response.addMethod("append_content", &adbase::http::Response::appendContent); 101 | response.addMethod("send_reply", &adbase::http::Response::sendReply); 102 | response.addMethod("get_code", &adbase::http::Response::getCode); 103 | response.addMethod("set_body_size", &adbase::http::Response::getBodySize); 104 | _context->storage->bindClass(engine); 105 | _context->appMetrics->bindClass(engine); 106 | } 107 | 108 | // }}} 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/Http/Index.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDP_HTTP_INDEX_HPP_ 2 | #define AIDP_HTTP_INDEX_HPP_ 3 | 4 | #include 5 | #include "HttpInterface.hpp" 6 | 7 | namespace adserver { 8 | namespace http { 9 | class Index : HttpInterface { 10 | public: 11 | Index(AdServerContext* context); 12 | void registerLocation(adbase::http::Server* http); 13 | void index(adbase::http::Request* request, adbase::http::Response* response, void*); 14 | std::weak_ptr& getRequest(); 15 | std::weak_ptr& getResponse(); 16 | private: 17 | void bindLuaClass(adbase::lua::Engine* engine); 18 | }; 19 | } 20 | } 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /src/Http/Server.cpp: -------------------------------------------------------------------------------- 1 | #include "Server.hpp" 2 | #include "Version.hpp" 3 | 4 | namespace adserver { 5 | namespace http { 6 | // {{{ Server::Server() 7 | 8 | Server::Server(AdServerContext* context) : 9 | HttpInterface(context) { 10 | } 11 | 12 | // }}} 13 | // {{{ void Server::registerLocation() 14 | 15 | void Server::registerLocation(adbase::http::Server* http) { 16 | ADSERVER_HTTP_ADD_API(http, Server, status) 17 | ADSERVER_HTTP_ADD_API(http, Server, metrics) 18 | } 19 | 20 | // }}} 21 | // {{{ void Server::status() 22 | 23 | void Server::status(adbase::http::Request* request, adbase::http::Response* response, void*) { 24 | (void)request; 25 | std::string result; 26 | // 如下 json 手动拼接为了减少对 json 库的依赖，在实际项目中推荐用 rapidjson 27 | std::unordered_map procs = adbase::procStats(); 28 | procs["version"] = VERSION; 29 | procs["git_sha1"] = GIT_SHA1; 30 | procs["git_dirty"] = GIT_DIRTY; 31 | procs["build_id"] = BUILD_ID; 32 | procs["build_type"] = BUILD_TYPE; 33 | std::string system = "{"; 34 | for (auto &t : procs) { 35 | system +="\"" + t.first + "\":\"" + t.second + "\","; 36 | } 37 | system = adbase::rightTrim(system, ","); 38 | 39 | // Metrics 40 | std::unordered_map gauges; 41 | std::unordered_map counters; 42 | std::unordered_map meters; 43 | std::unordered_map histograms; 44 | std::unordered_map timers; 45 | if (_context->metrics != nullptr) { 46 | gauges = _context->metrics->getGauges(); 47 | counters = _context->metrics->getCounter(); 48 | histograms = _context->metrics->getHistograms(); 49 | meters = _context->metrics->getMeters(); 50 | timers = _context->metrics->getTimers(); 51 | } 52 | 53 | std::unordered_map metricItems; 54 | for (auto &t : gauges) { 55 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 56 | std::unordered_map metricTags = name.tags; 57 | if (name.moduleName == "self") { 58 | continue; 59 | } 60 | 61 | metricTags["value"] = std::to_string(t.second); 62 | std::string item = "{"; 63 | for (auto &tag : metricTags) { 64 | item += "\"" + tag.first + "\":\"" + tag.second + "\","; 65 | } 66 | item = adbase::rightTrim(item, ","); 67 | item += "}"; 68 | metricItems[getKey(name.moduleName, name.metricName)] += item + ","; 69 | } 70 | 71 | 72 | for (auto &t : counters) { 73 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 74 | std::unordered_map metricTags = name.tags; 75 | if (name.moduleName == "self") { 76 | continue; 77 | } 78 | 79 | metricTags["value"] = std::to_string(t.second); 80 | std::string item = "{"; 81 | for (auto &tag : metricTags) { 82 | item += "\"" + tag.first + "\":\"" + tag.second + "\","; 83 | } 84 | item = adbase::rightTrim(item, ","); 85 | item += "}"; 86 | 87 | metricItems[getKey(name.moduleName, name.metricName)] += item + ","; 88 | } 89 | 90 | for (auto &t : meters) { 91 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 92 | std::unordered_map metricTags = name.tags; 93 | if (name.moduleName == "self") { 94 | continue; 95 | } 96 | 97 | metricTags["count"] = std::to_string(t.second.count); 98 | metricTags["mean"] = std::to_string(t.second.meanRate); 99 | metricTags["min1"] = std::to_string(t.second.min1Rate); 100 | metricTags["min5"] = std::to_string(t.second.min5Rate); 101 | metricTags["min15"] = std::to_string(t.second.min15Rate); 102 | std::string item = "{"; 103 | for (auto &tag : metricTags) { 104 | item += "\"" + tag.first + "\":\"" + tag.second + "\","; 105 | } 106 | item = adbase::rightTrim(item, ","); 107 | item += "}"; 108 | 109 | metricItems[getKey(name.moduleName, name.metricName)] += item + ","; 110 | } 111 | for (auto &t : histograms) { 112 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 113 | std::unordered_map metricTags = name.tags; 114 | if (name.moduleName == "self") { 115 | continue; 116 | } 117 | 118 | metricTags["min"] = std::to_string(t.second.min); 119 | metricTags["max"] = std::to_string(t.second.max); 120 | metricTags["stddev"] = std::to_string(t.second.stddev); 121 | metricTags["median"] = std::to_string(t.second.median); 122 | metricTags["percent75"] = std::to_string(t.second.percent75); 123 | metricTags["percent95"] = std::to_string(t.second.percent95); 124 | metricTags["percent98"] = std::to_string(t.second.percent98); 125 | metricTags["percent99"] = std::to_string(t.second.percent99); 126 | metricTags["percent999"] = std::to_string(t.second.percent999); 127 | std::string item = "{"; 128 | for (auto &tag : metricTags) { 129 | item += "\"" + tag.first + "\":" + tag.second + ","; 130 | } 131 | item = adbase::rightTrim(item, ","); 132 | item += "}"; 133 | 134 | metricItems[getKey(name.moduleName, name.metricName)] += item + ","; 135 | } 136 | for (auto &t : timers) { 137 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 138 | std::unordered_map metricTags = name.tags; 139 | 140 | metricTags["count"] = std::to_string(t.second.meter.count); 141 | metricTags["mean"] = std::to_string(t.second.meter.meanRate); 142 | metricTags["min1"] = std::to_string(t.second.meter.min1Rate); 143 | metricTags["min5"] = std::to_string(t.second.meter.min5Rate); 144 | metricTags["min15"] = std::to_string(t.second.meter.min15Rate); 145 | metricTags["min"] = std::to_string(t.second.histogram.min); 146 | metricTags["max"] = std::to_string(t.second.histogram.max); 147 | metricTags["stddev"] = std::to_string(t.second.histogram.stddev); 148 | metricTags["median"] = std::to_string(t.second.histogram.median); 149 | metricTags["percent75"] = std::to_string(t.second.histogram.percent75); 150 | metricTags["percent95"] = std::to_string(t.second.histogram.percent95); 151 | metricTags["percent98"] = std::to_string(t.second.histogram.percent98); 152 | metricTags["percent99"] = std::to_string(t.second.histogram.percent99); 153 | metricTags["percent999"] = std::to_string(t.second.histogram.percent999); 154 | std::string item = "{"; 155 | for (auto &tag : metricTags) { 156 | item += "\"" + tag.first + "\":\"" + tag.second + "\","; 157 | } 158 | item = adbase::rightTrim(item, ","); 159 | item += "}"; 160 | 161 | metricItems[getKey(name.moduleName, name.metricName)] += item + ","; 162 | } 163 | 164 | std::unordered_map modulesItems; 165 | modulesItems["system"] = system; 166 | for (auto &t : metricItems) { 167 | std::vector result = adbase::explode(t.first, 26); 168 | if (modulesItems.find(result[0]) == modulesItems.end()) { 169 | modulesItems[result[0]] = "{"; 170 | } 171 | 172 | modulesItems[result[0]] += "\"" + result[1] + "\":[" + adbase::rightTrim(t.second, ",") + "],"; 173 | } 174 | 175 | result = "{"; 176 | for (auto &t : modulesItems) { 177 | result += "\"" + t.first + "\":" + adbase::rightTrim(t.second, ",") + "},"; 178 | } 179 | result = adbase::rightTrim(result, ","); 180 | result += "}"; 181 | 182 | responseJson(response, result, 0, ""); 183 | } 184 | 185 | // }}} 186 | // {{{ void Server::metrics() 187 | 188 | void Server::metrics(adbase::http::Request* request, adbase::http::Response* response, void*) { 189 | (void)request; 190 | std::string result; 191 | std::string tagValue = request->getQuery("tags"); 192 | std::unordered_map tags; 193 | if (!tagValue.empty()) { 194 | std::vector tagStrs = adbase::explode(tagValue, '|', true); 195 | for (auto &t: tagStrs) { 196 | std::vector tagKV = adbase::explode(t, ':', true); 197 | if (tagKV.size() == 2) { 198 | tags[tagKV[0]] = tagKV[1]; 199 | } 200 | } 201 | } 202 | 203 | int count = 0; 204 | std::unordered_map procs = adbase::procStats(); 205 | tags["name"] = adbase::trim(procs["name"], "()"); 206 | 207 | 208 | tags["service"] = request->getServerAddress(); 209 | tags["version"] = VERSION; 210 | 211 | for (auto &t : procs) { 212 | std::string key = adbase::replace(".", "_", t.first, count); 213 | tags["metric_type"] = "gauges"; 214 | result += formatMetric(key, toUint64(t.second), tags); 215 | } 216 | 217 | // Metrics 218 | std::unordered_map gauges; 219 | std::unordered_map counters; 220 | std::unordered_map meters; 221 | std::unordered_map histograms; 222 | std::unordered_map timers; 223 | if (_context->metrics != nullptr) { 224 | gauges = _context->metrics->getGauges(); 225 | counters = _context->metrics->getCounter(); 226 | histograms = _context->metrics->getHistograms(); 227 | meters = _context->metrics->getMeters(); 228 | timers = _context->metrics->getTimers(); 229 | } 230 | 231 | tags["metric_type"] = "gauges"; 232 | for (auto &t : gauges) { 233 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 234 | std::unordered_map metricTags = name.tags; 235 | if (name.moduleName == "self") { 236 | continue; 237 | } 238 | 239 | for (auto &t : tags) { 240 | metricTags[t.first] = t.second; 241 | } 242 | std::string key = name.moduleName + "_" + name.metricName; 243 | key = adbase::replace(".", "_", key, count); 244 | result += formatMetric(key, t.second, metricTags); 245 | } 246 | tags["metric_type"] = "counters"; 247 | for (auto &t : counters) { 248 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 249 | std::unordered_map metricTags = name.tags; 250 | if (name.moduleName == "self") { 251 | continue; 252 | } 253 | std::string key = name.moduleName + "_" + name.metricName; 254 | key = adbase::replace(".", "_", key, count); 255 | for (auto &t : tags) { 256 | metricTags[t.first] = t.second; 257 | } 258 | result += formatMetric(key, t.second, metricTags); 259 | } 260 | 261 | tags["metric_type"] = "meters"; 262 | for (auto &t : meters) { 263 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 264 | std::unordered_map metricTags = name.tags; 265 | if (name.moduleName == "self") { 266 | continue; 267 | } 268 | std::string key = name.moduleName + "_" + name.metricName; 269 | key = adbase::replace(".", "_", key, count); 270 | for (auto &t : tags) { 271 | metricTags[t.first] = t.second; 272 | } 273 | metricTags["meter_type"] = "count"; 274 | result += formatMetric(key, t.second.count, metricTags); 275 | metricTags["meter_type"] = "mean"; 276 | result += formatMetric(key, toUint64(t.second.meanRate), metricTags); 277 | metricTags["meter_type"] = "min1"; 278 | result += formatMetric(key, toUint64(t.second.min1Rate), metricTags); 279 | metricTags["meter_type"] = "min5"; 280 | result += formatMetric(key, toUint64(t.second.min5Rate), metricTags); 281 | metricTags["meter_type"] = "min15"; 282 | result += formatMetric(key, toUint64(t.second.min15Rate), metricTags); 283 | } 284 | 285 | tags["metric_type"] = "histograms"; 286 | for (auto &t : histograms) { 287 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 288 | std::unordered_map metricTags = name.tags; 289 | if (name.moduleName == "self") { 290 | continue; 291 | } 292 | 293 | std::string key = name.moduleName + "_" + name.metricName; 294 | key = adbase::replace(".", "_", key, count); 295 | for (auto &t : tags) { 296 | metricTags[t.first] = t.second; 297 | } 298 | metricTags["histograms_type"] = "min"; 299 | result += formatMetric(key, toUint64(t.second.min), metricTags); 300 | metricTags["histograms_type"] = "max"; 301 | result += formatMetric(key, toUint64(t.second.max), metricTags); 302 | metricTags["histograms_type"] = "mean"; 303 | result += formatMetric(key, toUint64(t.second.mean), metricTags); 304 | metricTags["histograms_type"] = "stddev"; 305 | result += formatMetric(key, toUint64(t.second.stddev), metricTags); 306 | metricTags["histograms_type"] = "median"; 307 | result += formatMetric(key, toUint64(t.second.median), metricTags); 308 | metricTags["histograms_type"] = "percent75"; 309 | result += formatMetric(key, toUint64(t.second.percent75), metricTags); 310 | metricTags["histograms_type"] = "percent95"; 311 | result += formatMetric(key, toUint64(t.second.percent95), metricTags); 312 | metricTags["histograms_type"] = "percent98"; 313 | result += formatMetric(key, toUint64(t.second.percent98), metricTags); 314 | metricTags["histograms_type"] = "percent99"; 315 | result += formatMetric(key, toUint64(t.second.percent99), metricTags); 316 | metricTags["histograms_type"] = "percent999"; 317 | result += formatMetric(key, toUint64(t.second.percent999), metricTags); 318 | } 319 | 320 | tags["metric_type"] = "timers"; 321 | for (auto &t : timers) { 322 | adbase::metrics::MetricName name = adbase::metrics::Metrics::getMetricName(t.first); 323 | std::unordered_map metricTags = name.tags; 324 | 325 | std::string key = name.moduleName + "_" + name.metricName; 326 | key = adbase::replace(".", "_", key, count); 327 | for (auto &t : tags) { 328 | metricTags[t.first] = t.second; 329 | } 330 | metricTags["timers_type"] = "count"; 331 | result += formatMetric(key, t.second.meter.count, metricTags); 332 | metricTags["timers_type"] = "mean"; 333 | result += formatMetric(key, toUint64(t.second.meter.meanRate), metricTags); 334 | metricTags["timers_type"] = "min1"; 335 | result += formatMetric(key, toUint64(t.second.meter.min1Rate), metricTags); 336 | metricTags["timers_type"] = "min5"; 337 | result += formatMetric(key, toUint64(t.second.meter.min5Rate), metricTags); 338 | metricTags["timers_type"] = "min15"; 339 | result += formatMetric(key, toUint64(t.second.meter.min15Rate), metricTags); 340 | metricTags["timers_type"] = "min"; 341 | result += formatMetric(key, toUint64(t.second.histogram.min), metricTags); 342 | metricTags["timers_type"] = "max"; 343 | result += formatMetric(key, toUint64(t.second.histogram.max), metricTags); 344 | metricTags["timers_type"] = "mean"; 345 | result += formatMetric(key, toUint64(t.second.histogram.mean), metricTags); 346 | metricTags["timers_type"] = "stddev"; 347 | result += formatMetric(key, toUint64(t.second.histogram.stddev), metricTags); 348 | metricTags["timers_type"] = "median"; 349 | result += formatMetric(key, toUint64(t.second.histogram.median), metricTags); 350 | metricTags["timers_type"] = "percent75"; 351 | result += formatMetric(key, toUint64(t.second.histogram.percent75), metricTags); 352 | metricTags["timers_type"] = "percent95"; 353 | result += formatMetric(key, toUint64(t.second.histogram.percent95), metricTags); 354 | metricTags["timers_type"] = "percent98"; 355 | result += formatMetric(key, toUint64(t.second.histogram.percent98), metricTags); 356 | metricTags["timers_type"] = "percent99"; 357 | result += formatMetric(key, toUint64(t.second.histogram.percent99), metricTags); 358 | metricTags["timers_type"] = "percent999"; 359 | result += formatMetric(key, toUint64(t.second.histogram.percent999), metricTags); 360 | } 361 | 362 | responseHeader(response); 363 | response->setContent(result); 364 | response->sendReply(); 365 | } 366 | 367 | // }}} 368 | // {{{ const std::string Server::formatMetric() 369 | 370 | const std::string Server::formatMetric(std::string key, uint64_t value, std::unordered_map tags) { 371 | std::string result = "adbase_"; 372 | 373 | int count = 0; 374 | std::string keyFormat = adbase::replace(".", "_", key, count); 375 | result += keyFormat; 376 | if (!tags.empty()) { 377 | result += "{"; 378 | for (auto &t : tags) { 379 | result += t.first; 380 | result += "=\""; 381 | result += t.second; 382 | result += "\","; 383 | } 384 | result = adbase::rightTrim(result, ","); 385 | result += "} "; 386 | } 387 | 388 | result += " "; 389 | result += std::to_string(value) + "\n"; 390 | return result; 391 | } 392 | 393 | // }}} 394 | // {{{ uint64_t Server::toUint64() 395 | 396 | uint64_t Server::toUint64(std::string value) { 397 | errno = 0; 398 | uint64_t result = static_cast(strtoull(value.c_str(), nullptr, 10)); 399 | if (errno != 0) { 400 | return 0; 401 | } 402 | return result; 403 | } 404 | 405 | // }}} 406 | // {{{ uint64_t Server::toUint64() 407 | 408 | uint64_t Server::toUint64(double value) { 409 | return static_cast(value * 10000); 410 | } 411 | 412 | // }}} 413 | // {{{ const std::string Server::getKey() 414 | 415 | const std::string Server::getKey(const std::string& moduleName, const std::string& metricName) { 416 | std::string result = moduleName; 417 | result.append(1, 26); 418 | result.append(metricName); 419 | return result; 420 | } 421 | 422 | // }}} 423 | } 424 | } 425 | -------------------------------------------------------------------------------- /src/Http/Server.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDP_HTTP_SERVER_HPP_ 2 | #define AIDP_HTTP_SERVER_HPP_ 3 | 4 | #include "HttpInterface.hpp" 5 | 6 | namespace adserver { 7 | namespace http { 8 | class Server : HttpInterface { 9 | public: 10 | Server(AdServerContext* context); 11 | void registerLocation(adbase::http::Server* http); 12 | void status(adbase::http::Request* request, adbase::http::Response* response, void*); 13 | void metrics(adbase::http::Request* request, adbase::http::Response* response, void*); 14 | 15 | private: 16 | const std::string formatMetric(std::string key, uint64_t value, std::unordered_map tags); 17 | const std::string getKey(const std::string& moduleName, const std::string& metricName); 18 | uint64_t toUint64(std::string value); 19 | uint64_t toUint64(double value); 20 | }; 21 | } 22 | } 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /src/McProcessor.cpp: -------------------------------------------------------------------------------- 1 | #include "McProcessor.hpp" 2 | #include "App.hpp" 3 | 4 | // {{{ McProcessor::McProcessor() 5 | 6 | McProcessor::McProcessor(AdServerContext* context) : 7 | _context(context) { 8 | _configure = _context->config; 9 | } 10 | 11 | // }}} 12 | // {{{ McProcessor::~McProcessor() 13 | 14 | McProcessor::~McProcessor() { 15 | } 16 | 17 | // }}} 18 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::add() 19 | 20 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::add(const void* key, 21 | uint16_t keylen, 22 | const void *data, 23 | uint32_t datalen, 24 | uint32_t flags, 25 | uint32_t exptime, 26 | uint64_t* cas) { 27 | (void)flags; 28 | (void)exptime; 29 | (void)cas; 30 | std::string keyData(static_cast(key), static_cast(keylen)); 31 | adbase::Buffer bufferBody; 32 | bufferBody.append(static_cast(data), static_cast(datalen)); 33 | LOG_INFO << "Mc ADD key:" << keyData; 34 | LOG_INFO << "Mc ADD data size:" << bufferBody.readableBytes(); 35 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 36 | } 37 | 38 | // }}} 39 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::append() 40 | 41 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::append(const void* key, 42 | uint16_t keylen, 43 | const void *data, 44 | uint32_t datalen, 45 | uint64_t cas, 46 | uint64_t* resultCas) { 47 | (void)cas; 48 | (void)resultCas; 49 | std::string keyData(static_cast(key), static_cast(keylen)); 50 | adbase::Buffer bufferBody; 51 | bufferBody.append(static_cast(data), static_cast(datalen)); 52 | LOG_INFO << "Mc APPEND key:" << keyData; 53 | LOG_INFO << "Mc APPEND data size:" << bufferBody.readableBytes(); 54 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 55 | } 56 | 57 | // }}} 58 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::decrement() 59 | 60 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::decrement(const void* key, 61 | uint16_t keylen, 62 | uint64_t delta, 63 | uint64_t initial, 64 | uint32_t expiration, 65 | uint64_t* result, 66 | uint64_t* resultCas) { 67 | (void)delta; 68 | (void)initial; 69 | (void)expiration; 70 | (void)result; 71 | (void)resultCas; 72 | std::string keyData(static_cast(key), static_cast(keylen)); 73 | LOG_INFO << "Mc DECREMENT key:" << keyData; 74 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 75 | } 76 | 77 | // }}} 78 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::deleteOp() 79 | 80 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::deleteOp(const void* key, 81 | uint16_t keylen, 82 | uint64_t cas) { 83 | (void)cas; 84 | std::string keyData(static_cast(key), static_cast(keylen)); 85 | LOG_INFO << "Mc DELETE key:" << keyData; 86 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 87 | } 88 | 89 | // }}} 90 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::flush() 91 | 92 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::flush(uint32_t when) { 93 | LOG_INFO << "Mc FLUSH when:" << when; 94 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 95 | } 96 | 97 | // }}} 98 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::get() 99 | 100 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::get(const void* key, 101 | uint16_t keylen, 102 | adbase::Buffer *data) { 103 | std::string keyData(static_cast(key), static_cast(keylen)); 104 | data->append(static_cast(key), static_cast(keylen)); 105 | LOG_INFO << "Mc GET key:" << keyData; 106 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 107 | } 108 | 109 | // }}} 110 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::increment() 111 | 112 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::increment(const void* key, 113 | uint16_t keylen, 114 | uint64_t delta, 115 | uint64_t initial, 116 | uint32_t expiration, 117 | uint64_t* result, 118 | uint64_t* resultCas) { 119 | (void)delta; 120 | (void)initial; 121 | (void)expiration; 122 | (void)result; 123 | (void)resultCas; 124 | std::string keyData(static_cast(key), static_cast(keylen)); 125 | LOG_INFO << "Mc INCREMENT key:" << keyData; 126 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 127 | } 128 | 129 | // }}} 130 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::prepend() 131 | 132 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::prepend(const void* key, 133 | uint16_t keylen, 134 | const void *data, 135 | uint32_t datalen, 136 | uint64_t cas, 137 | uint64_t* resultCas) { 138 | (void)cas; 139 | (void)resultCas; 140 | std::string keyData(static_cast(key), static_cast(keylen)); 141 | adbase::Buffer bufferBody; 142 | bufferBody.append(static_cast(data), static_cast(datalen)); 143 | LOG_INFO << "Mc PREPEND key:" << keyData; 144 | LOG_INFO << "Mc PREPEND data size:" << bufferBody.readableBytes(); 145 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 146 | } 147 | 148 | // }}} 149 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::noop() 150 | 151 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::noop() { 152 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 153 | } 154 | 155 | // }}} 156 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::quit() 157 | 158 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::quit() { 159 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 160 | } 161 | 162 | // }}} 163 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::replace() 164 | 165 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::replace(const void* key, 166 | uint16_t keylen, 167 | const void *data, 168 | uint32_t datalen, 169 | uint32_t flags, 170 | uint32_t exptime, 171 | uint64_t cas, 172 | uint64_t* resultCas) { 173 | (void)flags; 174 | (void)exptime; 175 | (void)cas; 176 | (void)resultCas; 177 | std::string keyData(static_cast(key), static_cast(keylen)); 178 | adbase::Buffer bufferBody; 179 | bufferBody.append(static_cast(data), static_cast(datalen)); 180 | LOG_INFO << "Mc REPLACE key:" << keyData; 181 | LOG_INFO << "Mc REPLACE data size:" << bufferBody.readableBytes(); 182 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 183 | } 184 | 185 | // }}} 186 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::set() 187 | 188 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::set(const void* key, 189 | uint16_t keylen, 190 | const void *data, 191 | uint32_t datalen, 192 | uint32_t flags, 193 | uint32_t exptime, 194 | uint64_t cas, 195 | uint64_t* resultCas) { 196 | (void)flags; 197 | (void)exptime; 198 | (void)cas; 199 | (void)resultCas; 200 | std::string keyData(static_cast(key), static_cast(keylen)); 201 | adbase::Buffer bufferBody; 202 | bufferBody.append(static_cast(data), static_cast(datalen)); 203 | LOG_INFO << "Mc SET key:" << keyData; 204 | LOG_INFO << "Mc SET data size:" << bufferBody.readableBytes(); 205 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 206 | } 207 | 208 | // }}} 209 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::stat() 210 | 211 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::stat(const void* key, 212 | uint16_t keylen, 213 | adbase::Buffer *data) { 214 | std::string keyData(static_cast(key), static_cast(keylen)); 215 | data->append(static_cast(key), static_cast(keylen)); 216 | LOG_INFO << "Mc STAT key:" << keyData; 217 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 218 | } 219 | 220 | // }}} 221 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::version() 222 | 223 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::version(adbase::Buffer *data) { 224 | data->append("0.1.0"); 225 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 226 | } 227 | 228 | // }}} 229 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::verbosity() 230 | 231 | adbase::mc::ProtocolBinaryResponseStatus McProcessor::verbosity(uint32_t verbose) { 232 | LOG_INFO << "Mc FLUSH verbose:" << verbose; 233 | return adbase::mc::PROTOCOL_BINARY_RESPONSE_SUCCESS; 234 | } 235 | 236 | // }}} 237 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::preExecute() 238 | 239 | void McProcessor::preExecute() { 240 | LOG_TRACE << "CMD exec pre."; 241 | } 242 | 243 | // }}} 244 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::postExecute() 245 | 246 | void McProcessor::postExecute() { 247 | LOG_TRACE << "CMD exec post."; 248 | } 249 | 250 | // }}} 251 | // {{{ adbase::mc::ProtocolBinaryResponseStatus McProcessor::unknownExecute() 252 | 253 | void McProcessor::unknownExecute() { 254 | LOG_TRACE << "CMD exec unknown."; 255 | } 256 | 257 | // }}} -------------------------------------------------------------------------------- /src/McProcessor.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDP_MC_PROCESSOR_HPP_ 2 | #define AIDP_MC_PROCESSOR_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "AdbaseConfig.hpp" 8 | 9 | class McProcessor { 10 | public: 11 | McProcessor(AdServerContext* context); 12 | ~McProcessor(); 13 | adbase::mc::ProtocolBinaryResponseStatus add(const void* key, 14 | uint16_t keylen, 15 | const void *data, 16 | uint32_t datalen, 17 | uint32_t flags, 18 | uint32_t exptime, 19 | uint64_t* cas); 20 | adbase::mc::ProtocolBinaryResponseStatus append(const void* key, 21 | uint16_t keylen, 22 | const void *data, 23 | uint32_t datalen, 24 | uint64_t cas, 25 | uint64_t* resultCas); 26 | adbase::mc::ProtocolBinaryResponseStatus decrement(const void* key, 27 | uint16_t keylen, 28 | uint64_t delta, 29 | uint64_t initial, 30 | uint32_t expiration, 31 | uint64_t* result, 32 | uint64_t* resultCas); 33 | adbase::mc::ProtocolBinaryResponseStatus deleteOp(const void* key, 34 | uint16_t keylen, 35 | uint64_t cas); 36 | adbase::mc::ProtocolBinaryResponseStatus flush(uint32_t when); 37 | adbase::mc::ProtocolBinaryResponseStatus get(const void* key, 38 | uint16_t keylen, 39 | adbase::Buffer *data); 40 | adbase::mc::ProtocolBinaryResponseStatus increment(const void* key, 41 | uint16_t keylen, 42 | uint64_t delta, 43 | uint64_t initial, 44 | uint32_t expiration, 45 | uint64_t* result, 46 | uint64_t* resultCas); 47 | adbase::mc::ProtocolBinaryResponseStatus prepend(const void* key, 48 | uint16_t keylen, 49 | const void *data, 50 | uint32_t datalen, 51 | uint64_t cas, 52 | uint64_t* resultCas); 53 | adbase::mc::ProtocolBinaryResponseStatus noop(); 54 | adbase::mc::ProtocolBinaryResponseStatus quit(); 55 | adbase::mc::ProtocolBinaryResponseStatus replace(const void* key, 56 | uint16_t keylen, 57 | const void *data, 58 | uint32_t datalen, 59 | uint32_t flags, 60 | uint32_t exptime, 61 | uint64_t cas, 62 | uint64_t* resultCas); 63 | adbase::mc::ProtocolBinaryResponseStatus set(const void* key, 64 | uint16_t keylen, 65 | const void *data, 66 | uint32_t datalen, 67 | uint32_t flags, 68 | uint32_t exptime, 69 | uint64_t cas, 70 | uint64_t* resultCas); 71 | adbase::mc::ProtocolBinaryResponseStatus stat(const void* key, 72 | uint16_t keylen, 73 | adbase::Buffer *data); 74 | adbase::mc::ProtocolBinaryResponseStatus version(adbase::Buffer *data); 75 | adbase::mc::ProtocolBinaryResponseStatus verbosity(uint32_t); 76 | void preExecute(); 77 | void postExecute(); 78 | void unknownExecute(); 79 | 80 | private: 81 | /// 传输上下文指针 82 | AdServerContext* _context; 83 | AdbaseConfig* _configure; 84 | }; 85 | 86 | #endif -------------------------------------------------------------------------------- /src/Timer.cpp: -------------------------------------------------------------------------------- 1 | #include "Timer.hpp" 2 | #include "App.hpp" 3 | 4 | // {{{ macros 5 | 6 | #define ADD_AFTER_TIMER(interval, name) do { \ 7 | uint64_t timerId##name = _timer->runAfter((interval),\ 8 | std::bind(&Timer:: name, this, std::placeholders::_1), nullptr);\ 9 | _timerIds.push_back(timerId##name);\ 10 | } while (0); 11 | #define ADD_EVERY_TIMER(interval, name) do { \ 12 | uint64_t timerId##name = _timer->runEvery((interval),\ 13 | std::bind(&Timer:: name, this, std::placeholders::_1), nullptr);\ 14 | _timerIds.push_back(timerId##name);\ 15 | } while (0); 16 | 17 | // }}} 18 | // {{{ Timer::Timer() 19 | 20 | Timer::Timer(TimerContext* context) : 21 | _context(context) { 22 | _configure = _context->config; 23 | _timer = new adbase::Timer(_context->mainEventBase); 24 | } 25 | 26 | // }}} 27 | // {{{ adbase::Timer* Timer::getTimer() 28 | 29 | adbase::Timer* Timer::getTimer() { 30 | return _timer; 31 | } 32 | 33 | // }}} 34 | // {{{ Timer::~Timer() 35 | 36 | Timer::~Timer() { 37 | for (auto &t : _timerIds) { 38 | _timer->cancel(t); 39 | } 40 | delete _timer; 41 | } 42 | 43 | // }}} 44 | // {{{ void Timer::init() 45 | 46 | void Timer::init() { 47 | /// 间隔一段时间执行一次 48 | //ADD_EVERY_TIMER(1 * 1000, one); 49 | /// 一段时间后仅执行一次 50 | //ADD_AFTER_TIMER(1 * 1000, one); 51 | ADD_EVERY_TIMER(_configure->intervalClearStorage, clearStorage); 52 | ADD_EVERY_TIMER(10000, checkMessageQueue); 53 | ADD_EVERY_TIMER(_configure->mallocTrimInterval, mallocTrim); 54 | } 55 | 56 | // }}} 57 | // {{{ void Timer::clearStorage() 58 | 59 | void Timer::clearStorage(void*) { 60 | if (_context->storage != nullptr) { 61 | _context->storage->clear(); 62 | } 63 | } 64 | 65 | // }}} 66 | // {{{ void Timer::checkMessageQueue() 67 | 68 | void Timer::checkMessageQueue(void*) { 69 | if (_context->app != nullptr) { 70 | _context->app->checkQueue(); 71 | } 72 | } 73 | 74 | // }}} 75 | // {{{ void Timer::mallocTrim() 76 | 77 | void Timer::mallocTrim(void*) { 78 | adbase::mallocTrim(_configure->mallocTrimPad); 79 | } 80 | 81 | // }}} 82 | -------------------------------------------------------------------------------- /src/Timer.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AIDP_TIMER_HPP_ 2 | #define AIDP_TIMER_HPP_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "AdbaseConfig.hpp" 8 | 9 | class Timer { 10 | public: 11 | Timer(TimerContext* context); 12 | ~Timer(); 13 | void init(); 14 | adbase::Timer* getTimer(); 15 | void clearStorage(void* args); 16 | void checkMessageQueue(void* args); 17 | void mallocTrim(void* args); 18 | 19 | private: 20 | /// 传输上下文指针 21 | TimerContext* _context; 22 | AdbaseConfig* _configure; 23 | adbase::Timer* _timer; 24 | std::vector _timerIds; 25 | }; 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /src/Version.hpp: -------------------------------------------------------------------------------- 1 | /// 该程序自动生成，禁止修改 2 | #ifndef AIDP_VERSION_HPP_ 3 | #define AIDP_VERSION_HPP_ 4 | #define VERSION "1.0.0" 5 | #define SOVERSION "1" 6 | #define GIT_DIRTY "758" 7 | #define GIT_SHA1 "a3a4ad7c" 8 | #define BUILD_ID "bpdev-1503035195" 9 | #define BUILD_TYPE "Release" 10 | #endif 11 | -------------------------------------------------------------------------------- /src/Version.hpp.in: -------------------------------------------------------------------------------- 1 | /// 该程序自动生成，禁止修改 2 | #ifndef AIDP_VERSION_HPP_ 3 | #define AIDP_VERSION_HPP_ 4 | #define VERSION "@PROJECT_VERSION_MAJOR@.@PROJECT_VERSION_MINOR@.@PROJECT_VERSION_PATCH@" 5 | #define SOVERSION "${PROJECT_VERSION_MAJOR}" 6 | #define GIT_DIRTY "${GIT_DIRTY}" 7 | #define GIT_SHA1 "${GIT_SHA1}" 8 | #define BUILD_ID "${BUILD_ID}" 9 | #define BUILD_TYPE "${CMAKE_BUILD_TYPE}" 10 | #endif 11 | -------------------------------------------------------------------------------- /src/aidp.cpp: -------------------------------------------------------------------------------- 1 | /// 该程序自动生成，禁止修改 2 | #include "BootStrap.hpp" 3 | // {{{ global 4 | 5 | // 引导程序 6 | BootStrap* bootMain; 7 | 8 | // }}} 9 | // {{{ static void killSignal() 10 | 11 | static void killSignal(const int sig) { 12 | if (bootMain != nullptr) { 13 | bootMain->stop(sig); 14 | delete bootMain; 15 | } 16 | } 17 | 18 | // }}} 19 | // {{{ static void reloadConf() 20 | 21 | static void reloadConf(const int sig) { 22 | (void)sig; 23 | LOG_ERROR << "Start reload config..."; 24 | bootMain->reload(); 25 | } 26 | 27 | // }}} 28 | // {{{ static void registerSignal() 29 | 30 | static void registerSignal() { 31 | /* 忽略Broken Pipe信号 */ 32 | signal(SIGPIPE, SIG_IGN); 33 | /* 处理kill信号 */ 34 | signal(SIGINT, killSignal); 35 | signal(SIGKILL, killSignal); 36 | signal(SIGQUIT, killSignal); 37 | signal(SIGTERM, killSignal); 38 | signal(SIGHUP, killSignal); 39 | signal(SIGSEGV, killSignal); 40 | signal(SIGUSR1, reloadConf); 41 | } 42 | 43 | // }}} 44 | // {{{ int main() 45 | 46 | int main(int argc, char **argv) { 47 | try { 48 | // 初始化 BootStrap 49 | bootMain = new BootStrap(); 50 | bootMain->init(argc, argv); 51 | // 注册信号处理 52 | registerSignal(); 53 | // 启动运行 54 | bootMain->run(); 55 | } catch (std::exception &e) { 56 | LOG_SYSFATAL << "Main exception: " << e.what(); 57 | } catch (...) { 58 | LOG_SYSFATAL << "System exception."; 59 | } 60 | 61 | return 0; 62 | } 63 | 64 | // }}} 65 | --------------------------------------------------------------------------------