├── README.md
├── async_http_post
├── async_http_post.pro
├── async_http_post.pro.user
└── main.cpp
└── build-async-http-post-Debug
├── Makefile
└── default.conf
/README.md:
--------------------------------------------------------------------------------
1 | # async-http-post
2 | 多线程异步http post数据发送程序，使用redis做队列，可接收php/java等其他语言的异步任务。
3 |
4 | ## 使用场景
5 |
6 | 例如：用户信息的修改后需将修改后的用户信息同步到N个不同机房的服务器
7 | 异步模式：用户修改信息接口将要发送的数据压入redis队列，async-http-post在后台监听redis队列，当有消息时，按队列顺序多线程发送数据至N个不同机房的服务器。
8 |
9 | ## 特点
10 |
11 | - 消息必达：当接收的服务器中有任何一个未返回http状态码200时，消息重新压入队列，等待下一次重试。
12 | - 消息不丢失：没一次待发送的数据都有缓存到本地，即便进程被kill，下次启动程序数据将从缓存重新压入队列。
13 | - 多线程：每一个url将启动一个单独的线程来处理，性能更高。
14 | - 多进程：支持多个进程同时运行，当数据量太大时，只需多运行几次，产生多个进程即可。
15 | - 自定义配置文件：每个进程可指定一个配置文件，可通过不同的配置文件监听不同的队列与发送至不同的服务器。
16 |
17 | ## 依赖库 (libraries)
18 | 使用本人开发的C++ 敏捷开发框架开发 [EasyCpp](https://github.com/onanying/easycpp)
19 |
20 | ## 安装 (install)
21 |
22 | ```
23 | # make
24 | ```
25 |
26 | ## 运行 (run)
27 |
28 | ```
29 | # ./async_http_post default.conf
30 | ```
31 |
32 | ## 配置 (config)
33 |
34 | > 每个配置文件会生成一个 .DATA 的数据文件夹，里面有log与cache两个文件夹
35 |
36 | redis_list_key: redis内list类型的key (“queue_user_token” 实际存储key为 “list:queue_user_token”)
37 | http_timeout: http请求的超时时间，单位:秒
38 | post_urls: 接收数据的url清单，POST请求
39 |
40 | ```
41 | {
42 |
43 | "redis_host" : "127.0.0.1",
44 |
45 | "redis_port" : 6379,
46 |
47 | "redis_auth" : "123456",
48 |
49 | "redis_list_key" : "queue_user_token",
50 |
51 | "http_timeout" : 30,
52 |
53 | "post_urls" : [
54 | {"url" : "http://114.119.4.115:8888/test.php"},
55 | {"url" : "http://114.119.4.115:8888/test.php"}
56 | ]
57 |
58 | }
59 | ```
60 |
61 | ## 日志 (log)
62 |
63 | 日志在log目录里，按日期每天生成一个日志文件，请定期手动清理日志。
64 | 每次http请求与响应占一行记录，error.log记录错误日志，info.log记录所有日志。
65 |
66 | ## 缓存 (cache)
67 |
68 | 缓存在cache目录里，程序会自动管理，请勿删除，否则会丢失数据。
69 |
70 |
71 |
72 |
73 |
--------------------------------------------------------------------------------
/async_http_post/async_http_post.pro:
--------------------------------------------------------------------------------
1 | #-------------------------------------------------
2 | #
3 | # Project created by QtCreator 2016-12-15T15:05:04
4 | #
5 | #-------------------------------------------------
6 |
7 | QT += core
8 |
9 | QT -= gui
10 |
11 | TARGET = async_http_post
12 | CONFIG += console
13 | CONFIG -= app_bundle
14 |
15 | TEMPLATE = app
16 |
17 |
18 | SOURCES += main.cpp
19 |
20 | INCLUDEPATH += /usr/local/include/
21 |
22 | LIBS += -lboost_system -lboost_thread -lboost_filesystem -lhiredis -levent -leasycpp
23 |
--------------------------------------------------------------------------------
/async_http_post/async_http_post.pro.user:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | ProjectExplorer.Project.ActiveTarget
7 | 0
8 |
9 |
10 | ProjectExplorer.Project.EditorSettings
11 |
12 | true
13 | false
14 | true
15 |
16 | Cpp
17 |
18 | CppGlobal
19 |
20 |
21 |
22 | QmlJS
23 |
24 | QmlJSGlobal
25 |
26 |
27 | 2
28 | UTF-8
29 | false
30 | 4
31 | false
32 | true
33 | 1
34 | true
35 | 0
36 | true
37 | 0
38 | 8
39 | true
40 | 1
41 | true
42 | true
43 | true
44 | false
45 |
46 |
47 |
48 | ProjectExplorer.Project.PluginSettings
49 |
50 |
51 |
52 | ProjectExplorer.Project.Target.0
53 |
54 | 桌面
55 | 桌面
56 | {1e7fc4e4-8378-4d85-a7e3-17ef5899eac2}
57 | 0
58 | 0
59 | 0
60 |
61 |
62 |
63 | true
64 | qmake
65 |
66 | QtProjectManager.QMakeBuildStep
67 | false
68 | true
69 |
70 | false
71 |
72 |
73 | true
74 | Make
75 |
76 | Qt4ProjectManager.MakeStep
77 |
78 | -w
79 | -r
80 |
81 | false
82 |
83 |
84 |
85 | 2
86 | 构建
87 |
88 | ProjectExplorer.BuildSteps.Build
89 |
90 |
91 |
92 | true
93 | Make
94 |
95 | Qt4ProjectManager.MakeStep
96 |
97 | -w
98 | -r
99 |
100 | true
101 | clean
102 |
103 |
104 | 1
105 | 清理
106 |
107 | ProjectExplorer.BuildSteps.Clean
108 |
109 | 2
110 | false
111 |
112 | Debug
113 |
114 | Qt4ProjectManager.Qt4BuildConfiguration
115 | 2
116 | /home/github/async-http-post/build-async-http-post-Debug
117 | true
118 |
119 |
120 |
121 |
122 | true
123 | qmake
124 |
125 | QtProjectManager.QMakeBuildStep
126 | false
127 | true
128 |
129 | false
130 |
131 |
132 | true
133 | Make
134 |
135 | Qt4ProjectManager.MakeStep
136 |
137 | -w
138 | -r
139 |
140 | false
141 |
142 |
143 |
144 | 2
145 | 构建
146 |
147 | ProjectExplorer.BuildSteps.Build
148 |
149 |
150 |
151 | true
152 | Make
153 |
154 | Qt4ProjectManager.MakeStep
155 |
156 | -w
157 | -r
158 |
159 | true
160 | clean
161 |
162 |
163 | 1
164 | 清理
165 |
166 | ProjectExplorer.BuildSteps.Clean
167 |
168 | 2
169 | false
170 |
171 | Release
172 |
173 | Qt4ProjectManager.Qt4BuildConfiguration
174 | 0
175 | /home/github/async-http-post/build-Release
176 | true
177 |
178 | 2
179 |
180 |
181 | 0
182 | 部署
183 |
184 | ProjectExplorer.BuildSteps.Deploy
185 |
186 | 1
187 | 在本地部署
188 |
189 | ProjectExplorer.DefaultDeployConfiguration
190 |
191 | 1
192 |
193 |
194 | true
195 |
196 | false
197 | false
198 | false
199 | false
200 | true
201 | 0.01
202 | 10
203 | true
204 | 25
205 |
206 | true
207 | valgrind
208 |
209 | 0
210 | 1
211 | 2
212 | 3
213 | 4
214 | 5
215 | 6
216 | 7
217 | 8
218 | 9
219 | 10
220 | 11
221 | 12
222 | 13
223 | 14
224 |
225 | 2
226 |
227 | async_http_post
228 |
229 | Qt4ProjectManager.Qt4RunConfiguration:/home/github/async-http-post/async_http_post/async_http_post.pro
230 |
231 | async_http_post.pro
232 | false
233 | true
234 |
235 | 3768
236 | true
237 | false
238 | false
239 | false
240 | true
241 |
242 | 1
243 |
244 |
245 |
246 | ProjectExplorer.Project.TargetCount
247 | 1
248 |
249 |
250 | ProjectExplorer.Project.Updater.EnvironmentId
251 | {9f863264-3678-4123-b9ff-8977c67f50db}
252 |
253 |
254 | ProjectExplorer.Project.Updater.FileVersion
255 | 14
256 |
257 |
258 |
--------------------------------------------------------------------------------
/async_http_post/main.cpp:
--------------------------------------------------------------------------------
1 | #include
2 | #include
3 |
4 | #include
5 | #include
6 | #include
7 |
8 | #include
9 | #include
10 | #include
11 | #include
12 | #include
13 | #include
14 | #include
15 |
16 | using namespace std;
17 | using namespace easycpp;
18 |
19 | /// 变量定义
20 | string redisHost;
21 | int redisPort;
22 | string redisAuth;
23 | string redisListKey;
24 | vector postUrls;
25 | int httpTimeout;
26 | string nowPid;
27 | string cacheDirPath;
28 | string logDirPath;
29 |
30 | /// 获取配置文件
31 | void init_config(string file)
32 | {
33 | string confStr = helpers::file_get_contents(file);
34 | if(confStr == ""){
35 | string msg = "配置文件 " + file + " 读取错误";
36 | throw libraries::Exception(msg);
37 | }
38 | try{
39 | libraries::JsonObject jsonObj = helpers::json_init(confStr);
40 | redisHost = helpers::json_get_string(jsonObj, "redis_host");
41 | redisPort = helpers::json_get_int(jsonObj, "redis_port");
42 | redisAuth = helpers::json_get_string(jsonObj, "redis_auth");
43 | redisListKey = helpers::json_get_string(jsonObj, "redis_list_key");
44 | postUrls = helpers::json_get_array(jsonObj, "post_urls");
45 | httpTimeout = helpers::json_get_int(jsonObj, "http_timeout");
46 | } catch (exception& /* ex */) {
47 | string msg = "配置文件 " + file + " 参数错误";
48 | throw libraries::Exception(msg);
49 | }
50 | }
51 |
52 | /// post数据
53 | void post(string url, libraries::JsonObject params, string paramsStr, int * postErrCode)
54 | {
55 | string response;
56 | int errCode = helpers::http_post(url, params, response, httpTimeout);
57 |
58 | // 写入log文件
59 | string msg = url + " | " + paramsStr + " | " + response;
60 | helpers::log_info("http_post", msg, logDirPath);
61 |
62 | if (errCode == 200) {
63 | // 成功
64 | *postErrCode = 200;
65 | } else {
66 | // 失败
67 | // 写入log文件
68 | string msg = url + " | " + paramsStr + " | " + response;
69 | helpers::log_error("post", msg, logDirPath);
70 | }
71 | }
72 |
73 | /// 启动
74 | void start()
75 | {
76 | vector postThread;
77 | vector postErrCode;
78 |
79 | try {
80 | models::RedisModel rs(redisHost, redisPort, redisAuth);
81 | // 将当前进程的缓存重新压入队列
82 | string cacheStr = helpers::file_get_contents(cacheDirPath + "/" + nowPid);
83 | if(!cacheStr.empty()){
84 | rs.pushList(redisListKey, cacheStr); // 数据重新压入队列
85 | }
86 | // 从redis取出数据
87 | while (true) {
88 | string paramsStr = rs.pullList(redisListKey, 3600); // 堵塞1小时
89 | if(!paramsStr.empty()){
90 | // 存入缓存
91 | helpers::file_put_contents(cacheDirPath + "/" + nowPid, paramsStr, FILE_REPLACE);
92 | // 转换为json
93 | libraries::JsonObject params = helpers::json_init(paramsStr);
94 | // 循环发送给push_list里面的url
95 | BOOST_FOREACH (libraries::JsonObject &item, postUrls) {
96 | string url = helpers::json_get_string(item, "url");
97 | // 启动多线程
98 | postErrCode.push_back(new int(-3));
99 | postThread.push_back(new boost::thread(boost::bind(&post, url, params, paramsStr, postErrCode.back())));
100 | }
101 | // 等待全部线程完成
102 | BOOST_FOREACH (boost::thread *item, postThread) {
103 | item->join();
104 | }
105 | // 检查执行结果
106 | bool allPostState = true;
107 | BOOST_FOREACH (int *value, postErrCode) {
108 | if(*value != 200){
109 | allPostState = false;
110 | }
111 | }
112 | // 释放内存
113 | BOOST_FOREACH (boost::thread *item, postThread) {
114 | delete item;
115 | }
116 | BOOST_FOREACH (int *value, postErrCode) {
117 | delete value;
118 | }
119 | // 清空容器
120 | postThread.clear();
121 | postErrCode.clear();
122 | // post失败
123 | if(!allPostState){
124 | rs.pushList(redisListKey, paramsStr); // 数据重新压入队列
125 | }
126 | }
127 | }
128 | } catch (exception& ex) {
129 | // 将错误写入log文件
130 | helpers::log_error("start", ex.what(), logDirPath);
131 | // 出错后堵塞一段时间
132 | sleep(60);
133 | }
134 | }
135 |
136 | /// 读取所有缓存
137 | void read_all_cache()
138 | {
139 | // 将所有缓存重新压入队列
140 | models::RedisModel rs(redisHost, redisPort, redisAuth);
141 | vector filenames = helpers::readdir(cacheDirPath);
142 | BOOST_FOREACH (string item, filenames) {
143 | // 读取缓存
144 | string filename = string(item.c_str());
145 | string cacheStr = helpers::file_get_contents(filename);
146 | if(!cacheStr.empty()){
147 | rs.pushList(redisListKey, cacheStr); // 数据重新压入队列
148 | }
149 | // 删除缓存
150 | boost::filesystem::remove(filename);
151 | }
152 | }
153 |
154 | int main(int argc, char *argv[])
155 | {
156 | // fork新进程， 使终端可关闭
157 | int pid = fork();
158 | if(pid == -1){
159 | // fork出错
160 | cout << "程序fork进程出错" << endl;
161 | return 1;
162 | }
163 | if(pid > 0){
164 | // 结束父进程
165 | cout << "程序在进程 " << pid << " 启动成功" << endl;
166 | return 0;
167 | }
168 |
169 | // 初始化参数
170 | try {
171 | init_config(argc >= 2 ? argv[1] : "default.conf");
172 | } catch (exception& ex) {
173 | cout << "INIT CONFIG ERROR: " << ex.what() << endl;
174 | return 1;
175 | }
176 |
177 | // 获取当前进程id
178 | nowPid = helpers::strval(getpid());
179 |
180 | // 配置目录
181 | string configFileName = argc >= 2 ? argv[1] : "default.conf";
182 | configFileName += ".DATA";
183 | cacheDirPath = configFileName + "/cache";
184 | logDirPath = configFileName + "/log";
185 |
186 | // 读取所有缓存
187 | try {
188 | read_all_cache();
189 | } catch (exception& ex) {
190 | cout << "READ ALL CACHE ERROR: " << ex.what() << endl;
191 | return 2;
192 | }
193 |
194 | // 循环启动
195 | while (true) {
196 | start();
197 | }
198 |
199 | return 0;
200 | }
201 |
--------------------------------------------------------------------------------
/build-async-http-post-Debug/Makefile:
--------------------------------------------------------------------------------
1 | #############################################################################
2 | # Makefile for building: async_http_post
3 | # Generated by qmake (2.01a) (Qt 4.8.7) on: ?? 12? 21 17:20:25 2016
4 | # Project: ../async_http_post/async_http_post.pro
5 | # Template: app
6 | # Command: /usr/lib/x86_64-linux-gnu/qt4/bin/qmake -spec /usr/share/qt4/mkspecs/linux-g++-64 CONFIG+=debug CONFIG+=declarative_debug -o Makefile ../async_http_post/async_http_post.pro
7 | #############################################################################
8 |
9 | ####### Compiler, tools and options
10 |
11 | CC = gcc
12 | CXX = g++
13 | DEFINES = -DQT_CORE_LIB -DQT_SHARED
14 | CFLAGS = -m64 -pipe -g -Wall -W -D_REENTRANT $(DEFINES)
15 | CXXFLAGS = -m64 -pipe -g -Wall -W -D_REENTRANT $(DEFINES)
16 | INCPATH = -I/usr/share/qt4/mkspecs/linux-g++-64 -I../async_http_post -I/usr/include/qt4/QtCore -I/usr/include/qt4 -I/usr/local/include -I. -I../async_http_post -I.
17 | LINK = g++
18 | LFLAGS = -m64
19 | LIBS = $(SUBLIBS) -L/usr/lib/x86_64-linux-gnu -lboost_system -lboost_thread -lboost_filesystem -lhiredis -levent -leasycpp -lQtCore -lpthread
20 | AR = ar cqs
21 | RANLIB =
22 | QMAKE = /usr/lib/x86_64-linux-gnu/qt4/bin/qmake
23 | TAR = tar -cf
24 | COMPRESS = gzip -9f
25 | COPY = cp -f
26 | SED = sed
27 | COPY_FILE = $(COPY)
28 | COPY_DIR = $(COPY) -r
29 | STRIP = strip
30 | INSTALL_FILE = install -m 644 -p
31 | INSTALL_DIR = $(COPY_DIR)
32 | INSTALL_PROGRAM = install -m 755 -p
33 | DEL_FILE = rm -f
34 | SYMLINK = ln -f -s
35 | DEL_DIR = rmdir
36 | MOVE = mv -f
37 | CHK_DIR_EXISTS= test -d
38 | MKDIR = mkdir -p
39 |
40 | ####### Output directory
41 |
42 | OBJECTS_DIR = ./
43 |
44 | ####### Files
45 |
46 | SOURCES = ../async_http_post/main.cpp
47 | OBJECTS = main.o
48 | QMAKE_TARGET = async_http_post
49 | DESTDIR =
50 | TARGET = async_http_post
51 |
52 | first: all
53 | ####### Implicit rules
54 |
55 | .SUFFIXES: .o .c .cpp .cc .cxx .C
56 |
57 | .cpp.o:
58 | $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<"
59 |
60 | .cc.o:
61 | $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<"
62 |
63 | .cxx.o:
64 | $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<"
65 |
66 | .C.o:
67 | $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<"
68 |
69 | .c.o:
70 | $(CC) -c $(CFLAGS) $(INCPATH) -o "$@" "$<"
71 |
72 | ####### Build rules
73 |
74 | all: Makefile $(TARGET)
75 |
76 | $(TARGET): $(OBJECTS)
77 | $(LINK) $(LFLAGS) -o $(TARGET) $(OBJECTS) $(OBJCOMP) $(LIBS)
78 |
79 | qmake: FORCE
80 | @$(QMAKE) -spec /usr/share/qt4/mkspecs/linux-g++-64 CONFIG+=debug CONFIG+=declarative_debug -o Makefile ../async_http_post/async_http_post.pro
81 |
82 | dist:
83 | @$(CHK_DIR_EXISTS) .tmp/async_http_post1.0.0 || $(MKDIR) .tmp/async_http_post1.0.0
84 | $(COPY_FILE) --parents $(SOURCES) $(DIST) .tmp/async_http_post1.0.0/ && $(COPY_FILE) --parents ../async_http_post/main.cpp .tmp/async_http_post1.0.0/ && (cd `dirname .tmp/async_http_post1.0.0` && $(TAR) async_http_post1.0.0.tar async_http_post1.0.0 && $(COMPRESS) async_http_post1.0.0.tar) && $(MOVE) `dirname .tmp/async_http_post1.0.0`/async_http_post1.0.0.tar.gz . && $(DEL_FILE) -r .tmp/async_http_post1.0.0
85 |
86 |
87 | clean:compiler_clean
88 | -$(DEL_FILE) $(OBJECTS)
89 | -$(DEL_FILE) *~ core *.core
90 |
91 |
92 | ####### Sub-libraries
93 |
94 | distclean: clean
95 | -$(DEL_FILE) $(TARGET)
96 | -$(DEL_FILE) Makefile
97 |
98 |
99 | check: first
100 |
101 | ####### Compile
102 |
103 | main.o: ../async_http_post/main.cpp
104 | $(CXX) -c $(CXXFLAGS) $(INCPATH) -o main.o ../async_http_post/main.cpp
105 |
106 | ####### Install
107 |
108 | install: FORCE
109 |
110 | uninstall: FORCE
111 |
112 | FORCE:
113 |
114 |
--------------------------------------------------------------------------------
/build-async-http-post-Debug/default.conf:
--------------------------------------------------------------------------------
1 | {
2 |
3 | "redis_host" : "127.0.0.1",
4 |
5 | "redis_port" : 6379,
6 |
7 | "redis_auth" : "123456",
8 |
9 | "redis_list_key" : "queue_user_token",
10 |
11 | "http_timeout" : 30,
12 |
13 | "post_urls" : [
14 | {"url" : "http://114.119.4.115:8888/test.php"},
15 | {"url" : "http://114.119.4.115:8888/test.php"}
16 | ]
17 |
18 | }
19 |
--------------------------------------------------------------------------------