├── CMakeLists.txt ├── README.md ├── build.sh ├── log ├── CMakeLists.txt ├── README.md ├── log.cpp ├── log.h ├── log_queue.h ├── pr.cpp ├── pr.h └── tests │ ├── CMakeLists.txt │ ├── test_log_async.cpp │ ├── test_log_level.cpp │ ├── test_log_sync.cpp │ ├── test_pr.cpp │ └── test_q.cpp ├── memory ├── README.md ├── chunk.cpp ├── chunk.h ├── data_buf.cpp ├── data_buf.h ├── mem_pool.cpp ├── mem_pool.h └── tests │ ├── CMakeLists.txt │ ├── test_buf.cpp │ └── test_mem.cpp ├── net ├── README.md ├── acceptor.cpp ├── acceptor.h ├── epoll.cpp ├── epoll.h ├── event_loop.cpp ├── event_loop.h ├── tcp_conn.cpp ├── tcp_conn.h ├── tcp_server.cpp ├── tcp_server.h └── tests │ ├── CMakeLists.txt │ ├── echo_client.sh │ ├── echo_server.cpp │ ├── http_for_bench.cpp │ └── install_client.sh ├── threadpool ├── README.md ├── tests │ ├── CMakeLists.txt │ └── test_threadpool.cpp └── threadpool.h ├── timer ├── README.md ├── hash_map.h ├── tests │ ├── CMakeLists.txt │ └── test_timer.cpp └── timer.h └── webbench-1.5 ├── COPYRIGHT ├── ChangeLog ├── Makefile ├── README.md ├── debian ├── changelog ├── control ├── copyright ├── dirs └── rules ├── socket.c ├── tags ├── webbench ├── webbench.1 ├── webbench.c └── webbench.o /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.16) 2 | project(MyServer CXX) 3 | 4 | set(CMAKE_CXX_STANDARD 20) 5 | 6 | if(CMAKE_DEBUG_FLAG) 7 | set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -g") 8 | endif() 9 | 10 | message(STATUS "CMAKE_CXX_FLAGS = " ${CMAKE_CXX_FLAGS}) 11 | 12 | add_subdirectory(log/tests) 13 | add_subdirectory(threadpool/tests) 14 | add_subdirectory(timer/tests) 15 | add_subdirectory(memory/tests) 16 | add_subdirectory(net/tests) 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Env 2 | - Linux version 5.3.0-28-generic 3 | - gcc version 7.5.0 4 | - Ubuntu 18.04.1 5 | - cmake version 3.17.0 6 | 7 | ## Build 8 |   ./build.sh 9 | ## Description 10 |   基于C++11、部分C++14/17特性的一个高性能并发网络服务器，包括目前已实现日志、线程池、内存池、定时器、网络io等模块。模块间低耦合高内聚，可作为整体也可单独提供服务。对各模块提供了单元测试。 11 |   网络io使用epoll LT触发模式，采用主从reactor设计。提供同步和异步日志，内存池使用哈希表、链表结合的管理，线程池支持任意任务参数和任务结果返回，定时器使用最小堆管理、支持多执行线程、支持在指定时间后执行任务、支持周期性执行任务、支持指定时间间隔重复执行指定次数任务、支持取消定时器等。 12 |   *具体设计参考各目录下readme。* 13 | ## TODO List 14 | > * http解析设计与实现 15 | > * 数据库连接池及缓存设计与实现 16 | > * C++20模块与协程引入 17 | > * 日志优化：参考[Nanolog](https://github.com/PlatformLab/NanoLog)设计，引入日志压缩、无锁队列、寄存器记录时间、减少缓存miss等方案 18 | > * 使用lua脚本语言，配置服务器ip、port，并检查参数合法性 19 | > * 引入gtest测试框架，整合单元测试 20 | > * test 21 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -x 4 | 5 | SOURCE_DIR=`pwd` 6 | BUILD_DIR=${BUILD_DIR:-../build} 7 | BUILD_TYPE=${BUILD_TYPE:-Debug} 8 | DEBUG_FLAG=${DEBUG_FLAG:-1} 9 | 10 | mkdir -p $BUILD_DIR/$BUILD_TYPE \ 11 | && cd $BUILD_DIR/$BUILD_TYPE \ 12 | && cmake \ 13 | -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ 14 | -DCMAKE_DEBUG_FLAG=$DEBUG_FLAG \ 15 | $SOURCE_DIR \ 16 | && make $* -------------------------------------------------------------------------------- /log/CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CandyConfident/HighPerformanceConcurrentServer/69fe1509347d173dad35a96d886863f8a886998a/log/CMakeLists.txt -------------------------------------------------------------------------------- /log/README.md: -------------------------------------------------------------------------------- 1 | ## 日志系统 2 |   日志系统包括pr模块和log模块，其中pr模块以区分日志级别的形式简单封装了printf；log模块实现了同步和异步日志，可选择向文件或者屏幕输出日志。 3 | ### log模块 4 | > * 日志类为单例模式 5 | > * 同步日志时使用日志的线程竞争锁向文件写入内容 6 | > * 异步日志时使用日志的线程向阻塞队列写入内容 7 | > * 异步日志时由单独的日志线程读取阻塞队列内容写入文件 8 | > * 日志文件按天分类 9 | > * 日志文件限制最大行数 10 | 11 | ### 日志测试 12 | > * 对阻塞队列的测试 13 | > * 对pr模块功能的测试 14 | > * 对log模块日志级别的测试 15 | > * 对log模块同步日志的多线程测试 16 | > * 对log模块异步日志的多线程测试 -------------------------------------------------------------------------------- /log/log.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "log.h" 7 | 8 | using namespace std; 9 | 10 | Logger::log_level g_log_level = Logger::LOG_LEVEL_INFO; 11 | 12 | const char* LogLevelName[Logger::NUM_LOG_LEVELS] = 13 | { 14 | "[ERROR]", 15 | "[WARN ]", 16 | "[INFO ]", 17 | "[DEBUG]", 18 | }; 19 | 20 | struct my_time 21 | { 22 | int year; 23 | char month; 24 | char day; 25 | char hour; 26 | char minute; 27 | char second; 28 | }; 29 | 30 | static my_time get_current_sys_time() 31 | { 32 | auto tt = chrono::system_clock::to_time_t(chrono::system_clock::now()); 33 | struct tm* ptm = localtime(&tt); 34 | my_time t = { ptm->tm_year + 1900, static_cast(ptm->tm_mon + 1), static_cast(ptm->tm_mday), 35 | static_cast(ptm->tm_hour), static_cast(ptm->tm_min), static_cast(ptm->tm_sec)}; 36 | return t; 37 | } 38 | 39 | Logger::Logger() 40 | { 41 | PR_DEBUG("log class constructed\n"); 42 | l_count = 0; 43 | } 44 | 45 | Logger::~Logger() 46 | { 47 | if(l_asyncw_thread) 48 | { 49 | is_thread_stop = true; 50 | if(l_asyncw_thread->joinable()) 51 | { 52 | l_buffer_queue->notify(); 53 | l_asyncw_thread->join(); 54 | } 55 | delete l_asyncw_thread; 56 | } 57 | 58 | lock_guard lck (l_mutex); 59 | if (l_fp != NULL) 60 | { 61 | fclose(l_fp); 62 | } 63 | 64 | if(l_buf) 65 | { 66 | delete [] l_buf; 67 | } 68 | 69 | if(l_buffer_queue) 70 | { 71 | delete l_buffer_queue; 72 | } 73 | } 74 | 75 | bool Logger::init(const char *file_name, int buffer_queue_size, 76 | Logger::log_level level, int buffer_size, int split_lines) 77 | { 78 | if(l_inited) 79 | { 80 | PR_WARN("Logger has been initialized, do not try again!\n"); 81 | return false; 82 | } 83 | 84 | if(!file_name) 85 | { 86 | l_is_stdout = true; 87 | } 88 | 89 | if( !l_is_stdout && strlen(file_name)>=128 ) 90 | { 91 | PR_ERROR("file name must be less than 128 bytes!\n"); 92 | // exit(-1); 93 | throw invalid_argument("file name must be less than 128 bytes!");; 94 | } 95 | 96 | set_log_level(level); 97 | 98 | if (buffer_queue_size >= 1) 99 | { 100 | l_is_async = true; 101 | l_buffer_queue = new buffer_queue(buffer_queue_size); 102 | l_asyncw_thread = new thread(&Logger::async_flush); 103 | } 104 | 105 | l_buf_size = buffer_size; 106 | l_buf = new char[l_buf_size]; 107 | memset(l_buf, '\0', l_buf_size); 108 | l_split_lines = split_lines; 109 | 110 | my_time tm = get_current_sys_time(); 111 | l_today = tm.day; 112 | 113 | if(l_is_stdout) 114 | { 115 | l_inited = true; 116 | l_fp = stdout; 117 | PR_DEBUG("succeed in using stdout as log output\n"); 118 | PR_DEBUG("log init finished!\n"); 119 | return true; 120 | } 121 | 122 | const char *p = strrchr(file_name, '/'); 123 | char log_file_fullname[268] = {0}; 124 | 125 | if (p == NULL) 126 | { 127 | PR_ERROR("log file name should behind '/'\n"); 128 | return false; 129 | } 130 | else 131 | { 132 | strcpy(l_file_name, p + 1); 133 | strncpy(l_dir_name, file_name, p - file_name + 1); 134 | snprintf(log_file_fullname, 267, "%s%04d_%02d_%02d_%s", l_dir_name, 135 | tm.year, tm.month, tm.day, l_file_name); 136 | 137 | l_fp = fopen(log_file_fullname, "a"); 138 | } 139 | 140 | if (l_fp == NULL) 141 | { 142 | PR_ERROR("open %s failed!\n", log_file_fullname); 143 | return false; 144 | } 145 | 146 | l_inited = true; 147 | PR_DEBUG("succeed in using file %s as log output\n", log_file_fullname); 148 | PR_DEBUG("log init finished!\n"); 149 | 150 | return true; 151 | } 152 | 153 | void Logger::write_log(const char* file_name, const char* tn_callbackname, int line_no, log_level level, const char *format, ...) 154 | { 155 | my_time my_tm = get_current_sys_time(); 156 | 157 | { 158 | lock_guard lck (l_mutex); 159 | l_count++; 160 | 161 | if (l_today != my_tm.day || l_count % l_split_lines == 0) 162 | { 163 | PR_DEBUG("start to create a new log file\n"); 164 | char new_file_name[301] = {0}; 165 | fflush(l_fp); 166 | fclose(l_fp); 167 | char prefix[24] = {0}; 168 | 169 | snprintf(prefix, 23, "%04d_%02d_%02d_", my_tm.year, my_tm.month, my_tm.day); 170 | 171 | if (l_today != my_tm.day) 172 | { 173 | snprintf(new_file_name, 300, "%s%s%s", l_dir_name, prefix, l_file_name); 174 | l_today = my_tm.day; 175 | l_count = 0; 176 | } 177 | else 178 | { 179 | snprintf(new_file_name, 300, "%s%s%s.%lld", l_dir_name, prefix, l_file_name, l_count / l_split_lines); 180 | } 181 | l_fp = fopen(new_file_name, "a"); 182 | } 183 | } 184 | 185 | va_list valst; 186 | va_start(valst, format); 187 | 188 | string log_str; 189 | { 190 | lock_guard lck (l_mutex);; 191 | 192 | int n = snprintf(l_buf, 300, "%04d-%02d-%02d %02d:%02d:%02d %s [%s:%s:%d] ", 193 | my_tm.year, my_tm.month, my_tm.day, 194 | my_tm.hour, my_tm.minute, my_tm.second, LogLevelName[level], 195 | file_name, tn_callbackname, line_no); 196 | 197 | int m = vsnprintf(l_buf + n, l_buf_size - 1, format, valst); 198 | l_buf[n + m] = '\n'; 199 | l_buf[n + m + 1] = '\0'; 200 | log_str = l_buf; 201 | } 202 | va_end(valst); 203 | 204 | 205 | if (l_is_async) 206 | { 207 | while (!l_buffer_queue->push(log_str) && !is_thread_stop) //FIXME: use tm_condvar replacing busy loop 208 | { 209 | } 210 | 211 | } 212 | else 213 | { 214 | lock_guard lck (l_mutex); 215 | fputs(log_str.c_str(), l_fp); 216 | } 217 | } 218 | 219 | void Logger::flush(void) 220 | { 221 | lock_guard lck (l_mutex); 222 | fflush(l_fp); 223 | } 224 | -------------------------------------------------------------------------------- /log/log.h: -------------------------------------------------------------------------------- 1 | #ifndef __LOG_H__ 2 | #define __LOG_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "log_queue.h" 15 | #include "pr.h" 16 | 17 | using namespace std; 18 | 19 | class Logger 20 | { 21 | public: 22 | typedef enum 23 | { 24 | LOG_LEVEL_ERROR, 25 | LOG_LEVEL_WARN, 26 | LOG_LEVEL_INFO, 27 | LOG_LEVEL_DEBUG, 28 | NUM_LOG_LEVELS, 29 | } log_level; 30 | 31 | static Logger *get_instance() 32 | { 33 | static Logger instance; 34 | return &instance; 35 | } 36 | 37 | static void async_flush(void) 38 | { 39 | Logger::get_instance()->async_write(); 40 | } 41 | 42 | static log_level get_log_level(); 43 | 44 | static log_level set_log_level(log_level level); 45 | 46 | // this function is not reentrant, should be called at main thread before subthread started. 47 | bool init(const char *file_name, int buffer_queue_size = 0, Logger::log_level = Logger::LOG_LEVEL_INFO, 48 | int buffer_size = 8192, int split_lines = 5000); 49 | 50 | bool is_inited() 51 | { 52 | return l_inited; 53 | } 54 | 55 | void write_log(const char* file_name, const char* tn_callbackname, int line_no, log_level level, const char *format, ...); 56 | 57 | void flush(void); 58 | 59 | private: 60 | Logger(); 61 | Logger(const Logger&); 62 | ~Logger(); 63 | void *async_write() 64 | { 65 | string single_line; 66 | //@ assuming that no thread pushes elements in, once async writing thread starts, it falls in sleep 67 | // and will never be awaken. To deal with that, an interface "notify" is introduced in buffer_queue. 68 | // In destruction func, notify is called to wake up this thread and let it stop naturally. 69 | //@ assuming that pop func returns false under spurious wakeup, the loop will be over as well as the async writing thread, 70 | // which is not what we expected. So the variable is_thread_stop is introduced. Only in destruction func will it be setted, 71 | // combining with the notify func, the thread can finish normally. Anothor function of is_thread_stop is to terminate the 72 | // busy loop of pushing operation in write_log func when this class comes to end. 73 | while(l_buffer_queue->pop(single_line) && !is_thread_stop) 74 | { 75 | lock_guard lck (l_mutex); 76 | fputs(single_line.c_str(), l_fp); 77 | } 78 | } 79 | 80 | private: 81 | char l_dir_name[128]; 82 | char l_file_name[128]; 83 | int l_split_lines; 84 | int l_buf_size; 85 | long long l_count; 86 | int l_today; 87 | FILE *l_fp; 88 | char *l_buf = nullptr; 89 | bool l_inited = false; 90 | buffer_queue *l_buffer_queue = nullptr; 91 | bool l_is_async; 92 | bool l_is_stdout = false; 93 | atomic is_thread_stop = false; 94 | mutex l_mutex; // TODO: add mutexes for different critical resources 95 | thread *l_asyncw_thread = nullptr; 96 | }; 97 | 98 | 99 | extern Logger::log_level g_log_level; 100 | 101 | inline Logger::log_level Logger::get_log_level() 102 | { 103 | return g_log_level; 104 | } 105 | 106 | inline Logger::log_level Logger::set_log_level(Logger::log_level level) 107 | { 108 | assert(level>=Logger::LOG_LEVEL_ERROR && level<=Logger::LOG_LEVEL_DEBUG); 109 | Logger::log_level old_level = g_log_level; 110 | g_log_level = level; 111 | return old_level; 112 | } 113 | 114 | #define LOG_DEBUG(format, ...) \ 115 | do{ \ 116 | if(!Logger::get_instance()->is_inited()) \ 117 | { \ 118 | PR_ERROR("logger must be inited before user!\n"); \ 119 | } \ 120 | if(Logger::LOG_LEVEL_DEBUG <= Logger::get_log_level()) \ 121 | { \ 122 | Logger::get_instance()->write_log(__FILE__, __FUNCTION__, \ 123 | __LINE__, Logger::LOG_LEVEL_DEBUG, format, ##__VA_ARGS__); \ 124 | Logger::get_instance()->flush(); \ 125 | } \ 126 | } while(0) 127 | 128 | #define LOG_INFO(format, ...) \ 129 | do{ \ 130 | if(!Logger::get_instance()->is_inited()) \ 131 | { \ 132 | PR_ERROR("logger must be inited before user!\n"); \ 133 | } \ 134 | if(Logger::LOG_LEVEL_INFO <= Logger::get_log_level()) \ 135 | { \ 136 | Logger::get_instance()->write_log(__FILE__, __FUNCTION__, \ 137 | __LINE__, Logger::LOG_LEVEL_INFO, format, ##__VA_ARGS__); \ 138 | Logger::get_instance()->flush(); \ 139 | } \ 140 | } while(0) 141 | 142 | #define LOG_WARN(format, ...) \ 143 | do{ \ 144 | if(!Logger::get_instance()->is_inited()) \ 145 | { \ 146 | PR_ERROR("logger must be inited before user!\n"); \ 147 | } \ 148 | if(Logger::LOG_LEVEL_WARN <= Logger::get_log_level()) \ 149 | { \ 150 | Logger::get_instance()->write_log(__FILE__, __FUNCTION__, \ 151 | __LINE__, Logger::LOG_LEVEL_WARN, format, ##__VA_ARGS__); \ 152 | Logger::get_instance()->flush(); \ 153 | } \ 154 | } while(0) 155 | 156 | #define LOG_ERROR(format, ...) \ 157 | do{ \ 158 | if(!Logger::get_instance()->is_inited()) \ 159 | { \ 160 | PR_ERROR("logger must be inited before user!\n"); \ 161 | } \ 162 | if(Logger::LOG_LEVEL_ERROR <= Logger::get_log_level()) \ 163 | { \ 164 | Logger::get_instance()->write_log(__FILE__, __FUNCTION__, \ 165 | __LINE__, Logger::LOG_LEVEL_ERROR, format, ##__VA_ARGS__); \ 166 | Logger::get_instance()->flush(); \ 167 | } \ 168 | } while(0) 169 | 170 | #endif 171 | -------------------------------------------------------------------------------- /log/log_queue.h: -------------------------------------------------------------------------------- 1 | #ifndef __Logger_BUFFER_H__ 2 | #define __Logger_BUFFER_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "pr.h" 12 | 13 | using namespace std; 14 | 15 | template 16 | class buffer_queue 17 | { 18 | public: 19 | buffer_queue() : b_array(new T[capacity]) 20 | { 21 | 22 | } 23 | 24 | explicit buffer_queue(int max_size, bool debug=false) 25 | { 26 | if (max_size <= 0) 27 | { 28 | PR_ERROR("max_size is illegal!\n"); 29 | // exit(-1); 30 | throw invalid_argument("max_size is illegal!"); 31 | } 32 | 33 | b_debug = debug; 34 | capacity = max_size; 35 | b_array = new T[capacity]; 36 | } 37 | 38 | void clear() 39 | { 40 | lock_guard lck (b_mutex); 41 | if(b_debug) 42 | { 43 | b_rcnt = 0; 44 | b_wcnt = 0; 45 | } 46 | b_first = -1; 47 | b_last = -1; 48 | b_size = 0; 49 | } 50 | 51 | ~buffer_queue() 52 | { 53 | lock_guard lck (b_mutex); 54 | if (b_array != NULL) 55 | delete [] b_array; 56 | } 57 | 58 | int get_rcnt() const 59 | { 60 | lock_guard lck (b_mutex); 61 | if(b_debug) 62 | { 63 | return b_rcnt; 64 | } 65 | PR_WARN("this method is invalid when q_debug is false!\n"); 66 | return -1; 67 | } 68 | 69 | int get_wcnt() const 70 | { 71 | lock_guard lck (b_mutex); 72 | if(b_debug) 73 | { 74 | return b_wcnt; 75 | } 76 | PR_WARN("this method is invalid when q_debug is false!\n"); 77 | return -1; 78 | } 79 | 80 | int get_size() const 81 | { 82 | lock_guard lck (b_mutex); 83 | assert(b_size >= 0 && b_size <= capacity); 84 | return b_size; 85 | } 86 | 87 | bool is_full() const 88 | { 89 | lock_guard lck (b_mutex); 90 | return b_size == capacity ? true : false; 91 | } 92 | 93 | bool is_empty() const 94 | { 95 | lock_guard lck (b_mutex); 96 | return b_size == 0 ? true :false; 97 | } 98 | 99 | int get_capacity() const 100 | { 101 | return capacity; 102 | } 103 | 104 | bool front(T &value) 105 | { 106 | lock_guard lck (b_mutex); 107 | if (0 == b_size) 108 | { 109 | return false; 110 | } 111 | value = b_array[b_first+1]; 112 | return true; 113 | } 114 | 115 | bool back(T &value) 116 | { 117 | lock_guard lck (b_mutex); 118 | if (0 == b_size) 119 | { 120 | return false; 121 | } 122 | value = b_array[b_last]; 123 | return true; 124 | } 125 | 126 | bool push(const T &item) 127 | { 128 | unique_lock lck (b_mutex);; 129 | if (b_size >= capacity) 130 | { 131 | b_cond.notify_all(); 132 | return false; 133 | } 134 | 135 | b_last = (b_last + 1) % capacity; 136 | b_array[b_last] = item; 137 | 138 | b_size++; 139 | if(b_debug) 140 | { 141 | b_wcnt++; 142 | } 143 | b_cond.notify_all(); 144 | 145 | return true; 146 | } 147 | 148 | void notify() 149 | { 150 | b_cond.notify_all(); 151 | } 152 | 153 | bool pop(T &item) 154 | { 155 | 156 | unique_lock lck (b_mutex); 157 | if (b_size <= 0) 158 | { 159 | b_cond.wait(lck); 160 | if(b_size <= 0) 161 | { 162 | return false; 163 | } 164 | } 165 | 166 | b_first = (b_first + 1) % capacity; 167 | item = b_array[b_first]; 168 | b_size--; 169 | if(b_debug) 170 | { 171 | b_rcnt++; 172 | } 173 | 174 | return true; 175 | } 176 | 177 | bool pop(T &item, int ms_timeout) 178 | { 179 | unique_lock lck (b_mutex); 180 | if (b_size <= 0) 181 | { 182 | if(b_cond.wait_for(lck, chrono::milliseconds(ms_timeout)) == cv_status::timeout) 183 | return false; 184 | } 185 | 186 | if (b_size <= 0) 187 | { 188 | return false; 189 | } 190 | 191 | b_first = (b_first + 1) % capacity; 192 | item = b_array[b_first]; 193 | b_size--; 194 | if(b_debug) 195 | { 196 | b_rcnt++; 197 | } 198 | 199 | return true; 200 | } 201 | 202 | private: 203 | mutable mutex b_mutex; 204 | condition_variable b_cond; // TODO: add condition_variable for full, 205 | // to reduce busy waiting when pushing. 206 | 207 | T *b_array; 208 | long long b_rcnt = 0; 209 | long long b_wcnt = 0; 210 | bool b_debug = false; 211 | int b_first = -1; 212 | int b_last = -1; 213 | int b_size = 0; 214 | int capacity = 1000; 215 | }; 216 | 217 | #endif 218 | -------------------------------------------------------------------------------- /log/pr.cpp: -------------------------------------------------------------------------------- 1 | #include "pr.h" 2 | 3 | int pr_level = PR_LEVEL_INFO; 4 | 5 | int pr_set_level(int level) 6 | { 7 | if(level > PR_LEVEL_DEBUG) 8 | { 9 | level = PR_LEVEL_DEBUG; 10 | } 11 | else if(level < PR_LEVEL_ERROR) 12 | { 13 | level = PR_LEVEL_ERROR; 14 | } 15 | 16 | int old_level = pr_level; 17 | pr_level = level; 18 | 19 | return old_level; 20 | } 21 | 22 | string tid_to_string(const thread::id& tid) 23 | { 24 | ostringstream oss; 25 | oss << tid << endl; 26 | return oss.str(); 27 | } 28 | 29 | long long tid_to_ll(const thread::id& tid) 30 | { 31 | return atoll(tid_to_string(tid).c_str()); 32 | } -------------------------------------------------------------------------------- /log/pr.h: -------------------------------------------------------------------------------- 1 | #ifndef __PR_H__ 2 | #define __PR_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | using namespace std; 10 | 11 | #define PR_LEVEL_ERROR 0 12 | #define PR_LEVEL_WARN 1 13 | #define PR_LEVEL_INFO 2 14 | #define PR_LEVEL_DEBUG 3 15 | 16 | extern int pr_level; 17 | 18 | #define PR(level, val, fmt, ...) \ 19 | do { \ 20 | if( level <= pr_level ) \ 21 | printf("[%-5s]" "[%s:%d] " fmt, val, \ 22 | __FUNCTION__, __LINE__, ##__VA_ARGS__); \ 23 | } while(0) 24 | 25 | #define PR_DEBUG(fmt, ...) \ 26 | PR(PR_LEVEL_DEBUG, "debug", fmt, ##__VA_ARGS__) 27 | 28 | #define PR_INFO(fmt, ...) \ 29 | PR(PR_LEVEL_INFO, "info", fmt, ##__VA_ARGS__) 30 | 31 | #define PR_WARN(fmt, ...) \ 32 | PR(PR_LEVEL_WARN, "warn", fmt, ##__VA_ARGS__) 33 | 34 | #define PR_ERROR(fmt, ...) \ 35 | PR(PR_LEVEL_ERROR, "error", fmt, ##__VA_ARGS__) 36 | 37 | int pr_set_level(int level); 38 | 39 | string tid_to_string(const thread::id& tid); 40 | 41 | long long tid_to_ll(const thread::id& tid); 42 | 43 | #endif -------------------------------------------------------------------------------- /log/tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(SRCS 2 | test_pr.cpp 3 | ../pr.cpp 4 | ) 5 | include_directories(${PROJECT_SOURCE_DIR}/..) 6 | 7 | add_executable(pr_test ${SRCS}) 8 | 9 | set(SRCS 10 | test_q.cpp 11 | ../pr.cpp 12 | ) 13 | add_executable(q_test ${SRCS}) 14 | target_link_libraries(q_test pthread) 15 | 16 | set(SRCS 17 | test_log_level.cpp 18 | ../pr.cpp 19 | ../log.cpp 20 | ) 21 | add_executable(log_level_test ${SRCS}) 22 | target_link_libraries(log_level_test pthread) 23 | 24 | set(SRCS 25 | test_log_sync.cpp 26 | ../pr.cpp 27 | ../log.cpp 28 | ) 29 | add_executable(log_sync_test ${SRCS}) 30 | target_link_libraries(log_sync_test pthread) 31 | 32 | set(SRCS 33 | test_log_async.cpp 34 | ../pr.cpp 35 | ../log.cpp 36 | ) 37 | add_executable(log_async_test ${SRCS}) 38 | target_link_libraries(log_async_test pthread) -------------------------------------------------------------------------------- /log/tests/test_log_async.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "../log.h" 7 | #include "../pr.h" 8 | 9 | using namespace std; 10 | 11 | const int g_item_num =11000; 12 | const int g_t_num = 5; 13 | atomic g_t_cnt = 0; 14 | 15 | void log_async_test() 16 | { 17 | long long tid = tid_to_ll(this_thread::get_id()); 18 | PR_INFO("[threa_%d]tid is: %lld\n", g_t_cnt.fetch_add(1), tid); 19 | 20 | for(int i=0; iinit("./log_async.txt", 10); 29 | 30 | vector threads; 31 | 32 | PR_INFO("async log test started\n"); 33 | PR_INFO("start to log\n"); 34 | auto start = chrono::steady_clock::now(); 35 | 36 | for(int i=0; i(end - start); 47 | PR_INFO("end logging\n"); 48 | PR_INFO("totally write %d items into files\n", g_item_num * g_t_num); 49 | PR_INFO("costed time: %d ms\n" ,static_cast(duration.count())); 50 | return 0; 51 | } -------------------------------------------------------------------------------- /log/tests/test_log_level.cpp: -------------------------------------------------------------------------------- 1 | #include "../log.h" 2 | #include "../pr.h" 3 | 4 | void log_level_test() 5 | { 6 | PR_INFO("\n------------------set log level as debug----------------\n"); 7 | Logger::set_log_level(Logger::LOG_LEVEL_DEBUG); 8 | LOG_ERROR("*****************log error\n"); 9 | LOG_WARN("*****************log warn\n"); 10 | LOG_INFO("*****************log info\n"); 11 | LOG_DEBUG("*****************log debug\n"); 12 | 13 | PR_INFO("\n------------------set log level as info----------------\n"); 14 | Logger::set_log_level(Logger::LOG_LEVEL_INFO); 15 | LOG_ERROR("*****************log error\n"); 16 | LOG_WARN("*****************log warn\n"); 17 | LOG_INFO("*****************log info\n"); 18 | LOG_DEBUG("*****************log debug\n"); 19 | 20 | PR_INFO("\n------------------set log level as warn----------------\n"); 21 | Logger::set_log_level(Logger::LOG_LEVEL_WARN); 22 | LOG_ERROR("*****************log error\n"); 23 | LOG_WARN("*****************log warn\n"); 24 | LOG_INFO("*****************log info\n"); 25 | LOG_DEBUG("*****************log debug\n"); 26 | 27 | PR_INFO("\n------------------set log level as error----------------\n"); 28 | Logger::set_log_level(Logger::LOG_LEVEL_ERROR); 29 | LOG_ERROR("*****************log error\n"); 30 | LOG_WARN("*****************log warn\n"); 31 | LOG_INFO("*****************log info\n"); 32 | LOG_DEBUG("*****************log debug\n"); 33 | } 34 | 35 | int main() 36 | { 37 | Logger::get_instance()->init(NULL); 38 | log_level_test(); 39 | 40 | return 0; 41 | } -------------------------------------------------------------------------------- /log/tests/test_log_sync.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "../log.h" 6 | #include "../pr.h" 7 | 8 | using namespace std; 9 | 10 | const int g_item_num = 11000; 11 | const int g_t_num = 5; 12 | atomic g_t_cnt = 0; 13 | 14 | void log_sync_test() 15 | { 16 | long long tid = tid_to_ll(this_thread::get_id()); 17 | PR_INFO("[threa_%d]tid is: %lld\n", g_t_cnt.fetch_add(1), tid); 18 | for(int i=0; iinit("./log_sync.txt"); 28 | 29 | vector threads; 30 | 31 | PR_INFO("sync log test started\n"); 32 | PR_INFO("start to log\n"); 33 | auto start = chrono::steady_clock::now(); 34 | 35 | for(int i=0; i(end - start); 46 | PR_INFO("end logging\n"); 47 | PR_INFO("totally write %d items into files\n", g_t_num * g_item_num); 48 | PR_INFO("costed time: %d ms\n" ,static_cast(duration.count())); 49 | return 0; 50 | } -------------------------------------------------------------------------------- /log/tests/test_pr.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "../pr.h" 3 | 4 | void pr_test(const char* title) 5 | { 6 | printf("------------level %s-----------\r\n", title); 7 | PR_DEBUG("pr: %s\r\n", "debug"); 8 | PR_INFO("pr: %s\r\n", "info"); 9 | PR_WARN("pr: %s\r\n", "warn"); 10 | PR_ERROR("pr: %s\r\n", "error"); 11 | printf("\r\n"); 12 | } 13 | 14 | int main() 15 | { 16 | ostringstream oss; 17 | string tid = tid_to_string(this_thread::get_id()); 18 | PR_INFO("tid is:%s\n", tid.c_str()); 19 | 20 | pr_set_level(PR_LEVEL_DEBUG); 21 | pr_test("debug"); 22 | 23 | pr_set_level(PR_LEVEL_INFO); 24 | pr_test("info"); 25 | 26 | pr_set_level(PR_LEVEL_WARN); 27 | pr_test("warn"); 28 | 29 | pr_set_level(PR_LEVEL_ERROR); 30 | pr_test("error"); 31 | 32 | return 0; 33 | } -------------------------------------------------------------------------------- /log/tests/test_q.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "../pr.h" 7 | #include "../log_queue.h" 8 | 9 | using namespace std; 10 | 11 | const int g_size = 10; 12 | typedef int qType; 13 | auto g_q = buffer_queue(g_size, true); 14 | atomic g_p_cnt = 0; 15 | atomic g_c_cnt = 0; 16 | 17 | void produer(int p_num) 18 | { 19 | long long tid = tid_to_ll(this_thread::get_id()); 20 | PR_INFO("[tid:%lld] %d elements should be produced\n", tid, p_num); 21 | int i = 0; 22 | while(i& q, bool test=false) 53 | { 54 | for(int i=0; i(i)); 56 | 57 | if(test) 58 | { 59 | qType f_val; 60 | q.front(f_val); 61 | assert(f_val==0); 62 | 63 | for(int i=0; i(end - start); 111 | PR_INFO("cond wait_for duration: %d ms\n" ,static_cast(duration.count())); 112 | assert(!ret); 113 | 114 | PR_INFO("buffer queue function test passed!\n\n\n"); 115 | 116 | g_q.clear(); 117 | fill_q(g_q); 118 | int origin_num = g_q.get_size(); 119 | assert(origin_num==g_size); 120 | PR_INFO("------------------------------------------\n"); 121 | PR_INFO("buffer queue capacity is %d\n", g_size); 122 | PR_INFO("origin element num is %d\n", origin_num); 123 | PR_INFO("------------------------------------------\n"); 124 | int p_n1 = 100; 125 | int p_n2 = 50; 126 | int c_n1 = 160; 127 | thread p1(produer, p_n1); 128 | thread p2(produer, p_n2); 129 | thread c1(consumer, c_n1); 130 | 131 | p1.join(); 132 | p2.join(); 133 | c1.join(); 134 | 135 | PR_INFO("\n"); 136 | PR_INFO("origin element num is %d\n", origin_num); 137 | PR_INFO("totally produced %d elements\n", g_p_cnt.load()); 138 | PR_INFO("totally consumed %d elements\n", g_c_cnt.load()); 139 | 140 | assert( g_q.get_size() == ( p_n1 + p_n2 + g_size - c_n1 ) ); 141 | assert( g_q.get_rcnt() == ( c_n1 ) ); 142 | assert( g_q.get_wcnt() == ( g_size + p_n1 + p_n2 ) ); 143 | PR_INFO("buffer queue mutithread test passed!\n"); 144 | 145 | return 0; 146 | } -------------------------------------------------------------------------------- /memory/README.md: -------------------------------------------------------------------------------- 1 | ## 内存池 2 |   内存池包括memory pool，chunk和data_buf。 3 | ### memory pool 4 | > * 使用单例的模式 5 | > * 以哈希表的结构管理不同大小的chunk组成的链表 6 | > * 分配内存时，找到距离最近的chunk进行分配 7 | > * 回收时把chunk挂回对应链表的头部 8 | > * 当链表上没有chunk可用时申请新的chunk分配出去 9 | > * 分配内存时向上取整 10 | > * unique_lock和lock_guard最大的不同是unique_lock不需要始终拥有关联的mutex，而lock_guard始终拥有mutex。 11 | > * std::unique_lock 与std::lock_guard都能实现自动加锁与解锁功能，但是std::unique_lock要比std::lock_guard更灵活，但是更灵活的代价是占用空间相对更大一点且相对更慢一点。 12 | > * std::unique_lock相对std::lock_guard更灵活的地方在于在等待中的线程如果在等待期间需要解锁mutex，并在之后重新将其锁定。而std::lock_guard却不具备这样的功能。 13 | 14 | ### chunk 15 | > * 内存池分配内存的单位 16 | > * 内存池管理的链表中的一个节点 17 | ### data_buf 18 | > * 应用层缓冲区的数据结构 19 | > * 实际上是一个由内存池管理的chunk 20 | > * 支持数据到data_buf，data_buf到socket文件的双向流动 21 | ### 内存池测试 22 | > * 对memory pool分配回收chunk块的测试 23 | > * 对数据经过data_buf到文件fd的双向流动测试 -------------------------------------------------------------------------------- /memory/chunk.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "chunk.h" 6 | 7 | Chunk::Chunk(int size) : capacity(size), data(new char[size]) 8 | { 9 | assert(data); 10 | } 11 | 12 | Chunk::~Chunk() 13 | { 14 | if( data ) 15 | { 16 | delete [] data; 17 | } 18 | } 19 | 20 | void Chunk::clear() { 21 | length = head = 0; 22 | } 23 | 24 | void Chunk::adjust() { 25 | if (head != 0) { 26 | if (length != 0) { 27 | memmove(data, data+head, length); 28 | } 29 | head = 0; 30 | } 31 | } 32 | 33 | void Chunk::copy(const Chunk *other) { 34 | memcpy(data, other->data + other->head, other->length); 35 | head = 0; 36 | length = other->length; 37 | } 38 | 39 | void Chunk::pop(int len) { 40 | length -= len; 41 | head += len; 42 | } 43 | 44 | void Chunk::print_data() 45 | { 46 | for(int i=head; i 2 | #include 3 | #include 4 | #include 5 | 6 | #include "data_buf.h" 7 | #include "pr.h" 8 | 9 | BufferBase::BufferBase() 10 | { 11 | } 12 | 13 | BufferBase::~BufferBase() 14 | { 15 | clear(); 16 | } 17 | 18 | const int BufferBase::length() const 19 | { 20 | return data_buf != nullptr ? data_buf->length : 0; 21 | } 22 | 23 | void BufferBase::pop(int len) 24 | { 25 | assert(data_buf != nullptr && len <= data_buf->length); 26 | 27 | data_buf->pop(len); 28 | if(data_buf->length == 0) { 29 | Mempool::get_instance().retrieve(data_buf); 30 | data_buf = nullptr; 31 | } 32 | } 33 | 34 | void BufferBase::clear() 35 | { 36 | if (data_buf != nullptr) { 37 | Mempool::get_instance().retrieve(data_buf); 38 | data_buf = nullptr; 39 | } 40 | } 41 | 42 | 43 | int InputBuffer::read_from_fd(int fd) 44 | { 45 | int need_read; 46 | // FIONREAD: get readable bytes num in kernel buffer 47 | if (ioctl(fd, FIONREAD, &need_read) == -1) { 48 | PR_ERROR("ioctl FIONREAD error\n"); 49 | return -1; 50 | } 51 | 52 | if (data_buf == nullptr) { 53 | data_buf = Mempool::get_instance().alloc_chunk(need_read); 54 | if (data_buf == nullptr) { 55 | PR_INFO("no free buf for alloc\n"); 56 | return -1; 57 | } 58 | } 59 | else { 60 | assert(data_buf->head == 0); 61 | if (data_buf->capacity - data_buf->length < (int)need_read) { 62 | Chunk *new_buf = Mempool::get_instance().alloc_chunk(need_read + data_buf->length); 63 | if (new_buf == nullptr) { 64 | PR_INFO("no free buf for alloc\n"); 65 | return -1; 66 | } 67 | new_buf->copy(data_buf); 68 | Mempool::get_instance().retrieve(data_buf); 69 | data_buf = new_buf; 70 | } 71 | } 72 | 73 | int already_read = 0; 74 | do { 75 | if(need_read == 0) { 76 | already_read = read(fd, data_buf->data + data_buf->length, m4K); 77 | } else { 78 | already_read = read(fd, data_buf->data + data_buf->length, need_read); 79 | } 80 | } while (already_read == -1 && errno == EINTR); 81 | if (already_read > 0) { 82 | if (need_read != 0) { 83 | assert(already_read == need_read); 84 | } 85 | data_buf->length += already_read; 86 | } 87 | 88 | return already_read; 89 | } 90 | 91 | const char *InputBuffer::get_from_buf() const 92 | { 93 | return data_buf != nullptr ? data_buf->data + data_buf->head : nullptr; 94 | } 95 | 96 | void InputBuffer::adjust() 97 | { 98 | if (data_buf != nullptr) { 99 | data_buf->adjust(); 100 | } 101 | } 102 | 103 | 104 | int OutputBuffer::write2buf(const char *data, int len) 105 | { 106 | if (data_buf == nullptr) { 107 | data_buf = Mempool::get_instance().alloc_chunk(len); 108 | if (data_buf == nullptr) { 109 | PR_INFO("no free buf for alloc\n"); 110 | return -1; 111 | } 112 | } 113 | else { 114 | assert(data_buf->head == 0); 115 | if (data_buf->capacity - data_buf->length < len) { 116 | Chunk *new_buf = Mempool::get_instance().alloc_chunk(len + data_buf->length); 117 | if (new_buf == nullptr) { 118 | PR_INFO("no free buf for alloc\n"); 119 | return -1; 120 | } 121 | new_buf->copy(data_buf); 122 | Mempool::get_instance().retrieve(data_buf); 123 | data_buf = new_buf; 124 | } 125 | } 126 | 127 | memcpy(data_buf->data + data_buf->length, data, len); 128 | data_buf->length += len; 129 | 130 | return 0; 131 | } 132 | 133 | int OutputBuffer::write2fd(int fd) 134 | { 135 | assert(data_buf != nullptr && data_buf->head == 0); 136 | 137 | int already_write = 0; 138 | 139 | do { 140 | already_write = write(fd, data_buf->data, data_buf->length); 141 | } while (already_write == -1 && errno == EINTR); 142 | 143 | if (already_write > 0) { 144 | data_buf->pop(already_write); 145 | data_buf->adjust(); 146 | } 147 | 148 | if (already_write == -1 && errno == EAGAIN) { 149 | already_write = 0; 150 | } 151 | 152 | return already_write; 153 | } -------------------------------------------------------------------------------- /memory/data_buf.h: -------------------------------------------------------------------------------- 1 | #ifndef __DATA_BUF_H__ 2 | #define __DATA_NIF_H__ 3 | 4 | #include "chunk.h" 5 | #include "mem_pool.h" 6 | 7 | class BufferBase { 8 | public: 9 | BufferBase(); 10 | ~BufferBase(); 11 | 12 | const int length() const; 13 | void pop(int len); 14 | void clear(); 15 | 16 | protected: 17 | Chunk *data_buf{ nullptr }; 18 | }; 19 | 20 | class InputBuffer : public BufferBase 21 | { 22 | public: 23 | int read_from_fd(int fd); 24 | 25 | const char *get_from_buf() const; 26 | 27 | void adjust(); 28 | }; 29 | 30 | class OutputBuffer : public BufferBase 31 | { 32 | public: 33 | int write2buf(const char *data, int len); 34 | 35 | int write2fd(int fd); 36 | }; 37 | 38 | #endif -------------------------------------------------------------------------------- /memory/mem_pool.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "../log/pr.h" 4 | #include "mem_pool.h" 5 | 6 | void Mempool::mem_init(MEM_CAP size, int chunk_num) 7 | { 8 | Chunk *prev; 9 | mp_pool[size] = new (std::nothrow) Chunk(size); 10 | if (mp_pool[size] == nullptr) { 11 | PR_ERROR("new chunk %d error", static_cast(size)); 12 | exit(1); 13 | } 14 | prev = mp_pool[size]; 15 | 16 | for (int i = 1; i < chunk_num; i ++) { 17 | prev->next = new (std::nothrow)Chunk(size); 18 | if (prev->next == nullptr) { 19 | PR_ERROR("new chunk m4K error"); 20 | exit(1); 21 | } 22 | prev = prev->next; 23 | } 24 | mp_total_size_kb += size/1024 * chunk_num; 25 | } 26 | 27 | Mempool::Mempool() : mp_total_size_kb(0), mp_left_size_kb(0) 28 | { 29 | mem_init(m4K, 2000); 30 | mem_init(m16K, 500); 31 | mem_init(m64K, 250); 32 | mem_init(m256K, 100); 33 | mem_init(m1M, 25); 34 | mem_init(m4M, 10); 35 | mp_left_size_kb = mp_total_size_kb; 36 | } 37 | 38 | Chunk *Mempool::alloc_chunk(int n) 39 | { 40 | int index; 41 | bool found = false; 42 | for(index = mLow; index <= mUp; index = index * MEM_CAP_MULTI_POWER) 43 | { 44 | if(n <= index) 45 | { 46 | found = true; 47 | break; 48 | } 49 | } 50 | 51 | if(!found) 52 | { 53 | return nullptr; 54 | } 55 | 56 | lock_guard lck(mp_mutex); 57 | if (mp_pool[index] == nullptr) { 58 | if (mp_total_size_kb + index/1024 >= MAX_POOL_SIZE) { 59 | PR_ERROR("beyond the limit size of memory!\n"); 60 | exit(1); 61 | } 62 | 63 | Chunk *new_buf = new (std::nothrow) Chunk(index); 64 | if (new_buf == nullptr) { 65 | PR_ERROR("new chunk error\n"); 66 | exit(1); 67 | } 68 | mp_total_size_kb += index/1024; 69 | return new_buf; 70 | } 71 | 72 | Chunk *target = mp_pool[index]; 73 | mp_pool[index] = target->next; 74 | target->next = nullptr; 75 | mp_left_size_kb -= index/1024; 76 | 77 | return target; 78 | } 79 | 80 | void Mempool::retrieve(Chunk *block) 81 | { 82 | int index = block->capacity; 83 | block->length = 0; 84 | block->head = 0; 85 | 86 | lock_guard lck(mp_mutex); 87 | assert(mp_pool.find(index) != mp_pool.end()); 88 | 89 | block->next = mp_pool[index]; 90 | mp_pool[index] = block; 91 | mp_left_size_kb += block->capacity/1024; 92 | } 93 | 94 | int Mempool::get_list_size_byte(MEM_CAP index) 95 | { 96 | int size = 0; 97 | lock_guard lck(mp_mutex); 98 | assert(mp_pool.find(index) != mp_pool.end()); 99 | Chunk *node = mp_pool[index]; 100 | 101 | while(node) 102 | { 103 | size += node->capacity; 104 | node = node->next; 105 | } 106 | 107 | return size; 108 | } 109 | 110 | void Mempool::print_list_content(MEM_CAP index) 111 | { 112 | lock_guard lck(mp_mutex); 113 | int cnt = 0; 114 | printf("***************start to print %dkb chunk_size list data*******************\n", index/1024); 115 | assert(mp_pool.find(index) != mp_pool.end()); 116 | Chunk *node = mp_pool[index]; 117 | 118 | while (node) 119 | { 120 | if(cnt <= 5) 121 | node->print_data(); 122 | cnt++; 123 | node = node->next; 124 | } 125 | printf("...\n"); 126 | printf("******************end, node cnt is %d************************\n\n", cnt); 127 | } -------------------------------------------------------------------------------- /memory/mem_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef __MEM_POOL_H__ 2 | #define __MEM_POOL_H__ 3 | 4 | #include 5 | #include 6 | 7 | #include "chunk.h" 8 | 9 | using namespace std; 10 | 11 | typedef unordered_map pool_t; 12 | 13 | #define MEM_CAP_MULTI_POWER (4) 14 | 15 | typedef enum { 16 | mLow = 4096, 17 | m4K = mLow, 18 | m16K = m4K * MEM_CAP_MULTI_POWER, 19 | m64K = m16K * MEM_CAP_MULTI_POWER, 20 | m256K = m64K * MEM_CAP_MULTI_POWER, 21 | m1M = m256K * MEM_CAP_MULTI_POWER, 22 | m4M = m1M * MEM_CAP_MULTI_POWER, 23 | mUp = m4M 24 | } MEM_CAP; 25 | 26 | #define MAX_POOL_SIZE (4U *1024 *1024) 27 | 28 | class Mempool 29 | { 30 | public: 31 | static Mempool& get_instance() { 32 | static Mempool mp_instance; 33 | return mp_instance; 34 | } 35 | 36 | Chunk *alloc_chunk(int n); 37 | Chunk *alloc_chunk() { return alloc_chunk(m4K); } 38 | 39 | void retrieve(Chunk *block); 40 | 41 | // FIXME: use smart ptr to manage chunk or add destroy interface to recycle memory. 42 | // static void destroy(); 43 | 44 | ~Mempool() = default; 45 | 46 | // api for debug 47 | [[deprecated("mem pool debug api deprecated!")]] 48 | int get_total_size_kb(){ return mp_total_size_kb; } 49 | [[deprecated("mem pool debug api deprecated!")]] 50 | int get_left_size_kb(){ return mp_left_size_kb; } 51 | [[deprecated("mem pool debug api deprecated!")]] 52 | int get_list_size_byte(MEM_CAP index); 53 | [[deprecated("mem pool debug api deprecated!")]] 54 | void print_list_content(MEM_CAP index); 55 | 56 | private: 57 | Mempool(); 58 | Mempool(const Mempool&) = delete; 59 | Mempool(Mempool&&) = delete; 60 | Mempool& operator=(const Mempool&) = delete; 61 | Mempool& operator=(Mempool&&) = delete; 62 | 63 | void mem_init(MEM_CAP size, int chunk_num); 64 | 65 | pool_t mp_pool; 66 | uint64_t mp_total_size_kb; 67 | uint64_t mp_left_size_kb; 68 | mutex mp_mutex; 69 | }; 70 | 71 | #endif -------------------------------------------------------------------------------- /memory/tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | aux_source_directory(.. native_source) 2 | set(SRCS 3 | test_mem.cpp 4 | ../../log/pr.cpp 5 | ../../log/log.cpp 6 | ) 7 | list(APPEND SRCS ${native_source}) 8 | 9 | set(INCS 10 | ../ 11 | ../../log 12 | ) 13 | include_directories(${INCS}) 14 | 15 | add_executable(mem_test ${SRCS}) 16 | target_link_libraries(mem_test pthread) 17 | 18 | list(REMOVE_ITEM SRCS test_mem.cpp) 19 | list(APPEND SRCS test_buf.cpp) 20 | add_executable(buf_test ${SRCS}) 21 | target_link_libraries(buf_test pthread) 22 | 23 | -------------------------------------------------------------------------------- /memory/tests/test_buf.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "data_buf.h" 5 | #include "log.h" 6 | 7 | int main() 8 | { 9 | Logger::get_instance()->init(NULL); 10 | 11 | FILE *fp = fopen("./test_buf.txt", "w+t"); 12 | if(fp == NULL) 13 | { 14 | LOG_ERROR("open file failed\n"); 15 | return -1; 16 | } 17 | int fd = fileno(fp); 18 | 19 | const char *content = "hello"; 20 | fputs(content, fp); 21 | rewind(fp); 22 | LOG_INFO("content of file: %s", content); 23 | 24 | InputBuffer ib; 25 | OutputBuffer ob; 26 | 27 | int r_cnt = ib.read_from_fd(fd); 28 | LOG_INFO("read %d bytes from file by InputBuffer\n", r_cnt); 29 | const char *r_data = ib.get_from_buf(); 30 | LOG_INFO("data get from InputBuffer: %s", r_data); 31 | ib.clear(); 32 | 33 | const char * w_data = "world"; 34 | ob.write2buf(w_data, strlen(w_data)); 35 | int w_cnt= ob.write2fd(fd); 36 | LOG_INFO("write %d bytes to file by OutputBuffer\n", w_cnt); 37 | LOG_INFO("data written to file by OutputBuffer: %s", w_data); 38 | 39 | rewind(fp); 40 | r_cnt = ib.read_from_fd(fd); 41 | LOG_INFO("read %d bytes from file by InputBuffer\n", r_cnt); 42 | r_data = ib.get_from_buf(); 43 | LOG_INFO("data get from buf: %s", r_data); 44 | 45 | fclose(fp); 46 | 47 | return 0; 48 | } -------------------------------------------------------------------------------- /memory/tests/test_mem.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "mem_pool.h" 6 | #include "log.h" 7 | 8 | using namespace std; 9 | 10 | void print_total_size() 11 | { 12 | int total_size = Mempool::get_instance().get_total_size_kb(); 13 | LOG_INFO("total size of mem pool is %dkb\n", total_size); 14 | } 15 | 16 | void print_left_size() 17 | { 18 | int left_size = Mempool::get_instance().get_left_size_kb(); 19 | LOG_INFO("left size of mem pool is %dkb\n", left_size); 20 | } 21 | 22 | void print_list_size() 23 | { 24 | int list_size; 25 | for(int index = mLow; index <= mUp; index = index * MEM_CAP_MULTI_POWER) 26 | { 27 | list_size = Mempool::get_instance().get_list_size_byte((MEM_CAP)index); 28 | LOG_INFO("total size of pool's %dkb list is %dkb\n", index/1024, list_size/1024); 29 | } 30 | } 31 | 32 | void print_list_content() 33 | { 34 | for(int index = mLow; index <= mUp; index = index * MEM_CAP_MULTI_POWER) 35 | { 36 | Mempool::get_instance().print_list_content((MEM_CAP)index); 37 | } 38 | } 39 | 40 | void alloc_and_fill(shared_ptr& sp_chunk, vector& chunks, MEM_CAP index, int num) 41 | { 42 | for(int i=0; icopy(sp_chunk.get()); 46 | chunks.push_back(b); 47 | } 48 | } 49 | 50 | void retrieve_chunks(vector& chunks) 51 | { 52 | for(auto i = chunks.begin(); i!=chunks.end(); ++i) 53 | { 54 | Mempool::get_instance().retrieve(*i); 55 | } 56 | } 57 | 58 | int main() 59 | { 60 | Logger::get_instance()->init(NULL); 61 | 62 | LOG_INFO("===================before alloc...\n"); 63 | print_total_size(); 64 | print_left_size(); 65 | print_list_size(); 66 | print_list_content(); 67 | 68 | vector chunks; 69 | 70 | shared_ptr sp_chunk = make_shared(10); 71 | string s = "hello"; 72 | strcpy(sp_chunk->data, s.c_str()); 73 | sp_chunk->length = strlen(s.c_str()); 74 | 75 | LOG_INFO("===================start to alloc...\n"); 76 | alloc_and_fill(sp_chunk, chunks, m4K, 1000); 77 | LOG_INFO("alloc 1000 chunks, chunk size: %dkb\n", m4K/1024); 78 | alloc_and_fill(sp_chunk, chunks, m16K, 1000); 79 | LOG_INFO("alloc 1000 chunks, chunk size: %dkb\n", m16K/1024); 80 | alloc_and_fill(sp_chunk, chunks, m64K, 500); 81 | LOG_INFO("alloc 500 chunks, chunk size: %dkb\n", m64K/1024); 82 | alloc_and_fill(sp_chunk, chunks, m256K, 30); 83 | LOG_INFO("alloc 30 chunks, chunk size: %dkb\n", m256K/1024); 84 | alloc_and_fill(sp_chunk, chunks, m1M, 30); 85 | LOG_INFO("alloc 30 chunks, chunk size: %dkb\n", m1M/1024); 86 | alloc_and_fill(sp_chunk, chunks, m4M, 30); 87 | LOG_INFO("alloc 30 chunks, chunk size: %dkb\n", m4M/1024); 88 | 89 | LOG_INFO("===================after alloc...\n"); 90 | print_total_size(); 91 | print_left_size(); 92 | print_list_size(); 93 | print_list_content(); 94 | 95 | LOG_INFO("===================start to retrieve...\n"); 96 | retrieve_chunks(chunks); 97 | 98 | LOG_INFO("===================after retrieve...\n"); 99 | print_total_size(); 100 | print_left_size(); 101 | print_list_size(); 102 | print_list_content(); 103 | 104 | return 0; 105 | } -------------------------------------------------------------------------------- /net/README.md: -------------------------------------------------------------------------------- 1 | ## 网络io 2 |   使用epoll LT触发模式，主从reactor设计。包括epoll，event loop，tcp connection，acceptor，tcp server。 3 | ### epoll 4 | > * 对linux中epoll的封装 5 | > * 实现对所监听fd集合及事件、回调函数的增删改 6 | > * 实现对所监听fd注册事件的监视及回调触发 7 | ### event loop 8 | > * 包含一个epoll，在loop循环中对sock fd集合进行监听 9 | > * 支持同线程和跨线程添加任务 10 | > * 通过event fd实现异步添加任务到loop循环中执行 11 | ### tcp connection 12 | > * 一个tcp connection代表一个与客户端通信的连接 13 | > * 一个tcp connection属于一个event loop，包含所属event loop的指针 14 | > * 一个tcp connection属于一个tcp server，包含所属tcp server的指针 15 | > * 一个tcp connection包含data_buf，作为应用层缓冲区收发数据 16 | > * 可以给tcp connection设置事件和回调函数，这些将被注册到所属eventloop的epoll中被监听和触发 17 | > * tcp connection包含定时器id，当有新的消息到来，tcp server可以通过id更新定时器中该tcp connection的时间，实现剔除超时连接 18 | > * tcp connection中包含std::any的对象，用于对应用层协议对象状态的保存和获取，以实现对各种应用层协议的支持 19 | ### acceptor 20 | > * 实现bind，listen，accept功能 21 | > * 属于一个单独的event loop，在其中执行accept任务 22 | ### tcp server 23 | > * 使用acceptor进行bind，listen，accept 24 | > * 拥有线程池，每个线程池中执行一个event loop，用于对tcp conn事件的监听处理 25 | > * 拥有定时器，对tcp conn进行超时剔除 26 | > * 使用round robin的方式，选取event loop为新来的tcp连接服务 27 | 28 | ### 测试 29 | > * echo客户端 30 | > * 在tcp server的基础上，实现的echo server 31 | -------------------------------------------------------------------------------- /net/acceptor.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "../log/pr.h" 12 | #include "../log/log.h" 13 | #include "event_loop.h" 14 | #include "tcp_server.h" 15 | #include "acceptor.h" 16 | 17 | using namespace std; 18 | 19 | Acceptor::Acceptor(TcpServer* server, EventLoop* loop, const char *ip, uint16_t port) 20 | : ac_server(server), 21 | ac_loop(loop), 22 | ac_listening(false), 23 | ac_idle_fd(open("/dev/null", O_RDONLY | O_CLOEXEC)), 24 | ac_listen_fd(socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP)) 25 | { 26 | LOG_INFO("create one acceptor, listen fd is %d\n", ac_listen_fd); 27 | assert(ac_listen_fd >= 0); 28 | assert(ac_idle_fd >= 0); 29 | 30 | int op = 1; 31 | if (setsockopt(ac_listen_fd, SOL_SOCKET, SO_REUSEADDR, &op, sizeof(op)) < 0) { 32 | PR_ERROR("set listen socket SO_REUSEADDR failed!\n"); 33 | } 34 | 35 | memset(&ac_server_addr, 0, sizeof(ac_server_addr)); 36 | ac_server_addr.sin_family = AF_INET; 37 | inet_aton(ip, &ac_server_addr.sin_addr); 38 | ac_server_addr.sin_port = htons(port); 39 | 40 | LOG_INFO("acceptor bind, ip is %s, port is %d\n", ip, (int)port); 41 | if (::bind(ac_listen_fd, (const struct sockaddr*)&ac_server_addr, sizeof(ac_server_addr)) < 0) { 42 | PR_ERROR("bind server port error!\n"); 43 | exit(1); 44 | } 45 | } 46 | 47 | Acceptor::~Acceptor() 48 | { 49 | close(ac_listen_fd); 50 | close(ac_idle_fd); 51 | } 52 | 53 | void Acceptor::listen() 54 | { 55 | LOG_INFO("acceptor execute listen, listen fd is %d\n", ac_listen_fd); 56 | if (::listen(ac_listen_fd, 1000) == -1) { 57 | PR_ERROR("server listen error\n"); 58 | exit(1); 59 | } 60 | ac_listening = true; 61 | ac_loop->add_to_poller(ac_listen_fd, EPOLLIN, [this](){ this->do_accept(); }); 62 | } 63 | 64 | void Acceptor::do_accept() 65 | { 66 | int connfd; 67 | struct sockaddr_in conn_addr; 68 | socklen_t conn_addrlen = sizeof conn_addr; 69 | while(true) { 70 | if (connfd = accept(ac_listen_fd, (struct sockaddr*)&conn_addr, &conn_addrlen); connfd == -1) { 71 | if (errno == EINTR) { 72 | PR_ERROR("accept fail, errno=EINTR, continue\n"); 73 | continue; 74 | } 75 | else if (errno == EMFILE) { 76 | PR_WARN("accept fail, errno=EMFILE, use idle fd\n"); 77 | close(ac_idle_fd); 78 | ac_idle_fd = accept(ac_listen_fd, NULL, NULL); 79 | close(ac_idle_fd); 80 | ac_idle_fd = open("/dev/null", O_RDONLY | O_CLOEXEC); 81 | } 82 | else if (errno == EAGAIN) { 83 | PR_ERROR("accept fail, errno=EAGAIN, break\n"); 84 | break; 85 | } 86 | else { 87 | PR_ERROR("accept fail, error no:%d, error str:%s\n", errno,strerror(errno)); 88 | continue; 89 | } 90 | } 91 | else { 92 | LOG_INFO("accepted one connection, sock fd is %d\n", connfd); 93 | EventLoop* sub_loop = ac_server->get_next_loop(); 94 | 95 | TcpConnSP conn = make_shared(ac_server, sub_loop, connfd, conn_addr, conn_addrlen); 96 | conn->set_connected_cb(ac_server->ts_connected_cb); 97 | conn->set_message_cb(ac_server->ts_message_cb); 98 | conn->set_close_cb(ac_server->ts_close_cb); 99 | conn->add_task(); 100 | 101 | lock_guard lck(ac_server->ts_mutex); 102 | ac_server->add_new_tcp_conn(conn); 103 | } 104 | } 105 | } 106 | 107 | -------------------------------------------------------------------------------- /net/acceptor.h: -------------------------------------------------------------------------------- 1 | #ifndef __ACCEPTOR_H__ 2 | #define __ACCEPTOR_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace std; 9 | 10 | class EventLoop; 11 | class TcpServer; 12 | 13 | class Acceptor 14 | { 15 | public: 16 | Acceptor(TcpServer* server, EventLoop* loop, const char *ip, uint16_t port); 17 | ~Acceptor(); 18 | 19 | bool is_listenning() const { return ac_listening; } 20 | void listen(); 21 | 22 | private: 23 | void do_accept(); 24 | 25 | TcpServer *ac_server; 26 | int ac_listen_fd; 27 | EventLoop *ac_loop; 28 | bool ac_listening; 29 | int ac_idle_fd; 30 | sockaddr_in ac_server_addr; 31 | }; 32 | 33 | #endif -------------------------------------------------------------------------------- /net/epoll.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "epoll.h" 7 | #include "../log/log.h" 8 | 9 | using namespace std; 10 | 11 | const int EVENTSNUM = 4096; 12 | const int EPOLLWAIT_TIME = 20000; 13 | 14 | Epoll::Epoll() : ep_epoll_fd(epoll_create1(EPOLL_CLOEXEC)), ep_events(EVENTSNUM) { 15 | assert(ep_epoll_fd > 0); 16 | } 17 | 18 | Epoll::~Epoll() {} 19 | 20 | void Epoll::epoll_add(int fd, int event, const EventCallback& cb) { 21 | int final_events; 22 | int op; 23 | 24 | if (auto ret = ep_event_map.find(fd); ret == ep_event_map.end()) { 25 | final_events = event; 26 | op = EPOLL_CTL_ADD; 27 | } 28 | else { 29 | final_events = ret->second.event | event; 30 | op = EPOLL_CTL_MOD; 31 | } 32 | 33 | if (event & EPOLLIN) { 34 | ep_event_map[fd].read_callback = cb; 35 | } 36 | else if (event & EPOLLOUT) { 37 | ep_event_map[fd].write_callback = cb; 38 | } 39 | 40 | ep_event_map[fd].event = final_events; 41 | 42 | struct epoll_event ev; 43 | ev.events = final_events; 44 | ev.data.fd = fd; 45 | if (epoll_ctl(ep_epoll_fd, op, fd, &ev) == -1) { 46 | PR_ERROR("epoll ctl error for fd %d\n", fd); 47 | return; 48 | } 49 | LOG_INFO("epoll add, fd is %d, event is %d\n", fd, final_events); 50 | ep_listen_fds.insert(fd); 51 | } 52 | 53 | 54 | void Epoll::epoll_del(int fd, int event) { 55 | 56 | ep_event_map_iter ret; 57 | if (ret = ep_event_map.find(fd); ret == ep_event_map.end()) { 58 | return ; 59 | } 60 | 61 | int &target_event = ret->second.event; 62 | 63 | if (target_event = target_event & (~event); target_event == 0) { 64 | this->epoll_del(fd); 65 | } 66 | else { 67 | struct epoll_event ev; 68 | ev.events = target_event; 69 | ev.data.fd = fd; 70 | if (epoll_ctl(ep_epoll_fd, EPOLL_CTL_MOD, fd, &ev) == -1) { 71 | PR_ERROR("epoll ctl error for fd %d\n", fd); 72 | return; 73 | } 74 | } 75 | } 76 | 77 | void Epoll::epoll_del(int fd) { 78 | 79 | ep_event_map.erase(fd); 80 | 81 | ep_listen_fds.erase(fd); 82 | 83 | epoll_ctl(ep_epoll_fd, EPOLL_CTL_DEL, fd, NULL); 84 | } 85 | 86 | int Epoll::poll() { 87 | while (true) { 88 | int event_count = 89 | epoll_wait(ep_epoll_fd, &*ep_events.begin(), ep_events.size(), EPOLLWAIT_TIME); 90 | if (event_count < 0) 91 | { 92 | PR_ERROR("epoll wait return val <0! error no:%d, error str:%s\n", errno, strerror(errno)); 93 | continue; 94 | } 95 | 96 | execute_cbs(event_count); 97 | return event_count; 98 | } 99 | } 100 | 101 | inline void Epoll::execute_cbs(int event_count) { 102 | for (int i = 0; i < event_count; i++) { 103 | auto ev_ret = ep_event_map.find(ep_events[i].data.fd); 104 | assert(ev_ret != ep_event_map.end()); 105 | 106 | io_event *ev = &(ev_ret->second); 107 | 108 | if (ep_events[i].events & EPOLLIN) { 109 | LOG_INFO("execute read cb\n"); 110 | if(ev->read_callback) ev->read_callback(); 111 | } 112 | else if (ep_events[i].events & EPOLLOUT) { 113 | LOG_INFO("execute write cb\n"); 114 | if(ev->write_callback) ev->write_callback(); 115 | } 116 | else if (ep_events[i].events & (EPOLLHUP|EPOLLERR)) { 117 | if (ev->read_callback) { 118 | ev->read_callback(); 119 | } 120 | else if (ev->write_callback) { 121 | ev->write_callback(); 122 | } 123 | else { 124 | LOG_INFO("get error, delete fd %d from epoll\n", ep_events[i].data.fd); 125 | epoll_del(ep_events[i].data.fd); 126 | } 127 | } 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /net/epoll.h: -------------------------------------------------------------------------------- 1 | #ifndef __EPOLL_H__ 2 | #define __EPOLL_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | using namespace std; 11 | 12 | class Epoll { 13 | public: 14 | 15 | typedef function EventCallback; 16 | 17 | struct io_event 18 | { 19 | int event; 20 | EventCallback read_callback; 21 | EventCallback write_callback; 22 | }; 23 | 24 | Epoll(); 25 | 26 | ~Epoll(); 27 | 28 | void epoll_add(int fd, int event, const EventCallback& cb); 29 | 30 | void epoll_del(int fd, int event); 31 | 32 | void epoll_del(int fd); 33 | 34 | int poll(); 35 | 36 | int get_epoll_fd() { return ep_epoll_fd; } 37 | 38 | void get_listen_fds(set &fd_set) { 39 | fd_set = ep_listen_fds; 40 | } 41 | 42 | private: 43 | void execute_cbs(int event_count); 44 | 45 | static const int MAXFDS = 100000; 46 | int ep_epoll_fd; 47 | set ep_listen_fds; 48 | 49 | typedef unordered_map::iterator ep_event_map_iter; 50 | unordered_map ep_event_map; 51 | vector ep_events; 52 | }; 53 | 54 | #endif -------------------------------------------------------------------------------- /net/event_loop.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "event_loop.h" 8 | #include "../log/pr.h" 9 | #include "../log/log.h" 10 | 11 | using namespace std; 12 | 13 | EventLoop::EventLoop() : el_epoller(new Epoll()) { 14 | if(el_evfd = { eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC) }; el_evfd < 0) 15 | { 16 | PR_ERROR("fail to create event_fd\n"); 17 | exit(1); 18 | } 19 | LOG_INFO("create one eventloop, event fd is %d\n", el_evfd); 20 | el_epoller->epoll_add(el_evfd, EPOLLIN /*| EPOLLET*/, [this](){ this->evfd_read(); }); 21 | } 22 | 23 | EventLoop::~EventLoop() { 24 | close(el_evfd); 25 | } 26 | 27 | void EventLoop::evfd_wakeup() { 28 | uint64_t one = 1; 29 | if(auto n = write(el_evfd, &one, sizeof one); n != sizeof one) 30 | { 31 | PR_ERROR("write %ld bytes to event_fd instead of 8\n", n); 32 | } 33 | } 34 | 35 | void EventLoop::evfd_read() { 36 | uint64_t one = 1; 37 | if(auto n = read(el_evfd, &one, sizeof one); n != sizeof one) 38 | { 39 | PR_ERROR("read %ld bytes from event_fd instead of 8\n", n); 40 | } 41 | } 42 | 43 | void EventLoop::add_task(Task&& cb) { 44 | LOG_INFO("eventloop, add one task\n"); 45 | if (is_in_loop_thread()) 46 | { 47 | cb(); 48 | } 49 | else 50 | { 51 | lock_guard lock(el_mutex); 52 | el_task_funcs.emplace_back(move(cb)); 53 | } 54 | 55 | if (!is_in_loop_thread() || el_dealing_task_funcs) { evfd_wakeup(); } 56 | } 57 | 58 | void EventLoop::loop() { 59 | el_quit = false; 60 | while (!el_quit) { 61 | auto cnt = el_epoller->poll(); 62 | LOG_INFO("eventloop, tid %lld, loop once, epoll event cnt %d\n", tid_to_ll(this_thread::get_id()), cnt); 63 | execute_task_funcs(); 64 | } 65 | } 66 | 67 | void EventLoop::execute_task_funcs() { 68 | std::vector functors; 69 | el_dealing_task_funcs = true; 70 | 71 | { 72 | lock_guard lock(el_mutex); 73 | functors.swap(el_task_funcs); 74 | } 75 | 76 | for (size_t i = 0; i < functors.size(); ++i) functors[i](); 77 | el_dealing_task_funcs = false; 78 | } 79 | 80 | void EventLoop::quit() { 81 | el_quit = true; 82 | if (!is_in_loop_thread()) { 83 | evfd_wakeup(); 84 | } 85 | } -------------------------------------------------------------------------------- /net/event_loop.h: -------------------------------------------------------------------------------- 1 | #ifndef __EVENT_LOOP_H__ 2 | #define __EVENT_LOOP_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "epoll.h" 12 | 13 | using namespace std; 14 | 15 | class EventLoop { 16 | public: 17 | typedef std::function Task; 18 | 19 | EventLoop(); 20 | 21 | ~EventLoop(); 22 | 23 | void loop(); 24 | void quit(); 25 | 26 | void add_task(Task&& cb); 27 | 28 | void add_to_poller(int fd, int event, const Epoll::EventCallback& cb) { 29 | el_epoller->epoll_add(fd, event, cb); 30 | } 31 | 32 | void del_from_poller(int fd, int event) { 33 | el_epoller->epoll_del(fd, event); 34 | } 35 | 36 | void del_from_poller(int fd) { 37 | el_epoller->epoll_del(fd); 38 | } 39 | 40 | bool is_in_loop_thread() const { return el_tid == this_thread::get_id(); } 41 | 42 | private: 43 | shared_ptr el_epoller; 44 | bool el_quit{ false }; 45 | 46 | const thread::id el_tid{ this_thread::get_id() }; 47 | mutex el_mutex; 48 | 49 | int el_evfd; 50 | std::vector el_task_funcs; 51 | bool el_dealing_task_funcs{ false }; 52 | 53 | void evfd_wakeup(); 54 | void evfd_read(); 55 | void execute_task_funcs(); 56 | }; 57 | 58 | #endif -------------------------------------------------------------------------------- /net/tcp_conn.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "tcp_conn.h" 9 | #include "tcp_server.h" 10 | #include "event_loop.h" 11 | #include "../log/pr.h" 12 | #include "../log/log.h" 13 | 14 | using namespace std; 15 | 16 | TcpConnection::TcpConnection(TcpServer *server, EventLoop* loop, int sockfd, struct sockaddr_in& addr, socklen_t& len) { 17 | tc_server = server; 18 | tc_peer_addr = addr; 19 | tc_peer_addrlen = len; 20 | tc_loop = loop; 21 | tc_fd = sockfd; 22 | 23 | set_sockfd(tc_fd); 24 | } 25 | 26 | void TcpConnection::add_task() { 27 | LOG_INFO("tcp connection add connected task to poller, conn fd is %d\n", tc_fd); 28 | tc_loop->add_task([shared_this=shared_from_this()](){ shared_this->connected(); }); 29 | LOG_INFO("tcp connection add do read to poller, conn fd is %d\n", tc_fd); 30 | tc_loop->add_to_poller(tc_fd, EPOLLIN, [shared_this=shared_from_this()](){ shared_this->do_read(); }); 31 | } 32 | 33 | TcpConnection::~TcpConnection() { 34 | LOG_INFO("TcpConnection descontructed, fd is %d\n", tc_fd); 35 | } 36 | 37 | void TcpConnection::set_sockfd(int& fd) { 38 | int flag = fcntl(fd, F_GETFL, 0); 39 | fcntl(fd, F_SETFL, O_NONBLOCK|flag); 40 | int op = 1; 41 | setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &op, sizeof(op)); 42 | } 43 | 44 | void TcpConnection::do_read() { 45 | if (int ret = tc_ibuf.read_from_fd(tc_fd); ret == -1) { 46 | PR_ERROR("read data from socket error\n"); 47 | this->do_close(); 48 | return; 49 | } 50 | else if (ret == 0) { 51 | LOG_INFO("connection closed by peer\n"); 52 | this->do_close(); 53 | return; 54 | } 55 | 56 | tc_message_cb(shared_from_this(), &tc_ibuf); 57 | 58 | return; 59 | } 60 | 61 | bool TcpConnection::send(const char *data, int len) { 62 | bool should_activate_epollout = false; 63 | if(tc_obuf.length() == 0) { 64 | should_activate_epollout = true; 65 | } 66 | 67 | if (int ret = tc_obuf.write2buf(data, len); ret != 0) { 68 | PR_ERROR("send data to output buf error\n"); 69 | return false; 70 | } 71 | 72 | if (should_activate_epollout == true) { 73 | tc_loop->add_to_poller(tc_fd,EPOLLOUT, [this](){ this->do_write(); }); 74 | } 75 | 76 | return true; 77 | } 78 | 79 | void TcpConnection::do_write() { 80 | while (tc_obuf.length()) { 81 | int ret; 82 | if (ret = tc_obuf.write2fd(tc_fd); ret == -1) { 83 | PR_ERROR("write2fd error, close conn!\n"); 84 | this->do_close(); 85 | return ; 86 | } 87 | if (ret == 0) { 88 | break; 89 | } 90 | } 91 | 92 | if (tc_obuf.length() == 0) { 93 | tc_loop->del_from_poller(tc_fd, EPOLLOUT); 94 | } 95 | 96 | return; 97 | } 98 | 99 | void TcpConnection::do_close() { 100 | if (tc_close_cb) { 101 | tc_close_cb(); 102 | } 103 | 104 | tc_loop->del_from_poller(tc_fd); 105 | tc_ibuf.clear(); 106 | tc_obuf.clear(); 107 | 108 | int fd = tc_fd; 109 | tc_fd = -1; 110 | close(fd); 111 | 112 | tc_server->do_clean(shared_from_this()); 113 | } 114 | 115 | void TcpConnection::connected() { 116 | if(tc_connected_cb) { 117 | LOG_INFO("execute connected callback, conn fd is %d\n", tc_fd); 118 | tc_connected_cb(shared_from_this()); 119 | } 120 | else { 121 | LOG_INFO("tcp connected callback is null\n"); 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /net/tcp_conn.h: -------------------------------------------------------------------------------- 1 | #ifndef __TCP_CONN_H__ 2 | #define __TCP_CONN_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "../memory/data_buf.h" 11 | 12 | using namespace std; 13 | 14 | class TcpConnection; 15 | 16 | typedef shared_ptr TcpConnSP; 17 | 18 | class EventLoop; 19 | class TcpServer; 20 | 21 | class TcpConnection : public enable_shared_from_this 22 | { 23 | public: 24 | typedef function ConnectionCallback; 25 | typedef function CloseCallback; 26 | typedef function MessageCallback; 27 | 28 | TcpConnection(TcpServer *server, EventLoop* loop, int sockfd, struct sockaddr_in& addr, socklen_t& len); 29 | ~TcpConnection(); 30 | 31 | EventLoop* getLoop() const { return tc_loop; } 32 | 33 | void add_task(); 34 | 35 | void set_context(const any& context) { tc_context = context; } 36 | auto get_context() { return &tc_context; } 37 | 38 | const char* get_peer_addr() { return inet_ntoa(tc_peer_addr.sin_addr);} 39 | auto get_fd() { return tc_fd; } 40 | 41 | bool send(const char *data, int len); 42 | 43 | void set_connected_cb(const ConnectionCallback& cb) { tc_connected_cb = cb; } 44 | void set_message_cb(const MessageCallback& cb) { tc_message_cb = cb; } 45 | void set_close_cb(const CloseCallback& cb) { tc_close_cb = cb; } 46 | 47 | void connected(); 48 | void active_close() { do_close(); } 49 | 50 | void set_timer_id(int id) {tc_timer_id = id; } 51 | int get_timer_id() { return tc_timer_id; } 52 | 53 | private: 54 | inline void set_sockfd(int& fd); 55 | void do_read(); 56 | void do_write(); 57 | void do_close(); 58 | 59 | TcpServer* tc_server; 60 | EventLoop* tc_loop; 61 | int tc_fd; 62 | int tc_timer_id{ -1 }; 63 | 64 | struct sockaddr_in tc_peer_addr; 65 | socklen_t tc_peer_addrlen; 66 | 67 | OutputBuffer tc_obuf; 68 | InputBuffer tc_ibuf; 69 | 70 | any tc_context; 71 | 72 | ConnectionCallback tc_connected_cb; 73 | MessageCallback tc_message_cb; 74 | CloseCallback tc_close_cb; 75 | }; 76 | 77 | #endif -------------------------------------------------------------------------------- /net/tcp_server.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "../log/pr.h" 6 | #include "../log/log.h" 7 | #include "../threadpool/threadpool.h" 8 | #include "acceptor.h" 9 | #include "tcp_server.h" 10 | #include "event_loop.h" 11 | 12 | TcpServer::TcpServer(EventLoop* loop, const char *ip, uint16_t port) { 13 | 14 | if (signal(SIGHUP, SIG_IGN) == SIG_ERR) { // SIGHUP: when terminal closing 15 | PR_ERROR("ignore SIGHUP signal error\n"); 16 | } 17 | if (signal(SIGPIPE, SIG_IGN) == SIG_ERR) { // SIGPIPE: write a closed client 18 | PR_ERROR("ignore SIGPIPE signal error\n"); 19 | } 20 | 21 | ts_acceptor_loop = loop; 22 | this->ip = ip; 23 | this->port = port; 24 | ts_message_cb = [this](const TcpConnSP& conn, InputBuffer* ibuf) { 25 | update_conn_timeout_time(conn); 26 | if(ts_msg_cb) 27 | { 28 | ts_msg_cb(conn, ibuf); 29 | } 30 | }; 31 | ts_acceptor = make_unique(this, loop, ip, port); 32 | } 33 | 34 | void TcpServer::start() { 35 | if (!ts_started) 36 | { 37 | ts_timer.run(); 38 | ts_started = true; 39 | LOG_INFO("tcp server create thread pool, thread num is %d\n", ts_thread_num); 40 | if(ts_thread_pool = make_unique(ts_thread_num); ts_thread_pool==nullptr) { 41 | PR_ERROR("tcp_server failed to create thread_pool\n"); 42 | exit(1); 43 | } 44 | for(int i=0; ipost_task([&ev]() { ev->loop(); }); 50 | } 51 | } 52 | 53 | if (!ts_acceptor->is_listenning()) 54 | { 55 | LOG_INFO("tcp server add listen task to accpetor eventloop\n"); 56 | ts_acceptor_loop->add_task([this](){ this->ts_acceptor->listen(); }); 57 | } 58 | } 59 | 60 | EventLoop* TcpServer::get_next_loop() { 61 | int size; 62 | if(size = ts_conn_loops.size(); size==0) { return nullptr; } 63 | 64 | ++ts_next_loop; 65 | ts_next_loop = ts_next_loop % size; 66 | return ts_conn_loops[ts_next_loop]; 67 | } 68 | 69 | void TcpServer::do_clean(const TcpConnSP& tcp_conn) { 70 | lock_guard lck(mutex); 71 | for(auto i=ts_tcp_connections.begin(), e=ts_tcp_connections.end(); i!=e; ++i) { 72 | if(tcp_conn==*i) { 73 | LOG_INFO("tcpserver do clean, erase tcp_conn\n"); 74 | ts_tcp_connections.erase(i); 75 | break; 76 | } 77 | } 78 | } 79 | 80 | TcpServer::~TcpServer() { 81 | 82 | } 83 | -------------------------------------------------------------------------------- /net/tcp_server.h: -------------------------------------------------------------------------------- 1 | #ifndef __TCP_SERVER_H__ 2 | #define __TCP_SERVER_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "tcp_conn.h" 10 | #include "../timer/timer.h" 11 | #include "../log/log.h" 12 | 13 | class EventLoop; 14 | class Threadpool; 15 | class Acceptor; 16 | 17 | class TcpServer 18 | { 19 | public: 20 | typedef TcpConnection::ConnectionCallback ConnectionCallback; 21 | typedef TcpConnection::CloseCallback CloseCallback; 22 | typedef TcpConnection::MessageCallback MessageCallback; 23 | 24 | friend class Acceptor; 25 | friend class TcpConnection; 26 | 27 | TcpServer(EventLoop* loop, const char *ip, uint16_t port); 28 | 29 | ~TcpServer(); 30 | 31 | void set_thread_num(int t_num) { ts_thread_num = t_num; } 32 | EventLoop* get_next_loop(); 33 | 34 | void start(); 35 | void do_clean(const TcpConnSP& tcp_conn); 36 | 37 | void set_tcp_conn_timeout_ms(int ms) { ts_tcp_conn_timout_ms = ms; } 38 | void set_connected_cb(const ConnectionCallback& cb) { ts_connected_cb = cb; } 39 | void set_message_cb(const MessageCallback& cb) { ts_msg_cb = cb; } 40 | void set_close_cb(const CloseCallback& cb) { ts_close_cb = cb; } 41 | 42 | private: 43 | void add_new_tcp_conn(const TcpConnSP& tcp_conn) { 44 | auto timer_id = ts_timer.run_after(ts_tcp_conn_timout_ms, false, [tcp_conn]{ 45 | LOG_INFO("tcp conn timeout!\n"); 46 | tcp_conn->active_close(); 47 | }); 48 | tcp_conn->set_timer_id(timer_id); 49 | ts_tcp_connections.emplace_back(tcp_conn); 50 | } 51 | 52 | void update_conn_timeout_time(const TcpConnSP& tcp_conn) { 53 | ts_timer.cancel(tcp_conn->get_timer_id()); 54 | add_new_tcp_conn(tcp_conn); 55 | } 56 | 57 | const char *ip; 58 | uint16_t port; 59 | unique_ptr ts_acceptor; 60 | 61 | EventLoop *ts_acceptor_loop; 62 | vector ts_conn_loops; 63 | unique_ptr ts_thread_pool; 64 | int ts_thread_num{ 1 }; 65 | int ts_next_loop{ -1 }; 66 | 67 | Timer ts_timer; 68 | int ts_tcp_conn_timout_ms { 6000 }; 69 | 70 | mutex ts_mutex; 71 | vector ts_tcp_connections; 72 | 73 | bool ts_started; 74 | 75 | ConnectionCallback ts_connected_cb; 76 | MessageCallback ts_msg_cb; 77 | MessageCallback ts_message_cb; 78 | CloseCallback ts_close_cb; 79 | }; 80 | 81 | #endif -------------------------------------------------------------------------------- /net/tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | aux_source_directory(.. native_source) 2 | aux_source_directory(../../memory memory_source) 3 | set(SRCS 4 | echo_server.cpp 5 | ../../log/pr.cpp 6 | ../../log/log.cpp 7 | ) 8 | list(APPEND SRCS ${native_source}) 9 | list(APPEND SRCS ${memory_source}) 10 | 11 | set(INCS 12 | ../ 13 | ../../log 14 | ../../threadpool 15 | ) 16 | include_directories(${INCS}) 17 | add_executable(echo_server ${SRCS}) 18 | target_link_libraries(echo_server pthread) 19 | 20 | execute_process(COMMAND sh install_client.sh WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) 21 | 22 | list(REMOVE_ITEM SRCS echo_server.cpp) 23 | list(APPEND SRCS http_for_bench.cpp) 24 | add_executable(http_for_bench ${SRCS}) 25 | target_link_libraries(http_for_bench pthread) -------------------------------------------------------------------------------- /net/tests/echo_client.sh: -------------------------------------------------------------------------------- 1 | telnet 127.0.0.1 8888 -------------------------------------------------------------------------------- /net/tests/echo_server.cpp: -------------------------------------------------------------------------------- 1 | #include "tcp_server.h" 2 | #include "event_loop.h" 3 | #include "pr.h" 4 | #include "log.h" 5 | 6 | class EchoServer 7 | { 8 | public: 9 | EchoServer(EventLoop* loop, const char *ip, uint16_t port) : 10 | es_loop(loop), es_server(loop, ip, port) 11 | { 12 | es_server.set_connected_cb([this](const TcpConnSP& conn){ this->echo_conneted_cb(conn); }); 13 | es_server.set_message_cb([this](const TcpConnSP& conn, InputBuffer* ibuf){ this->echo_message_cb(conn, ibuf); }); 14 | es_server.set_close_cb([this](){ this->echo_close_cb(); }); 15 | }; 16 | 17 | ~EchoServer() {}; 18 | 19 | void start(int thread_num) { es_server.set_thread_num(thread_num); es_server.start(); } 20 | 21 | void set_tcp_cn_timeout_ms(int ms) { es_server.set_tcp_conn_timeout_ms(ms); } 22 | 23 | private: 24 | void echo_conneted_cb(const TcpConnSP& conn) { 25 | PR_INFO("one connected! peer addr is %s, local socket fd is %d\n", conn->get_peer_addr(), conn->get_fd()); 26 | } 27 | 28 | void echo_message_cb(const TcpConnSP& conn, InputBuffer* ibuf) { 29 | 30 | const char *msg = ibuf->get_from_buf(); 31 | string msg_str(msg, msg+ibuf->length()); 32 | ibuf->pop(ibuf->length()); 33 | ibuf->adjust(); 34 | 35 | PR_INFO("socket fd %d recv message:%s", conn->get_fd(), msg_str.c_str()); 36 | 37 | conn->send(msg_str.c_str(), msg_str.length()); 38 | } 39 | 40 | void echo_close_cb() { PR_INFO("one connection closed in echo server!\n"); } 41 | 42 | TcpServer es_server; 43 | EventLoop *es_loop; 44 | }; 45 | 46 | 47 | int main() 48 | { 49 | Logger::get_instance()->init(NULL); 50 | 51 | EventLoop base_loop; 52 | EchoServer server(&base_loop, "127.0.0.1", 8888); 53 | server.set_tcp_cn_timeout_ms(8000); 54 | server.start(2); 55 | base_loop.loop(); 56 | 57 | return 0; 58 | } -------------------------------------------------------------------------------- /net/tests/http_for_bench.cpp: -------------------------------------------------------------------------------- 1 | #include "tcp_server.h" 2 | #include "event_loop.h" 3 | #include "pr.h" 4 | #include "log.h" 5 | 6 | class EchoServer 7 | { 8 | public: 9 | EchoServer(EventLoop* loop, const char *ip, uint16_t port) : 10 | es_loop(loop), es_server(loop, ip, port) 11 | { 12 | es_server.set_connected_cb([this](const TcpConnSP& conn){ this->echo_conneted_cb(conn); }); 13 | es_server.set_message_cb([this](const TcpConnSP& conn, InputBuffer* ibuf){ this->echo_message_cb(conn, ibuf); }); 14 | es_server.set_close_cb([this](){ this->echo_close_cb(); }); 15 | }; 16 | 17 | ~EchoServer() {}; 18 | 19 | void start(int thread_num) { es_server.set_thread_num(thread_num); es_server.start(); } 20 | 21 | void set_tcp_cn_timeout_ms(int ms) { es_server.set_tcp_conn_timeout_ms(ms); } 22 | 23 | private: 24 | void echo_conneted_cb(const TcpConnSP& conn) { 25 | PR_INFO("one connected! peer addr is %s, local socket fd is %d\n", conn->get_peer_addr(), conn->get_fd()); 26 | } 27 | 28 | void echo_message_cb(const TcpConnSP& conn, InputBuffer* ibuf) { 29 | 30 | const char *msg = ibuf->get_from_buf(); 31 | string msg_str(msg, msg+ibuf->length()); 32 | ibuf->pop(ibuf->length()); 33 | ibuf->adjust(); 34 | 35 | PR_INFO("socket fd %d recv message:%s", conn->get_fd(), msg_str.c_str()); 36 | 37 | string return_msg("HTTP/1.1 200 OK\r\n\r\n" 38 | "my titleHello World!"); 39 | 40 | conn->send(return_msg.c_str(), return_msg.length()); 41 | } 42 | 43 | void echo_close_cb() { PR_INFO("one connection closed in echo server!\n"); } 44 | 45 | TcpServer es_server; 46 | EventLoop *es_loop; 47 | }; 48 | 49 | 50 | int main() 51 | { 52 | Logger::get_instance()->init(NULL); 53 | 54 | EventLoop base_loop; 55 | EchoServer server(&base_loop, "127.0.0.1", 8889); 56 | server.set_tcp_cn_timeout_ms(8000); 57 | server.start(4); 58 | base_loop.loop(); 59 | 60 | return 0; 61 | } -------------------------------------------------------------------------------- /net/tests/install_client.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | echo "installing echo_client..." 4 | cp echo_client.sh ../../../build/Debug/net/tests/ 5 | chmod +x ../../../build/Debug/net/tests/echo_client.sh -------------------------------------------------------------------------------- /threadpool/README.md: -------------------------------------------------------------------------------- 1 | ## 线程池 2 | - 不支持 拷贝/移动 构造/赋值 函数，支持非自动增长和自动增长两者模式 3 | ### thread pool 4 | > * 使用队列结构和条件变量实现任务队列 5 | > * 使用模板，支持对可变参数任务的添加 6 | > * 自动增长模式在添加任务时，如果没有空闲执行线程，会为线程池新增一个执行线程 7 | > * 线程池执行线程竞争从任务队列获取任务执行 8 | > * 支持任务结果返回，使用std::future< T>对任务的结果进行返回 9 | ### 测试 10 | > * 使用普通函数、类普通成员函数、lambda对象、类静态成员函数等作为任务，对线程池进行测试，对future返回结果验证 -------------------------------------------------------------------------------- /threadpool/tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(SRCS 2 | test_threadpool.cpp 3 | ../../log/pr.cpp 4 | ../../log/log.cpp 5 | ) 6 | set(INCS 7 | ../ 8 | ../../log 9 | ) 10 | include_directories(${INCS}) 11 | add_executable(threadpool_test ${SRCS}) 12 | target_link_libraries(threadpool_test pthread) -------------------------------------------------------------------------------- /threadpool/tests/test_threadpool.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "threadpool.h" 6 | #include "pr.h" 7 | #include "log.h" 8 | 9 | using namespace std; 10 | 11 | random_device rd; 12 | 13 | mt19937 mt(rd()); 14 | 15 | uniform_int_distribution dist(0, 1000); 16 | 17 | auto rnd = bind(dist, mt); 18 | 19 | void func(int slp) 20 | { 21 | long long tid = tid_to_ll(this_thread::get_id()); 22 | LOG_INFO("[tid:%lld] hello, func !\n", tid); 23 | int slp_t = slp + rnd(); 24 | 25 | if (slp > 0) { 26 | LOG_INFO("[tid:%lld] ======= func sleep %d ========= \n",tid, slp_t); 27 | this_thread::sleep_for(chrono::milliseconds(slp_t)); 28 | } 29 | } 30 | 31 | struct stc_tn_callbackobj { 32 | stc_tn_callbackobj(int val) : ret_val(val) 33 | {} 34 | 35 | int operator()(int n) { 36 | long long tid = tid_to_ll(this_thread::get_id()); 37 | LOG_INFO("[tid:%lld] hello, stc_tn_callbackobj! ret value will be %d \n", tid, ret_val); 38 | return ret_val; 39 | } 40 | 41 | private: 42 | int ret_val; 43 | }; 44 | 45 | class cls_static_func { 46 | public: 47 | cls_static_func() 48 | { 49 | } 50 | 51 | static int cls_static_tn_callbackint(int n = 0) { 52 | long long tid = tid_to_ll(this_thread::get_id()); 53 | LOG_INFO("[tid:%lld] hello, cls_static_tn_callbackint! ret value will be %d \n", tid, n); 54 | return n; 55 | } 56 | 57 | static string_view cls_static_tn_callbackstr(int n, string_view str) { 58 | long long tid = tid_to_ll(this_thread::get_id()); 59 | LOG_INFO("[tid:%lld] hello, cls_static_tn_callbackstr !\n", tid); 60 | return str; 61 | } 62 | }; 63 | 64 | 65 | int main() 66 | { 67 | long long tid = tid_to_ll(this_thread::get_id()); 68 | 69 | try { 70 | 71 | Logger::get_instance()->init(NULL); 72 | 73 | Threadpool th_pool{ 50 }; 74 | cls_static_func stc_func; 75 | 76 | auto tn_callbackf = th_pool.post_task(func, 0); 77 | 78 | int param_strc_func = 100; 79 | auto strc_func = th_pool.post_task(stc_tn_callbackobj{param_strc_func}, 0); 80 | 81 | int param_cls_tn_callbackint = 999; 82 | auto cls_tn_callbackint = th_pool.post_task(stc_func.cls_static_tn_callbackint, param_cls_tn_callbackint); 83 | 84 | string_view param_cls_tn_callbackstr("multi args"); 85 | auto cls_tn_callbackstr = th_pool.post_task(cls_static_func::cls_static_tn_callbackstr, rnd(), param_cls_tn_callbackstr); 86 | 87 | auto lmda_func = th_pool.post_task([]()->string { 88 | long long tid = tid_to_ll(this_thread::get_id()); 89 | LOG_INFO("[tid:%lld] hello, lamda func !\n", tid); 90 | return "hello, lamda func !"; 91 | }); 92 | 93 | LOG_INFO("[tid:%lld] ======= sleep ========= \n", tid); 94 | this_thread::sleep_for(chrono::microseconds(1000)); 95 | 96 | for (int i = 0; i < 50; i++) { 97 | th_pool.post_task(func, i*100 ); 98 | } 99 | int idl_cnt = th_pool.idl_thread_cnt(); 100 | LOG_INFO("[tid:%lld] ======= post_task all 1 ========= idl_thread_cnt=%d\n", tid, idl_cnt); 101 | LOG_INFO("[tid:%lld] ======= sleep ========= \n", tid); 102 | this_thread::sleep_for(chrono::seconds(3)); 103 | 104 | auto ret_strc_func = strc_func.get(); 105 | auto ret_cls_tn_callbackint = cls_tn_callbackint.get(); 106 | auto ret_cls_tn_callbackstr1 = cls_tn_callbackstr.get(); 107 | 108 | LOG_INFO("[tid:%lld] ret_val of struct_func: %d\n", tid, ret_strc_func); 109 | assert(ret_strc_func == param_strc_func); 110 | 111 | LOG_INFO("[tid:%lld] ret_val of cls_tn_callbackint: %d\n", tid, ret_cls_tn_callbackint); 112 | assert(ret_cls_tn_callbackint == param_cls_tn_callbackint); 113 | 114 | LOG_INFO("[tid:%lld] ret_val of cls_tn_callbackstr: %s", tid, ret_cls_tn_callbackstr1.data()); 115 | assert(ret_cls_tn_callbackstr1 == param_cls_tn_callbackstr); 116 | 117 | LOG_INFO("[tid:%lld] ======= sleep ========= \n", tid); 118 | this_thread::sleep_for(chrono::seconds(3)); 119 | 120 | LOG_INFO("[tid:%lld] ======= end ========= \n\n", tid); 121 | 122 | 123 | Threadpool pool(4); 124 | vector< future > results; 125 | 126 | for (int i = 0; i < 8; ++i) { 127 | results.emplace_back( 128 | pool.post_task([i]() ->int { 129 | long long tid = tid_to_ll(this_thread::get_id()); 130 | LOG_INFO("[tid:%lld] hello, world %d\n", tid, i); 131 | this_thread::sleep_for(chrono::seconds(1)); 132 | return i*i; 133 | }) 134 | ); 135 | } 136 | 137 | idl_cnt = th_pool.idl_thread_cnt(); 138 | LOG_INFO("[tid:%lld] ======= post_task all 2 ========= idl_thread_cnt=%d\n", tid, idl_cnt); 139 | 140 | for (auto && result : results) 141 | { 142 | LOG_INFO("[tid:%lld] get result: %d\n",tid, result.get()); 143 | } 144 | } 145 | catch (exception& e) { 146 | LOG_ERROR("[tid:%lld] %s", tid, e.what()); 147 | } 148 | 149 | return 0; 150 | } -------------------------------------------------------------------------------- /threadpool/threadpool.h: -------------------------------------------------------------------------------- 1 | #ifndef __THREADPOOL_H__ 2 | #define __THREADPOOL_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #define THREADPOOL_MAX_NUM 64 15 | //#define THREADPOOL_AUTO_GROW 16 | 17 | using namespace std; 18 | 19 | class Threadpool 20 | { 21 | public: 22 | typedef function Task; 23 | 24 | inline Threadpool(unsigned short size = 4) { 25 | assert(size <= THREADPOOL_MAX_NUM); 26 | add_thread(size); 27 | } 28 | inline ~Threadpool() 29 | { 30 | tp_run=false; 31 | tp_task_cv.notify_all(); 32 | for (thread& thread : tp_pool) { 33 | if(thread.joinable()) 34 | thread.join(); 35 | } 36 | } 37 | 38 | template 39 | decltype(auto) post_task(F&& f, Args&&... args) 40 | { 41 | if (!tp_run) 42 | throw runtime_error("post_task on Threadpool has been stopped."); 43 | 44 | using return_type = typename std::result_of_t; 45 | auto task = make_shared>( 46 | bind(forward(f), forward(args)...) 47 | ); 48 | future res = task->get_future(); 49 | { 50 | lock_guard lock{ tp_lock }; 51 | tp_tasks.emplace([task](){ 52 | (*task)(); 53 | }); 54 | } 55 | #ifdef THREADPOOL_AUTO_GROW 56 | if (tp_idl_tnum < 1 && tp_pool.size() < THREADPOOL_MAX_NUM) 57 | add_thread(1); 58 | #endif 59 | tp_task_cv.notify_one(); 60 | 61 | return res; 62 | } 63 | 64 | 65 | int idl_thread_cnt() { return tp_idl_tnum; } 66 | 67 | int thread_cnt() { return tp_pool.size(); } 68 | 69 | #ifndef THREADPOOL_AUTO_GROW 70 | private: 71 | #endif 72 | 73 | void add_thread(unsigned short size) 74 | { 75 | for (; tp_pool.size() < THREADPOOL_MAX_NUM && size > 0; --size) 76 | { 77 | tp_pool.emplace_back( [this]{ 78 | while (tp_run) 79 | { 80 | Task task; 81 | { 82 | unique_lock lock{ tp_lock }; 83 | tp_task_cv.wait(lock, [this]{ 84 | return !tp_run || !tp_tasks.empty(); 85 | }); 86 | if (!tp_run && tp_tasks.empty()) 87 | return; 88 | task = move(tp_tasks.front()); 89 | tp_tasks.pop(); 90 | } 91 | tp_idl_tnum--; 92 | task(); 93 | tp_idl_tnum++; 94 | } 95 | }); 96 | tp_idl_tnum++; 97 | } 98 | } 99 | 100 | private: 101 | Threadpool(const Threadpool &) = delete; 102 | 103 | Threadpool(Threadpool &&) = delete; 104 | 105 | Threadpool & operator=(const Threadpool &) = delete; 106 | 107 | Threadpool & operator=(Threadpool &&) = delete; 108 | 109 | vector tp_pool; 110 | queue tp_tasks; 111 | mutex tp_lock; 112 | condition_variable tp_task_cv; 113 | atomic tp_run{ true }; 114 | atomic tp_idl_tnum{ 0 }; 115 | }; 116 | 117 | #endif -------------------------------------------------------------------------------- /timer/README.md: -------------------------------------------------------------------------------- 1 | ## 定时器 2 | - timer节点的管理使用最小堆。使得距离下一次调度时间最近得Task将会被调度。 3 | - 支持多线程执行到期任务，支持在指定时间后执行任务、周期性执行任务、指定时间间隔重复执行指定次数任务、取消定时器等功能。包括多线程安全的hash map模块和timer模块 4 | ### hash map 5 | > * 使用std::unordered_map作为底层数据结构，配合std::mutex，实现多线程安全的hash map 6 | ### timer 7 | > * tick计时使用std::condition_variable带过期时间的wait_for函数 8 | > * 使用原子变量分配定时器id，记录在hash map结构中，取消定时器即删除map中对应的item 9 | > * 使用阻塞队列存放timer节点，tick线程将到期节点的任务回调函数放入线程池，由线程池执行线程执行 10 | ### 测试 11 | > * 使用普通函数、类普通成员函数、lambda对象、类静态成员函数等作为到期任务，测试指定时间后执行任务、周期性执行任务、指定时间间隔重复执行指定次数任务、取消定时器等功能 -------------------------------------------------------------------------------- /timer/hash_map.h: -------------------------------------------------------------------------------- 1 | #ifndef __HASH_hm_mapH__ 2 | #define __HASH_hm_mapH__ 3 | 4 | #include 5 | #include 6 | 7 | using namespace std; 8 | 9 | template 10 | class hash_map{ 11 | public: 12 | void emplace(const K& key, const V& v) { 13 | unique_lock lck(hm_mutex); 14 | hm_map[key] = v; 15 | } 16 | 17 | void emplace(const K& key, V&& v) { 18 | unique_lock lck(hm_mutex); 19 | hm_map[key] = move(v); 20 | } 21 | 22 | void erase(const K& key) { 23 | unique_lock lck(hm_mutex); 24 | if (hm_map.find(key) != hm_map.end()) { 25 | hm_map.erase(key); 26 | } 27 | } 28 | 29 | bool get_val(const K& key, V& value) { 30 | unique_lock lck(hm_mutex); 31 | if (hm_map.find(key) != hm_map.end()) { 32 | value = hm_map[key]; 33 | return true; 34 | } 35 | return false; 36 | } 37 | 38 | bool is_key_exist(const K& key) { 39 | unique_lock lck(hm_mutex); 40 | return hm_map.find(key) != hm_map.end(); 41 | } 42 | 43 | size_t size() { 44 | unique_lock lck(hm_mutex); 45 | return hm_map.size(); 46 | } 47 | 48 | private: 49 | unordered_map hm_map; 50 | mutex hm_mutex; 51 | }; 52 | 53 | #endif -------------------------------------------------------------------------------- /timer/tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(SRCS 2 | test_timer.cpp 3 | ../../log/pr.cpp 4 | ../../log/log.cpp 5 | ) 6 | set(INCS 7 | ../ 8 | ../../log 9 | ../../threadpool 10 | ) 11 | include_directories(${INCS}) 12 | add_executable(timer_test ${SRCS}) 13 | target_link_libraries(timer_test pthread) -------------------------------------------------------------------------------- /timer/tests/test_timer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "timer.h" 4 | #include "log.h" 5 | #include "pr.h" 6 | 7 | using namespace std; 8 | 9 | void test_repeated_func(chrono::time_point t1, int num) 10 | { 11 | static int cnt = num; 12 | long long tid = tid_to_ll(this_thread::get_id()); 13 | 14 | auto t2 = chrono::steady_clock::now(); 15 | int tm_diff = static_cast( chrono::duration(t2-t1).count() ); 16 | LOG_INFO("[tid:%lld] hello, repeated_func! total repeated_cnt = %d, left cnt: %d, time diff is %d ms\n", 17 | tid, num, --cnt, tm_diff); 18 | } 19 | 20 | void test_run_at_func(chrono::time_point t1, int ms) 21 | { 22 | long long tid = tid_to_ll(this_thread::get_id()); 23 | 24 | auto t2 = chrono::steady_clock::now(); 25 | int tm_diff = static_cast( chrono::duration(t2-t1).count() ); 26 | LOG_INFO("[tid:%lld] hello, run_at_func ! run at %d ms, time diff is %d ms\n", tid, ms, tm_diff); 27 | } 28 | 29 | class period_cls 30 | { 31 | public: 32 | void operator()(chrono::time_point t1, int a) 33 | { 34 | long long tid = tid_to_ll(this_thread::get_id()); 35 | 36 | auto t2 = chrono::steady_clock::now(); 37 | int tm_diff = static_cast( chrono::duration(t2-t1).count() ); 38 | LOG_INFO("[tid:%lld] hello, period_cls_func with param %d! period_cls_run_cnt = %d, time diff is %d ms\n", 39 | tid, a, period_cls_cnt.load(), tm_diff); 40 | period_cls_cnt.fetch_add(1); 41 | return; 42 | } 43 | period_cls() = default; 44 | period_cls(const period_cls&){} 45 | period_cls(period_cls&&){} 46 | private: 47 | atomic period_cls_cnt = 0; 48 | }; 49 | 50 | void test_timer() { 51 | 52 | long long tid = tid_to_ll(this_thread::get_id()); 53 | 54 | Timer t; 55 | t.run(); 56 | 57 | /* test repeated */ 58 | auto t1 = std::chrono::steady_clock::now(); 59 | int repeated_cnt = 5; 60 | int repeated_timeout_ms = 800; 61 | LOG_INFO("[tid:%lld] start to test repeated run, repeated cnt is %d, timeout is %d ms\n", tid, repeated_cnt, repeated_timeout_ms); 62 | auto repeated_id = t.run_repeated(repeated_timeout_ms, repeated_cnt, test_repeated_func, t1, repeated_cnt); 63 | 64 | /* test run at certain time */ 65 | int run_at_ms = 1250; 66 | t1 = std::chrono::steady_clock::now(); 67 | auto now = chrono::high_resolution_clock::now(); 68 | LOG_INFO("[tid:%lld] start to test run at certain time, run at %d ms after now\n", tid, run_at_ms); 69 | auto certion_id = t.run_at(now + std::chrono::milliseconds(run_at_ms), test_run_at_func, t1, run_at_ms); 70 | 71 | /* test run once after certain time */ 72 | int run_after_ms = 2150; 73 | t1 = chrono::steady_clock::now(); 74 | LOG_INFO("[tid:%lld] start to test run once after certain time, run after %d ms\n", tid, run_after_ms); 75 | auto run_after_lamda = [t1, run_after_ms]{ 76 | long long tid = tid_to_ll(this_thread::get_id()); 77 | auto t2 = chrono::steady_clock::now(); 78 | int tm_diff = static_cast( chrono::duration(t2-t1).count() ); 79 | LOG_INFO("[tid:%lld] hello, run_after_once_lambda_func! time diff is %d ms\n", 80 | tid, tm_diff); 81 | }; 82 | auto once_id = t.run_after(run_after_ms, false, run_after_lamda); 83 | 84 | /* test run after certain time periodically */ 85 | int run_after_ms_period = 500; 86 | period_cls p_cls; 87 | int p_param = 99; 88 | t1 = chrono::steady_clock::now(); 89 | LOG_INFO("[tid:%lld] start to run after certain time periodically, period is %d ms\n", tid, run_after_ms_period); 90 | auto period_id = t.run_after(run_after_ms_period, true, [t1, &p_cls](int val){ p_cls(t1, val); }, p_param); 91 | // auto period_id = t.run_after(run_after_ms_period, true, bind(period_cls(), placeholders::_1), p_param); 92 | 93 | 94 | this_thread::sleep_for(chrono::seconds(5)); 95 | /* test cancel */ 96 | t.cancel(period_id); 97 | LOG_INFO("[tid:%lld] cancel periodically running\n", tid); 98 | } 99 | 100 | int main() 101 | { 102 | Logger::get_instance()->init(NULL); 103 | test_timer(); 104 | return 0; 105 | } -------------------------------------------------------------------------------- /timer/timer.h: -------------------------------------------------------------------------------- 1 | #ifndef __TIMER_H__ 2 | #define __TIMER_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "../threadpool/threadpool.h" 14 | #include "hash_map.h" 15 | 16 | using namespace std; 17 | 18 | #define DEFAULT_TIMER_THREAD_POOL_SIZE 2 19 | 20 | class Timer { 21 | public: 22 | struct TimerNode { 23 | chrono::time_point tn_tm_point; 24 | function tn_callback; 25 | int repeated_id; 26 | int tn_id; 27 | int tn_repeated_num; 28 | bool tn_is_period{ false }; 29 | bool tn_is_repeated{ false }; 30 | chrono::milliseconds tn_period; 31 | bool operator<(const TimerNode& b) const { return tn_tm_point > b.tn_tm_point; } 32 | }; 33 | 34 | enum class IdState{ Running = 0 }; 35 | 36 | Timer() : tm_thread_pool(DEFAULT_TIMER_THREAD_POOL_SIZE) { 37 | // cout <<"create timer" <=0; } 58 | 59 | int size() { return tm_queue.size(); } 60 | 61 | template 62 | int run_after(int ms_time, bool is_period, F&& f, Args&&... args) { 63 | TimerNode s; 64 | s.tn_id = tm_id.fetch_add(1); 65 | tm_id_state_map.emplace(s.tn_id, IdState::Running); 66 | s.tn_is_period = is_period; 67 | s.tn_period = chrono::milliseconds(ms_time); 68 | s.tn_tm_point = chrono::high_resolution_clock::now() + s.tn_period; 69 | // TODO: use lamda, tuple and apply of C++17 replacing bind 70 | s.tn_callback = bind(forward(f), forward(args)...); 71 | unique_lock lock(tm_mutex); 72 | tm_queue.push(s); 73 | tm_cond.notify_all(); 74 | // cout <<"push to time queue and notify all"< 80 | int run_at(const chrono::time_point& time_point, F&& f, 81 | Args&&... args) { 82 | TimerNode s; 83 | s.tn_id = tm_id.fetch_add(1); 84 | tm_id_state_map.emplace(s.tn_id, IdState::Running); 85 | s.tn_is_period = false; 86 | s.tn_tm_point = time_point; 87 | // TODO: use lamda, tuple and apply of C++17 replacing bind 88 | s.tn_callback = bind(forward(f), forward(args)...); 89 | unique_lock lock(tm_mutex); 90 | tm_queue.push(s); 91 | tm_cond.notify_all(); 92 | 93 | return s.tn_id; 94 | } 95 | 96 | template 97 | int run_repeated(int ms_time, int repeated_num, F&& f, Args&&... args) 98 | { 99 | TimerNode s; 100 | s.tn_id = tm_id.fetch_add(1); 101 | tm_id_state_map.emplace(s.tn_id, IdState::Running); 102 | s.tn_is_repeated = true; 103 | s.tn_repeated_num = repeated_num; 104 | s.tn_period = chrono::milliseconds(ms_time); 105 | s.tn_tm_point = chrono::high_resolution_clock::now() + s.tn_period; 106 | // TODO: use lamda, tuple and apply of C++17 replacing bind 107 | s.tn_callback = bind(forward(f), forward(args)...); 108 | unique_lock lock(tm_mutex); 109 | tm_queue.push(s); 110 | tm_cond.notify_all(); 111 | 112 | return s.tn_id; 113 | } 114 | 115 | void cancel(int id) 116 | { 117 | if(tm_id_state_map.is_key_exist(id)) 118 | { 119 | tm_id_state_map.erase(id); 120 | } 121 | } 122 | 123 | private: 124 | void run_local() 125 | { 126 | while (tm_running.load()) { 127 | unique_lock lock(tm_mutex); 128 | if (tm_queue.empty()) { 129 | tm_cond.wait(lock); 130 | continue; 131 | } 132 | auto s = tm_queue.top(); 133 | auto diff = s.tn_tm_point - chrono::high_resolution_clock::now(); 134 | if (chrono::duration_cast(diff).count() > 0) { 135 | tm_cond.wait_for(lock, diff); 136 | continue; 137 | } else { 138 | tm_queue.pop(); 139 | if(!tm_id_state_map.is_key_exist(s.tn_id)) 140 | { 141 | continue; 142 | } 143 | if(s.tn_is_period) 144 | { 145 | s.tn_tm_point = chrono::high_resolution_clock::now() + s.tn_period; 146 | tm_queue.push(s); 147 | } 148 | else if(s.tn_is_repeated && s.tn_repeated_num>0) 149 | { 150 | s.tn_tm_point = chrono::high_resolution_clock::now() + s.tn_period; 151 | s.tn_repeated_num--; 152 | tm_queue.push(s); 153 | } 154 | lock.unlock(); 155 | tm_thread_pool.post_task(move(s.tn_callback)); 156 | } 157 | } 158 | } 159 | 160 | priority_queue tm_queue; 161 | atomic tm_running; 162 | mutex tm_mutex; 163 | condition_variable tm_cond; 164 | thread tm_tick_thread; 165 | 166 | Threadpool tm_thread_pool; 167 | atomic tm_id; 168 | hash_map tm_id_state_map; 169 | }; 170 | 171 | #endif -------------------------------------------------------------------------------- /webbench-1.5/COPYRIGHT: -------------------------------------------------------------------------------- 1 | debian/copyright -------------------------------------------------------------------------------- /webbench-1.5/ChangeLog: -------------------------------------------------------------------------------- 1 | debian/changelog -------------------------------------------------------------------------------- /webbench-1.5/Makefile: -------------------------------------------------------------------------------- 1 | CFLAGS?= -Wall -ggdb -W -O 2 | CC?= gcc 3 | LIBS?= 4 | LDFLAGS?= 5 | PREFIX?= /usr/local 6 | VERSION=1.5 7 | TMPDIR=/tmp/webbench-$(VERSION) 8 | 9 | all: webbench tags 10 | 11 | tags: *.c 12 | -ctags *.c 13 | 14 | install: webbench 15 | install -s webbench $(DESTDIR)$(PREFIX)/bin 16 | install -m 644 webbench.1 $(DESTDIR)$(PREFIX)/man/man1 17 | install -d $(DESTDIR)$(PREFIX)/share/doc/webbench 18 | install -m 644 debian/copyright $(DESTDIR)$(PREFIX)/share/doc/webbench 19 | install -m 644 debian/changelog $(DESTDIR)$(PREFIX)/share/doc/webbench 20 | 21 | webbench: webbench.o Makefile 22 | $(CC) $(CFLAGS) $(LDFLAGS) -o webbench webbench.o $(LIBS) 23 | 24 | clean: 25 | -rm -f *.o webbench *~ core *.core tags 26 | 27 | tar: clean 28 | -debian/rules clean 29 | rm -rf $(TMPDIR) 30 | install -d $(TMPDIR) 31 | cp -p Makefile webbench.c socket.c webbench.1 $(TMPDIR) 32 | install -d $(TMPDIR)/debian 33 | -cp -p debian/* $(TMPDIR)/debian 34 | ln -sf debian/copyright $(TMPDIR)/COPYRIGHT 35 | ln -sf debian/changelog $(TMPDIR)/ChangeLog 36 | -cd $(TMPDIR) && cd .. && tar cozf webbench-$(VERSION).tar.gz webbench-$(VERSION) 37 | 38 | webbench.o: webbench.c socket.c Makefile 39 | 40 | .PHONY: clean install all tar 41 | -------------------------------------------------------------------------------- /webbench-1.5/README.md: -------------------------------------------------------------------------------- 1 | ## WebBench 2 |   由[Lionbridge](http://www.lionbridge.com)公司开发的web服务器压测工具。 3 | ### 功能 4 |   测试相同硬件上不同服务的性能和不同硬件上同一服务的运行状况。 5 |   指标：每秒钟响应请求数和每秒钟传输数据量。 6 | 7 | ### 使用示例 8 |   ./webbench --help 9 |   ./webbench -c 5000 -t 120 http://www.163.com -------------------------------------------------------------------------------- /webbench-1.5/debian/changelog: -------------------------------------------------------------------------------- 1 | webbench (1.5) unstable; urgency=low 2 | 3 | * allow building with both Gnu and BSD make 4 | 5 | -- Radim Kolar Fri, Jun 25 12:00:20 CEST 2004 6 | 7 | webbench (1.4) unstable; urgency=low 8 | 9 | * check if url is not too long 10 | * report correct program version number 11 | * use yield() when waiting for test start 12 | * corrected error codes 13 | * check availability of test server first 14 | * do not abort test if first request failed 15 | * report when some childrens are dead. 16 | * use alarm, not time() for lower syscal use by bench 17 | * use mode 644 for installed doc 18 | * makefile cleaned for better freebsd ports integration 19 | 20 | -- Radim Kolar Thu, 15 Jan 2004 11:15:52 +0100 21 | 22 | webbench (1.3) unstable; urgency=low 23 | 24 | * Build fixes for freeBSD 25 | * Default benchmark time 60 -> 30 26 | * generate tar with subdirectory 27 | * added to freeBSD ports collection 28 | 29 | -- Radim Kolar Mon, 12 Jan 2004 17:00:24 +0100 30 | 31 | webbench (1.2) unstable; urgency=low 32 | 33 | * Only debian-related bugfixes 34 | * Updated Debian/rules 35 | * Adapted to fit new directory system 36 | * moved from debstd to dh_* 37 | 38 | -- Radim Kolar Fri, 18 Jan 2002 12:33:04 +0100 39 | 40 | webbench (1.1) unstable; urgency=medium 41 | 42 | * Program debianized 43 | * added support for multiple methods (GET, HEAD, OPTIONS, TRACE) 44 | * added support for multiple HTTP versions (0.9 -- 1.1) 45 | * added long options 46 | * added multiple clients 47 | * wait for start of second before test 48 | * test time can be specified 49 | * better error checking when reading reply from server 50 | * FIX: tests was one second longer than expected 51 | 52 | -- Radim Kolar Thu, 16 Sep 1999 18:48:00 +0200 53 | 54 | Local variables: 55 | mode: debian-changelog 56 | End: 57 | -------------------------------------------------------------------------------- /webbench-1.5/debian/control: -------------------------------------------------------------------------------- 1 | Source: webbench 2 | Section: web 3 | Priority: extra 4 | Maintainer: Radim Kolar 5 | Build-Depends: debhelper (>> 3.0.0) 6 | Standards-Version: 3.5.2 7 | 8 | Package: webbench 9 | Architecture: any 10 | Depends: ${shlibs:Depends} 11 | Description: Simple forking Web benchmark 12 | webbench is very simple program for benchmarking WWW or Proxy servers. 13 | Uses fork() for simulating multiple clients load. Can use HTTP 0.9 - 1.1 14 | requests, but Keep-Alive connections are not supported. 15 | -------------------------------------------------------------------------------- /webbench-1.5/debian/copyright: -------------------------------------------------------------------------------- 1 | Webbench was written by Radim Kolar 1997-2004 (hsn@netmag.cz). 2 | 3 | UNIX sockets code (socket.c) taken from popclient 1.5 4/1/94 4 | public domain code, created by Virginia Tech Computing Center. 5 | 6 | Copyright: GPL (see /usr/share/common-licenses/GPL) 7 | -------------------------------------------------------------------------------- /webbench-1.5/debian/dirs: -------------------------------------------------------------------------------- 1 | usr/bin -------------------------------------------------------------------------------- /webbench-1.5/debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # Sample debian/rules that uses debhelper. 3 | # GNU copyright 1997 to 1999 by Joey Hess. 4 | 5 | # Uncomment this to turn on verbose mode. 6 | #export DH_VERBOSE=1 7 | 8 | # This is the debhelper compatability version to use. 9 | export DH_COMPAT=3 10 | 11 | configure: configure-stamp 12 | configure-stamp: 13 | dh_testdir 14 | touch configure-stamp 15 | 16 | build: configure-stamp build-stamp 17 | build-stamp: 18 | dh_testdir 19 | $(MAKE) 20 | touch build-stamp 21 | 22 | clean: 23 | dh_testdir 24 | rm -f build-stamp configure-stamp 25 | 26 | # Add here commands to clean up after the build process. 27 | -$(MAKE) clean 28 | 29 | dh_clean 30 | 31 | install: build 32 | dh_testdir 33 | dh_testroot 34 | dh_clean -k 35 | dh_installdirs 36 | 37 | # Add here commands to install the package into debian/webbench. 38 | $(MAKE) install DESTDIR=$(CURDIR)/debian/webbench 39 | 40 | 41 | # Build architecture-independent files here. 42 | binary-indep: build install 43 | # We have nothing to do by default. 44 | 45 | # Build architecture-dependent files here. 46 | binary-arch: build install 47 | dh_testdir 48 | dh_testroot 49 | dh_installdocs 50 | dh_installman webbench.1 51 | dh_installchangelogs 52 | dh_link 53 | dh_strip 54 | dh_compress 55 | dh_fixperms 56 | # dh_makeshlibs 57 | dh_installdeb 58 | dh_shlibdeps 59 | dh_gencontrol 60 | dh_md5sums 61 | dh_builddeb 62 | 63 | binary: binary-indep binary-arch 64 | .PHONY: build clean binary-indep binary-arch binary install configure 65 | -------------------------------------------------------------------------------- /webbench-1.5/socket.c: -------------------------------------------------------------------------------- 1 | /* $Id: socket.c 1.1 1995/01/01 07:11:14 cthuang Exp $ 2 | * 3 | * This module has been modified by Radim Kolar for OS/2 emx 4 | */ 5 | 6 | /*********************************************************************** 7 | module: socket.c 8 | program: popclient 9 | SCCS ID: @(#)socket.c 1.5 4/1/94 10 | programmer: Virginia Tech Computing Center 11 | compiler: DEC RISC C compiler (Ultrix 4.1) 12 | environment: DEC Ultrix 4.3 13 | description: UNIX sockets code. 14 | ***********************************************************************/ 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | int Socket(const char *host, int clientPort) 30 | { 31 | int sock; 32 | unsigned long inaddr; 33 | struct sockaddr_in ad; 34 | struct hostent *hp; 35 | 36 | memset(&ad, 0, sizeof(ad)); 37 | ad.sin_family = AF_INET; 38 | 39 | inaddr = inet_addr(host); 40 | if (inaddr != INADDR_NONE) 41 | memcpy(&ad.sin_addr, &inaddr, sizeof(inaddr)); 42 | else 43 | { 44 | hp = gethostbyname(host); 45 | if (hp == NULL) 46 | return -1; 47 | memcpy(&ad.sin_addr, hp->h_addr, hp->h_length); 48 | } 49 | ad.sin_port = htons(clientPort); 50 | 51 | sock = socket(AF_INET, SOCK_STREAM, 0); 52 | if (sock < 0) 53 | return sock; 54 | if (connect(sock, (struct sockaddr *)&ad, sizeof(ad)) < 0) 55 | return -1; 56 | return sock; 57 | } 58 | 59 | -------------------------------------------------------------------------------- /webbench-1.5/tags: -------------------------------------------------------------------------------- 1 | !_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;" to lines/ 2 | !_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/ 3 | !_TAG_PROGRAM_AUTHOR Darren Hiebert /dhiebert@users.sourceforge.net/ 4 | !_TAG_PROGRAM_NAME Exuberant Ctags // 5 | !_TAG_PROGRAM_URL http://ctags.sourceforge.net /official site/ 6 | !_TAG_PROGRAM_VERSION 5.9~svn20110310 // 7 | METHOD_GET webbench.c 35;" d file: 8 | METHOD_HEAD webbench.c 36;" d file: 9 | METHOD_OPTIONS webbench.c 37;" d file: 10 | METHOD_TRACE webbench.c 38;" d file: 11 | PROGRAM_VERSION webbench.c 39;" d file: 12 | REQUEST_SIZE webbench.c 50;" d file: 13 | Socket socket.c /^int Socket(const char *host, int clientPort)$/;" f 14 | alarm_handler webbench.c /^static void alarm_handler(int signal)$/;" f file: 15 | bench webbench.c /^static int bench(void)$/;" f file: 16 | benchcore webbench.c /^void benchcore(const char *host,const int port,const char *req)$/;" f 17 | benchtime webbench.c /^int benchtime=30;$/;" v 18 | build_request webbench.c /^void build_request(const char *url)$/;" f 19 | bytes webbench.c /^int bytes=0;$/;" v 20 | clients webbench.c /^int clients=1;$/;" v 21 | failed webbench.c /^int failed=0;$/;" v 22 | force webbench.c /^int force=0;$/;" v 23 | force_reload webbench.c /^int force_reload=0;$/;" v 24 | host webbench.c /^char host[MAXHOSTNAMELEN];$/;" v 25 | http10 webbench.c /^int http10=1; \/* 0 - http\/0.9, 1 - http\/1.0, 2 - http\/1.1 *\/$/;" v 26 | long_options webbench.c /^static const struct option long_options[]=$/;" v typeref:struct:option file: 27 | main webbench.c /^int main(int argc, char *argv[])$/;" f 28 | method webbench.c /^int method=METHOD_GET;$/;" v 29 | mypipe webbench.c /^int mypipe[2];$/;" v 30 | proxyhost webbench.c /^char *proxyhost=NULL;$/;" v 31 | proxyport webbench.c /^int proxyport=80;$/;" v 32 | request webbench.c /^char request[REQUEST_SIZE];$/;" v 33 | speed webbench.c /^int speed=0;$/;" v 34 | timerexpired webbench.c /^volatile int timerexpired=0;$/;" v 35 | usage webbench.c /^static void usage(void)$/;" f file: 36 | -------------------------------------------------------------------------------- /webbench-1.5/webbench: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CandyConfident/HighPerformanceConcurrentServer/69fe1509347d173dad35a96d886863f8a886998a/webbench-1.5/webbench -------------------------------------------------------------------------------- /webbench-1.5/webbench.1: -------------------------------------------------------------------------------- 1 | .TH WEBBENCH 1 "14 Jan 2004" 2 | .\" NAME should be all caps, SECTION should be 1-8, maybe w/ subsection 3 | .\" other parms are allowed: see man(7), man(1) 4 | .SH NAME 5 | webbench \- simple forking web benchmark 6 | .SH SYNOPSIS 7 | .B webbench 8 | .I "[options] URL" 9 | .br 10 | .SH "AUTHOR" 11 | This program and manual page was written by Radim Kolar, 12 | for the 13 | .B Supreme Personality of Godhead 14 | (but may be used by others). 15 | .SH "DESCRIPTION" 16 | .B webbench 17 | is simple program for benchmarking HTTP servers or any 18 | other servers, which can be accessed via HTTP proxy. Unlike others 19 | benchmarks, 20 | .B webbench 21 | uses multiple processes for simulating traffic 22 | generated by multiple users. This allows better operating 23 | on SMP systems and on systems with slow or buggy implementation 24 | of select(). 25 | .SH OPTIONS 26 | The programs follow the usual GNU command line syntax, with long 27 | options starting with two dashes (`-'). 28 | A summary of options are included below. 29 | .TP 30 | .B \-?, \-h, \-\-help 31 | Show summary of options. 32 | .TP 33 | .B \-v, \-\-version 34 | Show version of program. 35 | .TP 36 | .B \-f, \-\-force 37 | Do not wait for any response from server. Close connection after 38 | request is send. This option produce quite a good denial of service 39 | attack. 40 | .TP 41 | .B \-9, \-\-http09 42 | Use HTTP/0.9 protocol, if possible. 43 | .TP 44 | .B \-1, \-\-http10 45 | Use HTTP/1.0 protocol, if possible. 46 | .TP 47 | .B \-2, \-\-http11 48 | Use HTTP/1.1 protocol (without 49 | .I Keep-Alive 50 | ), if possible. 51 | .TP 52 | .B \-r, \-\-reload 53 | Forces proxy to reload document. If proxy is not 54 | set, option has no effect. 55 | .TP 56 | .B \-t, \-\-time 57 | Run benchmark for 58 | .I 59 | seconds. Default value is 30. 60 | .TP 61 | .B \-p, \-\-proxy 62 | Send request via proxy server. Needed for supporting others protocols 63 | than HTTP. 64 | .TP 65 | .B \-\-get 66 | Use GET request method. 67 | .TP 68 | .B \-\-head 69 | Use HEAD request method. 70 | .TP 71 | .B \-\-options 72 | Use OPTIONS request method. 73 | .TP 74 | .B \-\-trace 75 | Use TRACE request method. 76 | .TP 77 | .B \-c, \-\-clients 78 | Use 79 | .I 80 | multiple clients for benchmark. Default value 81 | is 1. 82 | .SH "EXIT STATUS" 83 | .TP 84 | 0 - sucess 85 | .TP 86 | 1 - benchmark failed, can not connect to server 87 | .TP 88 | 2 - bad command line argument(s) 89 | .TP 90 | 3 - internal error, i.e. fork failed 91 | .SH "TODO" 92 | Include support for using 93 | .I Keep-Alive 94 | HTTP/1.1 connections. 95 | .SH "COPYING" 96 | Webbench is distributed under GPL. Copyright 1997-2004 97 | Radim Kolar (hsn@netmag.cz). 98 | UNIX sockets code taken from popclient 1.5 4/1/94 99 | public domain code, created by Virginia Tech Computing Center. 100 | .BR 101 | This man page is public domain. 102 | -------------------------------------------------------------------------------- /webbench-1.5/webbench.c: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Radim Kolar 1997-2004 3 | * This is free software, see GNU Public License version 2 for 4 | * details. 5 | * 6 | * Simple forking WWW Server benchmark: 7 | * 8 | * Usage: 9 | * webbench --help 10 | * 11 | * Return codes: 12 | * 0 - sucess 13 | * 1 - benchmark failed (server is not on-line) 14 | * 2 - bad param 15 | * 3 - internal error, fork failed 16 | * 17 | */ 18 | #include "socket.c" 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | /* values */ 28 | volatile int timerexpired=0; 29 | int speed=0; 30 | int failed=0; 31 | int bytes=0; 32 | /* globals */ 33 | int http10=1; /* 0 - http/0.9, 1 - http/1.0, 2 - http/1.1 */ 34 | /* Allow: GET, HEAD, OPTIONS, TRACE */ 35 | #define METHOD_GET 0 36 | #define METHOD_HEAD 1 37 | #define METHOD_OPTIONS 2 38 | #define METHOD_TRACE 3 39 | #define PROGRAM_VERSION "1.5" 40 | int method=METHOD_GET; 41 | int clients=1; 42 | int force=0; 43 | int force_reload=0; 44 | int proxyport=80; 45 | char *proxyhost=NULL; 46 | int benchtime=30; 47 | /* internal */ 48 | int mypipe[2]; 49 | char host[MAXHOSTNAMELEN]; 50 | #define REQUEST_SIZE 2048 51 | char request[REQUEST_SIZE]; 52 | 53 | static const struct option long_options[]= 54 | { 55 | {"force",no_argument,&force,1}, 56 | {"reload",no_argument,&force_reload,1}, 57 | {"time",required_argument,NULL,'t'}, 58 | {"help",no_argument,NULL,'?'}, 59 | {"http09",no_argument,NULL,'9'}, 60 | {"http10",no_argument,NULL,'1'}, 61 | {"http11",no_argument,NULL,'2'}, 62 | {"get",no_argument,&method,METHOD_GET}, 63 | {"head",no_argument,&method,METHOD_HEAD}, 64 | {"options",no_argument,&method,METHOD_OPTIONS}, 65 | {"trace",no_argument,&method,METHOD_TRACE}, 66 | {"version",no_argument,NULL,'V'}, 67 | {"proxy",required_argument,NULL,'p'}, 68 | {"clients",required_argument,NULL,'c'}, 69 | {NULL,0,NULL,0} 70 | }; 71 | 72 | /* prototypes */ 73 | static void benchcore(const char* host,const int port, const char *request); 74 | static int bench(void); 75 | static void build_request(const char *url); 76 | 77 | static void alarm_handler(int signal) 78 | { 79 | timerexpired=1; 80 | } 81 | 82 | static void usage(void) 83 | { 84 | fprintf(stderr, 85 | "webbench [option]... URL\n" 86 | " -f|--force Don't wait for reply from server.\n" 87 | " -r|--reload Send reload request - Pragma: no-cache.\n" 88 | " -t|--time Run benchmark for seconds. Default 30.\n" 89 | " -p|--proxy Use proxy server for request.\n" 90 | " -c|--clients Run HTTP clients at once. Default one.\n" 91 | " -9|--http09 Use HTTP/0.9 style requests.\n" 92 | " -1|--http10 Use HTTP/1.0 protocol.\n" 93 | " -2|--http11 Use HTTP/1.1 protocol.\n" 94 | " --get Use GET request method.\n" 95 | " --head Use HEAD request method.\n" 96 | " --options Use OPTIONS request method.\n" 97 | " --trace Use TRACE request method.\n" 98 | " -?|-h|--help This information.\n" 99 | " -V|--version Display program version.\n" 100 | ); 101 | }; 102 | int main(int argc, char *argv[]) 103 | { 104 | int opt=0; 105 | int options_index=0; 106 | char *tmp=NULL; 107 | 108 | if(argc==1) 109 | { 110 | usage(); 111 | return 2; 112 | } 113 | 114 | while((opt=getopt_long(argc,argv,"912Vfrt:p:c:?h",long_options,&options_index))!=EOF ) 115 | { 116 | switch(opt) 117 | { 118 | case 0 : break; 119 | case 'f': force=1;break; 120 | case 'r': force_reload=1;break; 121 | case '9': http10=0;break; 122 | case '1': http10=1;break; 123 | case '2': http10=2;break; 124 | case 'V': printf(PROGRAM_VERSION"\n");exit(0); 125 | case 't': benchtime=atoi(optarg);break; 126 | case 'p': 127 | /* proxy server parsing server:port */ 128 | tmp=strrchr(optarg,':'); 129 | proxyhost=optarg; 130 | if(tmp==NULL) 131 | { 132 | break; 133 | } 134 | if(tmp==optarg) 135 | { 136 | fprintf(stderr,"Error in option --proxy %s: Missing hostname.\n",optarg); 137 | return 2; 138 | } 139 | if(tmp==optarg+strlen(optarg)-1) 140 | { 141 | fprintf(stderr,"Error in option --proxy %s Port number is missing.\n",optarg); 142 | return 2; 143 | } 144 | *tmp='\0'; 145 | proxyport=atoi(tmp+1);break; 146 | case ':': 147 | case 'h': 148 | case '?': usage();return 2;break; 149 | case 'c': clients=atoi(optarg);break; 150 | } 151 | } 152 | 153 | if(optind==argc) { 154 | fprintf(stderr,"webbench: Missing URL!\n"); 155 | usage(); 156 | return 2; 157 | } 158 | 159 | if(clients==0) clients=1; 160 | if(benchtime==0) benchtime=60; 161 | /* Copyright */ 162 | fprintf(stderr,"Webbench - Simple Web Benchmark "PROGRAM_VERSION"\n" 163 | "Copyright (c) Radim Kolar 1997-2004, GPL Open Source Software.\n" 164 | ); 165 | build_request(argv[optind]); 166 | /* print bench info */ 167 | printf("\nBenchmarking: "); 168 | switch(method) 169 | { 170 | case METHOD_GET: 171 | default: 172 | printf("GET");break; 173 | case METHOD_OPTIONS: 174 | printf("OPTIONS");break; 175 | case METHOD_HEAD: 176 | printf("HEAD");break; 177 | case METHOD_TRACE: 178 | printf("TRACE");break; 179 | } 180 | printf(" %s",argv[optind]); 181 | switch(http10) 182 | { 183 | case 0: printf(" (using HTTP/0.9)");break; 184 | case 2: printf(" (using HTTP/1.1)");break; 185 | } 186 | printf("\n"); 187 | if(clients==1) printf("1 client"); 188 | else 189 | printf("%d clients",clients); 190 | 191 | printf(", running %d sec", benchtime); 192 | if(force) printf(", early socket close"); 193 | if(proxyhost!=NULL) printf(", via proxy server %s:%d",proxyhost,proxyport); 194 | if(force_reload) printf(", forcing reload"); 195 | printf(".\n"); 196 | return bench(); 197 | } 198 | 199 | void build_request(const char *url) 200 | { 201 | char tmp[10]; 202 | int i; 203 | 204 | bzero(host,MAXHOSTNAMELEN); 205 | bzero(request,REQUEST_SIZE); 206 | 207 | if(force_reload && proxyhost!=NULL && http10<1) http10=1; 208 | if(method==METHOD_HEAD && http10<1) http10=1; 209 | if(method==METHOD_OPTIONS && http10<2) http10=2; 210 | if(method==METHOD_TRACE && http10<2) http10=2; 211 | 212 | switch(method) 213 | { 214 | default: 215 | case METHOD_GET: strcpy(request,"GET");break; 216 | case METHOD_HEAD: strcpy(request,"HEAD");break; 217 | case METHOD_OPTIONS: strcpy(request,"OPTIONS");break; 218 | case METHOD_TRACE: strcpy(request,"TRACE");break; 219 | } 220 | 221 | strcat(request," "); 222 | 223 | if(NULL==strstr(url,"://")) 224 | { 225 | fprintf(stderr, "\n%s: is not a valid URL.\n",url); 226 | exit(2); 227 | } 228 | if(strlen(url)>1500) 229 | { 230 | fprintf(stderr,"URL is too long.\n"); 231 | exit(2); 232 | } 233 | if(proxyhost==NULL) 234 | if (0!=strncasecmp("http://",url,7)) 235 | { fprintf(stderr,"\nOnly HTTP protocol is directly supported, set --proxy for others.\n"); 236 | exit(2); 237 | } 238 | /* protocol/host delimiter */ 239 | i=strstr(url,"://")-url+3; 240 | /* printf("%d\n",i); */ 241 | 242 | if(strchr(url+i,'/')==NULL) { 243 | fprintf(stderr,"\nInvalid URL syntax - hostname don't ends with '/'.\n"); 244 | exit(2); 245 | } 246 | if(proxyhost==NULL) 247 | { 248 | /* get port from hostname */ 249 | if(index(url+i,':')!=NULL && 250 | index(url+i,':')0) 275 | strcat(request,"User-Agent: WebBench "PROGRAM_VERSION"\r\n"); 276 | if(proxyhost==NULL && http10>0) 277 | { 278 | strcat(request,"Host: "); 279 | strcat(request,host); 280 | strcat(request,"\r\n"); 281 | } 282 | if(force_reload && proxyhost!=NULL) 283 | { 284 | strcat(request,"Pragma: no-cache\r\n"); 285 | } 286 | if(http10>1) 287 | strcat(request,"Connection: close\r\n"); 288 | /* add empty line at end */ 289 | if(http10>0) strcat(request,"\r\n"); 290 | // printf("Req=%s\n",request); 291 | } 292 | 293 | /* vraci system rc error kod */ 294 | static int bench(void) 295 | { 296 | int i,j,k; 297 | pid_t pid=0; 298 | FILE *f; 299 | 300 | /* check avaibility of target server */ 301 | i=Socket(proxyhost==NULL?host:proxyhost,proxyport); 302 | if(i<0) { 303 | fprintf(stderr,"\nConnect to server failed. Aborting benchmark.\n"); 304 | return 1; 305 | } 306 | close(i); 307 | /* create pipe */ 308 | if(pipe(mypipe)) 309 | { 310 | perror("pipe failed."); 311 | return 3; 312 | } 313 | 314 | /* not needed, since we have alarm() in childrens */ 315 | /* wait 4 next system clock tick */ 316 | /* 317 | cas=time(NULL); 318 | while(time(NULL)==cas) 319 | sched_yield(); 320 | */ 321 | 322 | /* fork childs */ 323 | for(i=0;i0) 418 | { 419 | /* fprintf(stderr,"Correcting failed by signal\n"); */ 420 | failed--; 421 | } 422 | return; 423 | } 424 | s=Socket(host,port); 425 | if(s<0) { failed++;continue;} 426 | if(rlen!=write(s,req,rlen)) {failed++;close(s);continue;} 427 | if(http10==0) 428 | if(shutdown(s,1)) { failed++;close(s);continue;} 429 | if(force==0) 430 | { 431 | /* read all available data from socket */ 432 | while(1) 433 | { 434 | if(timerexpired) break; 435 | i=read(s,buf,1500); 436 | /* fprintf(stderr,"%d\n",i); */ 437 | if(i<0) 438 | { 439 | failed++; 440 | close(s); 441 | goto nexttry; 442 | } 443 | else 444 | if(i==0) break; 445 | else 446 | bytes+=i; 447 | } 448 | } 449 | if(close(s)) {failed++;continue;} 450 | speed++; 451 | } 452 | } 453 | -------------------------------------------------------------------------------- /webbench-1.5/webbench.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CandyConfident/HighPerformanceConcurrentServer/69fe1509347d173dad35a96d886863f8a886998a/webbench-1.5/webbench.o --------------------------------------------------------------------------------