├── client.c ├── client_mutlport_epoll.c ├── server.c ├── server_mulport_epoll.c └── server_mulport_mulepoll.c /client.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #define MAX_PORTS 100 11 | 12 | int main(int argc, char **argv){ 13 | if(argc <= 2){ 14 | printf("Usage: %s ip port\n", argv[0]); 15 | exit(0); 16 | } 17 | 18 | struct sockaddr_in addr; 19 | const char *ip = argv[1]; 20 | int base_port = atoi(argv[2]); 21 | int opt = 1; 22 | int bufsize; 23 | socklen_t optlen; 24 | int connections = 0; 25 | 26 | memset(&addr, sizeof(addr), 0); 27 | addr.sin_family = AF_INET; 28 | inet_pton(AF_INET, ip, &addr.sin_addr); 29 | 30 | char tmp_data[10]; 31 | int index = 0; 32 | while(1){ 33 | if(++index >= MAX_PORTS){ 34 | index = 0; 35 | } 36 | int port = base_port + index; 37 | //printf("connect to %s:%d\n", ip, port); 38 | 39 | addr.sin_port = htons((short)port); 40 | 41 | int sock; 42 | if((sock = socket(AF_INET, SOCK_STREAM, 0)) == -1){ 43 | goto sock_err; 44 | } 45 | if(connect(sock, (struct sockaddr *)&addr, sizeof(addr)) == -1){ 46 | goto sock_err; 47 | } 48 | 49 | connections ++; 50 | 51 | if(connections % 1000 == 999){ 52 | //printf("press Enter to continue: "); 53 | //getchar(); 54 | printf("connections: %d, fd: %d\n", connections, sock); 55 | } 56 | usleep(1 * 1000); 57 | 58 | bufsize = 5000; 59 | setsockopt(sock, SOL_SOCKET, SO_SNDBUF, &bufsize, sizeof(bufsize)); 60 | setsockopt(sock, SOL_SOCKET, SO_RCVBUF, &bufsize, sizeof(bufsize)); 61 | } 62 | 63 | return 0; 64 | sock_err: 65 | printf("connections: %d\n", connections); 66 | printf("error: %s\n", strerror(errno)); 67 | return 0; 68 | } 69 | -------------------------------------------------------------------------------- /client_mutlport_epoll.c: -------------------------------------------------------------------------------- 1 | 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | 17 | #define MAX_BUFFER 128 18 | #define MAX_EPOLLSIZE (384*1024) 19 | #define MAX_PORT 4 20 | 21 | #define TIME_SUB_MS(tv1, tv2) ((tv1.tv_sec - tv2.tv_sec) * 1000 + (tv1.tv_usec - tv2.tv_usec) / 1000) 22 | 23 | int isContinue = 0; 24 | 25 | static int ntySetNonblock(int fd) { 26 | int flags; 27 | 28 | flags = fcntl(fd, F_GETFL, 0); 29 | if (flags < 0) return flags; 30 | flags |= O_NONBLOCK; 31 | if (fcntl(fd, F_SETFL, flags) < 0) return -1; 32 | return 0; 33 | } 34 | 35 | static int ntySetReUseAddr(int fd) { 36 | int reuse = 1; 37 | return setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(reuse)); 38 | } 39 | 40 | 41 | 42 | int main(int argc, char **argv) { 43 | if (argc <= 2) { 44 | printf("Usage: %s ip port\n", argv[0]); 45 | exit(0); 46 | } 47 | 48 | const char *ip = argv[1]; 49 | int port = atoi(argv[2]); 50 | int connections = 0; 51 | char buffer[128] = {0}; 52 | int i = 0, index = 0; 53 | 54 | struct epoll_event events[MAX_EPOLLSIZE]; 55 | 56 | int epoll_fd = epoll_create(MAX_EPOLLSIZE); 57 | 58 | strcpy(buffer, " Data From MulClient\n"); 59 | 60 | struct sockaddr_in addr; 61 | memset(&addr, 0, sizeof(struct sockaddr_in)); 62 | 63 | addr.sin_family = AF_INET; 64 | addr.sin_addr.s_addr = inet_addr(ip); 65 | 66 | struct timeval tv_begin; 67 | gettimeofday(&tv_begin, NULL); 68 | 69 | while (1) { 70 | //if (++index >= MAX_PORT) index = 0; 71 | 72 | struct epoll_event ev; 73 | int sockfd = 0; 74 | 75 | if (connections < 380000 && !isContinue) { 76 | sockfd = socket(AF_INET, SOCK_STREAM, 0); 77 | if (sockfd == -1) { 78 | perror("socket"); 79 | goto err; 80 | } 81 | 82 | //ntySetReUseAddr(sockfd); 83 | addr.sin_port = htons(port+index); 84 | 85 | if (connect(sockfd, (struct sockaddr*)&addr, sizeof(struct sockaddr_in)) < 0) { 86 | perror("connect"); 87 | goto err; 88 | } 89 | ntySetNonblock(sockfd); 90 | ntySetReUseAddr(sockfd); 91 | 92 | sprintf(buffer, "Hello Server: client --> %d\n", connections); 93 | send(sockfd, buffer, strlen(buffer), 0); 94 | 95 | ev.data.fd = sockfd; 96 | ev.events = EPOLLIN | EPOLLOUT; 97 | epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sockfd, &ev); 98 | 99 | connections ++; 100 | } 101 | //connections ++; 102 | if (connections % 1000 == 999 || connections >= 380000) { 103 | struct timeval tv_cur; 104 | memcpy(&tv_cur, &tv_begin, sizeof(struct timeval)); 105 | 106 | gettimeofday(&tv_begin, NULL); 107 | 108 | int time_used = TIME_SUB_MS(tv_begin, tv_cur); 109 | printf("connections: %d, sockfd:%d, time_used:%d\n", connections, sockfd, time_used); 110 | 111 | int nfds = epoll_wait(epoll_fd, events, connections, 100); 112 | for (i = 0;i < nfds;i ++) { 113 | int clientfd = events[i].data.fd; 114 | 115 | if (events[i].events & EPOLLOUT) { 116 | sprintf(buffer, "data from %d\n", clientfd); 117 | send(sockfd, buffer, strlen(buffer), 0); 118 | } else if (events[i].events & EPOLLIN) { 119 | char rBuffer[MAX_BUFFER] = {0}; 120 | ssize_t length = recv(sockfd, rBuffer, MAX_BUFFER, 0); 121 | if (length > 0) { 122 | printf(" RecvBuffer:%s\n", rBuffer); 123 | 124 | if (!strcmp(rBuffer, "quit")) { 125 | isContinue = 0; 126 | } 127 | 128 | } else if (length == 0) { 129 | printf(" Disconnect clientfd:%d\n", clientfd); 130 | connections --; 131 | close(clientfd); 132 | } else { 133 | if (errno == EINTR) continue; 134 | 135 | printf(" Error clientfd:%d, errno:%d\n", clientfd, errno); 136 | close(clientfd); 137 | } 138 | } else { 139 | printf(" clientfd:%d, errno:%d\n", clientfd, errno); 140 | close(clientfd); 141 | } 142 | } 143 | } 144 | 145 | usleep(1 * 1000); 146 | } 147 | 148 | return 0; 149 | 150 | err: 151 | printf("error : %s\n", strerror(errno)); 152 | return 0; 153 | 154 | } 155 | 156 | 157 | 158 | -------------------------------------------------------------------------------- /server.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #define MAX_PORTS 100 12 | 13 | int main(int argc, char **argv){ 14 | if(argc < 2){ 15 | printf("Usage: %s port\n", argv[0]); 16 | exit(0); 17 | } 18 | 19 | struct sockaddr_in addr; 20 | const char *ip = "0.0.0.0"; 21 | int opt = 1; 22 | int bufsize; 23 | socklen_t optlen; 24 | int connections = 0; 25 | int base_port = 7000; 26 | if(argc >= 2){ 27 | base_port = atoi(argv[1]); 28 | } 29 | 30 | int server_socks[MAX_PORTS]; 31 | 32 | int i; 33 | int port; 34 | for(i=0; i maxfd){ 68 | maxfd = server_socks[i]; 69 | } 70 | } 71 | int ret = select(maxfd + 1, &readset, NULL, NULL, NULL); 72 | if(ret < 0){ 73 | if(errno == EINTR){ 74 | continue; 75 | }else{ 76 | printf("select error! %s\n", strerror(errno)); 77 | exit(0); 78 | } 79 | } 80 | if(ret == 0){ 81 | continue; 82 | } 83 | 84 | for(i=0; i 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | 21 | #define SERVER_PORT 8080 22 | #define SERVER_IP "127.0.0.1" 23 | #define MAX_BUFFER 128 24 | #define MAX_EPOLLSIZE 100000 25 | #define MAX_THREAD 80 26 | #define MAX_PORT 100 27 | 28 | #define CPU_CORES_SIZE 8 29 | 30 | #define TIME_SUB_MS(tv1, tv2) ((tv1.tv_sec - tv2.tv_sec) * 1000 + (tv1.tv_usec - tv2.tv_usec) / 1000) 31 | 32 | static int ntySetNonblock(int fd) { 33 | int flags; 34 | 35 | flags = fcntl(fd, F_GETFL, 0); 36 | if (flags < 0) return flags; 37 | flags |= O_NONBLOCK; 38 | if (fcntl(fd, F_SETFL, flags) < 0) return -1; 39 | return 0; 40 | } 41 | 42 | static int ntySetReUseAddr(int fd) { 43 | int reuse = 1; 44 | return setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(reuse)); 45 | } 46 | 47 | static int ntySetAlive(int fd) { 48 | int alive = 1; 49 | int idle = 60; 50 | int interval = 5; 51 | int count = 2; 52 | 53 | setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (void*)&alive, sizeof(alive)); 54 | setsockopt(fd, SOL_TCP, TCP_KEEPIDLE, (void*)&idle, sizeof(idle)); 55 | setsockopt(fd, SOL_TCP, TCP_KEEPINTVL, (void*)&interval, sizeof(interval)); 56 | setsockopt(fd, SOL_TCP, TCP_KEEPCNT, (void*)&count, sizeof(count)); 57 | } 58 | 59 | 60 | /** **** ******** **************** thread pool **************** ******** **** **/ 61 | 62 | #define LL_ADD(item, list) { \ 63 | item->prev = NULL; \ 64 | item->next = list; \ 65 | list = item; \ 66 | } 67 | 68 | #define LL_REMOVE(item, list) { \ 69 | if (item->prev != NULL) item->prev->next = item->next; \ 70 | if (item->next != NULL) item->next->prev = item->prev; \ 71 | if (list == item) list = item->next; \ 72 | item->prev = item->next = NULL; \ 73 | } 74 | 75 | typedef struct worker { 76 | pthread_t thread; 77 | int terminate; 78 | struct workqueue *workqueue; 79 | struct worker *prev; 80 | struct worker *next; 81 | } worker_t; 82 | 83 | typedef struct job { 84 | void (*job_function)(struct job *job); 85 | void *user_data; 86 | struct job *prev; 87 | struct job *next; 88 | } job_t; 89 | 90 | typedef struct workqueue { 91 | struct worker *workers; 92 | struct job *waiting_jobs; 93 | pthread_mutex_t jobs_mutex; 94 | pthread_cond_t jobs_cond; 95 | } workqueue_t; 96 | 97 | 98 | static void *worker_function(void *ptr) { 99 | worker_t *worker = (worker_t *)ptr; 100 | job_t *job; 101 | 102 | while (1) { 103 | pthread_mutex_lock(&worker->workqueue->jobs_mutex); 104 | while (worker->workqueue->waiting_jobs == NULL) { 105 | if (worker->terminate) break; 106 | pthread_cond_wait(&worker->workqueue->jobs_cond, &worker->workqueue->jobs_mutex); 107 | } 108 | if (worker->terminate) break; 109 | job = worker->workqueue->waiting_jobs; 110 | if (job != NULL) { 111 | LL_REMOVE(job, worker->workqueue->waiting_jobs); 112 | } 113 | pthread_mutex_unlock(&worker->workqueue->jobs_mutex); 114 | 115 | if (job == NULL) continue; 116 | 117 | /* Execute the job. */ 118 | job->job_function(job); 119 | } 120 | 121 | free(worker); 122 | pthread_exit(NULL); 123 | } 124 | 125 | int workqueue_init(workqueue_t *workqueue, int numWorkers) { 126 | int i; 127 | worker_t *worker; 128 | pthread_cond_t blank_cond = PTHREAD_COND_INITIALIZER; 129 | pthread_mutex_t blank_mutex = PTHREAD_MUTEX_INITIALIZER; 130 | 131 | if (numWorkers < 1) numWorkers = 1; 132 | 133 | memset(workqueue, 0, sizeof(*workqueue)); 134 | memcpy(&workqueue->jobs_mutex, &blank_mutex, sizeof(workqueue->jobs_mutex)); 135 | memcpy(&workqueue->jobs_cond, &blank_cond, sizeof(workqueue->jobs_cond)); 136 | 137 | for (i = 0; i < numWorkers; i++) { 138 | if ((worker = malloc(sizeof(worker_t))) == NULL) { 139 | perror("Failed to allocate all workers"); 140 | return 1; 141 | } 142 | 143 | memset(worker, 0, sizeof(*worker)); 144 | worker->workqueue = workqueue; 145 | 146 | if (pthread_create(&worker->thread, NULL, worker_function, (void *)worker)) { 147 | perror("Failed to start all worker threads"); 148 | free(worker); 149 | return 1; 150 | } 151 | 152 | LL_ADD(worker, worker->workqueue->workers); 153 | } 154 | 155 | return 0; 156 | } 157 | 158 | 159 | void workqueue_shutdown(workqueue_t *workqueue) { 160 | 161 | worker_t *worker = NULL; 162 | for (worker = workqueue->workers; worker != NULL; worker = worker->next) { 163 | worker->terminate = 1; 164 | } 165 | 166 | 167 | pthread_mutex_lock(&workqueue->jobs_mutex); 168 | workqueue->workers = NULL; 169 | workqueue->waiting_jobs = NULL; 170 | pthread_cond_broadcast(&workqueue->jobs_cond); 171 | pthread_mutex_unlock(&workqueue->jobs_mutex); 172 | 173 | } 174 | 175 | 176 | void workqueue_add_job(workqueue_t *workqueue, job_t *job) { 177 | 178 | pthread_mutex_lock(&workqueue->jobs_mutex); 179 | 180 | LL_ADD(job, workqueue->waiting_jobs); 181 | 182 | pthread_cond_signal(&workqueue->jobs_cond); 183 | pthread_mutex_unlock(&workqueue->jobs_mutex); 184 | 185 | } 186 | 187 | static workqueue_t workqueue; 188 | void threadpool_init(void) { 189 | workqueue_init(&workqueue, MAX_THREAD); 190 | } 191 | 192 | 193 | /** **** ******** **************** thread pool **************** ******** **** **/ 194 | 195 | typedef struct client { 196 | int fd; 197 | char rBuffer[MAX_BUFFER]; 198 | int length; 199 | } client_t; 200 | 201 | void *client_cb(void *arg) { 202 | int clientfd = *(int *)arg; 203 | char buffer[MAX_BUFFER] = {0}; 204 | 205 | int childpid = getpid(); 206 | 207 | while (1) { 208 | bzero(buffer, MAX_BUFFER); 209 | ssize_t length = recv(clientfd, buffer, MAX_BUFFER, 0); //bio 210 | if (length > 0) { 211 | //printf(" PID:%d --> buffer: %s\n", childpid, buffer); 212 | 213 | int sLen = send(clientfd, buffer, length, 0); 214 | //printf(" PID:%d --> sLen: %d\n", childpid, sLen); 215 | } else if (length == 0) { 216 | printf(" PID:%d client disconnect\n", childpid); 217 | break; 218 | } else { 219 | printf(" PID:%d errno:%d\n", childpid, errno); 220 | break; 221 | } 222 | } 223 | } 224 | 225 | 226 | static int nRecv(int sockfd, void *data, size_t length, int *count) { 227 | int left_bytes; 228 | int read_bytes; 229 | int res; 230 | int ret_code; 231 | 232 | unsigned char *p; 233 | 234 | struct pollfd pollfds; 235 | pollfds.fd = sockfd; 236 | pollfds.events = ( POLLIN | POLLERR | POLLHUP ); 237 | 238 | read_bytes = 0; 239 | ret_code = 0; 240 | p = (unsigned char *)data; 241 | left_bytes = length; 242 | 243 | while (left_bytes > 0) { 244 | 245 | read_bytes = recv(sockfd, p, left_bytes, 0); 246 | if (read_bytes > 0) { 247 | left_bytes -= read_bytes; 248 | p += read_bytes; 249 | continue; 250 | } else if (read_bytes < 0) { 251 | if (!(errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) { 252 | ret_code = (errno != 0 ? errno : EINTR); 253 | } 254 | } else { 255 | ret_code = ENOTCONN; 256 | break; 257 | } 258 | 259 | res = poll(&pollfds, 1, 5); 260 | if (pollfds.revents & POLLHUP) { 261 | ret_code = ENOTCONN; 262 | break; 263 | } 264 | if (res < 0) { 265 | if (errno == EINTR) { 266 | continue; 267 | } 268 | ret_code = (errno != 0 ? errno : EINTR); 269 | } else if (res == 0) { 270 | ret_code = ETIMEDOUT; 271 | break; 272 | } 273 | 274 | } 275 | 276 | if (count != NULL) { 277 | *count = length - left_bytes; 278 | } 279 | //printf("nRecv:%s, ret_code:%d, count:%d\n", (char*)data, ret_code, *count); 280 | 281 | 282 | return ret_code; 283 | } 284 | 285 | void epoll_et_loop(int sockfd) 286 | { 287 | char buffer[MAX_BUFFER]; 288 | int ret; 289 | 290 | while (1) 291 | { 292 | memset(buffer, 0, MAX_BUFFER); 293 | ret = recv(sockfd, buffer, MAX_BUFFER, 0); 294 | if (ret == -1) 295 | { 296 | if (errno == EAGAIN || errno == EWOULDBLOCK) 297 | { 298 | printf(" read all data\n"); 299 | break; 300 | } 301 | close(sockfd); 302 | break; 303 | } 304 | else if (ret == 0) 305 | { 306 | printf(" disconnect\n"); 307 | close(sockfd); 308 | break; 309 | } 310 | else 311 | printf("Recv:%s, %d Bytes\n", buffer, ret); 312 | } 313 | 314 | } 315 | 316 | static int nSend(int sockfd, const void *buffer, int length, int flags) { 317 | 318 | int wrotelen = 0; 319 | int writeret = 0; 320 | 321 | unsigned char *p = (unsigned char *)buffer; 322 | 323 | struct pollfd pollfds = {0}; 324 | pollfds.fd = sockfd; 325 | pollfds.events = ( POLLOUT | POLLERR | POLLHUP ); 326 | 327 | do { 328 | int result = poll( &pollfds, 1, 5); 329 | if (pollfds.revents & POLLHUP) { 330 | 331 | printf(" ntySend errno:%d, revent:%x\n", errno, pollfds.revents); 332 | return -1; 333 | } 334 | 335 | if (result < 0) { 336 | if (errno == EINTR) continue; 337 | 338 | printf(" ntySend errno:%d, result:%d\n", errno, result); 339 | return -1; 340 | } else if (result == 0) { 341 | 342 | printf(" ntySend errno:%d, socket timeout \n", errno); 343 | return -1; 344 | } 345 | 346 | writeret = send( sockfd, p + wrotelen, length - wrotelen, flags ); 347 | if( writeret <= 0 ) 348 | { 349 | break; 350 | } 351 | wrotelen += writeret ; 352 | 353 | } while (wrotelen < length); 354 | 355 | return wrotelen; 356 | } 357 | 358 | 359 | static int curfds = 1; 360 | static int nRun = 0; 361 | 362 | 363 | void client_job(job_t *job) { 364 | 365 | 366 | client_t *rClient = (client_t*)job->user_data; 367 | int clientfd = rClient->fd; 368 | 369 | char buffer[MAX_BUFFER]; 370 | bzero(buffer, MAX_BUFFER); 371 | 372 | int length = 0; 373 | int ret = nRecv(clientfd, buffer, MAX_BUFFER, &length); 374 | if (length > 0) { 375 | if (nRun || buffer[0] == 'a') { 376 | printf(" TcpRecv --> curfds : %d, buffer: %s\n", curfds, buffer); 377 | 378 | nSend(clientfd, buffer, strlen(buffer), 0); 379 | } 380 | 381 | } else if (ret == ENOTCONN) { 382 | curfds --; 383 | close(clientfd); 384 | } else { 385 | 386 | } 387 | 388 | free(rClient); 389 | free(job); 390 | } 391 | 392 | void client_data_process(int clientfd) { 393 | 394 | char buffer[MAX_BUFFER]; 395 | bzero(buffer, MAX_BUFFER); 396 | int length = 0; 397 | int ret = nRecv(clientfd, buffer, MAX_BUFFER, &length); 398 | if (length > 0) { 399 | if (nRun || buffer[0] == 'a') { 400 | printf(" TcpRecv --> curfds : %d, buffer: %s\n", curfds, buffer); 401 | 402 | nSend(clientfd, buffer, strlen(buffer), 0); 403 | } 404 | 405 | } else if (ret == ENOTCONN) { 406 | curfds --; 407 | close(clientfd); 408 | } else { 409 | 410 | } 411 | 412 | } 413 | 414 | 415 | 416 | int listenfd(int fd, int *fds) { 417 | int i = 0; 418 | 419 | for (i = 0;i < MAX_PORT;i ++) { 420 | if (fd == *(fds+i)) return *(fds+i); 421 | } 422 | return 0; 423 | } 424 | 425 | int main(void) { 426 | int i = 0; 427 | int sockfds[MAX_PORT] = {0}; 428 | 429 | printf("C1000K Server Start\n"); 430 | 431 | threadpool_init(); // 432 | 433 | int epoll_fd = epoll_create(MAX_EPOLLSIZE); 434 | 435 | for (i = 0;i < MAX_PORT;i ++) { 436 | 437 | int sockfd = socket(AF_INET, SOCK_STREAM, 0); 438 | if (sockfd < 0) { 439 | perror("socket"); 440 | return 1; 441 | } 442 | 443 | struct sockaddr_in addr; 444 | memset(&addr, 0, sizeof(struct sockaddr_in)); 445 | 446 | addr.sin_family = AF_INET; 447 | addr.sin_port = htons(SERVER_PORT+i); 448 | addr.sin_addr.s_addr = INADDR_ANY; 449 | 450 | if (bind(sockfd, (struct sockaddr*)&addr, sizeof(struct sockaddr_in)) < 0) { 451 | perror("bind"); 452 | return 2; 453 | } 454 | 455 | if (listen(sockfd, 5) < 0) { 456 | perror("listen"); 457 | return 3; 458 | } 459 | 460 | sockfds[i] = sockfd; 461 | printf("C1000K Server Listen on Port:%d\n", SERVER_PORT+i); 462 | 463 | struct epoll_event ev; 464 | 465 | ev.events = EPOLLIN | EPOLLET; //EPOLLLT 466 | ev.data.fd = sockfd; 467 | epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sockfd, &ev); 468 | } 469 | 470 | struct timeval tv_begin; 471 | gettimeofday(&tv_begin, NULL); 472 | 473 | struct epoll_event events[MAX_EPOLLSIZE]; 474 | 475 | while (1) { 476 | 477 | int nfds = epoll_wait(epoll_fd, events, curfds, 5); //是不是秘书给累死。 478 | if (nfds == -1) { 479 | perror("epoll_wait"); 480 | break; 481 | } 482 | for (i = 0;i < nfds;i ++) { 483 | 484 | int sockfd = listenfd(events[i].data.fd, sockfds); 485 | if (sockfd) { 486 | struct sockaddr_in client_addr; 487 | memset(&client_addr, 0, sizeof(struct sockaddr_in)); 488 | socklen_t client_len = sizeof(client_addr); 489 | 490 | int clientfd = accept(sockfd, (struct sockaddr*)&client_addr, &client_len); 491 | if (clientfd < 0) { 492 | perror("accept"); 493 | return 4; 494 | } 495 | 496 | if (curfds ++ > 1000 * 1000) { 497 | nRun = 1; 498 | } 499 | #if 0 500 | printf(" Client %d: %d.%d.%d.%d:%d \n", curfds, *(unsigned char*)(&client_addr.sin_addr.s_addr), *((unsigned char*)(&client_addr.sin_addr.s_addr)+1), 501 | *((unsigned char*)(&client_addr.sin_addr.s_addr)+2), *((unsigned char*)(&client_addr.sin_addr.s_addr)+3), 502 | client_addr.sin_port); 503 | #elif 0 504 | if(curfds % 1000 == 999) { 505 | printf("connections: %d, fd: %d\n", curfds, clientfd); 506 | } 507 | #else 508 | if (curfds % 1000 == 999) { 509 | struct timeval tv_cur; 510 | memcpy(&tv_cur, &tv_begin, sizeof(struct timeval)); 511 | 512 | gettimeofday(&tv_begin, NULL); 513 | 514 | int time_used = TIME_SUB_MS(tv_begin, tv_cur); 515 | printf("connections: %d, sockfd:%d, time_used:%d\n", curfds, clientfd, time_used); 516 | } 517 | #endif 518 | ntySetNonblock(clientfd); 519 | ntySetReUseAddr(clientfd); 520 | 521 | struct epoll_event ev; 522 | ev.events = EPOLLIN | EPOLLET | EPOLLOUT; 523 | ev.data.fd = clientfd; 524 | epoll_ctl(epoll_fd, EPOLL_CTL_ADD, clientfd, &ev); 525 | 526 | } else { 527 | 528 | int clientfd = events[i].data.fd; 529 | #if 0 530 | if (nRun) { 531 | printf(" New Data is Comming\n"); 532 | client_data_process(clientfd); 533 | } else { 534 | 535 | client_t *rClient = (client_t*)malloc(sizeof(client_t)); 536 | memset(rClient, 0, sizeof(client_t)); 537 | rClient->fd = clientfd; 538 | 539 | job_t *job = malloc(sizeof(job_t)); 540 | job->job_function = client_job; 541 | job->user_data = rClient; 542 | workqueue_add_job(&workqueue, job); 543 | 544 | 545 | } 546 | #else 547 | client_data_process(clientfd); 548 | 549 | #endif 550 | } 551 | } 552 | 553 | } 554 | } 555 | 556 | 557 | 558 | 559 | 560 | 561 | -------------------------------------------------------------------------------- /server_mulport_mulepoll.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * author : wangbojing 4 | * email : 1989wangbojing@163.com 5 | * 测试操作系统的并发量 6 | */ 7 | 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | #define SERVER_PORT 8080 27 | #define SERVER_IP "127.0.0.1" 28 | #define MAX_BUFFER 128 29 | #define MAX_EPOLLSIZE 100000 30 | #define MAX_THREAD 80 31 | #define MAX_PORT 100 32 | 33 | #define CPU_CORES_SIZE 8 34 | 35 | #define TIME_SUB_MS(tv1, tv2) ((tv1.tv_sec - tv2.tv_sec) * 1000 + (tv1.tv_usec - tv2.tv_usec) / 1000) 36 | 37 | static int ntySetNonblock(int fd) { 38 | int flags; 39 | 40 | flags = fcntl(fd, F_GETFL, 0); 41 | if (flags < 0) return flags; 42 | flags |= O_NONBLOCK; 43 | if (fcntl(fd, F_SETFL, flags) < 0) return -1; 44 | return 0; 45 | } 46 | 47 | static int ntySetReUseAddr(int fd) { 48 | int reuse = 1; 49 | return setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(reuse)); 50 | } 51 | 52 | static int ntySetAlive(int fd) { 53 | int alive = 1; 54 | int idle = 60; 55 | int interval = 5; 56 | int count = 2; 57 | 58 | setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (void*)&alive, sizeof(alive)); 59 | setsockopt(fd, SOL_TCP, TCP_KEEPIDLE, (void*)&idle, sizeof(idle)); 60 | setsockopt(fd, SOL_TCP, TCP_KEEPINTVL, (void*)&interval, sizeof(interval)); 61 | setsockopt(fd, SOL_TCP, TCP_KEEPCNT, (void*)&count, sizeof(count)); 62 | } 63 | 64 | 65 | /** **** ******** **************** thread pool **************** ******** **** **/ 66 | 67 | #define LL_ADD(item, list) { \ 68 | item->prev = NULL; \ 69 | item->next = list; \ 70 | list = item; \ 71 | } 72 | 73 | #define LL_REMOVE(item, list) { \ 74 | if (item->prev != NULL) item->prev->next = item->next; \ 75 | if (item->next != NULL) item->next->prev = item->prev; \ 76 | if (list == item) list = item->next; \ 77 | item->prev = item->next = NULL; \ 78 | } 79 | 80 | typedef struct worker { 81 | pthread_t thread; 82 | int terminate; 83 | struct workqueue *workqueue; 84 | struct worker *prev; 85 | struct worker *next; 86 | } worker_t; 87 | 88 | typedef struct job { 89 | void (*job_function)(struct job *job); 90 | void *user_data; 91 | struct job *prev; 92 | struct job *next; 93 | } job_t; 94 | 95 | typedef struct workqueue { 96 | struct worker *workers; 97 | struct job *waiting_jobs; 98 | pthread_mutex_t jobs_mutex; 99 | pthread_cond_t jobs_cond; 100 | } workqueue_t; 101 | 102 | 103 | static void *worker_function(void *ptr) { 104 | worker_t *worker = (worker_t *)ptr; 105 | job_t *job; 106 | 107 | while (1) { 108 | pthread_mutex_lock(&worker->workqueue->jobs_mutex); 109 | while (worker->workqueue->waiting_jobs == NULL) { 110 | if (worker->terminate) break; 111 | pthread_cond_wait(&worker->workqueue->jobs_cond, &worker->workqueue->jobs_mutex); 112 | } 113 | if (worker->terminate) break; 114 | job = worker->workqueue->waiting_jobs; 115 | if (job != NULL) { 116 | LL_REMOVE(job, worker->workqueue->waiting_jobs); 117 | } 118 | pthread_mutex_unlock(&worker->workqueue->jobs_mutex); 119 | 120 | if (job == NULL) continue; 121 | 122 | /* Execute the job. */ 123 | job->job_function(job); 124 | } 125 | 126 | free(worker); 127 | pthread_exit(NULL); 128 | } 129 | 130 | int workqueue_init(workqueue_t *workqueue, int numWorkers) { 131 | int i; 132 | worker_t *worker; 133 | pthread_cond_t blank_cond = PTHREAD_COND_INITIALIZER; 134 | pthread_mutex_t blank_mutex = PTHREAD_MUTEX_INITIALIZER; 135 | 136 | if (numWorkers < 1) numWorkers = 1; 137 | 138 | memset(workqueue, 0, sizeof(*workqueue)); 139 | memcpy(&workqueue->jobs_mutex, &blank_mutex, sizeof(workqueue->jobs_mutex)); 140 | memcpy(&workqueue->jobs_cond, &blank_cond, sizeof(workqueue->jobs_cond)); 141 | 142 | for (i = 0; i < numWorkers; i++) { 143 | if ((worker = malloc(sizeof(worker_t))) == NULL) { 144 | perror("Failed to allocate all workers"); 145 | return 1; 146 | } 147 | 148 | memset(worker, 0, sizeof(*worker)); 149 | worker->workqueue = workqueue; 150 | 151 | if (pthread_create(&worker->thread, NULL, worker_function, (void *)worker)) { 152 | perror("Failed to start all worker threads"); 153 | free(worker); 154 | return 1; 155 | } 156 | 157 | LL_ADD(worker, worker->workqueue->workers); 158 | } 159 | 160 | return 0; 161 | } 162 | 163 | 164 | void workqueue_shutdown(workqueue_t *workqueue) { 165 | 166 | worker_t *worker = NULL; 167 | for (worker = workqueue->workers; worker != NULL; worker = worker->next) { 168 | worker->terminate = 1; 169 | } 170 | 171 | 172 | pthread_mutex_lock(&workqueue->jobs_mutex); 173 | workqueue->workers = NULL; 174 | workqueue->waiting_jobs = NULL; 175 | pthread_cond_broadcast(&workqueue->jobs_cond); 176 | pthread_mutex_unlock(&workqueue->jobs_mutex); 177 | 178 | } 179 | 180 | 181 | void workqueue_add_job(workqueue_t *workqueue, job_t *job) { 182 | 183 | pthread_mutex_lock(&workqueue->jobs_mutex); 184 | 185 | LL_ADD(job, workqueue->waiting_jobs); 186 | 187 | pthread_cond_signal(&workqueue->jobs_cond); 188 | pthread_mutex_unlock(&workqueue->jobs_mutex); 189 | 190 | } 191 | 192 | static workqueue_t workqueue; 193 | void threadpool_init(void) { 194 | workqueue_init(&workqueue, MAX_THREAD); 195 | } 196 | 197 | 198 | /** **** ******** **************** thread pool **************** ******** **** **/ 199 | 200 | typedef struct client { 201 | int fd; 202 | char rBuffer[MAX_BUFFER]; 203 | int length; 204 | } client_t; 205 | 206 | void *client_cb(void *arg) { 207 | int clientfd = *(int *)arg; 208 | char buffer[MAX_BUFFER] = {0}; 209 | 210 | int childpid = getpid(); 211 | 212 | while (1) { 213 | bzero(buffer, MAX_BUFFER); 214 | ssize_t length = recv(clientfd, buffer, MAX_BUFFER, 0); //bio 215 | if (length > 0) { 216 | //printf(" PID:%d --> buffer: %s\n", childpid, buffer); 217 | 218 | int sLen = send(clientfd, buffer, length, 0); 219 | //printf(" PID:%d --> sLen: %d\n", childpid, sLen); 220 | } else if (length == 0) { 221 | printf(" PID:%d client disconnect\n", childpid); 222 | break; 223 | } else { 224 | printf(" PID:%d errno:%d\n", childpid, errno); 225 | break; 226 | } 227 | } 228 | } 229 | 230 | 231 | static int nRecv(int sockfd, void *data, size_t length, int *count) { 232 | int left_bytes; 233 | int read_bytes; 234 | int res; 235 | int ret_code; 236 | 237 | unsigned char *p; 238 | 239 | struct pollfd pollfds; 240 | pollfds.fd = sockfd; 241 | pollfds.events = ( POLLIN | POLLERR | POLLHUP ); 242 | 243 | read_bytes = 0; 244 | ret_code = 0; 245 | p = (unsigned char *)data; 246 | left_bytes = length; 247 | 248 | while (left_bytes > 0) { 249 | 250 | read_bytes = recv(sockfd, p, left_bytes, 0); 251 | if (read_bytes > 0) { 252 | left_bytes -= read_bytes; 253 | p += read_bytes; 254 | continue; 255 | } else if (read_bytes < 0) { 256 | if (!(errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) { 257 | ret_code = (errno != 0 ? errno : EINTR); 258 | } 259 | } else { 260 | ret_code = ENOTCONN; 261 | break; 262 | } 263 | 264 | res = poll(&pollfds, 1, 5); 265 | if (pollfds.revents & POLLHUP) { 266 | ret_code = ENOTCONN; 267 | break; 268 | } 269 | if (res < 0) { 270 | if (errno == EINTR) { 271 | continue; 272 | } 273 | ret_code = (errno != 0 ? errno : EINTR); 274 | } else if (res == 0) { 275 | ret_code = ETIMEDOUT; 276 | break; 277 | } 278 | 279 | } 280 | 281 | if (count != NULL) { 282 | *count = length - left_bytes; 283 | } 284 | //printf("nRecv:%s, ret_code:%d, count:%d\n", (char*)data, ret_code, *count); 285 | 286 | 287 | return ret_code; 288 | } 289 | 290 | void epoll_et_loop(int sockfd) 291 | { 292 | char buffer[MAX_BUFFER]; 293 | int ret; 294 | 295 | while (1) 296 | { 297 | memset(buffer, 0, MAX_BUFFER); 298 | ret = recv(sockfd, buffer, MAX_BUFFER, 0); 299 | if (ret == -1) 300 | { 301 | if (errno == EAGAIN || errno == EWOULDBLOCK) 302 | { 303 | printf(" read all data\n"); 304 | break; 305 | } 306 | close(sockfd); 307 | break; 308 | } 309 | else if (ret == 0) 310 | { 311 | printf(" disconnect\n"); 312 | close(sockfd); 313 | break; 314 | } 315 | else 316 | printf("Recv:%s, %d Bytes\n", buffer, ret); 317 | } 318 | 319 | } 320 | 321 | static int nSend(int sockfd, const void *buffer, int length, int flags) { 322 | 323 | int wrotelen = 0; 324 | int writeret = 0; 325 | 326 | unsigned char *p = (unsigned char *)buffer; 327 | 328 | struct pollfd pollfds = {0}; 329 | pollfds.fd = sockfd; 330 | pollfds.events = ( POLLOUT | POLLERR | POLLHUP ); 331 | 332 | do { 333 | int result = poll( &pollfds, 1, 5); 334 | if (pollfds.revents & POLLHUP) { 335 | 336 | printf(" ntySend errno:%d, revent:%x\n", errno, pollfds.revents); 337 | return -1; 338 | } 339 | 340 | if (result < 0) { 341 | if (errno == EINTR) continue; 342 | 343 | printf(" ntySend errno:%d, result:%d\n", errno, result); 344 | return -1; 345 | } else if (result == 0) { 346 | 347 | printf(" ntySend errno:%d, socket timeout \n", errno); 348 | return -1; 349 | } 350 | 351 | writeret = send( sockfd, p + wrotelen, length - wrotelen, flags ); 352 | if( writeret <= 0 ) 353 | { 354 | break; 355 | } 356 | wrotelen += writeret ; 357 | 358 | } while (wrotelen < length); 359 | 360 | return wrotelen; 361 | } 362 | 363 | 364 | static int curfds = 1; 365 | static int nRun = 0; 366 | int sockfds[MAX_PORT] = {0}; 367 | 368 | 369 | void client_job(job_t *job) { 370 | 371 | 372 | client_t *rClient = (client_t*)job->user_data; 373 | int clientfd = rClient->fd; 374 | 375 | char buffer[MAX_BUFFER]; 376 | bzero(buffer, MAX_BUFFER); 377 | 378 | int length = 0; 379 | int ret = nRecv(clientfd, buffer, MAX_BUFFER, &length); 380 | if (length > 0) { 381 | if (nRun || buffer[0] == 'a') { 382 | printf(" TcpRecv --> curfds : %d, buffer: %s\n", curfds, buffer); 383 | 384 | nSend(clientfd, buffer, strlen(buffer), 0); 385 | } 386 | 387 | } else if (ret == ENOTCONN) { 388 | curfds --; 389 | close(clientfd); 390 | } else { 391 | 392 | } 393 | 394 | free(rClient); 395 | free(job); 396 | } 397 | 398 | void client_data_process(int clientfd) { 399 | 400 | char buffer[MAX_BUFFER]; 401 | bzero(buffer, MAX_BUFFER); 402 | int length = 0; 403 | //int ret = nRecv(clientfd, buffer, MAX_BUFFER, &length); 404 | int ret = recv(clientfd, buffer, MAX_BUFFER, 0); 405 | if (length > 0) { 406 | if (nRun || buffer[0] == 'a') { 407 | printf(" TcpRecv --> curfds : %d, buffer: %s\n", curfds, buffer); 408 | 409 | //nSend(clientfd, buffer, strlen(buffer), 0); 410 | send(clientfd, buffer, strlen(buffer), 0); 411 | } 412 | 413 | } else if (ret == ENOTCONN) { 414 | curfds --; 415 | close(clientfd); 416 | } else { 417 | 418 | } 419 | 420 | } 421 | 422 | 423 | 424 | int listenfd(int fd, int *fds) { 425 | int i = 0; 426 | 427 | for (i = 0;i < MAX_PORT;i ++) { 428 | if (fd == *(fds+i)) return *(fds+i); 429 | } 430 | return 0; 431 | } 432 | 433 | struct timeval tv_begin; 434 | struct timeval tv_cur; 435 | 436 | void *listen_thread(void *arg) { 437 | 438 | int i = 0; 439 | int epoll_fd = *(int *)arg; 440 | struct epoll_event events[MAX_EPOLLSIZE]; 441 | 442 | 443 | gettimeofday(&tv_begin, NULL); 444 | 445 | while (1) { 446 | 447 | int nfds = epoll_wait(epoll_fd, events, curfds, 5); 448 | if (nfds == -1) { 449 | perror("epoll_wait"); 450 | break; 451 | } 452 | for (i = 0;i < nfds;i ++) { 453 | 454 | int sockfd = listenfd(events[i].data.fd, sockfds); 455 | if (sockfd) { 456 | struct sockaddr_in client_addr; 457 | memset(&client_addr, 0, sizeof(struct sockaddr_in)); 458 | socklen_t client_len = sizeof(client_addr); 459 | 460 | int clientfd = accept(sockfd, (struct sockaddr*)&client_addr, &client_len); 461 | if (clientfd < 0) { 462 | perror("accept"); 463 | return NULL; 464 | } 465 | 466 | if (curfds ++ > 1000 * 1000) { 467 | nRun = 1; 468 | } 469 | #if 0 470 | printf(" Client %d: %d.%d.%d.%d:%d \n", curfds, *(unsigned char*)(&client_addr.sin_addr.s_addr), *((unsigned char*)(&client_addr.sin_addr.s_addr)+1), 471 | *((unsigned char*)(&client_addr.sin_addr.s_addr)+2), *((unsigned char*)(&client_addr.sin_addr.s_addr)+3), 472 | client_addr.sin_port); 473 | #elif 0 474 | if(curfds % 1000 == 999) { 475 | printf("connections: %d, fd: %d\n", curfds, clientfd); 476 | } 477 | #else 478 | if (curfds % 1000 == 999) { 479 | 480 | memcpy(&tv_cur, &tv_begin, sizeof(struct timeval)); 481 | 482 | gettimeofday(&tv_begin, NULL); 483 | int time_used = TIME_SUB_MS(tv_begin, tv_cur); 484 | 485 | printf("connections: %d, sockfd:%d, time_used:%d\n", curfds, clientfd, time_used); 486 | } 487 | #endif 488 | ntySetNonblock(clientfd); 489 | ntySetReUseAddr(clientfd); 490 | 491 | struct epoll_event ev; 492 | ev.events = EPOLLIN | EPOLLET | EPOLLOUT; 493 | ev.data.fd = clientfd; 494 | epoll_ctl(epoll_fd, EPOLL_CTL_ADD, clientfd, &ev); 495 | 496 | } else { 497 | 498 | int clientfd = events[i].data.fd; 499 | #if 0 500 | if (nRun) { 501 | printf(" New Data is Comming\n"); 502 | client_data_process(clientfd); 503 | } else { 504 | 505 | client_t *rClient = (client_t*)malloc(sizeof(client_t)); 506 | memset(rClient, 0, sizeof(client_t)); 507 | rClient->fd = clientfd; 508 | 509 | job_t *job = malloc(sizeof(job_t)); 510 | job->job_function = client_job; 511 | job->user_data = rClient; 512 | workqueue_add_job(&workqueue, job); 513 | 514 | 515 | } 516 | #else 517 | client_data_process(clientfd); 518 | #endif 519 | } 520 | } 521 | 522 | } 523 | 524 | } 525 | 526 | int main(void) { 527 | int i = 0; 528 | 529 | printf("C1000K Server Start\n"); 530 | 531 | threadpool_init(); // 532 | 533 | 534 | #if 0 535 | int epoll_fd = epoll_create(MAX_EPOLLSIZE); 536 | #else 537 | 538 | int epoll_fds[CPU_CORES_SIZE] = {0}; 539 | pthread_t thread_id[CPU_CORES_SIZE] = {0}; 540 | 541 | for (i = 0;i < CPU_CORES_SIZE;i ++) { 542 | epoll_fds[i] = epoll_create(MAX_EPOLLSIZE); 543 | 544 | pthread_create(&thread_id[i], NULL, listen_thread, &epoll_fds[i]); 545 | } 546 | 547 | 548 | #endif 549 | for (i = 0;i < MAX_PORT;i ++) { 550 | 551 | int sockfd = socket(AF_INET, SOCK_STREAM, 0); 552 | if (sockfd < 0) { 553 | perror("socket"); 554 | return 1; 555 | } 556 | 557 | struct sockaddr_in addr; 558 | memset(&addr, 0, sizeof(struct sockaddr_in)); 559 | 560 | addr.sin_family = AF_INET; 561 | addr.sin_port = htons(SERVER_PORT+i); 562 | addr.sin_addr.s_addr = INADDR_ANY; 563 | 564 | if (bind(sockfd, (struct sockaddr*)&addr, sizeof(struct sockaddr_in)) < 0) { 565 | perror("bind"); 566 | return 2; 567 | } 568 | 569 | if (listen(sockfd, 5) < 0) { 570 | perror("listen"); 571 | return 3; 572 | } 573 | 574 | sockfds[i] = sockfd; 575 | printf("C1000K Server Listen on Port:%d\n", SERVER_PORT+i); 576 | 577 | struct epoll_event ev; 578 | 579 | ev.events = EPOLLIN | EPOLLET; //EPOLLLT 580 | ev.data.fd = sockfd; 581 | epoll_ctl(epoll_fds[i%CPU_CORES_SIZE], EPOLL_CTL_ADD, sockfd, &ev); 582 | } 583 | 584 | for (i = 0;i < CPU_CORES_SIZE;i ++) { 585 | pthread_join(thread_id[i], NULL); 586 | } 587 | 588 | 589 | getchar(); 590 | printf("end\n"); 591 | } 592 | 593 | 594 | 595 | 596 | 597 | 598 | --------------------------------------------------------------------------------