├── .gitignore ├── .gitmodules ├── .travis.yml ├── Makefile ├── README.md ├── blocking_forking.c ├── blocking_single.c ├── client.c ├── config.h ├── go-client.go ├── http_handler.c ├── http_handler.h ├── infrastructure ├── README.md ├── ansible.cfg ├── aws_variables.tf ├── inventory │ ├── ec2.ini │ └── ec2.py ├── main.tf ├── provision.yml ├── roles │ ├── base │ │ ├── defaults │ │ │ └── main.yml │ │ ├── files │ │ │ └── avd.pub │ │ └── tasks │ │ │ └── main.yml │ └── c10kserver │ │ ├── defaults │ │ └── main.yml │ │ ├── handlers │ │ └── main.yml │ │ ├── tasks │ │ └── main.yml │ │ └── templates │ │ └── systemd.j2 ├── site.yml └── variables.tf ├── libuv-client.c ├── logging.h ├── socket_io.c └── socket_io.h /.gitignore: -------------------------------------------------------------------------------- 1 | # Local .terraform directories 2 | **/.terraform/* 3 | 4 | # .tfstate files 5 | *.tfstate 6 | *.tfstate.* 7 | 8 | # .tfvars files 9 | *.tfvars 10 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "mongoose"] 2 | path = mongoose 3 | url = https://github.com/cesanta/mongoose.git 4 | [submodule "picohttpparser"] 5 | path = picohttpparser 6 | url = https://github.com/h2o/picohttpparser.git 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: c 2 | 3 | compiler: 4 | - clang 5 | - gcc 6 | 7 | install: 8 | - sudo apt-get update 9 | - sudo apt-get install ruby ruby-dev build-essential rpm 10 | - gem install --no-ri --no-rdoc fpm 11 | - sudo apt-get install libevent-2.0-5 12 | 13 | script: 14 | - make packages 15 | 16 | deploy: 17 | provider: releases 18 | skip_cleanup: true 19 | api_key: 20 | secure: noeh4ygQJKUwVlDsh6VsCY9K3+5ZiMDdXWfEJmWWDrvLC0qmocHjsKtuEPa5MKf24EBbL6iTZje99r0tSwqymVDOl6mNobUIlH4nOR5l+cqQQqm45WqUpEwXpMpdGgmNtFpMoAHnxo+cgqnJB4H5nXpetmZUQ11gqHkbhfdPaMs/fcgtWh1c2NP7om/m1Dl9gT1pYNgIQvLhoPQC0eaDdKYasJ/F6hcnkqFzzJEuRwY0AFyJWGeaMYSVHIOiA4oyPdfNTeUPZApPtMt7x3NOSmp7iTc/ZpSQ4MuwnhrdfR72bT6u/A48q5E6t0/ek2bb+Vx5ocrShzANRRe7psYJ7b63hpc9O38LsqN/S2nGidLvyrw7bCr+5U2+AeUaBLY+Nv06czYvalwsUNfcewrSkSJ0FFEIsvWyGXS+HyLHlVM/Jab0c+WGQgtwU0S2JD6f1fyFTXzjpwTjfa513gnvY21yHCCQv+uQrY7uFemtTGmJDXqbpLd2KJd4sfHbDK+8qrV6HXc8kygNJGOoJ+DDaTweCtFYflK7nnTN05rzZisLGX1l31T2XXHSwbA4GhEHmIzI2TzgD+1KTc0P/Q/hkq0/pBgJZPqZ3nXHpfVYOm7clapJcWn37CmQIZUtTAC62YybjqNimzk4z+HJ8lWd66X9JImhFivDbv35gbdut1w= 21 | file_glob: true 22 | file: package/* 23 | on: 24 | tags: true -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CC := gcc 2 | CFLAGS := -Wall -Werror -DDEBUG 3 | 4 | BUILDDIR := build 5 | BUILDDIR_SERVER := $(BUILDDIR)/server 6 | BUILDDIR_CLIENT := $(BUILDDIR)/client 7 | 8 | PACKAGEDIR := package 9 | SERVER_PACKAGE_NAME := c10k-servers 10 | SERVER_PACKAGE_VERSION := 1.1 11 | 12 | COMMON_CODE := socket_io.c http_handler.c mongoose/mongoose.c picohttpparser/picohttpparser.c 13 | 14 | all: dirs server client 15 | 16 | dirs: 17 | mkdir -p $(BUILDDIR_SERVER) $(BUILDDIR_CLIENT) $(PACKAGEDIR) 18 | 19 | server: dirs blocking-single blocking-forking 20 | client: dirs simple-client libuv-client 21 | 22 | packages: server-packages 23 | 24 | server-packages: server-deb server-rpm 25 | server-deb: server 26 | fpm -s dir -t deb -C $(BUILDDIR_SERVER) --prefix /usr/local/bin -f -n $(SERVER_PACKAGE_NAME) -p $(PACKAGEDIR)/c10k-servers_$(SERVER_PACKAGE_VERSION).deb 27 | 28 | server-rpm: server 29 | fpm -s dir -t rpm -C $(BUILDDIR_SERVER) --prefix /usr/local/bin -f -n $(SERVER_PACKAGE_NAME) -p $(PACKAGEDIR)/c10k-servers_$(SERVER_PACKAGE_VERSION).rpm 30 | 31 | blocking-single: blocking_single.c $(COMMON_CODE) 32 | $(CC) $(CFLAGS) $(EXTRA_CFLAGS) $^ -o $(BUILDDIR_SERVER)/$@ 33 | 34 | blocking-forking: blocking_forking.c $(COMMON_CODE) 35 | $(CC) $(CFLAGS) $(EXTRA_CFLAGS) $^ -o $(BUILDDIR_SERVER)/$@ 36 | 37 | simple-client: client.c $(COMMON_CODE) 38 | $(CC) $(CFLAGS) $(EXTRA_CFLAGS) $^ -o $(BUILDDIR_CLIENT)/$@ 39 | 40 | libuv-client: libuv-client.c 41 | $(CC) $(CFLAGS) $(EXTRA_CFLAGS) -luv $^ -o $(BUILDDIR_CLIENT)/$@ 42 | 43 | clean: 44 | rm -rf *.o $(BUILDDIR) $(PACKAGEDIR) 45 | 46 | # Simple test load - 100 concurent clients for 1000 requests 47 | .PHONY: test 48 | test: 49 | ab -n1000 -c 100 http://0.0.0.0:8282/ 50 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | c10k [![Build Status](https://travis-ci.org/dzeban/c10k.svg?branch=master)](https://travis-ci.org/dzeban/c10k) 2 | ==== 3 | 4 | Experiments on c10k problem solving. -------------------------------------------------------------------------------- /blocking_forking.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "config.h" 14 | #include "http_handler.h" 15 | #include "logging.h" 16 | 17 | static void sigchld_handler(int sig, siginfo_t *si, void *unused) 18 | { 19 | // Signals are not queued, meaning that SIGCHLD from multiple child 20 | // processes will be discarded. To prevent zombies, we have to call waitpid 21 | // in a loop for all pids 22 | while (waitpid(-1, NULL, WNOHANG) > 0); 23 | } 24 | 25 | int main(int argc, const char *argv[]) 26 | { 27 | int sock_listen; 28 | int rc = EXIT_SUCCESS; 29 | struct sockaddr_in address, peer_address; 30 | socklen_t peer_address_len; 31 | struct sigaction sa; 32 | 33 | memset(&address, 0, sizeof(address)); 34 | address.sin_family = AF_INET; 35 | address.sin_port = htons(PORT); 36 | address.sin_addr.s_addr = htonl(INADDR_ANY); 37 | 38 | // Setup signal handler to catch SIGCHLD and avoid zombie children 39 | sa.sa_flags = SA_SIGINFO; 40 | sigemptyset(&sa.sa_mask); 41 | sa.sa_sigaction = sigchld_handler; 42 | if (sigaction(SIGCHLD, &sa, NULL)) { 43 | perror("sigaction"); 44 | rc = EXIT_FAILURE; 45 | goto exit_rc; 46 | } 47 | 48 | // Make listen socket binded to port 8282 49 | sock_listen = socket(AF_INET, SOCK_STREAM, 0); 50 | if (sock_listen < 0) { 51 | perror("socket"); 52 | rc = EXIT_FAILURE; 53 | goto exit_rc; 54 | } 55 | 56 | int yes = 1; 57 | if (setsockopt(sock_listen, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)) == -1) { 58 | perror("setsockopt"); 59 | rc = EXIT_FAILURE; 60 | goto exit_socket; 61 | } 62 | 63 | 64 | if (bind(sock_listen, (struct sockaddr *)&address, sizeof(address))) { 65 | perror("bind"); 66 | rc = EXIT_FAILURE; 67 | goto exit_socket; 68 | } 69 | 70 | if (listen(sock_listen, BACKLOG)) { 71 | perror("listen"); 72 | rc = EXIT_FAILURE; 73 | goto exit_socket; 74 | } 75 | 76 | while (1) { 77 | int sock_client; 78 | pid_t child; 79 | 80 | sock_client = accept(sock_listen, (struct sockaddr *)&peer_address, &peer_address_len); 81 | if (sock_client < 0) { 82 | // "accept" is a "slow" system call, if signal handling happened 83 | // while accept was working we'll receive EINTR. And it's ok. 84 | // On Linux it can be automatically restarted by providing 85 | // SA_RESTART to signal handler and NOT specifying SO_RCVTIMEO. 86 | // But we opt to make more portable and restart it by hand. 87 | if (errno != EINTR) { 88 | perror("accept"); 89 | } 90 | continue; 91 | } 92 | debug("Accept from %s, sock %d\n", inet_ntoa(peer_address.sin_addr), sock_client); 93 | 94 | child = fork(); 95 | switch (child) { 96 | case 0: // Child process 97 | if (close(sock_listen)) { 98 | perror("close sock_listen"); 99 | } 100 | 101 | http_handler_loop(sock_client); 102 | 103 | if (close(sock_client)) { 104 | perror("close"); 105 | } 106 | 107 | debug("About to exit from %d\n", getpid()); 108 | exit(EXIT_SUCCESS); 109 | break; 110 | 111 | case -1: // Error 112 | perror("fork"); 113 | if (close(sock_client)) { 114 | perror("close sock_client on error"); 115 | } 116 | break; 117 | 118 | default: // Parent 119 | // Parent process 120 | if (close(sock_client)) { 121 | perror("close sock_client in parent"); 122 | } 123 | break; 124 | } 125 | } 126 | 127 | exit_socket: 128 | close(sock_listen); 129 | exit_rc: 130 | exit(rc); 131 | } 132 | -------------------------------------------------------------------------------- /blocking_single.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "config.h" 12 | #include "http_handler.h" 13 | #include "logging.h" 14 | 15 | int main(int argc, const char *argv[]) 16 | { 17 | int sock_listen; 18 | int rc = EXIT_SUCCESS; 19 | struct sockaddr_in address, peer_address; 20 | socklen_t peer_address_len; 21 | 22 | memset(&address, 0, sizeof(address)); 23 | address.sin_family = AF_INET; 24 | address.sin_port = htons(PORT); 25 | address.sin_addr.s_addr = htonl(INADDR_ANY); 26 | 27 | sock_listen = socket(AF_INET, SOCK_STREAM, 0); 28 | if (sock_listen < 0) { 29 | perror("socket"); 30 | rc = EXIT_FAILURE; 31 | goto exit_rc; 32 | } 33 | 34 | int yes = 1; 35 | if (setsockopt(sock_listen, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)) == -1) { 36 | perror("setsockopt"); 37 | rc = EXIT_FAILURE; 38 | goto exit_socket; 39 | } 40 | 41 | 42 | if (bind(sock_listen, (struct sockaddr *)&address, sizeof(address))) { 43 | perror("bind"); 44 | rc = EXIT_FAILURE; 45 | goto exit_socket; 46 | } 47 | 48 | if (listen(sock_listen, BACKLOG)) { 49 | perror("listen"); 50 | rc = EXIT_FAILURE; 51 | goto exit_socket; 52 | } 53 | 54 | while (1) { 55 | int sock_client; 56 | sock_client = accept(sock_listen, (struct sockaddr *)&peer_address, &peer_address_len); 57 | 58 | debug("Client accept\n"); 59 | 60 | http_handler_loop(sock_client); 61 | 62 | close(sock_client); 63 | } 64 | 65 | exit_socket: 66 | close(sock_listen); 67 | exit_rc: 68 | exit(rc); 69 | } 70 | -------------------------------------------------------------------------------- /client.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "config.h" 12 | #include "socket_io.h" 13 | 14 | #define REQUEST "GET / HTTP/1.1\r\nHost: 0.0.0.0:8282\r\nUser-Agent: curl/7.43.0\r\nAccept: */*\r\n\r\n" 15 | 16 | int main(int argc, const char *argv[]) 17 | { 18 | int sock; 19 | struct sockaddr_in addr_in; 20 | char *buf; 21 | const int bufsize = 4096; 22 | int rc = EXIT_SUCCESS; 23 | 24 | buf = malloc(bufsize); 25 | if (!buf) { 26 | perror("malloc"); 27 | exit(EXIT_FAILURE); 28 | } 29 | memset(buf, 0, bufsize); 30 | 31 | memset(&addr_in, 0, sizeof(addr_in)); 32 | addr_in.sin_family = AF_INET; 33 | addr_in.sin_port = htons(PORT); 34 | addr_in.sin_addr.s_addr = inet_addr("0.0.0.0"); 35 | 36 | sock = socket(AF_INET, SOCK_STREAM, 0); 37 | if (sock < 0) { 38 | perror("socket"); 39 | rc = EXIT_FAILURE; 40 | goto exit_free; 41 | } 42 | 43 | if (connect(sock, (struct sockaddr *)&addr_in, sizeof(addr_in)) < 0) { 44 | perror("connect"); 45 | rc = EXIT_FAILURE; 46 | goto exit_sock; 47 | } 48 | 49 | if (socket_write(sock, REQUEST, strlen(REQUEST))) { 50 | perror("socket_write"); 51 | rc = EXIT_FAILURE; 52 | goto exit_sock; 53 | } 54 | 55 | if (socket_read(sock, buf, bufsize) < 0) { 56 | perror("socket_read"); 57 | rc = EXIT_FAILURE; 58 | goto exit_sock; 59 | } 60 | 61 | puts(buf); 62 | 63 | exit_sock: 64 | close(sock); 65 | exit_free: 66 | free(buf); 67 | exit(rc); 68 | } 69 | -------------------------------------------------------------------------------- /config.h: -------------------------------------------------------------------------------- 1 | #ifndef _CONFIG_H 2 | #define _CONFIG_H 3 | 4 | #define ADDR "0.0.0.0" 5 | #define PORT 8282 6 | #define BACKLOG 20000 7 | 8 | #endif 9 | -------------------------------------------------------------------------------- /go-client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "net" 10 | "net/http" 11 | "sync" 12 | "time" 13 | ) 14 | 15 | func request(addr string, delay, requests int, wg *sync.WaitGroup) { 16 | conn, err := net.Dial("tcp", addr) 17 | if err != nil { 18 | log.Fatal("dial error ", err) 19 | } 20 | 21 | for i := 0; i < requests; i++ { 22 | time.Sleep(time.Duration(delay) * time.Second) 23 | 24 | req, err := http.NewRequest("GET", "/index.html", nil) 25 | if err != nil { 26 | log.Fatal("failed to create http request") 27 | } 28 | 29 | req.Host = "localhost" 30 | 31 | err = req.Write(conn) 32 | if err != nil { 33 | log.Fatal("failed to send http request") 34 | } 35 | 36 | respReader := bufio.NewReader(conn) 37 | resp, err := http.ReadResponse(respReader, req) 38 | if err != nil { 39 | log.Fatal("read reponse error ", err) 40 | } 41 | 42 | body, err := ioutil.ReadAll(resp.Body) 43 | if err != nil { 44 | log.Fatal("body read error ", err) 45 | } 46 | fmt.Println(resp.Status, string(body)) 47 | 48 | resp.Body.Close() 49 | } 50 | conn.Close() 51 | 52 | wg.Done() 53 | } 54 | 55 | func main() { 56 | var ( 57 | addr string 58 | n int 59 | requests int 60 | delay int 61 | ) 62 | 63 | flag.StringVar(&addr, "addr", "127.0.0.1:80", "Address to connect") 64 | flag.IntVar(&n, "n", 1, "Number of connections") 65 | flag.IntVar(&requests, "r", 1, "Number of requests per connection") 66 | flag.IntVar(&delay, "d", 5, "Delay in seconds") 67 | flag.Parse() 68 | 69 | var wg sync.WaitGroup 70 | wg.Add(n) 71 | for i := 0; i < n; i++ { 72 | go request(addr, delay, requests, &wg) 73 | } 74 | 75 | wg.Wait() 76 | } 77 | -------------------------------------------------------------------------------- /http_handler.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "picohttpparser/picohttpparser.h" 5 | #include "mongoose/mongoose.h" 6 | #include "http_handler.h" 7 | #include "socket_io.h" 8 | #include "logging.h" 9 | 10 | struct handler_ctx *handler_init() 11 | { 12 | struct handler_ctx *ctx = malloc(sizeof(*ctx)); 13 | if (!ctx) { 14 | perror("malloc"); 15 | return NULL; 16 | } 17 | 18 | ctx->bufsize = BUFSIZE; 19 | ctx->buf = malloc(ctx->bufsize); 20 | if (!ctx->buf) { 21 | perror("malloc"); 22 | free(ctx); 23 | return NULL; 24 | } 25 | 26 | return ctx; 27 | } 28 | 29 | void handler_destroy(struct handler_ctx *ctx) 30 | { 31 | free(ctx->buf); 32 | free(ctx); 33 | } 34 | 35 | int http_handler(int sock_client, struct handler_ctx *ctx) 36 | { 37 | struct http_message request; 38 | 39 | int nread; 40 | nread = socket_read(sock_client, ctx->buf, ctx->bufsize); 41 | if (nread < 0) { 42 | return -1; 43 | } 44 | debug("Read %d\n", nread); 45 | 46 | nread = mg_parse_http(ctx->buf, nread, &request, 1); 47 | if (nread <= 0) { 48 | fprintf(stderr, "Invalid HTTP request: nread %d, buf: %s\n", nread, ctx->buf); 49 | return -1; 50 | } 51 | debug("Successfully parsed request\n"); 52 | 53 | if (strncmp(request.method.p, GET, request.method.len) == 0) { 54 | return socket_write(sock_client, RESPONSE_BODY, strlen(RESPONSE_BODY)); 55 | } else { 56 | fprintf(stderr, "Invalid method: %s\n", request.method.p); 57 | return -2; 58 | } 59 | } 60 | 61 | int http_handler_loop(int sock_client) 62 | { 63 | struct http_message request; 64 | struct handler_ctx *ctx; 65 | int rc = 0; 66 | int nread; 67 | 68 | do { 69 | ctx = handler_init(); 70 | 71 | nread = socket_read(sock_client, ctx->buf, ctx->bufsize); 72 | if (nread < 0) { 73 | rc = -1; 74 | break; 75 | } 76 | 77 | if (nread == 0) { 78 | debug("EOF\n"); 79 | rc = 0; 80 | break; 81 | } 82 | 83 | debug("Read %d\n", nread); 84 | 85 | nread = mg_parse_http(ctx->buf, nread, &request, 1); 86 | if (nread <= 0) { 87 | rc = -2; 88 | break; 89 | } 90 | debug("Successfully parsed request\n"); 91 | 92 | if (strncmp(request.method.p, GET, request.method.len) == 0) { 93 | if (socket_write(sock_client, RESPONSE_BODY, strlen(RESPONSE_BODY))) { 94 | rc = -3; 95 | break; 96 | } 97 | } else { 98 | rc = -4; 99 | break; 100 | } 101 | handler_destroy(ctx); 102 | 103 | } while (nread > 0); 104 | 105 | handler_destroy(ctx); 106 | return rc; 107 | } 108 | -------------------------------------------------------------------------------- /http_handler.h: -------------------------------------------------------------------------------- 1 | #ifndef _HTTP_HANDLER_H 2 | #define _HTTP_HANDLER_H 3 | 4 | #define BUFSIZE 4096 5 | #define GET "GET" 6 | #define RESPONSE_BODY "HTTP/1.1 200 OK\r\nCache: no-cache\r\nContent-Type: text/html\r\nContent-Length: 44\r\n\r\n

It works!

" 7 | 8 | struct handler_ctx { 9 | char *buf; 10 | size_t bufsize; 11 | }; 12 | 13 | struct handler_ctx *handler_init(); 14 | void handler_destroy(struct handler_ctx *ctx); 15 | int http_handler(int sock_client, struct handler_ctx *ctx); 16 | int http_handler_loop(int sock_client); 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /infrastructure/README.md: -------------------------------------------------------------------------------- 1 | # c10k test rig 2 | 3 | This is the c10k test rig consisting of server and multiple clients all spawned 4 | in the cloud (only AWS for now). 5 | 6 | Infrastructure is created with Terraform and provisioned with Ansible. 7 | 8 | To spawn it, apply Terraform configuration and then launch `site.yml` playbook. 9 | 10 | $ terraform apply 11 | $ ansible-playbook site.yml 12 | 13 | To make these command successful, Terraform and Ansible require valid cloud 14 | provider configuration described hereafter. 15 | 16 | ## AWS setup 17 | 18 | AWS requires credentials in the form of access key id and secret access key. 19 | Both Terraform and Ansible may use shared credentials file ~/.aws/credentials, 20 | so it has to be configured. 21 | 22 | Nevertheless, you can overide this with env variables `AWS_ACCESS_KEY_ID` and 23 | `AWS_SECRET_ACCESS_KEY` 24 | 25 | -------------------------------------------------------------------------------- /infrastructure/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | inventory = ./inventory 3 | roles_path = ./roles 4 | host_key_checking = False 5 | allow_world_readable_tmpfiles = True 6 | remote_user = test 7 | 8 | [privilege_escalation] 9 | 10 | [paramiko_connection] 11 | 12 | [ssh_connection] 13 | ssh_args = -C -o ControlMaster=auto -o ControlPersist=10m -o ForwardAgent=yes 14 | control_path = %(directory)s/ansible-ssh-%%C 15 | pipelining = True 16 | 17 | [accelerate] 18 | 19 | [selinux] 20 | 21 | [colors] 22 | -------------------------------------------------------------------------------- /infrastructure/aws_variables.tf: -------------------------------------------------------------------------------- 1 | variable "aws_region" { 2 | default = "eu-west-1" 3 | } 4 | 5 | variable "aws_vpc_cidr" { 6 | default = "172.16.10.0/24" 7 | } 8 | 9 | variable "aws_ami_fedora27" { 10 | type = "map" 11 | 12 | default = { 13 | "us-east-1" = "ami-bcf84ec6" 14 | "us-west-1" = "ami-18c7f878" 15 | "us-west-2" = "ami-3594414d" 16 | "eu-west-1" = "ami-78389b01" 17 | "eu-central-1" = "ami-5f7cf830" 18 | "ap-southeast-1" = "ami-37286754" 19 | "ap-northeast-1" = "ami-0550fe63" 20 | "ap-southeast-2" = "ami-9d58b6ff" 21 | "sa-east-1" = "ami-0048336c" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /infrastructure/inventory/ec2.ini: -------------------------------------------------------------------------------- 1 | # Ansible EC2 external inventory script settings 2 | # 3 | 4 | [ec2] 5 | 6 | # to talk to a private eucalyptus instance uncomment these lines 7 | # and edit edit eucalyptus_host to be the host name of your cloud controller 8 | #eucalyptus = True 9 | #eucalyptus_host = clc.cloud.domain.org 10 | 11 | # AWS regions to make calls to. Set this to 'all' to make request to all regions 12 | # in AWS and merge the results together. Alternatively, set this to a comma 13 | # separated list of regions. E.g. 'us-east-1,us-west-1,us-west-2' and do not 14 | # provide the 'regions_exclude' option. If this is set to 'auto', AWS_REGION or 15 | # AWS_DEFAULT_REGION environment variable will be read to determine the region. 16 | regions = all 17 | regions_exclude = us-gov-west-1, cn-north-1 18 | 19 | # When generating inventory, Ansible needs to know how to address a server. 20 | # Each EC2 instance has a lot of variables associated with it. Here is the list: 21 | # http://docs.pythonboto.org/en/latest/ref/ec2.html#module-boto.ec2.instance 22 | # Below are 2 variables that are used as the address of a server: 23 | # - destination_variable 24 | # - vpc_destination_variable 25 | 26 | # This is the normal destination variable to use. If you are running Ansible 27 | # from outside EC2, then 'public_dns_name' makes the most sense. If you are 28 | # running Ansible from within EC2, then perhaps you want to use the internal 29 | # address, and should set this to 'private_dns_name'. The key of an EC2 tag 30 | # may optionally be used; however the boto instance variables hold precedence 31 | # in the event of a collision. 32 | destination_variable = public_dns_name 33 | 34 | # This allows you to override the inventory_name with an ec2 variable, instead 35 | # of using the destination_variable above. Addressing (aka ansible_ssh_host) 36 | # will still use destination_variable. Tags should be written as 'tag_TAGNAME'. 37 | #hostname_variable = tag_Name 38 | 39 | # For server inside a VPC, using DNS names may not make sense. When an instance 40 | # has 'subnet_id' set, this variable is used. If the subnet is public, setting 41 | # this to 'ip_address' will return the public IP address. For instances in a 42 | # private subnet, this should be set to 'private_ip_address', and Ansible must 43 | # be run from within EC2. The key of an EC2 tag may optionally be used; however 44 | # the boto instance variables hold precedence in the event of a collision. 45 | # WARNING: - instances that are in the private vpc, _without_ public ip address 46 | # will not be listed in the inventory until You set: 47 | # vpc_destination_variable = private_ip_address 48 | vpc_destination_variable = ip_address 49 | 50 | # The following two settings allow flexible ansible host naming based on a 51 | # python format string and a comma-separated list of ec2 tags. Note that: 52 | # 53 | # 1) If the tags referenced are not present for some instances, empty strings 54 | # will be substituted in the format string. 55 | # 2) This overrides both destination_variable and vpc_destination_variable. 56 | # 57 | #destination_format = {0}.{1}.example.com 58 | #destination_format_tags = Name,environment 59 | 60 | # To tag instances on EC2 with the resource records that point to them from 61 | # Route53, set 'route53' to True. 62 | route53 = False 63 | 64 | # To use Route53 records as the inventory hostnames, uncomment and set 65 | # to equal the domain name you wish to use. You must also have 'route53' (above) 66 | # set to True. 67 | # route53_hostnames = .example.com 68 | 69 | # To exclude RDS instances from the inventory, uncomment and set to False. 70 | #rds = False 71 | 72 | # To exclude ElastiCache instances from the inventory, uncomment and set to False. 73 | elasticache = False 74 | 75 | # Additionally, you can specify the list of zones to exclude looking up in 76 | # 'route53_excluded_zones' as a comma-separated list. 77 | # route53_excluded_zones = samplezone1.com, samplezone2.com 78 | 79 | # By default, only EC2 instances in the 'running' state are returned. Set 80 | # 'all_instances' to True to return all instances regardless of state. 81 | all_instances = False 82 | 83 | # By default, only EC2 instances in the 'running' state are returned. Specify 84 | # EC2 instance states to return as a comma-separated list. This 85 | # option is overridden when 'all_instances' is True. 86 | # instance_states = pending, running, shutting-down, terminated, stopping, stopped 87 | 88 | # By default, only RDS instances in the 'available' state are returned. Set 89 | # 'all_rds_instances' to True return all RDS instances regardless of state. 90 | all_rds_instances = False 91 | 92 | # Include RDS cluster information (Aurora etc.) 93 | include_rds_clusters = False 94 | 95 | # By default, only ElastiCache clusters and nodes in the 'available' state 96 | # are returned. Set 'all_elasticache_clusters' and/or 'all_elastic_nodes' 97 | # to True return all ElastiCache clusters and nodes, regardless of state. 98 | # 99 | # Note that all_elasticache_nodes only applies to listed clusters. That means 100 | # if you set all_elastic_clusters to false, no node will be return from 101 | # unavailable clusters, regardless of the state and to what you set for 102 | # all_elasticache_nodes. 103 | all_elasticache_replication_groups = False 104 | all_elasticache_clusters = False 105 | all_elasticache_nodes = False 106 | 107 | # API calls to EC2 are slow. For this reason, we cache the results of an API 108 | # call. Set this to the path you want cache files to be written to. Two files 109 | # will be written to this directory: 110 | # - ansible-ec2.cache 111 | # - ansible-ec2.index 112 | cache_path = ~/.ansible/tmp 113 | 114 | # The number of seconds a cache file is considered valid. After this many 115 | # seconds, a new API call will be made, and the cache file will be updated. 116 | # To disable the cache, set this value to 0 117 | cache_max_age = 300 118 | 119 | # Organize groups into a nested/hierarchy instead of a flat namespace. 120 | nested_groups = False 121 | 122 | # Replace - tags when creating groups to avoid issues with ansible 123 | replace_dash_in_groups = True 124 | 125 | # If set to true, any tag of the form "a,b,c" is expanded into a list 126 | # and the results are used to create additional tag_* inventory groups. 127 | expand_csv_tags = False 128 | 129 | # The EC2 inventory output can become very large. To manage its size, 130 | # configure which groups should be created. 131 | group_by_instance_id = True 132 | group_by_region = True 133 | group_by_availability_zone = True 134 | group_by_aws_account = False 135 | group_by_ami_id = True 136 | group_by_instance_type = True 137 | group_by_instance_state = False 138 | group_by_platform = True 139 | group_by_key_pair = True 140 | group_by_vpc_id = True 141 | group_by_security_group = True 142 | group_by_tag_keys = True 143 | group_by_tag_none = True 144 | group_by_route53_names = True 145 | group_by_rds_engine = True 146 | group_by_rds_parameter_group = True 147 | group_by_elasticache_engine = True 148 | group_by_elasticache_cluster = True 149 | group_by_elasticache_parameter_group = True 150 | group_by_elasticache_replication_group = True 151 | 152 | # If you only want to include hosts that match a certain regular expression 153 | # pattern_include = staging-* 154 | 155 | # If you want to exclude any hosts that match a certain regular expression 156 | # pattern_exclude = staging-* 157 | 158 | # Instance filters can be used to control which instances are retrieved for 159 | # inventory. For the full list of possible filters, please read the EC2 API 160 | # docs: http://docs.aws.amazon.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeInstances.html#query-DescribeInstances-filters 161 | # Filters are key/value pairs separated by '=', to list multiple filters use 162 | # a list separated by commas. To "AND" criteria together, use "&". Note that 163 | # the "AND" is not useful along with stack_filters and so such usage is not allowed. 164 | # See examples below. 165 | 166 | # If you want to apply multiple filters simultaneously, set stack_filters to 167 | # True. Default behaviour is to combine the results of all filters. Stacking 168 | # allows the use of multiple conditions to filter down, for example by 169 | # environment and type of host. 170 | stack_filters = False 171 | 172 | # Retrieve only instances with (key=value) env=staging tag 173 | # instance_filters = tag:env=staging 174 | 175 | # Retrieve only instances with role=webservers OR role=dbservers tag 176 | # instance_filters = tag:role=webservers,tag:role=dbservers 177 | 178 | # Retrieve only t1.micro instances OR instances with tag env=staging 179 | # instance_filters = instance-type=t1.micro,tag:env=staging 180 | 181 | # You can use wildcards in filter values also. Below will list instances which 182 | # tag Name value matches webservers1* 183 | # (ex. webservers15, webservers1a, webservers123 etc) 184 | # instance_filters = tag:Name=webservers1* 185 | 186 | # Retrieve only instances of type t1.micro that also have tag env=stage 187 | # instance_filters = instance-type=t1.micro&tag:env=stage 188 | 189 | # Retrieve instances of type t1.micro AND tag env=stage, as well as any instance 190 | # that are of type m3.large, regardless of env tag 191 | # instance_filters = instance-type=t1.micro&tag:env=stage,instance-type=m3.large 192 | 193 | # An IAM role can be assumed, so all requests are run as that role. 194 | # This can be useful for connecting across different accounts, or to limit user 195 | # access 196 | # iam_role = role-arn 197 | 198 | # A boto configuration profile may be used to separate out credentials 199 | # see http://boto.readthedocs.org/en/latest/boto_config_tut.html 200 | # boto_profile = some-boto-profile-name 201 | 202 | 203 | [credentials] 204 | 205 | # The AWS credentials can optionally be specified here. Credentials specified 206 | # here are ignored if the environment variable AWS_ACCESS_KEY_ID or 207 | # AWS_PROFILE is set, or if the boto_profile property above is set. 208 | # 209 | # Supplying AWS credentials here is not recommended, as it introduces 210 | # non-trivial security concerns. When going down this route, please make sure 211 | # to set access permissions for this file correctly, e.g. handle it the same 212 | # way as you would a private SSH key. 213 | # 214 | # Unlike the boto and AWS configure files, this section does not support 215 | # profiles. 216 | # 217 | # aws_access_key_id = AXXXXXXXXXXXXXX 218 | # aws_secret_access_key = XXXXXXXXXXXXXXXXXXX 219 | # aws_security_token = XXXXXXXXXXXXXXXXXXXXXXXXXXXX 220 | -------------------------------------------------------------------------------- /infrastructure/inventory/ec2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ''' 4 | EC2 external inventory script 5 | ================================= 6 | 7 | Generates inventory that Ansible can understand by making API request to 8 | AWS EC2 using the Boto library. 9 | 10 | NOTE: This script assumes Ansible is being executed where the environment 11 | variables needed for Boto have already been set: 12 | export AWS_ACCESS_KEY_ID='AK123' 13 | export AWS_SECRET_ACCESS_KEY='abc123' 14 | 15 | Optional region environment variable if region is 'auto' 16 | 17 | This script also assumes that there is an ec2.ini file alongside it. To specify a 18 | different path to ec2.ini, define the EC2_INI_PATH environment variable: 19 | 20 | export EC2_INI_PATH=/path/to/my_ec2.ini 21 | 22 | If you're using eucalyptus you need to set the above variables and 23 | you need to define: 24 | 25 | export EC2_URL=http://hostname_of_your_cc:port/services/Eucalyptus 26 | 27 | If you're using boto profiles (requires boto>=2.24.0) you can choose a profile 28 | using the --boto-profile command line argument (e.g. ec2.py --boto-profile prod) or using 29 | the AWS_PROFILE variable: 30 | 31 | AWS_PROFILE=prod ansible-playbook -i ec2.py myplaybook.yml 32 | 33 | For more details, see: http://docs.pythonboto.org/en/latest/boto_config_tut.html 34 | 35 | When run against a specific host, this script returns the following variables: 36 | - ec2_ami_launch_index 37 | - ec2_architecture 38 | - ec2_association 39 | - ec2_attachTime 40 | - ec2_attachment 41 | - ec2_attachmentId 42 | - ec2_block_devices 43 | - ec2_client_token 44 | - ec2_deleteOnTermination 45 | - ec2_description 46 | - ec2_deviceIndex 47 | - ec2_dns_name 48 | - ec2_eventsSet 49 | - ec2_group_name 50 | - ec2_hypervisor 51 | - ec2_id 52 | - ec2_image_id 53 | - ec2_instanceState 54 | - ec2_instance_type 55 | - ec2_ipOwnerId 56 | - ec2_ip_address 57 | - ec2_item 58 | - ec2_kernel 59 | - ec2_key_name 60 | - ec2_launch_time 61 | - ec2_monitored 62 | - ec2_monitoring 63 | - ec2_networkInterfaceId 64 | - ec2_ownerId 65 | - ec2_persistent 66 | - ec2_placement 67 | - ec2_platform 68 | - ec2_previous_state 69 | - ec2_private_dns_name 70 | - ec2_private_ip_address 71 | - ec2_publicIp 72 | - ec2_public_dns_name 73 | - ec2_ramdisk 74 | - ec2_reason 75 | - ec2_region 76 | - ec2_requester_id 77 | - ec2_root_device_name 78 | - ec2_root_device_type 79 | - ec2_security_group_ids 80 | - ec2_security_group_names 81 | - ec2_shutdown_state 82 | - ec2_sourceDestCheck 83 | - ec2_spot_instance_request_id 84 | - ec2_state 85 | - ec2_state_code 86 | - ec2_state_reason 87 | - ec2_status 88 | - ec2_subnet_id 89 | - ec2_tenancy 90 | - ec2_virtualization_type 91 | - ec2_vpc_id 92 | 93 | These variables are pulled out of a boto.ec2.instance object. There is a lack of 94 | consistency with variable spellings (camelCase and underscores) since this 95 | just loops through all variables the object exposes. It is preferred to use the 96 | ones with underscores when multiple exist. 97 | 98 | In addition, if an instance has AWS tags associated with it, each tag is a new 99 | variable named: 100 | - ec2_tag_[Key] = [Value] 101 | 102 | Security groups are comma-separated in 'ec2_security_group_ids' and 103 | 'ec2_security_group_names'. 104 | 105 | When destination_format and destination_format_tags are specified 106 | the destination_format can be built from the instance tags and attributes. 107 | The behavior will first check the user defined tags, then proceed to 108 | check instance attributes, and finally if neither are found 'nil' will 109 | be used instead. 110 | 111 | 'my_instance': { 112 | 'region': 'us-east-1', # attribute 113 | 'availability_zone': 'us-east-1a', # attribute 114 | 'private_dns_name': '172.31.0.1', # attribute 115 | 'ec2_tag_deployment': 'blue', # tag 116 | 'ec2_tag_clusterid': 'ansible', # tag 117 | 'ec2_tag_Name': 'webserver', # tag 118 | ... 119 | } 120 | 121 | Inside of the ec2.ini file the following settings are specified: 122 | ... 123 | destination_format: {0}-{1}-{2}-{3} 124 | destination_format_tags: Name,clusterid,deployment,private_dns_name 125 | ... 126 | 127 | These settings would produce a destination_format as the following: 128 | 'webserver-ansible-blue-172.31.0.1' 129 | ''' 130 | 131 | # (c) 2012, Peter Sankauskas 132 | # 133 | # This file is part of Ansible, 134 | # 135 | # Ansible is free software: you can redistribute it and/or modify 136 | # it under the terms of the GNU General Public License as published by 137 | # the Free Software Foundation, either version 3 of the License, or 138 | # (at your option) any later version. 139 | # 140 | # Ansible is distributed in the hope that it will be useful, 141 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 142 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 143 | # GNU General Public License for more details. 144 | # 145 | # You should have received a copy of the GNU General Public License 146 | # along with Ansible. If not, see . 147 | 148 | ###################################################################### 149 | 150 | import sys 151 | import os 152 | import argparse 153 | import re 154 | from time import time 155 | import boto 156 | from boto import ec2 157 | from boto import rds 158 | from boto import elasticache 159 | from boto import route53 160 | from boto import sts 161 | import six 162 | 163 | from ansible.module_utils import ec2 as ec2_utils 164 | 165 | HAS_BOTO3 = False 166 | try: 167 | import boto3 # noqa 168 | HAS_BOTO3 = True 169 | except ImportError: 170 | pass 171 | 172 | from six.moves import configparser 173 | from collections import defaultdict 174 | 175 | try: 176 | import json 177 | except ImportError: 178 | import simplejson as json 179 | 180 | DEFAULTS = { 181 | 'all_elasticache_clusters': 'False', 182 | 'all_elasticache_nodes': 'False', 183 | 'all_elasticache_replication_groups': 'False', 184 | 'all_instances': 'False', 185 | 'all_rds_instances': 'False', 186 | 'aws_access_key_id': None, 187 | 'aws_secret_access_key': None, 188 | 'aws_security_token': None, 189 | 'boto_profile': None, 190 | 'cache_max_age': '300', 191 | 'cache_path': '~/.ansible/tmp', 192 | 'destination_variable': 'public_dns_name', 193 | 'elasticache': 'True', 194 | 'eucalyptus': 'False', 195 | 'eucalyptus_host': None, 196 | 'expand_csv_tags': 'False', 197 | 'group_by_ami_id': 'True', 198 | 'group_by_availability_zone': 'True', 199 | 'group_by_aws_account': 'False', 200 | 'group_by_elasticache_cluster': 'True', 201 | 'group_by_elasticache_engine': 'True', 202 | 'group_by_elasticache_parameter_group': 'True', 203 | 'group_by_elasticache_replication_group': 'True', 204 | 'group_by_instance_id': 'True', 205 | 'group_by_instance_state': 'False', 206 | 'group_by_instance_type': 'True', 207 | 'group_by_key_pair': 'True', 208 | 'group_by_platform': 'True', 209 | 'group_by_rds_engine': 'True', 210 | 'group_by_rds_parameter_group': 'True', 211 | 'group_by_region': 'True', 212 | 'group_by_route53_names': 'True', 213 | 'group_by_security_group': 'True', 214 | 'group_by_tag_keys': 'True', 215 | 'group_by_tag_none': 'True', 216 | 'group_by_vpc_id': 'True', 217 | 'hostname_variable': None, 218 | 'iam_role': None, 219 | 'include_rds_clusters': 'False', 220 | 'nested_groups': 'False', 221 | 'pattern_exclude': None, 222 | 'pattern_include': None, 223 | 'rds': 'False', 224 | 'regions': 'all', 225 | 'regions_exclude': 'us-gov-west-1, cn-north-1', 226 | 'replace_dash_in_groups': 'True', 227 | 'route53': 'False', 228 | 'route53_excluded_zones': '', 229 | 'route53_hostnames': None, 230 | 'stack_filters': 'False', 231 | 'vpc_destination_variable': 'ip_address' 232 | } 233 | 234 | 235 | class Ec2Inventory(object): 236 | 237 | def _empty_inventory(self): 238 | return {"_meta": {"hostvars": {}}} 239 | 240 | def __init__(self): 241 | ''' Main execution path ''' 242 | 243 | # Inventory grouped by instance IDs, tags, security groups, regions, 244 | # and availability zones 245 | self.inventory = self._empty_inventory() 246 | 247 | self.aws_account_id = None 248 | 249 | # Index of hostname (address) to instance ID 250 | self.index = {} 251 | 252 | # Boto profile to use (if any) 253 | self.boto_profile = None 254 | 255 | # AWS credentials. 256 | self.credentials = {} 257 | 258 | # Read settings and parse CLI arguments 259 | self.parse_cli_args() 260 | self.read_settings() 261 | 262 | # Make sure that profile_name is not passed at all if not set 263 | # as pre 2.24 boto will fall over otherwise 264 | if self.boto_profile: 265 | if not hasattr(boto.ec2.EC2Connection, 'profile_name'): 266 | self.fail_with_error("boto version must be >= 2.24 to use profile") 267 | 268 | # Cache 269 | if self.args.refresh_cache: 270 | self.do_api_calls_update_cache() 271 | elif not self.is_cache_valid(): 272 | self.do_api_calls_update_cache() 273 | 274 | # Data to print 275 | if self.args.host: 276 | data_to_print = self.get_host_info() 277 | 278 | elif self.args.list: 279 | # Display list of instances for inventory 280 | if self.inventory == self._empty_inventory(): 281 | data_to_print = self.get_inventory_from_cache() 282 | else: 283 | data_to_print = self.json_format_dict(self.inventory, True) 284 | 285 | print(data_to_print) 286 | 287 | def is_cache_valid(self): 288 | ''' Determines if the cache files have expired, or if it is still valid ''' 289 | 290 | if os.path.isfile(self.cache_path_cache): 291 | mod_time = os.path.getmtime(self.cache_path_cache) 292 | current_time = time() 293 | if (mod_time + self.cache_max_age) > current_time: 294 | if os.path.isfile(self.cache_path_index): 295 | return True 296 | 297 | return False 298 | 299 | def read_settings(self): 300 | ''' Reads the settings from the ec2.ini file ''' 301 | 302 | scriptbasename = __file__ 303 | scriptbasename = os.path.basename(scriptbasename) 304 | scriptbasename = scriptbasename.replace('.py', '') 305 | 306 | defaults = { 307 | 'ec2': { 308 | 'ini_fallback': os.path.join(os.path.dirname(__file__), 'ec2.ini'), 309 | 'ini_path': os.path.join(os.path.dirname(__file__), '%s.ini' % scriptbasename) 310 | } 311 | } 312 | 313 | if six.PY3: 314 | config = configparser.ConfigParser(DEFAULTS) 315 | else: 316 | config = configparser.SafeConfigParser(DEFAULTS) 317 | ec2_ini_path = os.environ.get('EC2_INI_PATH', defaults['ec2']['ini_path']) 318 | ec2_ini_path = os.path.expanduser(os.path.expandvars(ec2_ini_path)) 319 | 320 | if not os.path.isfile(ec2_ini_path): 321 | ec2_ini_path = os.path.expanduser(defaults['ec2']['ini_fallback']) 322 | 323 | if os.path.isfile(ec2_ini_path): 324 | config.read(ec2_ini_path) 325 | 326 | # Add empty sections if they don't exist 327 | try: 328 | config.add_section('ec2') 329 | except configparser.DuplicateSectionError: 330 | pass 331 | 332 | try: 333 | config.add_section('credentials') 334 | except configparser.DuplicateSectionError: 335 | pass 336 | 337 | # is eucalyptus? 338 | self.eucalyptus = config.getboolean('ec2', 'eucalyptus') 339 | self.eucalyptus_host = config.get('ec2', 'eucalyptus_host') 340 | 341 | # Regions 342 | self.regions = [] 343 | configRegions = config.get('ec2', 'regions') 344 | if (configRegions == 'all'): 345 | if self.eucalyptus_host: 346 | self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name, **self.credentials) 347 | else: 348 | configRegions_exclude = config.get('ec2', 'regions_exclude') 349 | 350 | for regionInfo in ec2.regions(): 351 | if regionInfo.name not in configRegions_exclude: 352 | self.regions.append(regionInfo.name) 353 | else: 354 | self.regions = configRegions.split(",") 355 | if 'auto' in self.regions: 356 | env_region = os.environ.get('AWS_REGION') 357 | if env_region is None: 358 | env_region = os.environ.get('AWS_DEFAULT_REGION') 359 | self.regions = [env_region] 360 | 361 | # Destination addresses 362 | self.destination_variable = config.get('ec2', 'destination_variable') 363 | self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable') 364 | self.hostname_variable = config.get('ec2', 'hostname_variable') 365 | 366 | if config.has_option('ec2', 'destination_format') and \ 367 | config.has_option('ec2', 'destination_format_tags'): 368 | self.destination_format = config.get('ec2', 'destination_format') 369 | self.destination_format_tags = config.get('ec2', 'destination_format_tags').split(',') 370 | else: 371 | self.destination_format = None 372 | self.destination_format_tags = None 373 | 374 | # Route53 375 | self.route53_enabled = config.getboolean('ec2', 'route53') 376 | self.route53_hostnames = config.get('ec2', 'route53_hostnames') 377 | 378 | self.route53_excluded_zones = [] 379 | self.route53_excluded_zones = [a for a in config.get('ec2', 'route53_excluded_zones').split(',') if a] 380 | 381 | # Include RDS instances? 382 | self.rds_enabled = config.getboolean('ec2', 'rds') 383 | 384 | # Include RDS cluster instances? 385 | self.include_rds_clusters = config.getboolean('ec2', 'include_rds_clusters') 386 | 387 | # Include ElastiCache instances? 388 | self.elasticache_enabled = config.getboolean('ec2', 'elasticache') 389 | 390 | # Return all EC2 instances? 391 | self.all_instances = config.getboolean('ec2', 'all_instances') 392 | 393 | # Instance states to be gathered in inventory. Default is 'running'. 394 | # Setting 'all_instances' to 'yes' overrides this option. 395 | ec2_valid_instance_states = [ 396 | 'pending', 397 | 'running', 398 | 'shutting-down', 399 | 'terminated', 400 | 'stopping', 401 | 'stopped' 402 | ] 403 | self.ec2_instance_states = [] 404 | if self.all_instances: 405 | self.ec2_instance_states = ec2_valid_instance_states 406 | elif config.has_option('ec2', 'instance_states'): 407 | for instance_state in config.get('ec2', 'instance_states').split(','): 408 | instance_state = instance_state.strip() 409 | if instance_state not in ec2_valid_instance_states: 410 | continue 411 | self.ec2_instance_states.append(instance_state) 412 | else: 413 | self.ec2_instance_states = ['running'] 414 | 415 | # Return all RDS instances? (if RDS is enabled) 416 | self.all_rds_instances = config.getboolean('ec2', 'all_rds_instances') 417 | 418 | # Return all ElastiCache replication groups? (if ElastiCache is enabled) 419 | self.all_elasticache_replication_groups = config.getboolean('ec2', 'all_elasticache_replication_groups') 420 | 421 | # Return all ElastiCache clusters? (if ElastiCache is enabled) 422 | self.all_elasticache_clusters = config.getboolean('ec2', 'all_elasticache_clusters') 423 | 424 | # Return all ElastiCache nodes? (if ElastiCache is enabled) 425 | self.all_elasticache_nodes = config.getboolean('ec2', 'all_elasticache_nodes') 426 | 427 | # boto configuration profile (prefer CLI argument then environment variables then config file) 428 | self.boto_profile = self.args.boto_profile or \ 429 | os.environ.get('AWS_PROFILE') or \ 430 | config.get('ec2', 'boto_profile') 431 | 432 | # AWS credentials (prefer environment variables) 433 | if not (self.boto_profile or os.environ.get('AWS_ACCESS_KEY_ID') or 434 | os.environ.get('AWS_PROFILE')): 435 | 436 | aws_access_key_id = config.get('credentials', 'aws_access_key_id') 437 | aws_secret_access_key = config.get('credentials', 'aws_secret_access_key') 438 | aws_security_token = config.get('credentials', 'aws_security_token') 439 | 440 | if aws_access_key_id: 441 | self.credentials = { 442 | 'aws_access_key_id': aws_access_key_id, 443 | 'aws_secret_access_key': aws_secret_access_key 444 | } 445 | if aws_security_token: 446 | self.credentials['security_token'] = aws_security_token 447 | 448 | # Cache related 449 | cache_dir = os.path.expanduser(config.get('ec2', 'cache_path')) 450 | if self.boto_profile: 451 | cache_dir = os.path.join(cache_dir, 'profile_' + self.boto_profile) 452 | if not os.path.exists(cache_dir): 453 | os.makedirs(cache_dir) 454 | 455 | cache_name = 'ansible-ec2' 456 | cache_id = self.boto_profile or os.environ.get('AWS_ACCESS_KEY_ID', self.credentials.get('aws_access_key_id')) 457 | if cache_id: 458 | cache_name = '%s-%s' % (cache_name, cache_id) 459 | cache_name += '-' + str(abs(hash(__file__)))[1:7] 460 | self.cache_path_cache = os.path.join(cache_dir, "%s.cache" % cache_name) 461 | self.cache_path_index = os.path.join(cache_dir, "%s.index" % cache_name) 462 | self.cache_max_age = config.getint('ec2', 'cache_max_age') 463 | 464 | self.expand_csv_tags = config.getboolean('ec2', 'expand_csv_tags') 465 | 466 | # Configure nested groups instead of flat namespace. 467 | self.nested_groups = config.getboolean('ec2', 'nested_groups') 468 | 469 | # Replace dash or not in group names 470 | self.replace_dash_in_groups = config.getboolean('ec2', 'replace_dash_in_groups') 471 | 472 | # IAM role to assume for connection 473 | self.iam_role = config.get('ec2', 'iam_role') 474 | 475 | # Configure which groups should be created. 476 | 477 | group_by_options = [a for a in DEFAULTS if a.startswith('group_by')] 478 | for option in group_by_options: 479 | setattr(self, option, config.getboolean('ec2', option)) 480 | 481 | # Do we need to just include hosts that match a pattern? 482 | self.pattern_include = config.get('ec2', 'pattern_include') 483 | if self.pattern_include: 484 | self.pattern_include = re.compile(self.pattern_include) 485 | 486 | # Do we need to exclude hosts that match a pattern? 487 | self.pattern_exclude = config.get('ec2', 'pattern_exclude') 488 | if self.pattern_exclude: 489 | self.pattern_exclude = re.compile(self.pattern_exclude) 490 | 491 | # Do we want to stack multiple filters? 492 | self.stack_filters = config.getboolean('ec2', 'stack_filters') 493 | 494 | # Instance filters (see boto and EC2 API docs). Ignore invalid filters. 495 | self.ec2_instance_filters = [] 496 | 497 | if config.has_option('ec2', 'instance_filters'): 498 | filters = config.get('ec2', 'instance_filters') 499 | 500 | if self.stack_filters and '&' in filters: 501 | self.fail_with_error("AND filters along with stack_filter enabled is not supported.\n") 502 | 503 | filter_sets = [f for f in filters.split(',') if f] 504 | 505 | for filter_set in filter_sets: 506 | filters = {} 507 | filter_set = filter_set.strip() 508 | for instance_filter in filter_set.split("&"): 509 | instance_filter = instance_filter.strip() 510 | if not instance_filter or '=' not in instance_filter: 511 | continue 512 | filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)] 513 | if not filter_key: 514 | continue 515 | filters[filter_key] = filter_value 516 | self.ec2_instance_filters.append(filters.copy()) 517 | 518 | def parse_cli_args(self): 519 | ''' Command line argument processing ''' 520 | 521 | parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on EC2') 522 | parser.add_argument('--list', action='store_true', default=True, 523 | help='List instances (default: True)') 524 | parser.add_argument('--host', action='store', 525 | help='Get all the variables about a specific instance') 526 | parser.add_argument('--refresh-cache', action='store_true', default=False, 527 | help='Force refresh of cache by making API requests to EC2 (default: False - use cache files)') 528 | parser.add_argument('--profile', '--boto-profile', action='store', dest='boto_profile', 529 | help='Use boto profile for connections to EC2') 530 | self.args = parser.parse_args() 531 | 532 | def do_api_calls_update_cache(self): 533 | ''' Do API calls to each region, and save data in cache files ''' 534 | 535 | if self.route53_enabled: 536 | self.get_route53_records() 537 | 538 | for region in self.regions: 539 | self.get_instances_by_region(region) 540 | if self.rds_enabled: 541 | self.get_rds_instances_by_region(region) 542 | if self.elasticache_enabled: 543 | self.get_elasticache_clusters_by_region(region) 544 | self.get_elasticache_replication_groups_by_region(region) 545 | if self.include_rds_clusters: 546 | self.include_rds_clusters_by_region(region) 547 | 548 | self.write_to_cache(self.inventory, self.cache_path_cache) 549 | self.write_to_cache(self.index, self.cache_path_index) 550 | 551 | def connect(self, region): 552 | ''' create connection to api server''' 553 | if self.eucalyptus: 554 | conn = boto.connect_euca(host=self.eucalyptus_host, **self.credentials) 555 | conn.APIVersion = '2010-08-31' 556 | else: 557 | conn = self.connect_to_aws(ec2, region) 558 | return conn 559 | 560 | def boto_fix_security_token_in_profile(self, connect_args): 561 | ''' monkey patch for boto issue boto/boto#2100 ''' 562 | profile = 'profile ' + self.boto_profile 563 | if boto.config.has_option(profile, 'aws_security_token'): 564 | connect_args['security_token'] = boto.config.get(profile, 'aws_security_token') 565 | return connect_args 566 | 567 | def connect_to_aws(self, module, region): 568 | connect_args = self.credentials 569 | 570 | # only pass the profile name if it's set (as it is not supported by older boto versions) 571 | if self.boto_profile: 572 | connect_args['profile_name'] = self.boto_profile 573 | self.boto_fix_security_token_in_profile(connect_args) 574 | 575 | if self.iam_role: 576 | sts_conn = sts.connect_to_region(region, **connect_args) 577 | role = sts_conn.assume_role(self.iam_role, 'ansible_dynamic_inventory') 578 | connect_args['aws_access_key_id'] = role.credentials.access_key 579 | connect_args['aws_secret_access_key'] = role.credentials.secret_key 580 | connect_args['security_token'] = role.credentials.session_token 581 | 582 | conn = module.connect_to_region(region, **connect_args) 583 | # connect_to_region will fail "silently" by returning None if the region name is wrong or not supported 584 | if conn is None: 585 | self.fail_with_error("region name: %s likely not supported, or AWS is down. connection to region failed." % region) 586 | return conn 587 | 588 | def get_instances_by_region(self, region): 589 | ''' Makes an AWS EC2 API call to the list of instances in a particular 590 | region ''' 591 | 592 | try: 593 | conn = self.connect(region) 594 | reservations = [] 595 | if self.ec2_instance_filters: 596 | if self.stack_filters: 597 | filters_dict = {} 598 | for filters in self.ec2_instance_filters: 599 | filters_dict.update(filters) 600 | reservations.extend(conn.get_all_instances(filters=filters_dict)) 601 | else: 602 | for filters in self.ec2_instance_filters: 603 | reservations.extend(conn.get_all_instances(filters=filters)) 604 | else: 605 | reservations = conn.get_all_instances() 606 | 607 | # Pull the tags back in a second step 608 | # AWS are on record as saying that the tags fetched in the first `get_all_instances` request are not 609 | # reliable and may be missing, and the only way to guarantee they are there is by calling `get_all_tags` 610 | instance_ids = [] 611 | for reservation in reservations: 612 | instance_ids.extend([instance.id for instance in reservation.instances]) 613 | 614 | max_filter_value = 199 615 | tags = [] 616 | for i in range(0, len(instance_ids), max_filter_value): 617 | tags.extend(conn.get_all_tags(filters={'resource-type': 'instance', 'resource-id': instance_ids[i:i + max_filter_value]})) 618 | 619 | tags_by_instance_id = defaultdict(dict) 620 | for tag in tags: 621 | tags_by_instance_id[tag.res_id][tag.name] = tag.value 622 | 623 | if (not self.aws_account_id) and reservations: 624 | self.aws_account_id = reservations[0].owner_id 625 | 626 | for reservation in reservations: 627 | for instance in reservation.instances: 628 | instance.tags = tags_by_instance_id[instance.id] 629 | self.add_instance(instance, region) 630 | 631 | except boto.exception.BotoServerError as e: 632 | if e.error_code == 'AuthFailure': 633 | error = self.get_auth_error_message() 634 | else: 635 | backend = 'Eucalyptus' if self.eucalyptus else 'AWS' 636 | error = "Error connecting to %s backend.\n%s" % (backend, e.message) 637 | self.fail_with_error(error, 'getting EC2 instances') 638 | 639 | def tags_match_filters(self, tags): 640 | ''' return True if given tags match configured filters ''' 641 | if not self.ec2_instance_filters: 642 | return True 643 | 644 | for filters in self.ec2_instance_filters: 645 | for filter_name, filter_value in filters.items(): 646 | if filter_name[:4] != 'tag:': 647 | continue 648 | filter_name = filter_name[4:] 649 | if filter_name not in tags: 650 | if self.stack_filters: 651 | return False 652 | continue 653 | if isinstance(filter_value, list): 654 | if self.stack_filters and tags[filter_name] not in filter_value: 655 | return False 656 | if not self.stack_filters and tags[filter_name] in filter_value: 657 | return True 658 | if isinstance(filter_value, six.string_types): 659 | if self.stack_filters and tags[filter_name] != filter_value: 660 | return False 661 | if not self.stack_filters and tags[filter_name] == filter_value: 662 | return True 663 | 664 | return self.stack_filters 665 | 666 | def get_rds_instances_by_region(self, region): 667 | ''' Makes an AWS API call to the list of RDS instances in a particular 668 | region ''' 669 | 670 | if not HAS_BOTO3: 671 | self.fail_with_error("Working with RDS instances requires boto3 - please install boto3 and try again", 672 | "getting RDS instances") 673 | 674 | client = ec2_utils.boto3_inventory_conn('client', 'rds', region, **self.credentials) 675 | db_instances = client.describe_db_instances() 676 | 677 | try: 678 | conn = self.connect_to_aws(rds, region) 679 | if conn: 680 | marker = None 681 | while True: 682 | instances = conn.get_all_dbinstances(marker=marker) 683 | marker = instances.marker 684 | for index, instance in enumerate(instances): 685 | # Add tags to instances. 686 | instance.arn = db_instances['DBInstances'][index]['DBInstanceArn'] 687 | tags = client.list_tags_for_resource(ResourceName=instance.arn)['TagList'] 688 | instance.tags = {} 689 | for tag in tags: 690 | instance.tags[tag['Key']] = tag['Value'] 691 | if self.tags_match_filters(instance.tags): 692 | self.add_rds_instance(instance, region) 693 | if not marker: 694 | break 695 | except boto.exception.BotoServerError as e: 696 | error = e.reason 697 | 698 | if e.error_code == 'AuthFailure': 699 | error = self.get_auth_error_message() 700 | elif e.error_code == "OptInRequired": 701 | error = "RDS hasn't been enabled for this account yet. " \ 702 | "You must either log in to the RDS service through the AWS console to enable it, " \ 703 | "or set 'rds = False' in ec2.ini" 704 | elif not e.reason == "Forbidden": 705 | error = "Looks like AWS RDS is down:\n%s" % e.message 706 | self.fail_with_error(error, 'getting RDS instances') 707 | 708 | def include_rds_clusters_by_region(self, region): 709 | if not HAS_BOTO3: 710 | self.fail_with_error("Working with RDS clusters requires boto3 - please install boto3 and try again", 711 | "getting RDS clusters") 712 | 713 | client = ec2_utils.boto3_inventory_conn('client', 'rds', region, **self.credentials) 714 | 715 | marker, clusters = '', [] 716 | while marker is not None: 717 | resp = client.describe_db_clusters(Marker=marker) 718 | clusters.extend(resp["DBClusters"]) 719 | marker = resp.get('Marker', None) 720 | 721 | account_id = boto.connect_iam().get_user().arn.split(':')[4] 722 | c_dict = {} 723 | for c in clusters: 724 | # remove these datetime objects as there is no serialisation to json 725 | # currently in place and we don't need the data yet 726 | if 'EarliestRestorableTime' in c: 727 | del c['EarliestRestorableTime'] 728 | if 'LatestRestorableTime' in c: 729 | del c['LatestRestorableTime'] 730 | 731 | if not self.ec2_instance_filters: 732 | matches_filter = True 733 | else: 734 | matches_filter = False 735 | 736 | try: 737 | # arn:aws:rds:::: 738 | tags = client.list_tags_for_resource( 739 | ResourceName='arn:aws:rds:' + region + ':' + account_id + ':cluster:' + c['DBClusterIdentifier']) 740 | c['Tags'] = tags['TagList'] 741 | 742 | if self.ec2_instance_filters: 743 | for filters in self.ec2_instance_filters: 744 | for filter_key, filter_values in filters.items(): 745 | # get AWS tag key e.g. tag:env will be 'env' 746 | tag_name = filter_key.split(":", 1)[1] 747 | # Filter values is a list (if you put multiple values for the same tag name) 748 | matches_filter = any(d['Key'] == tag_name and d['Value'] in filter_values for d in c['Tags']) 749 | 750 | if matches_filter: 751 | # it matches a filter, so stop looking for further matches 752 | break 753 | 754 | if matches_filter: 755 | break 756 | 757 | except Exception as e: 758 | if e.message.find('DBInstanceNotFound') >= 0: 759 | # AWS RDS bug (2016-01-06) means deletion does not fully complete and leave an 'empty' cluster. 760 | # Ignore errors when trying to find tags for these 761 | pass 762 | 763 | # ignore empty clusters caused by AWS bug 764 | if len(c['DBClusterMembers']) == 0: 765 | continue 766 | elif matches_filter: 767 | c_dict[c['DBClusterIdentifier']] = c 768 | 769 | self.inventory['db_clusters'] = c_dict 770 | 771 | def get_elasticache_clusters_by_region(self, region): 772 | ''' Makes an AWS API call to the list of ElastiCache clusters (with 773 | nodes' info) in a particular region.''' 774 | 775 | # ElastiCache boto module doesn't provide a get_all_instances method, 776 | # that's why we need to call describe directly (it would be called by 777 | # the shorthand method anyway...) 778 | try: 779 | conn = self.connect_to_aws(elasticache, region) 780 | if conn: 781 | # show_cache_node_info = True 782 | # because we also want nodes' information 783 | response = conn.describe_cache_clusters(None, None, None, True) 784 | 785 | except boto.exception.BotoServerError as e: 786 | error = e.reason 787 | 788 | if e.error_code == 'AuthFailure': 789 | error = self.get_auth_error_message() 790 | elif e.error_code == "OptInRequired": 791 | error = "ElastiCache hasn't been enabled for this account yet. " \ 792 | "You must either log in to the ElastiCache service through the AWS console to enable it, " \ 793 | "or set 'elasticache = False' in ec2.ini" 794 | elif not e.reason == "Forbidden": 795 | error = "Looks like AWS ElastiCache is down:\n%s" % e.message 796 | self.fail_with_error(error, 'getting ElastiCache clusters') 797 | 798 | try: 799 | # Boto also doesn't provide wrapper classes to CacheClusters or 800 | # CacheNodes. Because of that we can't make use of the get_list 801 | # method in the AWSQueryConnection. Let's do the work manually 802 | clusters = response['DescribeCacheClustersResponse']['DescribeCacheClustersResult']['CacheClusters'] 803 | 804 | except KeyError as e: 805 | error = "ElastiCache query to AWS failed (unexpected format)." 806 | self.fail_with_error(error, 'getting ElastiCache clusters') 807 | 808 | for cluster in clusters: 809 | self.add_elasticache_cluster(cluster, region) 810 | 811 | def get_elasticache_replication_groups_by_region(self, region): 812 | ''' Makes an AWS API call to the list of ElastiCache replication groups 813 | in a particular region.''' 814 | 815 | # ElastiCache boto module doesn't provide a get_all_instances method, 816 | # that's why we need to call describe directly (it would be called by 817 | # the shorthand method anyway...) 818 | try: 819 | conn = self.connect_to_aws(elasticache, region) 820 | if conn: 821 | response = conn.describe_replication_groups() 822 | 823 | except boto.exception.BotoServerError as e: 824 | error = e.reason 825 | 826 | if e.error_code == 'AuthFailure': 827 | error = self.get_auth_error_message() 828 | if not e.reason == "Forbidden": 829 | error = "Looks like AWS ElastiCache [Replication Groups] is down:\n%s" % e.message 830 | self.fail_with_error(error, 'getting ElastiCache clusters') 831 | 832 | try: 833 | # Boto also doesn't provide wrapper classes to ReplicationGroups 834 | # Because of that we can't make use of the get_list method in the 835 | # AWSQueryConnection. Let's do the work manually 836 | replication_groups = response['DescribeReplicationGroupsResponse']['DescribeReplicationGroupsResult']['ReplicationGroups'] 837 | 838 | except KeyError as e: 839 | error = "ElastiCache [Replication Groups] query to AWS failed (unexpected format)." 840 | self.fail_with_error(error, 'getting ElastiCache clusters') 841 | 842 | for replication_group in replication_groups: 843 | self.add_elasticache_replication_group(replication_group, region) 844 | 845 | def get_auth_error_message(self): 846 | ''' create an informative error message if there is an issue authenticating''' 847 | errors = ["Authentication error retrieving ec2 inventory."] 848 | if None in [os.environ.get('AWS_ACCESS_KEY_ID'), os.environ.get('AWS_SECRET_ACCESS_KEY')]: 849 | errors.append(' - No AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY environment vars found') 850 | else: 851 | errors.append(' - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment vars found but may not be correct') 852 | 853 | boto_paths = ['/etc/boto.cfg', '~/.boto', '~/.aws/credentials'] 854 | boto_config_found = [p for p in boto_paths if os.path.isfile(os.path.expanduser(p))] 855 | if len(boto_config_found) > 0: 856 | errors.append(" - Boto configs found at '%s', but the credentials contained may not be correct" % ', '.join(boto_config_found)) 857 | else: 858 | errors.append(" - No Boto config found at any expected location '%s'" % ', '.join(boto_paths)) 859 | 860 | return '\n'.join(errors) 861 | 862 | def fail_with_error(self, err_msg, err_operation=None): 863 | '''log an error to std err for ansible-playbook to consume and exit''' 864 | if err_operation: 865 | err_msg = 'ERROR: "{err_msg}", while: {err_operation}'.format( 866 | err_msg=err_msg, err_operation=err_operation) 867 | sys.stderr.write(err_msg) 868 | sys.exit(1) 869 | 870 | def get_instance(self, region, instance_id): 871 | conn = self.connect(region) 872 | 873 | reservations = conn.get_all_instances([instance_id]) 874 | for reservation in reservations: 875 | for instance in reservation.instances: 876 | return instance 877 | 878 | def add_instance(self, instance, region): 879 | ''' Adds an instance to the inventory and index, as long as it is 880 | addressable ''' 881 | 882 | # Only return instances with desired instance states 883 | if instance.state not in self.ec2_instance_states: 884 | return 885 | 886 | # Select the best destination address 887 | # When destination_format and destination_format_tags are specified 888 | # the following code will attempt to find the instance tags first, 889 | # then the instance attributes next, and finally if neither are found 890 | # assign nil for the desired destination format attribute. 891 | if self.destination_format and self.destination_format_tags: 892 | dest_vars = [] 893 | inst_tags = getattr(instance, 'tags') 894 | for tag in self.destination_format_tags: 895 | if tag in inst_tags: 896 | dest_vars.append(inst_tags[tag]) 897 | elif hasattr(instance, tag): 898 | dest_vars.append(getattr(instance, tag)) 899 | else: 900 | dest_vars.append('nil') 901 | 902 | dest = self.destination_format.format(*dest_vars) 903 | elif instance.subnet_id: 904 | dest = getattr(instance, self.vpc_destination_variable, None) 905 | if dest is None: 906 | dest = getattr(instance, 'tags').get(self.vpc_destination_variable, None) 907 | else: 908 | dest = getattr(instance, self.destination_variable, None) 909 | if dest is None: 910 | dest = getattr(instance, 'tags').get(self.destination_variable, None) 911 | 912 | if not dest: 913 | # Skip instances we cannot address (e.g. private VPC subnet) 914 | return 915 | 916 | # Set the inventory name 917 | hostname = None 918 | if self.hostname_variable: 919 | if self.hostname_variable.startswith('tag_'): 920 | hostname = instance.tags.get(self.hostname_variable[4:], None) 921 | else: 922 | hostname = getattr(instance, self.hostname_variable) 923 | 924 | # set the hostname from route53 925 | if self.route53_enabled and self.route53_hostnames: 926 | route53_names = self.get_instance_route53_names(instance) 927 | for name in route53_names: 928 | if name.endswith(self.route53_hostnames): 929 | hostname = name 930 | 931 | # If we can't get a nice hostname, use the destination address 932 | if not hostname: 933 | hostname = dest 934 | # to_safe strips hostname characters like dots, so don't strip route53 hostnames 935 | elif self.route53_enabled and self.route53_hostnames and hostname.endswith(self.route53_hostnames): 936 | hostname = hostname.lower() 937 | else: 938 | hostname = self.to_safe(hostname).lower() 939 | 940 | # if we only want to include hosts that match a pattern, skip those that don't 941 | if self.pattern_include and not self.pattern_include.match(hostname): 942 | return 943 | 944 | # if we need to exclude hosts that match a pattern, skip those 945 | if self.pattern_exclude and self.pattern_exclude.match(hostname): 946 | return 947 | 948 | # Add to index 949 | self.index[hostname] = [region, instance.id] 950 | 951 | # Inventory: Group by instance ID (always a group of 1) 952 | if self.group_by_instance_id: 953 | self.inventory[instance.id] = [hostname] 954 | if self.nested_groups: 955 | self.push_group(self.inventory, 'instances', instance.id) 956 | 957 | # Inventory: Group by region 958 | if self.group_by_region: 959 | self.push(self.inventory, region, hostname) 960 | if self.nested_groups: 961 | self.push_group(self.inventory, 'regions', region) 962 | 963 | # Inventory: Group by availability zone 964 | if self.group_by_availability_zone: 965 | self.push(self.inventory, instance.placement, hostname) 966 | if self.nested_groups: 967 | if self.group_by_region: 968 | self.push_group(self.inventory, region, instance.placement) 969 | self.push_group(self.inventory, 'zones', instance.placement) 970 | 971 | # Inventory: Group by Amazon Machine Image (AMI) ID 972 | if self.group_by_ami_id: 973 | ami_id = self.to_safe(instance.image_id) 974 | self.push(self.inventory, ami_id, hostname) 975 | if self.nested_groups: 976 | self.push_group(self.inventory, 'images', ami_id) 977 | 978 | # Inventory: Group by instance type 979 | if self.group_by_instance_type: 980 | type_name = self.to_safe('type_' + instance.instance_type) 981 | self.push(self.inventory, type_name, hostname) 982 | if self.nested_groups: 983 | self.push_group(self.inventory, 'types', type_name) 984 | 985 | # Inventory: Group by instance state 986 | if self.group_by_instance_state: 987 | state_name = self.to_safe('instance_state_' + instance.state) 988 | self.push(self.inventory, state_name, hostname) 989 | if self.nested_groups: 990 | self.push_group(self.inventory, 'instance_states', state_name) 991 | 992 | # Inventory: Group by platform 993 | if self.group_by_platform: 994 | if instance.platform: 995 | platform = self.to_safe('platform_' + instance.platform) 996 | else: 997 | platform = self.to_safe('platform_undefined') 998 | self.push(self.inventory, platform, hostname) 999 | if self.nested_groups: 1000 | self.push_group(self.inventory, 'platforms', platform) 1001 | 1002 | # Inventory: Group by key pair 1003 | if self.group_by_key_pair and instance.key_name: 1004 | key_name = self.to_safe('key_' + instance.key_name) 1005 | self.push(self.inventory, key_name, hostname) 1006 | if self.nested_groups: 1007 | self.push_group(self.inventory, 'keys', key_name) 1008 | 1009 | # Inventory: Group by VPC 1010 | if self.group_by_vpc_id and instance.vpc_id: 1011 | vpc_id_name = self.to_safe('vpc_id_' + instance.vpc_id) 1012 | self.push(self.inventory, vpc_id_name, hostname) 1013 | if self.nested_groups: 1014 | self.push_group(self.inventory, 'vpcs', vpc_id_name) 1015 | 1016 | # Inventory: Group by security group 1017 | if self.group_by_security_group: 1018 | try: 1019 | for group in instance.groups: 1020 | key = self.to_safe("security_group_" + group.name) 1021 | self.push(self.inventory, key, hostname) 1022 | if self.nested_groups: 1023 | self.push_group(self.inventory, 'security_groups', key) 1024 | except AttributeError: 1025 | self.fail_with_error('\n'.join(['Package boto seems a bit older.', 1026 | 'Please upgrade boto >= 2.3.0.'])) 1027 | 1028 | # Inventory: Group by AWS account ID 1029 | if self.group_by_aws_account: 1030 | self.push(self.inventory, self.aws_account_id, hostname) 1031 | if self.nested_groups: 1032 | self.push_group(self.inventory, 'accounts', self.aws_account_id) 1033 | 1034 | # Inventory: Group by tag keys 1035 | if self.group_by_tag_keys: 1036 | for k, v in instance.tags.items(): 1037 | if self.expand_csv_tags and v and ',' in v: 1038 | values = map(lambda x: x.strip(), v.split(',')) 1039 | else: 1040 | values = [v] 1041 | 1042 | for v in values: 1043 | if v: 1044 | key = self.to_safe("tag_" + k + "=" + v) 1045 | else: 1046 | key = self.to_safe("tag_" + k) 1047 | self.push(self.inventory, key, hostname) 1048 | if self.nested_groups: 1049 | self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k)) 1050 | if v: 1051 | self.push_group(self.inventory, self.to_safe("tag_" + k), key) 1052 | 1053 | # Inventory: Group by Route53 domain names if enabled 1054 | if self.route53_enabled and self.group_by_route53_names: 1055 | route53_names = self.get_instance_route53_names(instance) 1056 | for name in route53_names: 1057 | self.push(self.inventory, name, hostname) 1058 | if self.nested_groups: 1059 | self.push_group(self.inventory, 'route53', name) 1060 | 1061 | # Global Tag: instances without tags 1062 | if self.group_by_tag_none and len(instance.tags) == 0: 1063 | self.push(self.inventory, 'tag_none', hostname) 1064 | if self.nested_groups: 1065 | self.push_group(self.inventory, 'tags', 'tag_none') 1066 | 1067 | # Global Tag: tag all EC2 instances 1068 | self.push(self.inventory, 'ec2', hostname) 1069 | 1070 | self.inventory["_meta"]["hostvars"][hostname] = self.get_host_info_dict_from_instance(instance) 1071 | self.inventory["_meta"]["hostvars"][hostname]['ansible_host'] = dest 1072 | 1073 | def add_rds_instance(self, instance, region): 1074 | ''' Adds an RDS instance to the inventory and index, as long as it is 1075 | addressable ''' 1076 | 1077 | # Only want available instances unless all_rds_instances is True 1078 | if not self.all_rds_instances and instance.status != 'available': 1079 | return 1080 | 1081 | # Select the best destination address 1082 | dest = instance.endpoint[0] 1083 | 1084 | if not dest: 1085 | # Skip instances we cannot address (e.g. private VPC subnet) 1086 | return 1087 | 1088 | # Set the inventory name 1089 | hostname = None 1090 | if self.hostname_variable: 1091 | if self.hostname_variable.startswith('tag_'): 1092 | hostname = instance.tags.get(self.hostname_variable[4:], None) 1093 | else: 1094 | hostname = getattr(instance, self.hostname_variable) 1095 | 1096 | # If we can't get a nice hostname, use the destination address 1097 | if not hostname: 1098 | hostname = dest 1099 | 1100 | hostname = self.to_safe(hostname).lower() 1101 | 1102 | # Add to index 1103 | self.index[hostname] = [region, instance.id] 1104 | 1105 | # Inventory: Group by instance ID (always a group of 1) 1106 | if self.group_by_instance_id: 1107 | self.inventory[instance.id] = [hostname] 1108 | if self.nested_groups: 1109 | self.push_group(self.inventory, 'instances', instance.id) 1110 | 1111 | # Inventory: Group by region 1112 | if self.group_by_region: 1113 | self.push(self.inventory, region, hostname) 1114 | if self.nested_groups: 1115 | self.push_group(self.inventory, 'regions', region) 1116 | 1117 | # Inventory: Group by availability zone 1118 | if self.group_by_availability_zone: 1119 | self.push(self.inventory, instance.availability_zone, hostname) 1120 | if self.nested_groups: 1121 | if self.group_by_region: 1122 | self.push_group(self.inventory, region, instance.availability_zone) 1123 | self.push_group(self.inventory, 'zones', instance.availability_zone) 1124 | 1125 | # Inventory: Group by instance type 1126 | if self.group_by_instance_type: 1127 | type_name = self.to_safe('type_' + instance.instance_class) 1128 | self.push(self.inventory, type_name, hostname) 1129 | if self.nested_groups: 1130 | self.push_group(self.inventory, 'types', type_name) 1131 | 1132 | # Inventory: Group by VPC 1133 | if self.group_by_vpc_id and instance.subnet_group and instance.subnet_group.vpc_id: 1134 | vpc_id_name = self.to_safe('vpc_id_' + instance.subnet_group.vpc_id) 1135 | self.push(self.inventory, vpc_id_name, hostname) 1136 | if self.nested_groups: 1137 | self.push_group(self.inventory, 'vpcs', vpc_id_name) 1138 | 1139 | # Inventory: Group by security group 1140 | if self.group_by_security_group: 1141 | try: 1142 | if instance.security_group: 1143 | key = self.to_safe("security_group_" + instance.security_group.name) 1144 | self.push(self.inventory, key, hostname) 1145 | if self.nested_groups: 1146 | self.push_group(self.inventory, 'security_groups', key) 1147 | 1148 | except AttributeError: 1149 | self.fail_with_error('\n'.join(['Package boto seems a bit older.', 1150 | 'Please upgrade boto >= 2.3.0.'])) 1151 | # Inventory: Group by tag keys 1152 | if self.group_by_tag_keys: 1153 | for k, v in instance.tags.items(): 1154 | if self.expand_csv_tags and v and ',' in v: 1155 | values = map(lambda x: x.strip(), v.split(',')) 1156 | else: 1157 | values = [v] 1158 | 1159 | for v in values: 1160 | if v: 1161 | key = self.to_safe("tag_" + k + "=" + v) 1162 | else: 1163 | key = self.to_safe("tag_" + k) 1164 | self.push(self.inventory, key, hostname) 1165 | if self.nested_groups: 1166 | self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k)) 1167 | if v: 1168 | self.push_group(self.inventory, self.to_safe("tag_" + k), key) 1169 | 1170 | # Inventory: Group by engine 1171 | if self.group_by_rds_engine: 1172 | self.push(self.inventory, self.to_safe("rds_" + instance.engine), hostname) 1173 | if self.nested_groups: 1174 | self.push_group(self.inventory, 'rds_engines', self.to_safe("rds_" + instance.engine)) 1175 | 1176 | # Inventory: Group by parameter group 1177 | if self.group_by_rds_parameter_group: 1178 | self.push(self.inventory, self.to_safe("rds_parameter_group_" + instance.parameter_group.name), hostname) 1179 | if self.nested_groups: 1180 | self.push_group(self.inventory, 'rds_parameter_groups', self.to_safe("rds_parameter_group_" + instance.parameter_group.name)) 1181 | 1182 | # Global Tag: instances without tags 1183 | if self.group_by_tag_none and len(instance.tags) == 0: 1184 | self.push(self.inventory, 'tag_none', hostname) 1185 | if self.nested_groups: 1186 | self.push_group(self.inventory, 'tags', 'tag_none') 1187 | 1188 | # Global Tag: all RDS instances 1189 | self.push(self.inventory, 'rds', hostname) 1190 | 1191 | self.inventory["_meta"]["hostvars"][hostname] = self.get_host_info_dict_from_instance(instance) 1192 | self.inventory["_meta"]["hostvars"][hostname]['ansible_host'] = dest 1193 | 1194 | def add_elasticache_cluster(self, cluster, region): 1195 | ''' Adds an ElastiCache cluster to the inventory and index, as long as 1196 | it's nodes are addressable ''' 1197 | 1198 | # Only want available clusters unless all_elasticache_clusters is True 1199 | if not self.all_elasticache_clusters and cluster['CacheClusterStatus'] != 'available': 1200 | return 1201 | 1202 | # Select the best destination address 1203 | if 'ConfigurationEndpoint' in cluster and cluster['ConfigurationEndpoint']: 1204 | # Memcached cluster 1205 | dest = cluster['ConfigurationEndpoint']['Address'] 1206 | is_redis = False 1207 | else: 1208 | # Redis sigle node cluster 1209 | # Because all Redis clusters are single nodes, we'll merge the 1210 | # info from the cluster with info about the node 1211 | dest = cluster['CacheNodes'][0]['Endpoint']['Address'] 1212 | is_redis = True 1213 | 1214 | if not dest: 1215 | # Skip clusters we cannot address (e.g. private VPC subnet) 1216 | return 1217 | 1218 | # Add to index 1219 | self.index[dest] = [region, cluster['CacheClusterId']] 1220 | 1221 | # Inventory: Group by instance ID (always a group of 1) 1222 | if self.group_by_instance_id: 1223 | self.inventory[cluster['CacheClusterId']] = [dest] 1224 | if self.nested_groups: 1225 | self.push_group(self.inventory, 'instances', cluster['CacheClusterId']) 1226 | 1227 | # Inventory: Group by region 1228 | if self.group_by_region and not is_redis: 1229 | self.push(self.inventory, region, dest) 1230 | if self.nested_groups: 1231 | self.push_group(self.inventory, 'regions', region) 1232 | 1233 | # Inventory: Group by availability zone 1234 | if self.group_by_availability_zone and not is_redis: 1235 | self.push(self.inventory, cluster['PreferredAvailabilityZone'], dest) 1236 | if self.nested_groups: 1237 | if self.group_by_region: 1238 | self.push_group(self.inventory, region, cluster['PreferredAvailabilityZone']) 1239 | self.push_group(self.inventory, 'zones', cluster['PreferredAvailabilityZone']) 1240 | 1241 | # Inventory: Group by node type 1242 | if self.group_by_instance_type and not is_redis: 1243 | type_name = self.to_safe('type_' + cluster['CacheNodeType']) 1244 | self.push(self.inventory, type_name, dest) 1245 | if self.nested_groups: 1246 | self.push_group(self.inventory, 'types', type_name) 1247 | 1248 | # Inventory: Group by VPC (information not available in the current 1249 | # AWS API version for ElastiCache) 1250 | 1251 | # Inventory: Group by security group 1252 | if self.group_by_security_group and not is_redis: 1253 | 1254 | # Check for the existence of the 'SecurityGroups' key and also if 1255 | # this key has some value. When the cluster is not placed in a SG 1256 | # the query can return None here and cause an error. 1257 | if 'SecurityGroups' in cluster and cluster['SecurityGroups'] is not None: 1258 | for security_group in cluster['SecurityGroups']: 1259 | key = self.to_safe("security_group_" + security_group['SecurityGroupId']) 1260 | self.push(self.inventory, key, dest) 1261 | if self.nested_groups: 1262 | self.push_group(self.inventory, 'security_groups', key) 1263 | 1264 | # Inventory: Group by engine 1265 | if self.group_by_elasticache_engine and not is_redis: 1266 | self.push(self.inventory, self.to_safe("elasticache_" + cluster['Engine']), dest) 1267 | if self.nested_groups: 1268 | self.push_group(self.inventory, 'elasticache_engines', self.to_safe(cluster['Engine'])) 1269 | 1270 | # Inventory: Group by parameter group 1271 | if self.group_by_elasticache_parameter_group: 1272 | self.push(self.inventory, self.to_safe("elasticache_parameter_group_" + cluster['CacheParameterGroup']['CacheParameterGroupName']), dest) 1273 | if self.nested_groups: 1274 | self.push_group(self.inventory, 'elasticache_parameter_groups', self.to_safe(cluster['CacheParameterGroup']['CacheParameterGroupName'])) 1275 | 1276 | # Inventory: Group by replication group 1277 | if self.group_by_elasticache_replication_group and 'ReplicationGroupId' in cluster and cluster['ReplicationGroupId']: 1278 | self.push(self.inventory, self.to_safe("elasticache_replication_group_" + cluster['ReplicationGroupId']), dest) 1279 | if self.nested_groups: 1280 | self.push_group(self.inventory, 'elasticache_replication_groups', self.to_safe(cluster['ReplicationGroupId'])) 1281 | 1282 | # Global Tag: all ElastiCache clusters 1283 | self.push(self.inventory, 'elasticache_clusters', cluster['CacheClusterId']) 1284 | 1285 | host_info = self.get_host_info_dict_from_describe_dict(cluster) 1286 | 1287 | self.inventory["_meta"]["hostvars"][dest] = host_info 1288 | 1289 | # Add the nodes 1290 | for node in cluster['CacheNodes']: 1291 | self.add_elasticache_node(node, cluster, region) 1292 | 1293 | def add_elasticache_node(self, node, cluster, region): 1294 | ''' Adds an ElastiCache node to the inventory and index, as long as 1295 | it is addressable ''' 1296 | 1297 | # Only want available nodes unless all_elasticache_nodes is True 1298 | if not self.all_elasticache_nodes and node['CacheNodeStatus'] != 'available': 1299 | return 1300 | 1301 | # Select the best destination address 1302 | dest = node['Endpoint']['Address'] 1303 | 1304 | if not dest: 1305 | # Skip nodes we cannot address (e.g. private VPC subnet) 1306 | return 1307 | 1308 | node_id = self.to_safe(cluster['CacheClusterId'] + '_' + node['CacheNodeId']) 1309 | 1310 | # Add to index 1311 | self.index[dest] = [region, node_id] 1312 | 1313 | # Inventory: Group by node ID (always a group of 1) 1314 | if self.group_by_instance_id: 1315 | self.inventory[node_id] = [dest] 1316 | if self.nested_groups: 1317 | self.push_group(self.inventory, 'instances', node_id) 1318 | 1319 | # Inventory: Group by region 1320 | if self.group_by_region: 1321 | self.push(self.inventory, region, dest) 1322 | if self.nested_groups: 1323 | self.push_group(self.inventory, 'regions', region) 1324 | 1325 | # Inventory: Group by availability zone 1326 | if self.group_by_availability_zone: 1327 | self.push(self.inventory, cluster['PreferredAvailabilityZone'], dest) 1328 | if self.nested_groups: 1329 | if self.group_by_region: 1330 | self.push_group(self.inventory, region, cluster['PreferredAvailabilityZone']) 1331 | self.push_group(self.inventory, 'zones', cluster['PreferredAvailabilityZone']) 1332 | 1333 | # Inventory: Group by node type 1334 | if self.group_by_instance_type: 1335 | type_name = self.to_safe('type_' + cluster['CacheNodeType']) 1336 | self.push(self.inventory, type_name, dest) 1337 | if self.nested_groups: 1338 | self.push_group(self.inventory, 'types', type_name) 1339 | 1340 | # Inventory: Group by VPC (information not available in the current 1341 | # AWS API version for ElastiCache) 1342 | 1343 | # Inventory: Group by security group 1344 | if self.group_by_security_group: 1345 | 1346 | # Check for the existence of the 'SecurityGroups' key and also if 1347 | # this key has some value. When the cluster is not placed in a SG 1348 | # the query can return None here and cause an error. 1349 | if 'SecurityGroups' in cluster and cluster['SecurityGroups'] is not None: 1350 | for security_group in cluster['SecurityGroups']: 1351 | key = self.to_safe("security_group_" + security_group['SecurityGroupId']) 1352 | self.push(self.inventory, key, dest) 1353 | if self.nested_groups: 1354 | self.push_group(self.inventory, 'security_groups', key) 1355 | 1356 | # Inventory: Group by engine 1357 | if self.group_by_elasticache_engine: 1358 | self.push(self.inventory, self.to_safe("elasticache_" + cluster['Engine']), dest) 1359 | if self.nested_groups: 1360 | self.push_group(self.inventory, 'elasticache_engines', self.to_safe("elasticache_" + cluster['Engine'])) 1361 | 1362 | # Inventory: Group by parameter group (done at cluster level) 1363 | 1364 | # Inventory: Group by replication group (done at cluster level) 1365 | 1366 | # Inventory: Group by ElastiCache Cluster 1367 | if self.group_by_elasticache_cluster: 1368 | self.push(self.inventory, self.to_safe("elasticache_cluster_" + cluster['CacheClusterId']), dest) 1369 | 1370 | # Global Tag: all ElastiCache nodes 1371 | self.push(self.inventory, 'elasticache_nodes', dest) 1372 | 1373 | host_info = self.get_host_info_dict_from_describe_dict(node) 1374 | 1375 | if dest in self.inventory["_meta"]["hostvars"]: 1376 | self.inventory["_meta"]["hostvars"][dest].update(host_info) 1377 | else: 1378 | self.inventory["_meta"]["hostvars"][dest] = host_info 1379 | 1380 | def add_elasticache_replication_group(self, replication_group, region): 1381 | ''' Adds an ElastiCache replication group to the inventory and index ''' 1382 | 1383 | # Only want available clusters unless all_elasticache_replication_groups is True 1384 | if not self.all_elasticache_replication_groups and replication_group['Status'] != 'available': 1385 | return 1386 | 1387 | # Skip clusters we cannot address (e.g. private VPC subnet or clustered redis) 1388 | if replication_group['NodeGroups'][0]['PrimaryEndpoint'] is None or \ 1389 | replication_group['NodeGroups'][0]['PrimaryEndpoint']['Address'] is None: 1390 | return 1391 | 1392 | # Select the best destination address (PrimaryEndpoint) 1393 | dest = replication_group['NodeGroups'][0]['PrimaryEndpoint']['Address'] 1394 | 1395 | # Add to index 1396 | self.index[dest] = [region, replication_group['ReplicationGroupId']] 1397 | 1398 | # Inventory: Group by ID (always a group of 1) 1399 | if self.group_by_instance_id: 1400 | self.inventory[replication_group['ReplicationGroupId']] = [dest] 1401 | if self.nested_groups: 1402 | self.push_group(self.inventory, 'instances', replication_group['ReplicationGroupId']) 1403 | 1404 | # Inventory: Group by region 1405 | if self.group_by_region: 1406 | self.push(self.inventory, region, dest) 1407 | if self.nested_groups: 1408 | self.push_group(self.inventory, 'regions', region) 1409 | 1410 | # Inventory: Group by availability zone (doesn't apply to replication groups) 1411 | 1412 | # Inventory: Group by node type (doesn't apply to replication groups) 1413 | 1414 | # Inventory: Group by VPC (information not available in the current 1415 | # AWS API version for replication groups 1416 | 1417 | # Inventory: Group by security group (doesn't apply to replication groups) 1418 | # Check this value in cluster level 1419 | 1420 | # Inventory: Group by engine (replication groups are always Redis) 1421 | if self.group_by_elasticache_engine: 1422 | self.push(self.inventory, 'elasticache_redis', dest) 1423 | if self.nested_groups: 1424 | self.push_group(self.inventory, 'elasticache_engines', 'redis') 1425 | 1426 | # Global Tag: all ElastiCache clusters 1427 | self.push(self.inventory, 'elasticache_replication_groups', replication_group['ReplicationGroupId']) 1428 | 1429 | host_info = self.get_host_info_dict_from_describe_dict(replication_group) 1430 | 1431 | self.inventory["_meta"]["hostvars"][dest] = host_info 1432 | 1433 | def get_route53_records(self): 1434 | ''' Get and store the map of resource records to domain names that 1435 | point to them. ''' 1436 | 1437 | if self.boto_profile: 1438 | r53_conn = route53.Route53Connection(profile_name=self.boto_profile) 1439 | else: 1440 | r53_conn = route53.Route53Connection() 1441 | all_zones = r53_conn.get_zones() 1442 | 1443 | route53_zones = [zone for zone in all_zones if zone.name[:-1] not in self.route53_excluded_zones] 1444 | 1445 | self.route53_records = {} 1446 | 1447 | for zone in route53_zones: 1448 | rrsets = r53_conn.get_all_rrsets(zone.id) 1449 | 1450 | for record_set in rrsets: 1451 | record_name = record_set.name 1452 | 1453 | if record_name.endswith('.'): 1454 | record_name = record_name[:-1] 1455 | 1456 | for resource in record_set.resource_records: 1457 | self.route53_records.setdefault(resource, set()) 1458 | self.route53_records[resource].add(record_name) 1459 | 1460 | def get_instance_route53_names(self, instance): 1461 | ''' Check if an instance is referenced in the records we have from 1462 | Route53. If it is, return the list of domain names pointing to said 1463 | instance. If nothing points to it, return an empty list. ''' 1464 | 1465 | instance_attributes = ['public_dns_name', 'private_dns_name', 1466 | 'ip_address', 'private_ip_address'] 1467 | 1468 | name_list = set() 1469 | 1470 | for attrib in instance_attributes: 1471 | try: 1472 | value = getattr(instance, attrib) 1473 | except AttributeError: 1474 | continue 1475 | 1476 | if value in self.route53_records: 1477 | name_list.update(self.route53_records[value]) 1478 | 1479 | return list(name_list) 1480 | 1481 | def get_host_info_dict_from_instance(self, instance): 1482 | instance_vars = {} 1483 | for key in vars(instance): 1484 | value = getattr(instance, key) 1485 | key = self.to_safe('ec2_' + key) 1486 | 1487 | # Handle complex types 1488 | # state/previous_state changed to properties in boto in https://github.com/boto/boto/commit/a23c379837f698212252720d2af8dec0325c9518 1489 | if key == 'ec2__state': 1490 | instance_vars['ec2_state'] = instance.state or '' 1491 | instance_vars['ec2_state_code'] = instance.state_code 1492 | elif key == 'ec2__previous_state': 1493 | instance_vars['ec2_previous_state'] = instance.previous_state or '' 1494 | instance_vars['ec2_previous_state_code'] = instance.previous_state_code 1495 | elif isinstance(value, (int, bool)): 1496 | instance_vars[key] = value 1497 | elif isinstance(value, six.string_types): 1498 | instance_vars[key] = value.strip() 1499 | elif value is None: 1500 | instance_vars[key] = '' 1501 | elif key == 'ec2_region': 1502 | instance_vars[key] = value.name 1503 | elif key == 'ec2__placement': 1504 | instance_vars['ec2_placement'] = value.zone 1505 | elif key == 'ec2_tags': 1506 | for k, v in value.items(): 1507 | if self.expand_csv_tags and ',' in v: 1508 | v = list(map(lambda x: x.strip(), v.split(','))) 1509 | key = self.to_safe('ec2_tag_' + k) 1510 | instance_vars[key] = v 1511 | elif key == 'ec2_groups': 1512 | group_ids = [] 1513 | group_names = [] 1514 | for group in value: 1515 | group_ids.append(group.id) 1516 | group_names.append(group.name) 1517 | instance_vars["ec2_security_group_ids"] = ','.join([str(i) for i in group_ids]) 1518 | instance_vars["ec2_security_group_names"] = ','.join([str(i) for i in group_names]) 1519 | elif key == 'ec2_block_device_mapping': 1520 | instance_vars["ec2_block_devices"] = {} 1521 | for k, v in value.items(): 1522 | instance_vars["ec2_block_devices"][os.path.basename(k)] = v.volume_id 1523 | else: 1524 | pass 1525 | # TODO Product codes if someone finds them useful 1526 | # print key 1527 | # print type(value) 1528 | # print value 1529 | 1530 | instance_vars[self.to_safe('ec2_account_id')] = self.aws_account_id 1531 | 1532 | return instance_vars 1533 | 1534 | def get_host_info_dict_from_describe_dict(self, describe_dict): 1535 | ''' Parses the dictionary returned by the API call into a flat list 1536 | of parameters. This method should be used only when 'describe' is 1537 | used directly because Boto doesn't provide specific classes. ''' 1538 | 1539 | # I really don't agree with prefixing everything with 'ec2' 1540 | # because EC2, RDS and ElastiCache are different services. 1541 | # I'm just following the pattern used until now to not break any 1542 | # compatibility. 1543 | 1544 | host_info = {} 1545 | for key in describe_dict: 1546 | value = describe_dict[key] 1547 | key = self.to_safe('ec2_' + self.uncammelize(key)) 1548 | 1549 | # Handle complex types 1550 | 1551 | # Target: Memcached Cache Clusters 1552 | if key == 'ec2_configuration_endpoint' and value: 1553 | host_info['ec2_configuration_endpoint_address'] = value['Address'] 1554 | host_info['ec2_configuration_endpoint_port'] = value['Port'] 1555 | 1556 | # Target: Cache Nodes and Redis Cache Clusters (single node) 1557 | if key == 'ec2_endpoint' and value: 1558 | host_info['ec2_endpoint_address'] = value['Address'] 1559 | host_info['ec2_endpoint_port'] = value['Port'] 1560 | 1561 | # Target: Redis Replication Groups 1562 | if key == 'ec2_node_groups' and value: 1563 | host_info['ec2_endpoint_address'] = value[0]['PrimaryEndpoint']['Address'] 1564 | host_info['ec2_endpoint_port'] = value[0]['PrimaryEndpoint']['Port'] 1565 | replica_count = 0 1566 | for node in value[0]['NodeGroupMembers']: 1567 | if node['CurrentRole'] == 'primary': 1568 | host_info['ec2_primary_cluster_address'] = node['ReadEndpoint']['Address'] 1569 | host_info['ec2_primary_cluster_port'] = node['ReadEndpoint']['Port'] 1570 | host_info['ec2_primary_cluster_id'] = node['CacheClusterId'] 1571 | elif node['CurrentRole'] == 'replica': 1572 | host_info['ec2_replica_cluster_address_' + str(replica_count)] = node['ReadEndpoint']['Address'] 1573 | host_info['ec2_replica_cluster_port_' + str(replica_count)] = node['ReadEndpoint']['Port'] 1574 | host_info['ec2_replica_cluster_id_' + str(replica_count)] = node['CacheClusterId'] 1575 | replica_count += 1 1576 | 1577 | # Target: Redis Replication Groups 1578 | if key == 'ec2_member_clusters' and value: 1579 | host_info['ec2_member_clusters'] = ','.join([str(i) for i in value]) 1580 | 1581 | # Target: All Cache Clusters 1582 | elif key == 'ec2_cache_parameter_group': 1583 | host_info["ec2_cache_node_ids_to_reboot"] = ','.join([str(i) for i in value['CacheNodeIdsToReboot']]) 1584 | host_info['ec2_cache_parameter_group_name'] = value['CacheParameterGroupName'] 1585 | host_info['ec2_cache_parameter_apply_status'] = value['ParameterApplyStatus'] 1586 | 1587 | # Target: Almost everything 1588 | elif key == 'ec2_security_groups': 1589 | 1590 | # Skip if SecurityGroups is None 1591 | # (it is possible to have the key defined but no value in it). 1592 | if value is not None: 1593 | sg_ids = [] 1594 | for sg in value: 1595 | sg_ids.append(sg['SecurityGroupId']) 1596 | host_info["ec2_security_group_ids"] = ','.join([str(i) for i in sg_ids]) 1597 | 1598 | # Target: Everything 1599 | # Preserve booleans and integers 1600 | elif isinstance(value, (int, bool)): 1601 | host_info[key] = value 1602 | 1603 | # Target: Everything 1604 | # Sanitize string values 1605 | elif isinstance(value, six.string_types): 1606 | host_info[key] = value.strip() 1607 | 1608 | # Target: Everything 1609 | # Replace None by an empty string 1610 | elif value is None: 1611 | host_info[key] = '' 1612 | 1613 | else: 1614 | # Remove non-processed complex types 1615 | pass 1616 | 1617 | return host_info 1618 | 1619 | def get_host_info(self): 1620 | ''' Get variables about a specific host ''' 1621 | 1622 | if len(self.index) == 0: 1623 | # Need to load index from cache 1624 | self.load_index_from_cache() 1625 | 1626 | if self.args.host not in self.index: 1627 | # try updating the cache 1628 | self.do_api_calls_update_cache() 1629 | if self.args.host not in self.index: 1630 | # host might not exist anymore 1631 | return self.json_format_dict({}, True) 1632 | 1633 | (region, instance_id) = self.index[self.args.host] 1634 | 1635 | instance = self.get_instance(region, instance_id) 1636 | return self.json_format_dict(self.get_host_info_dict_from_instance(instance), True) 1637 | 1638 | def push(self, my_dict, key, element): 1639 | ''' Push an element onto an array that may not have been defined in 1640 | the dict ''' 1641 | group_info = my_dict.setdefault(key, []) 1642 | if isinstance(group_info, dict): 1643 | host_list = group_info.setdefault('hosts', []) 1644 | host_list.append(element) 1645 | else: 1646 | group_info.append(element) 1647 | 1648 | def push_group(self, my_dict, key, element): 1649 | ''' Push a group as a child of another group. ''' 1650 | parent_group = my_dict.setdefault(key, {}) 1651 | if not isinstance(parent_group, dict): 1652 | parent_group = my_dict[key] = {'hosts': parent_group} 1653 | child_groups = parent_group.setdefault('children', []) 1654 | if element not in child_groups: 1655 | child_groups.append(element) 1656 | 1657 | def get_inventory_from_cache(self): 1658 | ''' Reads the inventory from the cache file and returns it as a JSON 1659 | object ''' 1660 | 1661 | with open(self.cache_path_cache, 'r') as f: 1662 | json_inventory = f.read() 1663 | return json_inventory 1664 | 1665 | def load_index_from_cache(self): 1666 | ''' Reads the index from the cache file sets self.index ''' 1667 | 1668 | with open(self.cache_path_index, 'rb') as f: 1669 | self.index = json.load(f) 1670 | 1671 | def write_to_cache(self, data, filename): 1672 | ''' Writes data in JSON format to a file ''' 1673 | 1674 | json_data = self.json_format_dict(data, True) 1675 | with open(filename, 'w') as f: 1676 | f.write(json_data) 1677 | 1678 | def uncammelize(self, key): 1679 | temp = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', key) 1680 | return re.sub('([a-z0-9])([A-Z])', r'\1_\2', temp).lower() 1681 | 1682 | def to_safe(self, word): 1683 | ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups ''' 1684 | regex = r"[^A-Za-z0-9\_" 1685 | if not self.replace_dash_in_groups: 1686 | regex += r"\-" 1687 | return re.sub(regex + "]", "_", word) 1688 | 1689 | def json_format_dict(self, data, pretty=False): 1690 | ''' Converts a dict to a JSON object and dumps it as a formatted 1691 | string ''' 1692 | 1693 | if pretty: 1694 | return json.dumps(data, sort_keys=True, indent=2) 1695 | else: 1696 | return json.dumps(data) 1697 | 1698 | 1699 | if __name__ == '__main__': 1700 | # Run the script 1701 | Ec2Inventory() 1702 | -------------------------------------------------------------------------------- /infrastructure/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "${var.aws_region}" 3 | } 4 | 5 | resource "aws_key_pair" "aws_keypair" { 6 | key_name = "terraform_test" 7 | public_key = "${file(var.ssh_key_public)}" 8 | } 9 | 10 | resource "aws_vpc" "vpc" { 11 | cidr_block = "${var.aws_vpc_cidr}" 12 | 13 | tags = { 14 | Name = "terraform_test_vpc" 15 | } 16 | } 17 | 18 | resource "aws_internet_gateway" "terraform_gw" { 19 | vpc_id = "${aws_vpc.vpc.id}" 20 | 21 | tags { 22 | Name = "Internet gateway for C10K" 23 | } 24 | } 25 | 26 | resource "aws_route_table" "route_table" { 27 | vpc_id = "${aws_vpc.vpc.id}" 28 | 29 | route { 30 | cidr_block = "0.0.0.0/0" 31 | gateway_id = "${aws_internet_gateway.terraform_gw.id}" 32 | } 33 | 34 | tags { 35 | Name = "C10K route table" 36 | } 37 | } 38 | 39 | resource "aws_subnet" "subnet" { 40 | vpc_id = "${aws_vpc.vpc.id}" 41 | cidr_block = "${aws_vpc.vpc.cidr_block}" 42 | 43 | # map_public_ip_on_launch = true 44 | tags = { 45 | Name = "terraform_test_subnet" 46 | } 47 | } 48 | 49 | resource "aws_route_table_association" "a" { 50 | subnet_id = "${aws_subnet.subnet.id}" 51 | route_table_id = "${aws_route_table.route_table.id}" 52 | } 53 | 54 | resource "aws_security_group" "server_sg" { 55 | vpc_id = "${aws_vpc.vpc.id}" 56 | 57 | # SSH ingress access for provisioning 58 | ingress { 59 | from_port = 22 60 | to_port = 22 61 | protocol = "tcp" 62 | cidr_blocks = ["0.0.0.0/0"] 63 | description = "Allow SSH access for provisioning" 64 | } 65 | 66 | ingress { 67 | from_port = "${var.c10k_server_port}" 68 | to_port = "${var.c10k_server_port}" 69 | protocol = "tcp" 70 | cidr_blocks = ["0.0.0.0/0"] 71 | description = "Allow access to c10k servers" 72 | } 73 | 74 | egress { 75 | from_port = 0 76 | to_port = 0 77 | protocol = "-1" 78 | cidr_blocks = ["0.0.0.0/0"] 79 | } 80 | } 81 | 82 | resource "aws_instance" "c10k_server" { 83 | ami = "${var.aws_ami_fedora27[var.aws_region]}" 84 | instance_type = "${var.server_instance_type}" 85 | subnet_id = "${aws_subnet.subnet.id}" 86 | vpc_security_group_ids = ["${aws_security_group.server_sg.id}"] 87 | key_name = "${aws_key_pair.aws_keypair.key_name}" 88 | associate_public_ip_address = true 89 | count = 1 90 | 91 | tags { 92 | Name = "c10k_server" 93 | } 94 | 95 | provisioner "remote-exec" { 96 | # Install Python for Ansible 97 | inline = ["sudo dnf -y install python libselinux-python"] 98 | 99 | connection { 100 | type = "ssh" 101 | user = "fedora" 102 | private_key = "${file(var.ssh_key_private)}" 103 | } 104 | } 105 | 106 | provisioner "local-exec" { 107 | command = "ansible-playbook -u fedora -i '${self.public_ip},' --private-key ${var.ssh_key_private} -T 300 provision.yml" 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /infrastructure/provision.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - hosts: all 4 | become: yes 5 | roles: 6 | - base 7 | -------------------------------------------------------------------------------- /infrastructure/roles/base/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | base_user: test 3 | base_group: test 4 | -------------------------------------------------------------------------------- /infrastructure/roles/base/files/avd.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDdYP1gM8Pk60KPzfMg6lwY9l+uZfs88Rd6bsAPKTgK/TSGiOjFd69zvhW5bqaRGE7nyNhGD/boFB2XMQvil++13nq5ZL1xC2xZVYFfu5HSTtU9+jU0SZWvcbZ9Z/MHAP9WfWm+//w3qB+nJJhuxs2w+mweFKNBptFKEibOXhgDeHJn1xZwKTf61GZn4LDEbw7dIkjzNLLT2nYXJbMYgnVGmXv065jrYPSy+KVmHX+RIljDzOW6gK8uIL8AylXlxlKp2klzxVvLiCSlWGM8Zu3v6B6crEqeZkm5C6PagMoUo0qr/eN6fK2GVGR5dD09ptMDVHjz0aQfvZi9vMLAWQyt avd@lenovo 2 | -------------------------------------------------------------------------------- /infrastructure/roles/base/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Make sure we have a 'wheel' group 4 | group: 5 | name: wheel 6 | state: present 7 | 8 | - name: Allow 'wheel' group to have passwordless sudo 9 | lineinfile: 10 | dest: /etc/sudoers 11 | state: present 12 | regexp: '^%wheel' 13 | line: '%wheel ALL=(ALL) NOPASSWD: ALL' 14 | validate: 'visudo -cf %s' 15 | 16 | - name: Ensure base group 17 | group: 18 | name: "{{ base_group }}" 19 | state: present 20 | 21 | - name: Ensure base user 22 | user: 23 | name: "{{ base_user }}" 24 | group: "{{ base_group }}" 25 | groups: wheel 26 | append: yes 27 | state: present 28 | 29 | - name: Set authorized key took from file 30 | authorized_key: 31 | user: "{{ base_user }}" 32 | state: present 33 | key: "{{ lookup('file', '{{ item }}') }}" 34 | with_fileglob: 35 | - "files/*.pub" 36 | 37 | -------------------------------------------------------------------------------- /infrastructure/roles/c10kserver/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | c10kserver_version: 1.1 4 | c10kserver_download_url: 5 | deb: "https://github.com/dzeban/c10k/releases/download/v{{ c10kserver_version }}/c10k-servers_{{c10kserver_version}}.deb" 6 | rpm: "https://github.com/dzeban/c10k/releases/download/v{{ c10kserver_version }}/c10k-servers_{{c10kserver_version}}.rpm" 7 | 8 | c10kservers: 9 | - blocking-single 10 | - blocking-forking 11 | -------------------------------------------------------------------------------- /infrastructure/roles/c10kserver/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: systemd reload 4 | systemd: 5 | daemon-reload: yes 6 | 7 | - name: systemd enable 8 | systemd: 9 | name: "{{ item }}" 10 | enabled: yes 11 | with_items: "{{ c10kservers }}" 12 | -------------------------------------------------------------------------------- /infrastructure/roles/c10kserver/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Ensure latest version of c10k server 4 | dnf: 5 | name: "{{ c10kserver_download_url['rpm'] }}" 6 | state: latest 7 | when: ansible_os_family == 'RedHat' 8 | 9 | - name: Ensure latest version of c10k server 10 | dnf: 11 | name: "{{ c10kserver_download_url['deb'] }}" 12 | state: latest 13 | when: ansible_os_family == 'Debian' 14 | 15 | - name: Render service template 16 | template: 17 | src: systemd.j2 18 | dest: "/etc/systemd/system/{{ item }}.service" 19 | owner: root 20 | group: root 21 | mode: 0644 22 | vars: 23 | program: "{{ item }}" 24 | with_items: "{{ c10kservers }}" 25 | changed_when: True 26 | notify: 27 | - systemd reload 28 | - systemd enable 29 | -------------------------------------------------------------------------------- /infrastructure/roles/c10kserver/templates/systemd.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=c10k server {{ program }} 3 | 4 | [Service] 5 | ExecStart=/usr/local/bin/{{ program }} 6 | 7 | [Install] 8 | WantedBy=multi-user.target 9 | -------------------------------------------------------------------------------- /infrastructure/site.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - hosts: tag_Name_c10k_server 4 | become: yes 5 | roles: 6 | - c10kserver 7 | -------------------------------------------------------------------------------- /infrastructure/variables.tf: -------------------------------------------------------------------------------- 1 | variable "ssh_key_public" { 2 | default = "~/.ssh/id_rsa.pub" 3 | description = "Path to the SSH public key for accessing cloud instances. Used for creating AWS keypair." 4 | } 5 | 6 | variable "ssh_key_private" { 7 | default = "~/.ssh/id_rsa" 8 | description = "Path to the SSH public key for accessing cloud instances. Used for creating AWS keypair." 9 | } 10 | 11 | variable "c10k_server_port" { 12 | default = 8282 13 | } 14 | 15 | variable "server_instance_type" { 16 | default = "t2.micro" 17 | } 18 | -------------------------------------------------------------------------------- /libuv-client.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #define log(fmt, ...) do { \ 11 | fprintf(stderr, "location=\"%s:%d:%s\" level=INFO msg=\"" fmt "\"\n", __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__); \ 12 | } while(0) 13 | 14 | #define err(fmt, ...) do { \ 15 | fprintf(stderr, "location=%s:%d:%s level=ERROR msg=\"" fmt "\"\n", __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__); \ 16 | } while(0) 17 | 18 | #define goto_uv_err(status, label) do { \ 19 | if (status < 0) { \ 20 | err("%s:%d: %s: %s\n", __FILE__, __LINE__, uv_err_name(status), uv_strerror(status)); \ 21 | goto label; \ 22 | } \ 23 | } while(0) 24 | 25 | #define return_uv_err(status) do { \ 26 | if (status < 0) { \ 27 | err("%s:%d: %s: %s\n", __FILE__, __LINE__, uv_err_name(status), uv_strerror(status)); \ 28 | return; \ 29 | } \ 30 | } while(0) 31 | 32 | #define return_rc_uv_err(status, rc) do { \ 33 | if (status < 0) { \ 34 | fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, uv_err_name(status), uv_strerror(status)); \ 35 | return rc; \ 36 | } \ 37 | } while(0) 38 | 39 | // Set "Connection: close" header to avoid HTTP keep-alive and hint the server 40 | // to close connection. Without it we'll wait for EOF until keepalive timeout. 41 | // The other option here is to use HTTP/1.0 42 | char *GET_REQUEST = "GET /index.html HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"; 43 | int DELAY = 5; 44 | 45 | static void free_close_cb(uv_handle_t *handle) 46 | { 47 | free(handle); 48 | } 49 | 50 | static void alloc_cb(uv_handle_t *handle __attribute__((unused)), size_t suggested_size, uv_buf_t *buf) 51 | { 52 | char *b = calloc(1, suggested_size); 53 | if (!b) { 54 | // Trigger UV_ENOBUFS error in read cb 55 | buf->base = NULL; 56 | buf->len = 0; 57 | } 58 | 59 | buf->base = b; 60 | buf->len = suggested_size; 61 | } 62 | 63 | static void on_read(uv_stream_t* stream, ssize_t nread, const uv_buf_t* buf) 64 | { 65 | if (nread > 0) { 66 | printf("%s", buf->base); 67 | } else if (nread == UV_EOF) { 68 | log("close stream"); 69 | uv_connect_t *conn = uv_handle_get_data((uv_handle_t *)stream); 70 | uv_close((uv_handle_t *)stream, free_close_cb); 71 | free(conn); 72 | } else { 73 | return_uv_err(nread); 74 | } 75 | 76 | free(buf->base); 77 | } 78 | 79 | static void on_write(uv_write_t *req, int status) 80 | { 81 | return_uv_err(status); 82 | 83 | log("start reading"); 84 | uv_read_start(req->handle, alloc_cb, on_read); 85 | free(req); 86 | } 87 | 88 | static void timer_cb(uv_timer_t *timer) 89 | { 90 | uv_connect_t *connection = uv_handle_get_data((uv_handle_t *)timer); 91 | if (!connection) { 92 | err("invalid timer data: no connection handle"); 93 | return; 94 | } 95 | 96 | uv_stream_t *stream = connection->handle; 97 | 98 | uv_buf_t bufs[] = { 99 | { .base = GET_REQUEST, .len = strlen(GET_REQUEST) }, 100 | }; 101 | 102 | log("send http request"); 103 | uv_write_t *write_req = calloc(1, sizeof(uv_write_t)); 104 | if (!write_req) { 105 | err("malloc write request failed: %s", strerror(errno)); 106 | return; 107 | } 108 | // Store connection handle in stream to free memory on EOF 109 | uv_handle_set_data((uv_handle_t *)stream, connection); 110 | 111 | uv_write(write_req, stream, bufs, 1, on_write); 112 | 113 | uv_close((uv_handle_t *)timer, free_close_cb); 114 | } 115 | 116 | static void on_connect(uv_connect_t *connection, int status) 117 | { 118 | int rc = 0; 119 | 120 | return_uv_err(status); 121 | log("connected"); 122 | 123 | uv_loop_t *loop = uv_default_loop(); 124 | if (!loop) { 125 | err("no loop in connection handle"); 126 | return; 127 | } 128 | 129 | // Setup timer for sleep 130 | uv_timer_t *sleep_timer = malloc(sizeof(*sleep_timer)); 131 | if (!sleep_timer) { 132 | perror("malloc"); 133 | return; 134 | } 135 | 136 | rc = uv_timer_init(loop, sleep_timer); 137 | return_uv_err(rc); 138 | 139 | uv_handle_set_data((uv_handle_t *)sleep_timer, connection); 140 | 141 | log("starting timer"); 142 | rc = uv_timer_start(sleep_timer, timer_cb, DELAY * 1000, 0); 143 | return_uv_err(rc); 144 | } 145 | 146 | struct connection { 147 | uv_tcp_t *socket; 148 | uv_connect_t *conn; 149 | struct sockaddr_in dest; 150 | }; 151 | 152 | void connections_destroy(struct connection *conns, int n) 153 | { 154 | for (int i = 0; i < n; i++) { 155 | struct connection c = conns[i]; 156 | if (c.conn) { 157 | free(c.conn); 158 | } 159 | 160 | if (c.socket) { 161 | if (!uv_is_closing((uv_handle_t*)c.socket)) { 162 | uv_close((uv_handle_t*)c.socket, free_close_cb); 163 | } 164 | free(c.socket); 165 | } 166 | } 167 | 168 | free(conns); 169 | } 170 | 171 | struct connection *connections_setup(struct sockaddr_in dest, int n, uv_loop_t *loop) 172 | { 173 | struct connection *conns = calloc(n, sizeof(*conns)); 174 | if (!conns) { 175 | err("malloc connections failed: %s", strerror(errno)); 176 | return NULL; 177 | } 178 | 179 | for (int i = 0; i < n; i++) { 180 | struct connection c = conns[i]; 181 | c.socket = malloc(sizeof(uv_tcp_t)); 182 | if (!c.socket) { 183 | err("malloc socket failed: %s", strerror(errno)); 184 | goto err; 185 | } 186 | 187 | int rc = uv_tcp_init(loop, c.socket); 188 | goto_uv_err(rc, err); 189 | 190 | c.conn = malloc(sizeof(uv_connect_t)); 191 | if (!c.conn) { 192 | err("malloc uv_connect_t failed: %s", strerror(errno)); 193 | goto err; 194 | } 195 | 196 | rc = uv_tcp_connect(c.conn, c.socket, (const struct sockaddr*)&dest, on_connect); 197 | goto_uv_err(rc, err); 198 | } 199 | 200 | return conns; 201 | 202 | err: 203 | connections_destroy(conns, n); 204 | return NULL; 205 | } 206 | 207 | int main(int argc, const char *argv[]) 208 | { 209 | if (argc != 3 && argc != 4) { 210 | fprintf(stderr, "Usage: %s [delay]\n", argv[0]); 211 | fprintf(stderr, "\n"); 212 | fprintf(stderr, " addr - addess to connect like 127.0.0.1\n"); 213 | fprintf(stderr, " n - number of connections\n"); 214 | fprintf(stderr, " delay - delay between connection and request in seconds (default is 5)\n"); 215 | return -1; 216 | } 217 | 218 | int n = strtol(argv[2], NULL, 0); 219 | 220 | if (argc == 4) { 221 | DELAY = strtol(argv[3], NULL, 0); 222 | } 223 | 224 | int rc = -1; 225 | 226 | struct sockaddr_in dest; 227 | rc = uv_ip4_addr(argv[1], 80, &dest); 228 | goto_uv_err(rc, exit); 229 | 230 | uv_loop_t *loop = uv_default_loop(); 231 | 232 | struct connection *conns = connections_setup(dest, n, loop); 233 | rc = uv_run(loop, UV_RUN_DEFAULT); 234 | 235 | exit: 236 | rc = uv_loop_close(loop); 237 | return_rc_uv_err(rc, 1); 238 | 239 | connections_destroy(conns, n); 240 | 241 | return rc; 242 | } 243 | -------------------------------------------------------------------------------- /logging.h: -------------------------------------------------------------------------------- 1 | #ifndef _LOGGING_H 2 | #define _LOGGING_H 3 | 4 | #include 5 | #include 6 | 7 | #ifdef DEBUG 8 | #define debug(M, ...) fprintf(stderr, "=%05d= DEBUG %s:%d " M, getpid(), __FILE__, __LINE__, ##__VA_ARGS__) 9 | #else 10 | #define debug(M, ...) 11 | #endif 12 | 13 | #define info(M, ...) fprintf(stderr, "=%05d= INFO %s:%d " M, getpid(), __FILE__, __LINE__, ##__VA_ARGS__) 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /socket_io.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "socket_io.h" 6 | #include "logging.h" 7 | 8 | int socket_read(int sock, char *buf, size_t bufsize) 9 | { 10 | int nleft, nread; 11 | char *ptr; 12 | 13 | debug("Reading from socket\n"); 14 | 15 | ptr = buf; 16 | nleft = bufsize; 17 | while (nleft > 0) { 18 | nread = read(sock, ptr, nleft); 19 | if (nread < 0) { 20 | debug("errno %s\n", strerror(errno)); 21 | if (errno == EINTR) { 22 | nread = 0; // call read again 23 | } else { 24 | perror("read"); 25 | return -1; 26 | } 27 | } else if (nread == 0) { 28 | // EOF 29 | debug("EOF\n"); 30 | break; 31 | } else { 32 | // Detect end of request 33 | if (strstr(ptr, "\r\n\r\n") != NULL) { 34 | debug("Request end\n"); 35 | break; 36 | } 37 | } 38 | 39 | nleft -= nread; 40 | ptr += nread; 41 | } 42 | 43 | debug("Successfully read %d\n", nread); 44 | return nread; 45 | } 46 | 47 | int socket_write(int sock, const char *buf, size_t bufsize) 48 | { 49 | int nleft, nwritten; 50 | const char *ptr; 51 | 52 | debug("Writing to socket\n"); 53 | 54 | ptr = buf; 55 | nleft = bufsize; 56 | while (nleft > 0) { 57 | nwritten = write(sock, ptr, nleft); 58 | if (nwritten < 0) { 59 | debug("errno %s\n", strerror(errno)); 60 | if (errno == EINTR) { 61 | nwritten = 0; // call write again 62 | } else { 63 | perror("write"); 64 | return -1; 65 | } 66 | } else if (nwritten == 0) { 67 | // From write (2) man page: 68 | // > If count is zero and fd refers to a file other than a regular 69 | // > file, the results are not specified. 70 | // We're writing to socket, so we can't say for sure what's wrong, 71 | // therefore we'll fail. 72 | fprintf(stderr, "nwritten == 0\n"); 73 | return -1; 74 | } 75 | 76 | nleft -= nwritten; 77 | ptr += nwritten; 78 | } 79 | 80 | debug("Successfully written %d\n", nwritten); 81 | return 0; 82 | } 83 | -------------------------------------------------------------------------------- /socket_io.h: -------------------------------------------------------------------------------- 1 | #ifndef _SOCKET_IO_H 2 | #define _SOCKET_IO_H 3 | 4 | int socket_read(int sock, char *buf, size_t bufsize); 5 | int socket_write(int sock, const char *buf, size_t bufsize); 6 | 7 | #endif 8 | --------------------------------------------------------------------------------