├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── docs └── tls │ ├── .gitignore │ ├── Dockerfile │ ├── README.md │ └── build.sh └── files ├── etc ├── logrotate.conf ├── rsyslogd.conf └── service │ ├── confd │ ├── finish │ └── run │ ├── dnsmasq │ └── run │ ├── filebeat │ └── run │ ├── haproxy │ └── run │ ├── rsyslog │ └── run │ └── vamp-gateway-agent │ └── run └── usr └── local ├── filebeat ├── filebeat.template-es2x.json └── filebeat.template.json └── vamp ├── confd-generate-templates.sh ├── haproxy-reload.sh ├── haproxy-validate.sh ├── haproxy.basic.cfg └── tokenrenewer.sh /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | Makefile.local 3 | local.sh 4 | .idea/ 5 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | 3 | # https://github.com/peterbourgon/runsvinit 4 | ENV RUNSVINIT_URL=https://github.com/peterbourgon/runsvinit/releases/download/v2.0.0/runsvinit-linux-amd64.tgz 5 | 6 | ENV HAPROXY_VER=1.8.8 7 | ENV HAPROXY_MD5=8633b6e661169d2fc6a44d82a3aceae5 8 | ENV HAPROXY_URL=http://www.haproxy.org/download/1.8/src/haproxy-${HAPROXY_VER}.tar.gz 9 | 10 | ENV CONFD_VER=0.16.1-vamp 11 | ENV CONFD_URL=https://github.com/magneticio/confd/releases/download/v${CONFD_VER}/confd-${CONFD_VER}-linux-amd64 12 | 13 | ENV FILEBEAT_VER=5.1.2 14 | ENV FILEBEAT_URL=https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${FILEBEAT_VER}-linux-x86_64.tar.gz 15 | 16 | ADD files/ / 17 | ADD version /usr/local/vamp/version 18 | 19 | RUN set -xe && \ 20 | apk update && \ 21 | apk add --no-cache bash curl musl pcre rsyslog runit zlib openssl jq logrotate && \ 22 | curl --location --silent --show-error $RUNSVINIT_URL --output - | tar zxf - -C /sbin && \ 23 | chown 0:0 /sbin/runsvinit && \ 24 | chmod 0775 /sbin/runsvinit && \ 25 | \ 26 | curl --location --silent --show-error --output /usr/bin/confd $CONFD_URL && \ 27 | chmod 0755 /usr/bin/confd && \ 28 | \ 29 | apk --no-cache --virtual=build-deps add gcc linux-headers make musl-dev pcre-dev zlib-dev openssl-dev && \ 30 | mkdir /usr/src && \ 31 | curl -fL $HAPROXY_URL > /usr/src/haproxy.tar.gz && \ 32 | echo "$HAPROXY_MD5 /usr/src/haproxy.tar.gz" > /usr/src/haproxy.md5 && md5sum -c /usr/src/haproxy.md5 && \ 33 | tar xzf /usr/src/haproxy.tar.gz -C /usr/src && \ 34 | cd /usr/src/haproxy-${HAPROXY_VER} && \ 35 | make TARGET=linux2628 USE_PCRE=1 USE_ZLIB=1 USE_OPENSSL=1 && \ 36 | make install-bin && \ 37 | cd .. && \ 38 | rm -rf /usr/src/haproxy-${HAPROXY_VER} /usr/src/haproxy.tar.gz /usr/src/haproxy.md5 && \ 39 | apk del build-deps && \ 40 | \ 41 | curl --location --silent --show-error $FILEBEAT_URL --output - | tar zxf - -C /tmp && \ 42 | mv /tmp/filebeat-${FILEBEAT_VER}-linux-x86_64/filebeat /usr/local/bin/ && \ 43 | rm -rf /tmp/filebeat-${FILEBEAT_VER}-linux-x86_64 && \ 44 | \ 45 | ALPINE_GLIBC_BASE_URL="https://github.com/sgerrand/alpine-pkg-glibc/releases/download" && \ 46 | ALPINE_GLIBC_PACKAGE_VERSION="2.23-r3" && \ 47 | ALPINE_GLIBC_BASE_PACKAGE_FILENAME="glibc-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \ 48 | ALPINE_GLIBC_BIN_PACKAGE_FILENAME="glibc-bin-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \ 49 | ALPINE_GLIBC_I18N_PACKAGE_FILENAME="glibc-i18n-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \ 50 | apk add --no-cache --virtual=.build-dependencies wget ca-certificates && \ 51 | wget \ 52 | "https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub" \ 53 | -O "/etc/apk/keys/sgerrand.rsa.pub" && \ 54 | wget \ 55 | "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \ 56 | "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \ 57 | "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_I18N_PACKAGE_FILENAME" && \ 58 | apk add --no-cache \ 59 | "$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \ 60 | "$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \ 61 | "$ALPINE_GLIBC_I18N_PACKAGE_FILENAME" && \ 62 | \ 63 | rm "/etc/apk/keys/sgerrand.rsa.pub" && \ 64 | /usr/glibc-compat/bin/localedef --force --inputfile POSIX --charmap UTF-8 C.UTF-8 || true && \ 65 | echo "export LANG=C.UTF-8" > /etc/profile.d/locale.sh && \ 66 | \ 67 | apk del glibc-i18n && \ 68 | \ 69 | rm "/root/.wget-hsts" && \ 70 | apk del .build-dependencies && \ 71 | rm \ 72 | "$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \ 73 | "$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \ 74 | "$ALPINE_GLIBC_I18N_PACKAGE_FILENAME" && \ 75 | \ 76 | chmod +x /usr/local/vamp/tokenrenewer.sh 77 | RUN echo "0 * * * * /usr/sbin/logrotate /etc/logrotate.conf" | crontab - 78 | 79 | ENV LANG=C.UTF-8 80 | 81 | EXPOSE 1988 82 | 83 | CMD crond && /sbin/runsvinit 84 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2014-2016 Magnetic.io B.V. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # See: http://clarkgrubb.com/makefile-style-guide 2 | SHELL := bash 3 | .SHELLFLAGS := -eu -o pipefail -c 4 | .DEFAULT_GOAL := default 5 | .DELETE_ON_ERROR : 6 | .SUFFIXES : 7 | 8 | PROJECT := vamp-gateway-agent 9 | TARGET := $(CURDIR)/target 10 | IMAGE_TAG := $(shell echo $$BRANCH_NAME) 11 | VERSION := $(shell git describe --tags) 12 | 13 | # if Makefile.local exists, include it. 14 | ifneq ("$(wildcard Makefile.local)", "") 15 | include Makefile.local 16 | endif 17 | 18 | ifeq ($(strip $(IMAGE_TAG)),) 19 | IMAGE_TAG := $(shell git rev-parse --abbrev-ref HEAD) 20 | endif 21 | 22 | .PHONY: tag 23 | tag: 24 | @echo $(IMAGE_TAG) 25 | 26 | .PHONY: version 27 | version: 28 | @echo $(VERSION) 29 | 30 | .PHONY: clean 31 | clean: 32 | rm -Rf $(TARGET) 33 | 34 | .PHONY: purge 35 | purge: clean 36 | docker rmi -f $$(docker images | grep $(PROJECT) | awk '{print $$3}') || true 37 | 38 | .PHONY: build 39 | build: 40 | mkdir -p $(TARGET) || true 41 | @echo "Creating docker build context" 42 | cp $(CURDIR)/Dockerfile $(TARGET)/Dockerfile 43 | cp -Rf $(CURDIR)/files $(TARGET) 44 | echo $(VERSION) > $(TARGET)/version 45 | cd $(TARGET) && \ 46 | docker build -t magneticio/$(PROJECT):$(IMAGE_TAG) . 47 | 48 | .PHONY: default 49 | default: clean build 50 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Vamp Gateway Agent 2 | 3 | [![Join the chat at https://gitter.im/magneticio/vamp](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/magneticio/vamp?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 4 | [![Docker](https://img.shields.io/badge/docker-images-blue.svg)](https://hub.docker.com/r/magneticio/vamp-gateway-agent/tags/) 5 | 6 | Based on Vamp gateways, Vamp generates HAProxy configuration and stores it to KV store. 7 | 8 | Vamp Gateway Agent: 9 | - reads the [HAProxy](http://www.haproxy.org/) configuration using [confd](https://github.com/kelseyhightower/confd) 10 | - appends it to the base configuration `haproxy.basic.cnf` 11 | - if new configuration is valid, VGA reloads HAProxy with as little client traffic interruption as possible 12 | 13 | In addition to this VGA also: 14 | - send HAProxy log to Elasticsearch using [Filebeat](https://www.elastic.co/products/beats/filebeat) 15 | - handle and recover from ZooKeeper, etcd, Consul and Vault outages without interrupting the haproxy process and client requests 16 | - does Vault token renewal if needed 17 | 18 | ## Usage 19 | 20 | Following environment variables are mandatory: 21 | 22 | - `VAMP_KEY_VALUE_STORE_TYPE <=> confd -backend` 23 | - `VAMP_KEY_VALUE_STORE_CONNECTION <=> confd -node` 24 | - `VAMP_KEY_VALUE_STORE_PATH <=> key used by confd` 25 | - `VAMP_ELASTICSEARCH_URL <=> http://elasticsearch:9200` 26 | 27 | Example: 28 | 29 | ``` 30 | docker run -e VAMP_KEY_VALUE_STORE_TYPE=zookeeper \ 31 | -e VAMP_KEY_VALUE_STORE_CONNECTION=localhost:2181 \ 32 | -e VAMP_KEY_VALUE_STORE_PATH=/vamp/gateways/haproxy/1.6 \ 33 | -e VAMP_ELASTICSEARCH_URL=http://localhost:9200 \ 34 | magneticio/vamp-gateway-agent:katana 35 | ``` 36 | 37 | Available Docker images can be found at [Docker Hub](https://hub.docker.com/r/magneticio/vamp-gateway-agent/). 38 | 39 | ### Domain name resolver 40 | 41 | To enable dnsmasq to resolve virtual hosts, pass the following environment variables to the Docker container: 42 | 43 | - `VAMP_VGA_DNS_ENABLE` Set to non-empty value to enable 44 | - `VAMP_VGA_DNS_PORT` Listening port, default: 5353 45 | 46 | ## Building Docker images 47 | 48 | `make` targets: 49 | - `version` - displaying version (tag) 50 | - `clean` - removing temporal build directory `./target` 51 | - `purge` - running `clean` and removing all images `magneticio/vamp-gateway-agent:*` 52 | - `build` - copying files to `./target` directory and building the image `magneticio/vamp-gateway-agent:${version}` 53 | - `default` - `clean build` 54 | 55 | 56 | ## Additional documentation and examples 57 | 58 | - [TLS](https://github.com/magneticio/vamp-gateway-agent/tree/master/docs/tls) 59 | -------------------------------------------------------------------------------- /docs/tls/.gitignore: -------------------------------------------------------------------------------- 1 | .tmp/ 2 | -------------------------------------------------------------------------------- /docs/tls/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM magneticio/vamp-gateway-agent:VGA_TAG 2 | 3 | ADD vga/vgaCertAndKey.crt /usr/local/vamp/ 4 | ADD client/clientRootCA.crt /usr/local/vamp/ -------------------------------------------------------------------------------- /docs/tls/README.md: -------------------------------------------------------------------------------- 1 | ## Vamp Gateway Agent & TLS Termination 2 | 3 | ### Mutual TLS in VGA with self-signed certificates 4 | 5 | Run `build.sh`, possible parameters (environment variables): 6 | 7 | - VGA_TAG=${VGA_TAG:-tls} 8 | - VGA_BASE_TAG=${VGA_BASE_TAG:-katana} 9 | - VGA_DN=${VGA_DN:-localhost} 10 | - CLIENT_DN=${CLIENT_DN:-localhost} 11 | 12 | Script steps: 13 | 14 | - creating VGA certificates 15 | - creating client certificates 16 | - building the new VGA Docker image with the certificate 17 | 18 | Intermediate files including certificates are in `./.tmp` directory. 19 | 20 | Another approach is to use official VGA (without building the custom image) and mounting Docker volume with right certificate(s). 21 | 22 | ### HAProxy configuration 23 | 24 | In order to avoid warnings add to global HAProxy configuration: 25 | ``` 26 | tune.ssl.default-dh-param 2048 27 | ``` 28 | 29 | Assuming `/usr/local/vamp/vgaCertAndKey.crt` VGA certificate path and `/usr/local/vamp/clientRootCA.crt` client CA certificate path, update virtual hosts section: 30 | ``` 31 | bind 0.0.0.0:80 32 | ⇒ bind 0.0.0.0:443 ssl crt /usr/local/vamp/vgaCertAndKey.crt ca-file /usr/local/vamp/clientRootCA.crt verify required 33 | ``` 34 | 35 | Also TLS termination can be done differently. For instance just to terminate and proxy to a gateway port, replace virtual hosts part with: 36 | ``` 37 | ### BEGIN - TLS TERMINATION 38 | 39 | frontend tls_termination 40 | 41 | bind 0.0.0.0:443 ssl crt /usr/local/vamp/vgaCertAndKey.crt ca-file /usr/local/vamp/clientRootCA.crt verify required 42 | mode http 43 | 44 | option httplog 45 | log-format """{\"ci\":\"%ci\",\"cp\":%cp,\"t\":\"%t\",\"ft\":\"%ft\",\"b\":\"%b\",\"s\":\"%s\",\"Tq\":%Tq,\"Tw\":%Tw,\"Tc\":%Tc,\"Tr\":%Tr,\"Tt\":%Tt,\"ST\":%ST,\"B\":%B,\"CC\":\"%CC\",\"CS\":\"%CS\",\"tsc\":\"%tsc\",\"ac\":%ac,\"fc\":%fc,\"bc\":%bc,\"sc\":%sc,\"rc\":%rc,\"sq\":%sq,\"bq\":%bq,\"hr\":\"%hr\",\"hs\":\"%hs\",\"r\":%{+Q}r}""" 46 | 47 | use_backend tls_termination 48 | 49 | backend tls_termination 50 | 51 | balance roundrobin 52 | mode http 53 | 54 | option forwardfor 55 | http-request set-header X-Forwarded-Port %[dst_port] 56 | 57 | # server: sava/80 58 | server tls_termination 127.0.0.1:80 59 | 60 | ### END - TLS TERMINATION 61 | ``` 62 | 63 | Note `127.0.0.1:80` where `80` is a Vamp gateway port. 64 | 65 | ### Mutual TLS example 66 | 67 | Let's assume the following: 68 | 69 | - VGA is accessible on `aaa-bbb-ccc.eu-west-1.elb.amazonaws.com`. 70 | - we have `sava` deployed (port 9050) and virtual hosts enabled 71 | - it should work: `curl -H 'Host: 9050.sava.vamp' http://aaa-bbb-ccc.eu-west-1.elb.amazonaws.com` 72 | 73 | Now, setup the mutual TLS using self-signed certificates: 74 | 75 | - run: `export VGA_DN=*.eu-west-1.elb.amazonaws.com && export CLIENT_DN={YOUR_IP_ADDRESS} && ./build.sh` 76 | - we just created (by default): `magneticio/vamp-gateway-agent:tls`, redeploy VGA using that image 77 | - it should work as before: `curl -H 'Host: 9050.sava.vamp' http://aaa-bbb-ccc.eu-west-1.elb.amazonaws.com` 78 | - go to Vamp VGA template and update `bind 0.0.0.0:80` to `bind 0.0.0.0:443 ssl crt /usr/local/vamp/vgaCertAndKey.crt ca-file /usr/local/vamp/clientRootCA.crt verify required` 79 | - check now (notice `https`): `curl --cacert .tmp/vga/vgaRootCA.crt --cert .tmp/client/client.crt --key .tmp/client/client.key -H 'Host: 9050.sava.vamp' https://aaa-bbb-ccc.eu-west-1.elb.amazonaws.com` -------------------------------------------------------------------------------- /docs/tls/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 4 | 5 | reset="$(tput sgr0)" 6 | red="$(tput setaf 1)" 7 | green="$(tput setaf 2)" 8 | yellow="$(tput setaf 3)" 9 | 10 | # Check dependencies 11 | hash openssl 2> /dev/null || { echo "${red}please install ${green}openssl${reset}"; exit 1; } 12 | hash docker 2> /dev/null || { echo "${red}please install ${green}docker ${yellow}https://www.docker.com${reset}"; exit 1; } 13 | 14 | VGA_TAG=${VGA_TAG:-tls} 15 | VGA_BASE_TAG=${VGA_BASE_TAG:-katana} 16 | VGA_DN=${VGA_DN:-localhost} 17 | CLIENT_DN=${CLIENT_DN:-localhost} 18 | 19 | TEMP_DIR=${DIR}/.tmp 20 | VGA_CERTS_DIR=${TEMP_DIR}/vga 21 | CLIENT_CERTS_DIR=${TEMP_DIR}/client 22 | 23 | set -e 24 | 25 | rm -Rf ${TEMP_DIR} && mkdir -p ${VGA_CERTS_DIR} && mkdir -p ${CLIENT_CERTS_DIR} 26 | 27 | function step { 28 | echo "${green}$1${yellow}$2" 29 | } 30 | 31 | function finish { 32 | step "Done." 33 | exit 0 34 | } 35 | 36 | function panic { 37 | echo "${red}ERROR: $1${reset}" 38 | exit 1 39 | } 40 | 41 | function terminated { 42 | printf ${reset} 43 | } 44 | 45 | trap terminated EXIT 46 | 47 | step "Creating VGA certificate" 48 | cat > ${VGA_CERTS_DIR}/vga_root.config <<-EOF 49 | [req] 50 | default_bits = 2048 51 | prompt = no 52 | default_md = sha256 53 | distinguished_name = dn 54 | x509_extensions = v3_ca 55 | 56 | [ dn ] 57 | CN = vga_root 58 | 59 | [v3_ca] 60 | subjectAltName = @alt_names 61 | 62 | [ alt_names ] 63 | DNS.1 = vga_root 64 | EOF 65 | 66 | cat > ${VGA_CERTS_DIR}/vga.config <<-EOF 67 | [req] 68 | default_bits = 2048 69 | prompt = no 70 | default_md = sha256 71 | distinguished_name = dn 72 | x509_extensions = v3_ca 73 | 74 | [ dn ] 75 | CN = ${VGA_DN} 76 | 77 | [v3_ca] 78 | subjectAltName = @alt_names 79 | 80 | [ alt_names ] 81 | DNS.1 = ${VGA_DN} 82 | EOF 83 | 84 | #Generate server certificate 85 | openssl genrsa -out ${VGA_CERTS_DIR}/vgaRootCA.key 2048 86 | openssl req -x509 -new -nodes -key ${VGA_CERTS_DIR}/vgaRootCA.key -sha256 -days 1024 -config <(cat ${VGA_CERTS_DIR}/vga_root.config) -out ${VGA_CERTS_DIR}/vgaRootCA.crt 87 | openssl genrsa -out ${VGA_CERTS_DIR}/vga.key 2048 88 | openssl req -new -key ${VGA_CERTS_DIR}/vga.key -config <(cat ${VGA_CERTS_DIR}/vga.config) -out ${VGA_CERTS_DIR}/vga.csr 89 | openssl x509 -req -in ${VGA_CERTS_DIR}/vga.csr -CA ${VGA_CERTS_DIR}/vgaRootCA.crt -CAkey ${VGA_CERTS_DIR}/vgaRootCA.key -CAcreateserial -out ${VGA_CERTS_DIR}/vga.crt -days 500 -sha256 90 | cat ${VGA_CERTS_DIR}/vga.crt ${VGA_CERTS_DIR}/vga.key | tee ${VGA_CERTS_DIR}/vgaCertAndKey.crt 91 | 92 | step "Creating client certificate" 93 | cat > ${CLIENT_CERTS_DIR}/client_root.config <<-EOF 94 | [req] 95 | default_bits = 2048 96 | prompt = no 97 | default_md = sha256 98 | distinguished_name = dn 99 | x509_extensions = v3_ca 100 | 101 | [ dn ] 102 | CN = client_root 103 | 104 | [v3_ca] 105 | subjectAltName = @alt_names 106 | 107 | [ alt_names ] 108 | DNS.1 = client_root 109 | 110 | EOF 111 | 112 | cat > ${CLIENT_CERTS_DIR}/client.config <<-EOF 113 | [req] 114 | default_bits = 2048 115 | prompt = no 116 | default_md = sha256 117 | distinguished_name = dn 118 | x509_extensions = v3_ca 119 | 120 | [ dn ] 121 | CN = ${CLIENT_DN} 122 | 123 | [v3_ca] 124 | subjectAltName = @alt_names 125 | 126 | [ alt_names ] 127 | DNS.1 = ${CLIENT_DN} 128 | EOF 129 | 130 | openssl genrsa -out ${CLIENT_CERTS_DIR}/clientRootCA.key 2048 131 | openssl req -x509 -new -nodes -key ${CLIENT_CERTS_DIR}/clientRootCA.key -sha256 -days 1024 -config <(cat ${CLIENT_CERTS_DIR}/client_root.config) -out ${CLIENT_CERTS_DIR}/clientRootCA.crt 132 | openssl genrsa -out ${CLIENT_CERTS_DIR}/client.key 2048 133 | openssl req -new -key ${CLIENT_CERTS_DIR}/client.key -config <(cat ${CLIENT_CERTS_DIR}/client.config) -out ${CLIENT_CERTS_DIR}/client.csr 134 | openssl x509 -req -in ${CLIENT_CERTS_DIR}/client.csr -CA ${CLIENT_CERTS_DIR}/clientRootCA.crt -CAkey ${CLIENT_CERTS_DIR}/clientRootCA.key -CAcreateserial -out ${CLIENT_CERTS_DIR}/client.crt -days 500 -sha256 135 | 136 | step "Building VGA Docker image" 137 | step "VGA base image : " ${VGA_BASE_TAG} 138 | step "VGA new image tag: " ${VGA_TAG} 139 | sed -e "s/VGA_TAG/${VGA_BASE_TAG}/g" ${DIR}/Dockerfile > ${TEMP_DIR}/Dockerfile 140 | docker build -t tyga/vamp-gateway-agent:${VGA_TAG} -f ${TEMP_DIR}/Dockerfile ${TEMP_DIR} 141 | 142 | finish 143 | -------------------------------------------------------------------------------- /files/etc/logrotate.conf: -------------------------------------------------------------------------------- 1 | /var/log/haproxy.log { 2 | copytruncate 3 | missingok 4 | rotate 2 5 | size 100M 6 | notifempty 7 | } -------------------------------------------------------------------------------- /files/etc/rsyslogd.conf: -------------------------------------------------------------------------------- 1 | $ModLoad imuxsock.so # provides support for local system logging (e.g. via logger command) 2 | $ModLoad imudp.so # provides UDP listener 3 | $ModLoad imtcp.so # provides TCP listener 4 | 5 | # Listen on port 514 TCP and UDP 6 | $UDPServerRun 514 7 | $InputTCPServerRun 514 8 | 9 | # default permissions for all log files. 10 | $FileOwner root 11 | $FileGroup adm 12 | $FileCreateMode 0640 13 | $DirCreateMode 0755 14 | $Umask 0022 15 | 16 | # HAproxy format set in haproxy.cfg, we just pass the raw string to the logs 17 | $template filebeat_json,"%msg:::%\n" 18 | 19 | # Vamp HAproxy logs 20 | local0.* /var/log/haproxy.log;filebeat_json 21 | -------------------------------------------------------------------------------- /files/etc/service/confd/finish: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | : "${VAMP_GATEWAY_AGENT_CONFD_INTERVAL:=5}" 4 | 5 | sleep $VAMP_GATEWAY_AGENT_CONFD_INTERVAL 6 | exit 0 7 | -------------------------------------------------------------------------------- /files/etc/service/confd/run: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | handle() { echo "confd/run: got signal"; exit; } 4 | trap handle SIGINT 5 | 6 | [[ -e /usr/local/vamp/good2go ]] || exit 1 7 | 8 | : "${VAMP_GATEWAY_AGENT_CONFD_LOGLEVEL:=warn}" 9 | 10 | if [ -e /usr/local/vamp/token ]; then 11 | VAMP_KEY_VALUE_STORE_TOKEN="$( cat /usr/local/vamp/token )" 12 | fi 13 | 14 | exec 2>&1 15 | 16 | if [[ -z ${VAMP_KEY_VALUE_STORE_TOKEN} ]] ; then 17 | exec /usr/bin/confd \ 18 | -backend ${VAMP_KEY_VALUE_STORE_TYPE} \ 19 | -node ${VAMP_KEY_VALUE_STORE_CONNECTION} \ 20 | -log-level="$VAMP_GATEWAY_AGENT_CONFD_LOGLEVEL" \ 21 | -onetime=true 22 | else 23 | exec /usr/bin/confd \ 24 | -backend ${VAMP_KEY_VALUE_STORE_TYPE} \ 25 | -node ${VAMP_KEY_VALUE_STORE_CONNECTION} \ 26 | -auth-type token -auth-token ${VAMP_KEY_VALUE_STORE_TOKEN} \ 27 | -log-level="$VAMP_GATEWAY_AGENT_CONFD_LOGLEVEL" \ 28 | -onetime=true 29 | fi 30 | -------------------------------------------------------------------------------- /files/etc/service/dnsmasq/run: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | handle() { echo "dnsmasq/run: got signal"; exit; } 4 | trap handle SIGINT 5 | 6 | [[ -e /usr/local/vamp/good2go ]] || exit 1 7 | 8 | if [[ -z $VAMP_VGA_DNS_ENABLE ]] ; then 9 | echo "dnsmasq/run: disabling" 10 | sv stop dnsmasq 11 | exit 12 | fi 13 | 14 | : "${VAMP_VGA_CONF_DIR:=/usr/local/vamp}" 15 | : "${VAMP_VGA_DNS_PORT:=5353}" 16 | 17 | # Add virtual hosts to /etc/hosts and reload dnsmasq 18 | if [[ -n $VAMP_VGA_DNS_ENABLE ]]; then 19 | # Overwrite default port of 5353, if specified 20 | if [[ -n $VAMP_VGA_DNS_PORT && $VAMP_VGA_DNS_PORT =~ ^-?[0-9]+$ ]]; then 21 | echo "port=${VAMP_VGA_DNS_PORT}" > /etc/dnsmasq.conf 22 | fi 23 | 24 | awk -v host="$( hostname -i )" \ 25 | 'BEGIN { print "127.0.0.1\tlocalhost\n::1\tlocalhost\n" }; 26 | /^ acl .* hdr\(host\) -i .*$/ { print host "\t" $NF }' "${configuration}" > /etc/hosts 27 | 28 | kill -s SIGHUP $( pidof dnsmasq ) || dnsmasq 29 | fi 30 | -------------------------------------------------------------------------------- /files/etc/service/filebeat/run: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | handle() { echo "filebeat/run: got signal"; exit; } 4 | trap handle SIGINT 5 | 6 | [[ -e /usr/local/vamp/good2go ]] || exit 1 7 | [[ -e /usr/local/filebeat/filebeat.yml ]] || exit 1 8 | 9 | exec 2>&1 10 | exec /usr/local/bin/filebeat -c /usr/local/filebeat/filebeat.yml 11 | 12 | -------------------------------------------------------------------------------- /files/etc/service/haproxy/run: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Attempt to reload HAproxy if it's not running 4 | 5 | handle() { echo "haproxy/run: got signal"; exit; } 6 | trap handle SIGINT 7 | 8 | [[ -e /usr/local/vamp/haproxy.cfg ]] || exit 1 9 | 10 | pidof haproxy &> /dev/null || /usr/local/vamp/haproxy-reload.sh 11 | -------------------------------------------------------------------------------- /files/etc/service/rsyslog/run: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | handle() { echo "confd/run: got signal"; exit; } 4 | trap handle SIGINT 5 | 6 | cmd="rsyslogd -n -f /etc/rsyslogd.conf -i /var/run/rsyslogd.pid" 7 | echo "rsyslogd/run: starting: $cmd" 8 | 9 | exec $cmd 10 | -------------------------------------------------------------------------------- /files/etc/service/vamp-gateway-agent/run: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | handle() { echo "vamp-gateway-agent/run: got signal"; exit; } 4 | trap handle SIGINT 5 | 6 | [[ -e /usr/local/vamp/good2go ]] && exit 0 7 | 8 | : "${VAMP_GATEWAY_AGENT_LOGO:=TRUE}" 9 | 10 | if [ "$VAMP_GATEWAY_AGENT_LOGO" = "TRUE" ] || [ "$VAMP_GATEWAY_AGENT_LOGO" = "1" ]; then 11 | echo " 12 | ██╗ ██╗ █████╗ ███╗ ███╗██████╗ ██████╗ █████╗ ████████╗███████╗██╗ ██╗ █████╗ ██╗ ██╗ 13 | ██║ ██║██╔══██╗████╗ ████║██╔══██╗ ██╔════╝ ██╔══██╗╚══██╔══╝██╔════╝██║ ██║██╔══██╗╚██╗ ██╔╝ 14 | ██║ ██║███████║██╔████╔██║██████╔╝ ██║ ███╗███████║ ██║ █████╗ ██║ █╗ ██║███████║ ╚████╔╝ 15 | ╚██╗ ██╔╝██╔══██║██║╚██╔╝██║██╔═══╝ ██║ ██║██╔══██║ ██║ ██╔══╝ ██║███╗██║██╔══██║ ╚██╔╝ 16 | ╚████╔╝ ██║ ██║██║ ╚═╝ ██║██║ ╚██████╔╝██║ ██║ ██║ ███████╗╚███╔███╔╝██║ ██║ ██║ 17 | ╚═══╝ ╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝ ╚══════╝ ╚══╝╚══╝ ╚═╝ ╚═╝ ╚═╝ 18 | " 19 | fi 20 | 21 | : "${VAMP_KEY_VALUE_STORE_PATH?not provided.}" 22 | : "${VAMP_KEY_VALUE_STORE_TYPE?not provided.}" 23 | : "${VAMP_KEY_VALUE_STORE_CONNECTION?not provided.}" 24 | : "${VAMP_ELASTICSEARCH_URL?not provided.}" 25 | 26 | for try in {1..9} ; do 27 | echo "vamp-gateway-agent/run: waiting for syslog to come online..." 28 | [[ -e /var/run/rsyslogd.pid ]] && break || sleep 1 29 | done 30 | 31 | if [ ! -z "$(ls -A /usr/local/share/ca-certificates)" ]; then 32 | echo "Adding custom CA certificate" 33 | update-ca-certificates 2>/dev/null || true 34 | fi 35 | 36 | printf "VAMP GATEWAY AGENT: " && cat /usr/local/vamp/version 37 | /usr/bin/confd -version 38 | 39 | echo "VAMP_KEY_VALUE_STORE_TYPE : ${VAMP_KEY_VALUE_STORE_TYPE}" 40 | echo "VAMP_KEY_VALUE_STORE_CONNECTION : ${VAMP_KEY_VALUE_STORE_CONNECTION}" 41 | echo "VAMP_KEY_VALUE_STORE_PATH : ${VAMP_KEY_VALUE_STORE_PATH}" 42 | echo "VAMP_ELASTICSEARCH_URL : ${VAMP_ELASTICSEARCH_URL}" 43 | 44 | /usr/local/vamp/confd-generate-templates.sh 45 | 46 | if [[ "${VAMP_KEY_VALUE_STORE_TYPE}" == "vault" && -n ${VAMP_VAULT_CERTIFICATE_NAME} ]]; then 47 | curl --cacert ${CONFD_CLIENT_CAKEYS} \ 48 | --cert ${CONFD_CLIENT_CERT} \ 49 | --key ${CONFD_CLIENT_KEY} \ 50 | --request POST --data "{\"name\": \"${VAMP_VAULT_CERTIFICATE_NAME}\"}" -s \ 51 | ${VAMP_KEY_VALUE_STORE_CONNECTION}/v1/auth/cert/login | jq -r .auth.client_token > /usr/local/vamp/token 52 | fi 53 | 54 | touch /usr/local/vamp/good2go 55 | 56 | if [ "${VAMP_KEY_VALUE_STORE_TYPE}" == "vault" ]; then 57 | /usr/local/vamp/tokenrenewer.sh & 58 | fi 59 | -------------------------------------------------------------------------------- /files/usr/local/filebeat/filebeat.template-es2x.json: -------------------------------------------------------------------------------- 1 | { 2 | "mappings": { 3 | "_default_": { 4 | "_all": { 5 | "norms": { 6 | "enabled": false 7 | } 8 | }, 9 | "_meta": { 10 | "version": "5.1.2" 11 | }, 12 | "dynamic_templates": [ 13 | { 14 | "strings_as_keyword": { 15 | "mapping": { 16 | "ignore_above": 1024, 17 | "index": "not_analyzed", 18 | "type": "string" 19 | }, 20 | "match_mapping_type": "string" 21 | } 22 | } 23 | ], 24 | "properties": { 25 | "@timestamp": { 26 | "type": "date" 27 | }, 28 | "beat": { 29 | "properties": { 30 | "hostname": { 31 | "ignore_above": 1024, 32 | "index": "not_analyzed", 33 | "type": "string" 34 | }, 35 | "name": { 36 | "ignore_above": 1024, 37 | "index": "not_analyzed", 38 | "type": "string" 39 | }, 40 | "version": { 41 | "ignore_above": 1024, 42 | "index": "not_analyzed", 43 | "type": "string" 44 | } 45 | } 46 | }, 47 | "input_type": { 48 | "ignore_above": 1024, 49 | "index": "not_analyzed", 50 | "type": "string" 51 | }, 52 | "message": { 53 | "index": "analyzed", 54 | "norms": { 55 | "enabled": false 56 | }, 57 | "type": "string" 58 | }, 59 | "meta": { 60 | "properties": { 61 | "cloud": { 62 | "properties": { 63 | "availability_zone": { 64 | "ignore_above": 1024, 65 | "index": "not_analyzed", 66 | "type": "string" 67 | }, 68 | "instance_id": { 69 | "ignore_above": 1024, 70 | "index": "not_analyzed", 71 | "type": "string" 72 | }, 73 | "machine_type": { 74 | "ignore_above": 1024, 75 | "index": "not_analyzed", 76 | "type": "string" 77 | }, 78 | "project_id": { 79 | "ignore_above": 1024, 80 | "index": "not_analyzed", 81 | "type": "string" 82 | }, 83 | "provider": { 84 | "ignore_above": 1024, 85 | "index": "not_analyzed", 86 | "type": "string" 87 | }, 88 | "region": { 89 | "ignore_above": 1024, 90 | "index": "not_analyzed", 91 | "type": "string" 92 | } 93 | } 94 | } 95 | } 96 | }, 97 | "offset": { 98 | "type": "long" 99 | }, 100 | "source": { 101 | "ignore_above": 1024, 102 | "index": "not_analyzed", 103 | "type": "string" 104 | }, 105 | "tags": { 106 | "ignore_above": 1024, 107 | "index": "not_analyzed", 108 | "type": "string" 109 | }, 110 | "type": { 111 | "ignore_above": 1024, 112 | "index": "not_analyzed", 113 | "type": "string" 114 | } 115 | } 116 | } 117 | }, 118 | "order": 0, 119 | "settings": { 120 | "index.refresh_interval": "5s" 121 | }, 122 | "template": "vamp-vga-*" 123 | } -------------------------------------------------------------------------------- /files/usr/local/filebeat/filebeat.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "mappings": { 3 | "_default_": { 4 | "_all": { 5 | "norms": false 6 | }, 7 | "_meta": { 8 | "version": "5.1.2" 9 | }, 10 | "dynamic_templates": [ 11 | { 12 | "strings_as_keyword": { 13 | "mapping": { 14 | "ignore_above": 1024, 15 | "type": "keyword" 16 | }, 17 | "match_mapping_type": "string" 18 | } 19 | } 20 | ], 21 | "properties": { 22 | "@timestamp": { 23 | "type": "date" 24 | }, 25 | "beat": { 26 | "properties": { 27 | "hostname": { 28 | "ignore_above": 1024, 29 | "type": "keyword" 30 | }, 31 | "name": { 32 | "ignore_above": 1024, 33 | "type": "keyword" 34 | }, 35 | "version": { 36 | "ignore_above": 1024, 37 | "type": "keyword" 38 | } 39 | } 40 | }, 41 | "input_type": { 42 | "ignore_above": 1024, 43 | "type": "keyword" 44 | }, 45 | "message": { 46 | "norms": false, 47 | "type": "text" 48 | }, 49 | "meta": { 50 | "properties": { 51 | "cloud": { 52 | "properties": { 53 | "availability_zone": { 54 | "ignore_above": 1024, 55 | "type": "keyword" 56 | }, 57 | "instance_id": { 58 | "ignore_above": 1024, 59 | "type": "keyword" 60 | }, 61 | "machine_type": { 62 | "ignore_above": 1024, 63 | "type": "keyword" 64 | }, 65 | "project_id": { 66 | "ignore_above": 1024, 67 | "type": "keyword" 68 | }, 69 | "provider": { 70 | "ignore_above": 1024, 71 | "type": "keyword" 72 | }, 73 | "region": { 74 | "ignore_above": 1024, 75 | "type": "keyword" 76 | } 77 | } 78 | } 79 | } 80 | }, 81 | "offset": { 82 | "type": "long" 83 | }, 84 | "source": { 85 | "ignore_above": 1024, 86 | "type": "keyword" 87 | }, 88 | "tags": { 89 | "ignore_above": 1024, 90 | "type": "keyword" 91 | }, 92 | "type": { 93 | "ignore_above": 1024, 94 | "type": "keyword" 95 | } 96 | } 97 | } 98 | }, 99 | "order": 0, 100 | "settings": { 101 | "index.refresh_interval": "5s" 102 | }, 103 | "template": "vamp-vga-*" 104 | } -------------------------------------------------------------------------------- /files/usr/local/vamp/confd-generate-templates.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Ensure we have our directories 4 | dir_confd="/etc/confd/conf.d" 5 | dir_templates="/etc/confd/templates" 6 | 7 | mkdir -p "$dir_confd" 8 | mkdir -p "$dir_templates" 9 | 10 | 11 | # Generate config and templates for HAproxy 12 | echo "creating confd configuration and template" 13 | cat < "${dir_confd}/haproxy.toml" 14 | [template] 15 | src = "haproxy.tmpl" 16 | dest = "/usr/local/vamp/haproxy.cfg" 17 | keys = [ "${VAMP_KEY_VALUE_STORE_PATH}" ] 18 | check_cmd = "/usr/local/vamp/haproxy-validate.sh {{.src}}" 19 | reload_cmd = "/usr/local/vamp/haproxy-reload.sh" 20 | EOT 21 | 22 | cp /usr/local/vamp/haproxy.basic.cfg "${dir_templates}/haproxy.tmpl" 23 | cat <> "${dir_templates}/haproxy.tmpl" 24 | {{getv "${VAMP_KEY_VALUE_STORE_PATH}"}} 25 | EOT 26 | 27 | 28 | # Generate config and template for Filebeat 29 | cat < "${dir_confd}/filebeat.toml" 30 | [template] 31 | src = "filebeat.tmpl" 32 | dest = "/usr/local/filebeat/filebeat.yml" 33 | EOT 34 | 35 | cat <> "${dir_templates}/filebeat.tmpl" 36 | filebeat.prospectors: 37 | - input_type: log 38 | paths: 39 | - /var/log/haproxy.log 40 | json.message_key: message 41 | json.add_error_key: false 42 | json.keys_under_root: true 43 | 44 | output.elasticsearch: 45 | hosts: ["$VAMP_ELASTICSEARCH_URL"] 46 | index: "vamp-vga-$VAMP_NAMESPACE-%{+yyyy-MM-dd}" 47 | template.path: \${path.config}/filebeat.template.json 48 | 49 | path.home: /usr/local/filebeat 50 | path.config: \${path.home} 51 | path.data: \${path.home}/data 52 | path.logs: /var/log 53 | EOT 54 | -------------------------------------------------------------------------------- /files/usr/local/vamp/haproxy-reload.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | [ -e /usr/local/vamp/good2go ] || exit 1 4 | 5 | haproxy_cfg="/usr/local/vamp/haproxy.cfg" 6 | haproxy_pid="/usr/local/vamp/haproxy.pid" 7 | haproxy_sock="/usr/local/vamp/haproxy.sock" 8 | 9 | [ -e ${haproxy_cfg} ] || (echo "haproxy-reload.sh: error: no such file: ${haproxy_cfg}" && exit 1) 10 | 11 | [ -e ${haproxy_pid} ] || touch ${haproxy_pid} 12 | 13 | if [ -e ${haproxy_sock} ]; then 14 | haproxy -f ${haproxy_cfg} -p ${haproxy_pid} -x ${haproxy_sock} -sf $(cat ${haproxy_pid}) 15 | else 16 | haproxy -f ${haproxy_cfg} -p ${haproxy_pid} -sf $(cat ${haproxy_pid}) 17 | fi 18 | -------------------------------------------------------------------------------- /files/usr/local/vamp/haproxy-validate.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | haproxy_cfg="/usr/local/vamp/haproxy.cfg" 4 | 5 | [[ -n $1 ]] && haproxy_cfg="$1" 6 | 7 | haproxy -c -f "$haproxy_cfg" 8 | -------------------------------------------------------------------------------- /files/usr/local/vamp/haproxy.basic.cfg: -------------------------------------------------------------------------------- 1 | global 2 | 3 | pidfile /usr/local/vamp/haproxy.pid 4 | server-state-file /usr/local/vamp/haproxy_server_state 5 | 6 | log 127.0.0.1:514 local0 7 | stats socket /usr/local/vamp/haproxy.sock expose-fd listeners 8 | 9 | daemon 10 | nbproc 1 11 | user root 12 | group root 13 | maxconn 4096 14 | spread-checks 5 15 | 16 | defaults 17 | log global 18 | mode http 19 | option dontlognull 20 | option redispatch 21 | option clitcpka 22 | option srvtcpka 23 | option http-keep-alive 24 | 25 | retries 3 26 | maxconn 500000 27 | 28 | # slowloris protection: clients should send their full http request in the configured time 29 | timeout http-request 5s 30 | timeout connect 5s 31 | timeout client 50s 32 | timeout server 50s 33 | timeout check 5s 34 | 35 | listen stats 36 | bind :1988 37 | mode http 38 | monitor-uri /health 39 | stats enable 40 | stats uri / 41 | stats refresh 5s 42 | stats realm Haproxy\ Stats 43 | stats auth haproxy:haproxy 44 | 45 | -------------------------------------------------------------------------------- /files/usr/local/vamp/tokenrenewer.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ -e /usr/local/vamp/token ]; then 4 | VAMP_KEY_VALUE_STORE_TOKEN="$( cat /usr/local/vamp/token )" 5 | fi 6 | 7 | TOKEN=${VAMP_KEY_VALUE_STORE_TOKEN} 8 | URL=${VAMP_KEY_VALUE_STORE_CONNECTION} 9 | 10 | echo "Token renewer started to check ${URL}" 11 | while true; do 12 | SLEEP_DURATION=0 13 | CREATION_TTL=0 14 | TTL=0 15 | if [[ -n ${CONFD_CLIENT_CAKEYS} ]]; then 16 | LOOKUP=$(curl --cacert ${CONFD_CLIENT_CAKEYS} -s --header "X-Vault-Token: ${TOKEN}" ${URL}/v1/auth/token/lookup-self) 17 | else 18 | LOOKUP=$(curl -s --header "X-Vault-Token: ${TOKEN}" ${URL}/v1/auth/token/lookup-self) 19 | fi 20 | CREATION_TTL=$(echo ${LOOKUP} | jq .data.creation_ttl) 21 | TTL=$(echo ${LOOKUP} | jq .data.ttl) 22 | echo "${TTL} seconds left for expiration, creation duration is ${CREATION_TTL} seconds" 23 | let "SLEEP_DURATION=${TTL}/2" 24 | if [ "${SLEEP_DURATION}" -lt 10 ] 25 | then 26 | echo "SLEEP_DURATION is set to 10" 27 | SLEEP_DURATION=10 28 | fi 29 | echo "{ \"increment\": \"${CREATION_TTL}\" }" > payload.json 30 | echo "Renewing the token" 31 | if [[ -n ${CONFD_CLIENT_CAKEYS} ]]; then 32 | RESULT=$(curl --cacert ${CONFD_CLIENT_CAKEYS} -s --header "X-Vault-Token: ${TOKEN}" --request POST --data @payload.json ${URL}/v1/auth/token/renew-self || CURL_RETURN_CODE=$?) 33 | else 34 | RESULT=$(curl -s --header "X-Vault-Token: ${TOKEN}" --request POST --data @payload.json ${URL}/v1/auth/token/renew-self || CURL_RETURN_CODE=$?) 35 | fi 36 | echo "New lease_duration: $(echo ${RESULT} | jq .auth.lease_duration) seconds" 37 | echo "Wait for ${SLEEP_DURATION} seconds" 38 | sleep ${SLEEP_DURATION} 39 | done 40 | --------------------------------------------------------------------------------