├── utils ├── file │ ├── testdata │ │ ├── config.zip │ │ ├── docker-empty.yml │ │ ├── docker-long.yml │ │ ├── docker-adhoc.yml │ │ ├── config │ │ │ └── docker-adhoc.yml │ │ └── yaml │ └── file_test.go ├── wait │ ├── types.go │ ├── wait_test.go │ └── wait.go ├── pod │ ├── testdata │ │ ├── docker-long.yml │ │ └── docker-adhoc.yml │ ├── service.go │ ├── types.go │ ├── service_test.go │ ├── util_test.go │ └── util.go └── http │ ├── http_test.go │ ├── transport.go │ └── http.go ├── examples ├── vagrant │ ├── mesos_config │ │ ├── etc_mesos-master │ │ │ ├── hostname │ │ │ ├── ip │ │ │ └── roles │ │ ├── etc_mesos-slave │ │ │ ├── ip │ │ │ ├── hostname │ │ │ └── modules │ │ └── README.md │ ├── config │ │ ├── daemon.json │ │ ├── general.yaml │ │ ├── config.yaml │ │ ├── config_example.yaml │ │ ├── docker-compose-executor.json │ │ └── aurora-scheduler.conf │ ├── marathon │ │ ├── marathon │ │ └── deploy.json │ ├── mesos-module │ │ ├── v1.0.0 │ │ │ └── libmesos_dce_hook.so │ │ └── v1.1.0 │ │ │ └── libmesos_dce_hook.so │ ├── clusters.json │ ├── nginx │ │ └── nginx.conf │ └── provision-dev-cluster.sh ├── sampleapp │ ├── docker-adhoc.yml │ ├── app │ │ ├── back.png │ │ ├── logo.png │ │ ├── Dockerfile │ │ ├── server.js │ │ └── index.html │ ├── docker-compose-healthcheck.yml │ ├── nginx │ │ ├── Dockerfile │ │ ├── sites-enabled │ │ │ └── nodejs_project │ │ ├── server.crt │ │ └── server.key │ └── docker-compose.yml ├── manifest │ ├── docker-compose-none.yml │ ├── docker-compose-bridge.yml │ ├── docker-compose-none-generated.yml │ ├── docker-compose-host.yml │ ├── docker-compose-host-generated.yml │ └── docker-compose-bridge-generated.yml └── client.go ├── docs ├── images │ ├── deletetask.png │ ├── launchtask.png │ └── podmonitor.png ├── environment.md └── how-to-use.md ├── .gitignore ├── pluginimpl ├── general │ ├── general.yaml │ ├── testdata │ │ ├── docker-extra-host.yml │ │ ├── docker-infra-container.yml │ │ ├── test.yml │ │ └── test.yml-generated.yml │ ├── impl_test.go │ ├── editor_test.go │ └── impl.go └── example │ └── impl.go ├── mesos-modules ├── hook │ ├── modules.json.in │ └── compose_pod_cleanup_hook.cpp ├── bootstrap ├── Makefile.am ├── README.rst ├── m4 │ ├── ax_cxx_compile_stdcxx_11.m4 │ └── ax_compare_version.m4 └── configure.ac ├── plugin ├── plugin.go ├── type.go └── extpoints.go ├── .github └── workflows │ └── ci.yml ├── config ├── config.yaml ├── config_test.go └── config.go ├── Makefile ├── Dockerfile-mesos-modules ├── dce └── monitor │ ├── monitor.go │ └── plugin │ └── default │ └── monitor.go ├── Vagrantfile ├── go.mod ├── README.md ├── types └── types.go └── LICENSE /utils/file/testdata/config.zip: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/file/testdata/docker-empty.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/vagrant/mesos_config/etc_mesos-master/hostname: -------------------------------------------------------------------------------- 1 | 192.168.33.8 -------------------------------------------------------------------------------- /examples/vagrant/mesos_config/etc_mesos-master/ip: -------------------------------------------------------------------------------- 1 | 192.168.33.8 2 | -------------------------------------------------------------------------------- /examples/vagrant/mesos_config/etc_mesos-master/roles: -------------------------------------------------------------------------------- 1 | aurora-role 2 | -------------------------------------------------------------------------------- /examples/vagrant/mesos_config/etc_mesos-slave/ip: -------------------------------------------------------------------------------- 1 | 192.168.33.8 2 | -------------------------------------------------------------------------------- /examples/vagrant/config/daemon.json: -------------------------------------------------------------------------------- 1 | { 2 | "live-restore": true 3 | } -------------------------------------------------------------------------------- /examples/vagrant/marathon/marathon: -------------------------------------------------------------------------------- 1 | MARATHON_DECLINE_OFFER_DURATION=5000 -------------------------------------------------------------------------------- /examples/vagrant/mesos_config/etc_mesos-slave/hostname: -------------------------------------------------------------------------------- 1 | 192.168.33.8 2 | -------------------------------------------------------------------------------- /utils/file/testdata/docker-long.yml: -------------------------------------------------------------------------------- 1 | version: "2.1" 2 | services: 3 | redis: 4 | image: redis -------------------------------------------------------------------------------- /docs/images/deletetask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/dce-go/HEAD/docs/images/deletetask.png -------------------------------------------------------------------------------- /docs/images/launchtask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/dce-go/HEAD/docs/images/launchtask.png -------------------------------------------------------------------------------- /docs/images/podmonitor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/dce-go/HEAD/docs/images/podmonitor.png -------------------------------------------------------------------------------- /examples/sampleapp/docker-adhoc.yml: -------------------------------------------------------------------------------- 1 | version: "2.1" 2 | services: 3 | adhoc: 4 | image: dcego/adhoc:1.0 -------------------------------------------------------------------------------- /examples/sampleapp/app/back.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/dce-go/HEAD/examples/sampleapp/app/back.png -------------------------------------------------------------------------------- /examples/sampleapp/app/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/dce-go/HEAD/examples/sampleapp/app/logo.png -------------------------------------------------------------------------------- /examples/manifest/docker-compose-none.yml: -------------------------------------------------------------------------------- 1 | version: "2.1" 2 | services: 3 | redis: 4 | image: redis 5 | network_mode: none -------------------------------------------------------------------------------- /utils/wait/types.go: -------------------------------------------------------------------------------- 1 | package wait 2 | 3 | import "sync" 4 | 5 | type LogCommandStatus struct { 6 | sync.RWMutex 7 | IsRunning bool 8 | } 9 | -------------------------------------------------------------------------------- /utils/pod/testdata/docker-long.yml: -------------------------------------------------------------------------------- 1 | version: "2.1" 2 | services: 3 | redis: 4 | image: redis 5 | container_name: redis_test 6 | network_mode: none -------------------------------------------------------------------------------- /examples/vagrant/mesos-module/v1.0.0/libmesos_dce_hook.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/dce-go/HEAD/examples/vagrant/mesos-module/v1.0.0/libmesos_dce_hook.so -------------------------------------------------------------------------------- /examples/vagrant/mesos-module/v1.1.0/libmesos_dce_hook.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paypal/dce-go/HEAD/examples/vagrant/mesos-module/v1.1.0/libmesos_dce_hook.so -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | dce.iml 3 | dce-go.iml 4 | .idea/ 5 | .vagrant/ 6 | .DS_Store 7 | dce/executor 8 | executor 9 | .gitmodules 10 | paypal-plugins/ 11 | -------------------------------------------------------------------------------- /examples/vagrant/config/general.yaml: -------------------------------------------------------------------------------- 1 | infracontainer: 2 | image: dcego/networkproxy:1.2 3 | container_name: networkproxy_0.1 4 | networks: 5 | pre_existing: false -------------------------------------------------------------------------------- /examples/vagrant/config/config.yaml: -------------------------------------------------------------------------------- 1 | launchtask: 2 | podmonitorinterval: 10s 3 | pullretry: 3 4 | maxretry: 10 5 | timeout: 500s 6 | plugins: 7 | pluginorder: general 8 | foldername: poddata -------------------------------------------------------------------------------- /pluginimpl/general/general.yaml: -------------------------------------------------------------------------------- 1 | infracontainer: 2 | image: dcego/networkproxy:1.2 3 | container_name: networkproxy_0.1 4 | networks: 5 | pre_existing: true 6 | name : net 7 | driver: bridge -------------------------------------------------------------------------------- /examples/sampleapp/app/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:0.10.38 2 | 3 | RUN mkdir /src 4 | 5 | WORKDIR /src 6 | ADD logo.png . 7 | ADD back.png . 8 | ADD index.html . 9 | ADD server.js . 10 | 11 | CMD ["node","server.js"] -------------------------------------------------------------------------------- /examples/vagrant/config/config_example.yaml: -------------------------------------------------------------------------------- 1 | launchtask: 2 | podmonitorinterval: 10s 3 | pullretry: 3 4 | maxretry: 10 5 | timeout: 500s 6 | plugins: 7 | pluginorder: general,example 8 | foldername: poddata -------------------------------------------------------------------------------- /pluginimpl/general/testdata/docker-extra-host.yml: -------------------------------------------------------------------------------- 1 | services: 2 | redis: 3 | image: redis:latest 4 | container_name: redis_20170310102123445 5 | extra_hosts: 6 | - "redisip:0.0.0.0" 7 | - "redisip1:0.0.0.1" -------------------------------------------------------------------------------- /examples/sampleapp/docker-compose-healthcheck.yml: -------------------------------------------------------------------------------- 1 | version: "2.1" 2 | services: 3 | web: 4 | healthcheck: 5 | test: curl -f http://localhost:8081/index.html 6 | interval: 10s 7 | timeout: 10s 8 | retries: 3 9 | 10 | 11 | -------------------------------------------------------------------------------- /examples/vagrant/clusters.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "name": "devcluster", 3 | "zk": "192.168.33.8", 4 | "scheduler_zk_path": "/aurora/scheduler", 5 | "auth_mechanism": "UNAUTHENTICATED", 6 | "slave_run_directory": "latest", 7 | "slave_root": "/var/lib/mesos" 8 | }] -------------------------------------------------------------------------------- /examples/vagrant/nginx/nginx.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80; 3 | 4 | server_name docker-compose.local; 5 | access_log /var/log/nginx/access.log; 6 | error_log /var/log/nginx/error.log; 7 | root /var/www; 8 | 9 | location / { 10 | } 11 | } -------------------------------------------------------------------------------- /pluginimpl/general/testdata/docker-infra-container.yml: -------------------------------------------------------------------------------- 1 | networks: 2 | default: 3 | external: 4 | name: net 5 | services: 6 | networkproxy: 7 | container_name: networkproxy_0.1 8 | image: dcego/networkproxy:1.2 9 | version: "2.1" 10 | -------------------------------------------------------------------------------- /utils/pod/service.go: -------------------------------------------------------------------------------- 1 | package pod 2 | 3 | import ( 4 | "github.com/paypal/dce-go/types" 5 | ) 6 | 7 | func GetServiceDetail() types.ServiceDetail { 8 | return ServiceDetail 9 | } 10 | 11 | func SetServiceDetail(sd types.ServiceDetail) { 12 | ServiceDetail = sd 13 | } 14 | -------------------------------------------------------------------------------- /utils/file/testdata/docker-adhoc.yml: -------------------------------------------------------------------------------- 1 | version: "2.1" 2 | services: 3 | redis: 4 | container_name: redis1 5 | image: redis 6 | healthcheck: 7 | test: cat /etc/passwd 8 | interval: 3s 9 | timeout: 1s 10 | retries: 5 11 | command: cat /etc/passwd -------------------------------------------------------------------------------- /utils/pod/testdata/docker-adhoc.yml: -------------------------------------------------------------------------------- 1 | version: "2.1" 2 | services: 3 | redis: 4 | container_name: redis1 5 | image: redis 6 | healthcheck: 7 | test: cat /etc/passwd 8 | interval: 3s 9 | timeout: 1s 10 | retries: 5 11 | command: cat /etc/passwd -------------------------------------------------------------------------------- /utils/file/testdata/config/docker-adhoc.yml: -------------------------------------------------------------------------------- 1 | version: "2.1" 2 | services: 3 | redis: 4 | container_name: redis1 5 | image: redis 6 | healthcheck: 7 | test: cat /etc/passwd 8 | interval: 3s 9 | timeout: 1s 10 | retries: 5 11 | command: cat /etc/passwd -------------------------------------------------------------------------------- /mesos-modules/hook/modules.json.in: -------------------------------------------------------------------------------- 1 | { 2 | "libraries": [ 3 | { 4 | "file": "/output/usr/lib/mesos/modules/libmesos_dce_hook.so", 5 | "modules": [ 6 | { 7 | "name": "org_apache_mesos_ComposePodCleanupHook" 8 | } 9 | ] 10 | } 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /examples/sampleapp/nginx/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM tutum/nginx 2 | RUN rm /etc/nginx/sites-enabled/default 3 | ADD sites-enabled/ /etc/nginx/sites-enabled 4 | RUN mkdir -p /etc/nginx/ssl/certs 5 | RUN mkdir -p /etc/nginx/ssl/private 6 | 7 | COPY server.key /etc/nginx/ssl/private/ 8 | COPY server.crt /etc/nginx/ssl/certs/ -------------------------------------------------------------------------------- /examples/vagrant/mesos_config/etc_mesos-slave/modules: -------------------------------------------------------------------------------- 1 | { 2 | "libraries": [ 3 | { 4 | "file": "/output/usr/local/lib/mesos/libmesos_dce_hook.so", 5 | "modules": [ 6 | { 7 | "name": "org_apache_mesos_ComposePodCleanupHook" 8 | } 9 | ] 10 | } 11 | ] 12 | } -------------------------------------------------------------------------------- /utils/pod/types.go: -------------------------------------------------------------------------------- 1 | package pod 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/paypal/dce-go/types" 7 | ) 8 | 9 | type Status struct { 10 | sync.RWMutex 11 | Status types.PodStatus 12 | // if set to true, indicates that the pod was launched successfully and task moved to RUNNING state 13 | Launched bool 14 | } 15 | -------------------------------------------------------------------------------- /examples/sampleapp/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2.1" 2 | services: 3 | web: 4 | image: dcego/web:1.0 5 | volumes: 6 | - "./app:/src/app" 7 | ports: 8 | - "8081:8081" 9 | nginx: 10 | image: dcego/nginx:1.0 11 | ports: 12 | - "80:80" 13 | - "443:443" 14 | volumes: 15 | - /www/public 16 | volumes_from: 17 | - web 18 | -------------------------------------------------------------------------------- /examples/manifest/docker-compose-bridge.yml: -------------------------------------------------------------------------------- 1 | # single port and map port 2 | version: "2.1" 3 | services: 4 | web: 5 | image: dcego/web:1.0 6 | volumes: 7 | - "./app:/src/app" 8 | ports: 9 | - "8081" 10 | nginx: 11 | image: dcego/nginx:1.0 12 | ports: 13 | - "80:80" 14 | - "443:443" 15 | volumes: 16 | - /www/public 17 | volumes_from: 18 | - web -------------------------------------------------------------------------------- /examples/manifest/docker-compose-none-generated.yml: -------------------------------------------------------------------------------- 1 | services: 2 | redis: 3 | cgroup_parent: /mesos/0b7e302e-103c-4fbe-a457-460befbae5a6 4 | image: redis 5 | labels: 6 | com.company.label: awesome 7 | executorId: marathon-docker-compose-demo.4da44f8e-463e-11e7-b4d3-866758312cc8 8 | taskId: docker-compose-demo.4da44f8e-463e-11e7-b4d3-866758312cc8 9 | network_mode: none 10 | version: "2.1" -------------------------------------------------------------------------------- /examples/manifest/docker-compose-host.yml: -------------------------------------------------------------------------------- 1 | # keep host mode...no concept of pod...not recommend 2 | # don't support multi network mode in one pod...make sure network mode of services are same(TBD) 3 | version: "2.1" 4 | services: 5 | web: 6 | image: dcego/web:2.0 7 | volumes: 8 | - "./app:/src/app" 9 | network_mode: host 10 | nginx: 11 | image: dcego/nginx:2.0 12 | network_mode: host 13 | volumes: 14 | - /www/public 15 | volumes_from: 16 | - web -------------------------------------------------------------------------------- /examples/vagrant/marathon/deploy.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "docker-compose-demo", 3 | "cmd": " ", 4 | "cpus": 0.5, 5 | "mem": 64.0, 6 | "ports":[0,0,0], 7 | "instances": 1, 8 | "executor":"./executor", 9 | "labels": { 10 | "fileName": "sample-app/docker-compose.yml" 11 | }, 12 | "uris":["https://github.com/mesos/docker-compose-executor/releases/download/0.1.0/sample-app.tar.gz","http://192.168.33.7:8080/config.yaml","http://192.168.33.7:8080/general.yaml","http://192.168.33.7:8080/executor"] 13 | } -------------------------------------------------------------------------------- /pluginimpl/general/testdata/test.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | services: 3 | mysql: 4 | image: mysql 5 | restart: always 6 | container_name: mysql 7 | ports: 8 | - "81:3306" 9 | environment: 10 | - MYSQL_ROOT_PASSWORD=root 11 | - MYSQL_DATABASE=ghost 12 | - MYSQL_USER=ghost 13 | - MYSQL_PASSWORD=password 14 | extra_hosts: 15 | - ip:0.0.0.0 16 | ghost: 17 | image: dockerhub.paypalcorp.com/kkrishna/ghostlocal:1.0 18 | container_name: ghostlocal 19 | depends_on: 20 | - mysql 21 | ports: 22 | - "80:2368" 23 | labels: 24 | - "primary.container=true" -------------------------------------------------------------------------------- /examples/vagrant/mesos_config/README.md: -------------------------------------------------------------------------------- 1 | # Configuring the mesos package provided by mesosphere 2 | 3 | The mesosphere mesos package for debian is configured by creating files whose names match command 4 | line argument and environment variables. 5 | 6 | When setting up our environment, the files under directories `etc_mesos-master` and 7 | `etc_mesos-slave` are copied to `/etc/mesos-master` and `/etc/mesos-slave`, respectively. 8 | 9 | For more details, and the script that reads these files, see 10 | [here](https://github.com/mesosphere/mesos-deb-packaging/blob/fd3c3866a847d07a8beb0cf8811f173406f910df/mesos-init-wrapper#L12-L48). 11 | -------------------------------------------------------------------------------- /plugin/plugin.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package plugin 16 | 17 | func GetOrderedExtpoints(plugins []string) []ComposePlugin { 18 | return ComposePlugins.Select(plugins) 19 | } 20 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ develop ] 6 | pull_request: 7 | branches: [ develop ] 8 | 9 | jobs: 10 | 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: 1.21 20 | 21 | - name: Install goimports 22 | run: go get golang.org/x/tools/cmd/goimports 23 | 24 | - name: Set env with list of directories in repo containin go code 25 | run: echo GO_USR_DIRS=$(go list -f {{.Dir}} ./... | grep -E -v "/gen-go/|/vendor/") >> $GITHUB_ENV 26 | 27 | - name: Run goimports check 28 | run: test -z "`for d in $GO_USR_DIRS; do goimports -d $d/*.go | tee /dev/stderr; done`" 29 | 30 | - name: Build 31 | run: go build -o executor dce/main.go 32 | -------------------------------------------------------------------------------- /pluginimpl/general/testdata/test.yml-generated.yml: -------------------------------------------------------------------------------- 1 | services: 2 | ghost: 3 | cgroup_parent: /mesos/general 4 | container_name: taskId_ghostlocal 5 | depends_on: 6 | - mysql 7 | environment: 8 | PYTHONUNBUFFERED: 1 9 | image: dockerhub.paypalcorp.com/kkrishna/ghostlocal:1.0 10 | labels: 11 | - primary.container=true 12 | - executorId=executorId 13 | - taskId=taskId 14 | network_mode: service:networkproxy 15 | mysql: 16 | cgroup_parent: /mesos/general 17 | container_name: taskId_mysql 18 | environment: 19 | - MYSQL_ROOT_PASSWORD=root 20 | - MYSQL_DATABASE=ghost 21 | - MYSQL_USER=ghost 22 | - MYSQL_PASSWORD=password 23 | - PYTHONUNBUFFERED=1 24 | image: mysql 25 | labels: 26 | executorId: executorId 27 | taskId: taskId 28 | network_mode: service:networkproxy 29 | version: "2.1" 30 | -------------------------------------------------------------------------------- /examples/manifest/docker-compose-host-generated.yml: -------------------------------------------------------------------------------- 1 | services: 2 | nginx: 3 | cgroup_parent: /mesos/b9d0efac-6ba0-4a34-ab5f-a8e3cfd4d0a3 4 | image: dcego/nginx:2.0 5 | labels: 6 | com.company.label: awesome 7 | executorId: marathon-docker-compose-demo.ddefa95d-463d-11e7-b4d3-866758312cc8 8 | taskId: docker-compose-demo.ddefa95d-463d-11e7-b4d3-866758312cc8 9 | network_mode: host 10 | volumes: 11 | - /www/public 12 | volumes_from: 13 | - web 14 | web: 15 | cgroup_parent: /mesos/b9d0efac-6ba0-4a34-ab5f-a8e3cfd4d0a3 16 | image: dcego/web:2.0 17 | labels: 18 | com.company.label: awesome 19 | executorId: marathon-docker-compose-demo.ddefa95d-463d-11e7-b4d3-866758312cc8 20 | taskId: docker-compose-demo.ddefa95d-463d-11e7-b4d3-866758312cc8 21 | network_mode: host 22 | volumes: 23 | - ./app:/src/app 24 | version: "2.1" -------------------------------------------------------------------------------- /config/config.yaml: -------------------------------------------------------------------------------- 1 | launchtask: 2 | podmonitorinterval: 10s 3 | composehttptimeout: 300 4 | pullretry: 3 5 | maxretry: 3 6 | retryinterval: 10s 7 | timeout: 500s 8 | skippull: true 9 | composetrace: true 10 | debug: false 11 | httptimeout: 20s 12 | plugins: 13 | pluginorder: general 14 | podStatusHooks: 15 | task_launch: [] 16 | task_failed: [] 17 | task_finished: [] 18 | task_running: [] 19 | cleanpod: 20 | cleanfailtask: false 21 | timeout: 20s 22 | unhealthy: true 23 | cleanvolumeandcontaineronmesoskill: true 24 | cleanimageonmesoskill: false 25 | dockerdump: 26 | enable: false 27 | dumppath: /home/ubuntu 28 | dockerpidfile: /var/run/docker.pid 29 | containerpidfile: /run/docker/libcontainerd/docker-containerd.pid 30 | dockerlogpath: /var/log/upstart/docker.log 31 | dockercomposeverbose: false 32 | podMonitor: 33 | monitorName: default 34 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | #Licensed under the Apache License, Version 2.0 (the "License"); \ 2 | you may not use this file except in compliance with the License. \ 3 | You may obtain a copy of the License at \ 4 | http://www.apache.org/licenses/LICENSE-2.0 \ 5 | Unless required by applicable law or agreed to in writing, software \ 6 | distributed under the License is distributed on an "AS IS" BASIS, \ 7 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. \ 8 | See the License for the specific language governing permissions and \ 9 | limitations under the License. 10 | 11 | # set relative path to dockerfile 12 | BUILD_PATH := /home/vagrant/go/src/github.com/paypal/dce-go/dce 13 | 14 | 15 | all: build 16 | @echo "build binary file" 17 | 18 | build: 19 | go build -o executor ${BUILD_PATH}/main.go 20 | sudo mv executor /home/vagrant 21 | 22 | upload: 23 | sudo cp /home/vagrant/executor /usr/share/nginx/html -------------------------------------------------------------------------------- /examples/sampleapp/nginx/sites-enabled/nodejs_project: -------------------------------------------------------------------------------- 1 | server { 2 | 3 | listen 443 ssl; 4 | server_name example.com; 5 | 6 | # 7 | # SSL Handling 8 | # 9 | ssl on; 10 | ssl_protocols TLSv1 TLSv1.1 TLSv1.2; 11 | ssl_certificate /etc/nginx/ssl/certs/server.crt; 12 | ssl_certificate_key /etc/nginx/ssl/private/server.key; 13 | ssl_ciphers HIGH:!aNULL:!MD5:!kEDH; 14 | ssl_prefer_server_ciphers on; 15 | ssl_session_cache shared:SSL:10m; 16 | ssl_session_timeout 10m; 17 | 18 | location /public { 19 | alias /src/app/public; 20 | } 21 | 22 | location / { 23 | proxy_set_header Host $host; 24 | proxy_set_header X-Real-IP $remote_addr; 25 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 26 | proxy_set_header X-Forwarded-Proto $scheme; 27 | 28 | proxy_pass http://localhost:8081; 29 | proxy_read_timeout 90; 30 | proxy_redirect http://localhost:8081 https://example.com; 31 | } 32 | 33 | } -------------------------------------------------------------------------------- /docs/environment.md: -------------------------------------------------------------------------------- 1 | ## Installing environment 2 | 3 | This vagrant box helps you quickly create DCE environment which has mesos, marathon, aurora, golang1.7, docker, docker-compose and DCE installed. DCE will be compiled as a binary file called "executor" and you can find it at /home/vagrant. 4 | 5 | ### Requirements 6 | 7 | * Linux/Unix/Mac OS X 8 | * VirtualBox 9 | * Vagrant 10 | * Git 11 | 12 | ### Steps 13 | 1. [Install VirtualBox](https://www.virtualbox.org/wiki/Downloads) 14 | 15 | 2. [Install Vagrant](https://www.vagrantup.com/downloads.html) 16 | 3. Git clone this repository 17 | ``` 18 | git clone https://github.com/paypal/dce-go 19 | 20 | cd dce-go 21 | ``` 22 | 4. Start vagrant box 23 | 24 | ``` 25 | vagrant up 26 | ``` 27 | 5. Validate installation 28 | 29 | ``` 30 | mesos endpoint : http://192.168.33.8:5050 31 | 32 | marathon endpoint: http://192.168.33.8:8080/ui 33 | 34 | aurora endpoint: http://192.168.33.8:8081 35 | 36 | ``` 37 | 6. ssh to the vagrant box 38 | 39 | ``` 40 | vagrant ssh 41 | ``` 42 | 7. Next: [How to use](how-to-use.md) 43 | 44 | -------------------------------------------------------------------------------- /utils/pod/service_test.go: -------------------------------------------------------------------------------- 1 | package pod 2 | 3 | import ( 4 | "testing" 5 | 6 | "gopkg.in/yaml.v2" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestUpdateServiceDetail(t *testing.T) { 12 | // prepare taskInfo 13 | t.Run("manual-set", func(t *testing.T) { 14 | s := GetServiceDetail() 15 | assert.Empty(t, s) 16 | 17 | s["a1"] = make(map[string]interface{}) 18 | s["a1"]["b1"] = "c1" 19 | s["a1"]["b2"] = 2 20 | s["a1"]["b3"] = []interface{}{"c1, c2"} 21 | 22 | SetServiceDetail(s) 23 | 24 | s1 := GetServiceDetail() 25 | assert.EqualValues(t, s, s1) 26 | }) 27 | 28 | t.Run("parse-yaml", func(t *testing.T) { 29 | 30 | m := make(map[string]interface{}) 31 | sd := GetServiceDetail() 32 | 33 | var yamlFile = []byte(` 34 | Hacker: true 35 | name: steve 36 | hobbies: 37 | - skateboarding 38 | - snowboarding 39 | - go 40 | clothing: 41 | jacket: leather 42 | trousers: denim 43 | age: 35 44 | eyes : brown 45 | beard: true 46 | `) 47 | 48 | err := yaml.Unmarshal(yamlFile, &m) 49 | assert.NoError(t, err) 50 | 51 | sd["f1"] = m 52 | 53 | SetServiceDetail(sd) 54 | 55 | s1 := GetServiceDetail() 56 | assert.EqualValues(t, sd, s1) 57 | }) 58 | 59 | } 60 | -------------------------------------------------------------------------------- /examples/vagrant/config/docker-compose-executor.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "executor": { 4 | "command": { 5 | "value": "./executor -log_dir=/var/log -logtostderr=true -v=1", 6 | "shell": "true", 7 | "uris": [ 8 | { 9 | "cache": false, 10 | "executable": true, 11 | "extract": false, 12 | "value": "http://192.168.33.8/executor" 13 | }, 14 | { 15 | "cache": false, 16 | "executable": false, 17 | "extract": false, 18 | "value": "http://192.168.33.8/config.yaml" 19 | }, 20 | { 21 | "cache": false, 22 | "executable": false, 23 | "extract": false, 24 | "value": "http://192.168.33.8/general.yaml" 25 | } 26 | ] 27 | }, 28 | "name": "docker-compose-executor", 29 | "resources": [ 30 | { 31 | "name": "cpus", 32 | "scalar": { 33 | "value": 0.25 34 | }, 35 | "type": "SCALAR" 36 | }, 37 | { 38 | "name": "mem", 39 | "scalar": { 40 | "value": 256 41 | }, 42 | "type": "SCALAR" 43 | } 44 | ] 45 | }, 46 | "task_prefix": "compose-" 47 | } 48 | ] 49 | -------------------------------------------------------------------------------- /Dockerfile-mesos-modules: -------------------------------------------------------------------------------- 1 | #Mesos build instructions from https://mesos.apache.org/gettingstarted/ 2 | FROM ubuntu:trusty 3 | 4 | RUN apt-get update 5 | ENV DEBIAN_FRONTEND noninteractive 6 | RUN apt-get install -y tar wget git 7 | RUN apt-get install -y openjdk-7-jdk 8 | RUN apt-get install -y autoconf libtool 9 | RUN apt-get install -y build-essential python-dev libcurl4-nss-dev libsasl2-dev libsasl2-modules maven libapr1-dev libsvn-dev 10 | RUN apt-get install -y libcurl3 libcurl3-nss 11 | 12 | ARG MESOS_VERSION=master 13 | ARG MESOS_REPO=https://git-wip-us.apache.org/repos/asf/mesos.git 14 | ENV MESOS_VERSION=${MESOS_VERSION} 15 | ENV MESOS_REPO=${MESOS_REPO} 16 | 17 | RUN git clone -b ${MESOS_VERSION} ${MESOS_REPO} /mesos 18 | WORKDIR /mesos 19 | RUN ./bootstrap 20 | RUN mkdir build 21 | 22 | WORKDIR /mesos/build 23 | 24 | RUN ../configure --enable-optimize --enable-install-module-dependencies --disable-java --disable-optimize && make -j2 V=0 && make install 25 | 26 | COPY mesos-modules /mesos-modules 27 | WORKDIR /mesos-modules 28 | RUN ./bootstrap && \ 29 | rm -Rf build && \ 30 | mkdir build 31 | WORKDIR /mesos-modules/build 32 | RUN PATH=/usr/local/lib/mesos/3rdparty/bin:$PATH ../configure --with-mesos-root=/usr/local/lib/mesos --with-mesos-build-dir=/usr/local/lib/mesos/3rdparty && make 33 | 34 | VOLUME ["/output"] 35 | 36 | CMD "DESTDIR=/output make install" -------------------------------------------------------------------------------- /examples/sampleapp/nginx/server.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDmjCCAoICCQDAqCg4DmQtmjANBgkqhkiG9w0BAQsFADCBjjELMAkGA1UEBhMC 3 | VVMxEzARBgNVBAgTCkNhbGlmb3JuaWExEDAOBgNVBAcTB1Nhbkpvc2UxEDAOBgNV 4 | BAoTB2NvbXBhbnkxEDAOBgNVBAsTB3NlY3Rpb24xDDAKBgNVBAMTA01pYTEmMCQG 5 | CSqGSIb3DQEJARYXamlhbWluLnNpZ251cEBnbWFpbC5jb20wHhcNMTcwNTAzMTgz 6 | NzMyWhcNMTgwNTAzMTgzNzMyWjCBjjELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNh 7 | bGlmb3JuaWExEDAOBgNVBAcTB1Nhbkpvc2UxEDAOBgNVBAoTB2NvbXBhbnkxEDAO 8 | BgNVBAsTB3NlY3Rpb24xDDAKBgNVBAMTA01pYTEmMCQGCSqGSIb3DQEJARYXamlh 9 | bWluLnNpZ251cEBnbWFpbC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK 10 | AoIBAQDHF5icm2t/2vu6x43xI8G63uUA8TEMliqo+KXhmnmeCpxs3YiWIRy58HUx 11 | N+a+W8KVRRsyvB2GrSqOKSGC1GwhU+/mS7XXpDJCkwQY0mOLF7ZKkfsg8ByeB40/ 12 | 7b3kTeN05qogAIUiWPkfJA9H+jgvszQ5crA08D6X5z1qSH+rCGgVXjvcVm/5FrRU 13 | hC+d/8kFyZXQv5aP0pyvT3DSJK5GTyBL62CXxSSDWXY88V/owdYk2lERkEkTek0k 14 | BNqWrh31hHy6lnqUiHm/vGsKO7LsuB+/pl7GTEV1uyacGf6X9bKupEiwOwIEm42T 15 | zzesXrWOE0vJxXdHlE6BEeg8CguxAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAFMe 16 | Wm1/UnaJcUQaKVN5oqurl4wbt012QPmkDC2s/EJPMHAdWXlmmwWo86+iLbVdqFEA 17 | x5rK4r30B7HH+q127eBU5HEHQUCFKsWHssc6Bjh0biMGAu29WeEe3zRoEFnc5fuz 18 | bXftl7WXIjS8hiTCcYhef738EmZ4WWTt8Hiv4isP7JOisG3Y6JGfimAhHOFNoOap 19 | oUO9RU2ySJpmDiC9eiPHNnuR2Eyswy8TWp2vq1wIoHdXR3M6d2GilAuPgJ7mcLkW 20 | tePDKDqy/qPy605mvCGNELdljSEpYv3MPbMnFZ9C9xZ9QMgMEb+1YGoGiyVAHXDi 21 | QLTBBIc7ohTML+1jfQM= 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /dce/monitor/monitor.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | // Monitor pod status 16 | package monitor 17 | 18 | import ( 19 | "context" 20 | 21 | "github.com/paypal/dce-go/config" 22 | "github.com/paypal/dce-go/plugin" 23 | "github.com/paypal/dce-go/types" 24 | "github.com/pkg/errors" 25 | log "github.com/sirupsen/logrus" 26 | ) 27 | 28 | // MonitorPoller inspects pod status periodically 29 | func MonitorPoller(ctx context.Context) (types.PodStatus, error) { 30 | logger := log.WithFields(log.Fields{ 31 | "func": "monitor.MonitorPoller", 32 | }) 33 | 34 | logger.Println("====================Pod Monitor Poller====================") 35 | 36 | name := config.GetConfig().GetString("podMonitor.monitorName") 37 | monitor := plugin.Monitors.Lookup(name) 38 | if monitor == nil { 39 | return types.POD_FAILED, errors.Errorf("monitor plugin %s doesn't exist", name) 40 | } 41 | return monitor.Start(ctx) 42 | } 43 | -------------------------------------------------------------------------------- /mesos-modules/bootstrap: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Make sure that we are in the right directory. 4 | if test ! -f configure.ac; then 5 | cat >&2 <<__EOF__ 6 | 7 | You must run bootstrap from the root of the distribution. 8 | 9 | __EOF__ 10 | exit 1 11 | fi 12 | 13 | if [ -n "$AUTOMAKE" ] || [ -n "$ACLOCAL" ] ; then 14 | if [ -z "$ACLOCAL" ] || [ -z "$AUTOMAKE" ] ; then 15 | _present="AUTOMAKE" 16 | _missing="ACLOCAL" 17 | 18 | [ -n "$ACLOCAL" ] && _present="ACLOCAL" && _missing="AUTOMAKE" 19 | 20 | cat >&2 <<__EOF__ 21 | 22 | [ERROR]: You are providing the path to ${_present} 23 | through your environment but no reference to ${_missing}. 24 | To fix this error please specify ${_missing} too. 25 | 26 | As an example, if you are using automake-1.12 and have 27 | available aclocal-1.12 you will want to do the following: 28 | 29 | AUTOMAKE="/usr/local/bin/automake-1.12" \\ 30 | ACLOCAL="/usr/local/bin/aclocal-1.12" \\ 31 | ./bootstrap 32 | 33 | Your current environment has: 34 | AUTOMAKE="$AUTOMAKE" 35 | ACLOCAL="$ACLOCAL" 36 | 37 | __EOF__ 38 | exit 1 39 | fi 40 | else 41 | AUTOMAKE="$(which automake)" 42 | fi 43 | 44 | 45 | # Note that we don't use '--no-recursive' because older versions of 46 | # autoconf/autoreconf bail with that option. Unfortunately this means 47 | # that we'll modify a lot of files in 3rdparty/libprocess, but that 48 | # may change in the future. 49 | 50 | autoreconf --install --force -Wall --verbose "${@}" 51 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! 18 | VAGRANTFILE_API_VERSION = "2" 19 | 20 | # 1.5.0 is required to use vagrant cloud images. 21 | # https://www.vagrantup.com/blog/vagrant-1-5-and-vagrant-cloud.html 22 | Vagrant.require_version ">= 1.5.0" 23 | 24 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 25 | config.vm.hostname = "docker-compose.local" 26 | config.vm.box = "ubuntu/trusty64" 27 | 28 | config.vm.define "devcluster" do |dev| 29 | dev.vm.network :private_network, ip: "192.168.33.8" 30 | dev.vm.provider :virtualbox do |vb| 31 | vb.customize ["modifyvm", :id, "--memory", "3072"] 32 | vb.customize ["modifyvm", :id, "--cpus", 2] 33 | vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] 34 | end 35 | dev.vm.provision "shell", path: "examples/vagrant/provision-dev-cluster.sh" 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /mesos-modules/Makefile.am: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License 16 | 17 | AUTOMAKE_OPTIONS = subdir-objects 18 | ACLOCAL_AMFLAGS = -I m4 19 | 20 | # We want to install modules in mesos directory. 21 | pkglibdir = $(libdir)/mesos 22 | 23 | # Initialize variables here so we can use += operator everywhere else. 24 | pkglib_LTLIBRARIES = 25 | 26 | # Add compiler and linker flags for pthreads. 27 | AM_CXXFLAGS = $(PTHREAD_CFLAGS) 28 | AM_LIBS = $(PTHREAD_LIBS) 29 | 30 | # Setup CPPFLAGS that are used for most source files. 31 | AM_CPPFLAGS = $(MESOS_CPPFLAGS) -Wall -Werror 32 | 33 | # Library containing test hook module. 34 | pkglib_LTLIBRARIES += libmesos_dce_hook.la 35 | libmesos_dce_hook_la_SOURCES = hook/compose_pod_cleanup_hook.cpp 36 | libmesos_dce_hook_la_LDFLAGS = -release $(PACKAGE_VERSION) -shared $(MESOS_LDFLAGS) 37 | -------------------------------------------------------------------------------- /examples/manifest/docker-compose-bridge-generated.yml: -------------------------------------------------------------------------------- 1 | # docker-compose-bridge.yml-generated.yml 2 | services: 3 | nginx: 4 | cgroup_parent: /mesos/9dfb4f24-cdd9-42c1-8324-c0bb3fc470d6 5 | image: dcego/nginx:1.0 6 | labels: 7 | com.company.label: awesome 8 | executorId: marathon-docker-compose-demo.fa8db2e8-4639-11e7-b4d3-866758312cc8 9 | taskId: docker-compose-demo.fa8db2e8-4639-11e7-b4d3-866758312cc8 10 | network_mode: service:networkproxy 11 | volumes: 12 | - /www/public 13 | volumes_from: 14 | - web 15 | web: 16 | cgroup_parent: /mesos/9dfb4f24-cdd9-42c1-8324-c0bb3fc470d6 17 | image: dcego/web:1.0 18 | labels: 19 | com.company.label: awesome 20 | executorId: marathon-docker-compose-demo.fa8db2e8-4639-11e7-b4d3-866758312cc8 21 | taskId: docker-compose-demo.fa8db2e8-4639-11e7-b4d3-866758312cc8 22 | network_mode: service:networkproxy 23 | volumes: 24 | - ./app:/src/app 25 | version: "2.1" 26 | 27 | # docker-infra-container.yml-generated.yml 28 | networks: 29 | default: 30 | driver: bridge 31 | services: 32 | networkproxy: 33 | cgroup_parent: /mesos/9dfb4f24-cdd9-42c1-8324-c0bb3fc470d6 34 | container_name: docker-compose-demo.fa8db2e8-4639-11e7-b4d3-866758312cc8_networkproxy_0.1 35 | image: dcego/networkproxy:1.2 36 | labels: 37 | com.company.label: awesome 38 | executorId: marathon-docker-compose-demo.fa8db2e8-4639-11e7-b4d3-866758312cc8 39 | taskId: docker-compose-demo.fa8db2e8-4639-11e7-b4d3-866758312cc8 40 | networks: 41 | - default 42 | ports: 43 | - 32783:8081 44 | - 31545:80 45 | - 31546:443 46 | version: "2.1" -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/paypal/dce-go 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/apache/thrift v0.16.0 7 | github.com/fsnotify/fsnotify v1.4.9 // indirect 8 | github.com/gogo/protobuf v1.3.2 // indirect 9 | github.com/google/uuid v1.3.0 // indirect 10 | github.com/mesos/mesos-go v0.0.11 11 | github.com/mitchellh/mapstructure v1.3.2 // indirect 12 | github.com/paypal/gorealis v1.22.4 13 | github.com/pborman/uuid v1.2.0 // indirect 14 | github.com/pelletier/go-toml v1.8.0 // indirect 15 | github.com/pkg/errors v0.9.1 16 | github.com/samuel/go-zookeeper v0.0.0-20200724154423-2164a8ac840e // indirect 17 | github.com/sirupsen/logrus v1.9.3 18 | github.com/spf13/afero v1.9.2 // indirect 19 | github.com/spf13/cast v1.3.1 // indirect 20 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 21 | github.com/spf13/pflag v1.0.5 // indirect 22 | github.com/spf13/viper v1.7.1 23 | github.com/stretchr/testify v1.8.3 24 | golang.org/x/net v0.10.0 // indirect 25 | golang.org/x/sys v0.16.0 // indirect 26 | golang.org/x/text v0.9.0 // indirect 27 | gopkg.in/ini.v1 v1.57.0 // indirect 28 | gopkg.in/yaml.v2 v2.4.0 29 | gopkg.in/yaml.v3 v3.0.1 // indirect 30 | ) 31 | 32 | require ( 33 | github.com/davecgh/go-spew v1.1.1 // indirect 34 | github.com/golang/glog v1.1.0 // indirect 35 | github.com/hashicorp/hcl v1.0.0 // indirect 36 | github.com/magiconair/properties v1.8.1 // indirect 37 | github.com/pmezard/go-difflib v1.0.0 // indirect 38 | github.com/subosito/gotenv v1.2.0 // indirect 39 | ) 40 | 41 | replace ( 42 | github.com/apache/thrift => github.com/ridv/thrift v0.13.2 43 | google.golang.org/genproto => google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 44 | ) 45 | -------------------------------------------------------------------------------- /examples/sampleapp/app/server.js: -------------------------------------------------------------------------------- 1 | var http = require('http'); 2 | var fs = require('fs'); 3 | var url = require('url'); 4 | 5 | // Create a server 6 | http.createServer( function (request, response) { 7 | // Parse the request containing file name 8 | var pathname = url.parse(request.url).pathname; 9 | // Print the name of the file for which request is made. 10 | console.log("Request for " + pathname + " received."); 11 | 12 | if (pathname == '/back.png') { 13 | var img = fs.readFileSync('./back.png'); 14 | response.writeHead(200, {'Content-Type': 'image/png' }); 15 | response.end(img, 'binary'); 16 | } 17 | 18 | if (pathname == '/logo.png') { 19 | var img = fs.readFileSync('./logo.png'); 20 | response.writeHead(200, {'Content-Type': 'image/png' }); 21 | response.end(img, 'binary'); 22 | } 23 | 24 | // Read the requested file content from file system 25 | fs.readFile(pathname.substr(1), function (err, data) { 26 | if (err) { 27 | console.log(err); 28 | // HTTP Status: 404 : NOT FOUND 29 | // Content Type: text/plain 30 | response.writeHead(404, {'Content-Type': 'text/html'}); 31 | }else { 32 | //Page found 33 | // HTTP Status: 200 : OK 34 | // Content Type: text/plain 35 | response.writeHead(200, {'Content-Type': 'text/html'}); 36 | 37 | // Write the content of the file to response body 38 | response.write(data.toString()); 39 | } 40 | // Send the response body 41 | response.end(); 42 | }); 43 | }).listen(8081); 44 | 45 | // Console will print the message 46 | console.log('Server running at http://127.0.0.1:8081/'); 47 | -------------------------------------------------------------------------------- /examples/sampleapp/nginx/server.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpQIBAAKCAQEAxxeYnJtrf9r7useN8SPBut7lAPExDJYqqPil4Zp5ngqcbN2I 3 | liEcufB1MTfmvlvClUUbMrwdhq0qjikhgtRsIVPv5ku116QyQpMEGNJjixe2SpH7 4 | IPAcngeNP+295E3jdOaqIACFIlj5HyQPR/o4L7M0OXKwNPA+l+c9akh/qwhoFV47 5 | 3FZv+Ra0VIQvnf/JBcmV0L+Wj9Kcr09w0iSuRk8gS+tgl8Ukg1l2PPFf6MHWJNpR 6 | EZBJE3pNJATalq4d9YR8upZ6lIh5v7xrCjuy7Lgfv6ZexkxFdbsmnBn+l/WyrqRI 7 | sDsCBJuNk883rF61jhNLycV3R5ROgRHoPAoLsQIDAQABAoIBAEE9tkD7nUqUlBEs 8 | +5KdyQpXqGuanhwlyWz9rj4zxM7JY2E5Z1zrFOLJrV2nl/GhWC5aKwOBKZvMezmC 9 | uPyKZ7S0MNUi0kAMCnxOePU5XG1kI9Rj6gttI7OVffAJFEUQUQ0p2W4LPZV32osd 10 | 4fQQRwE6lE6PCrcOUzSsDjAZ+dwtnKUrzVI2fWekVPp4EijmFgu5tpMJNljTWce/ 11 | qW4Unw0zVR1slo59kvG/MKBNTTXGh7siUxD8URgK1CmhS+Ehae/zd+ljYiBpWzIJ 12 | zkeLDe1gBICEEIeXM6qGhGdN/rn9stdVlRlRffGqov4mQu1VVS0txG/VdpzBu8Qb 13 | uF1KiQECgYEA9ANS3hMrS9BX93WMIPdrI2904f/L3xYMkKerF21+P1reBYuFdTtZ 14 | cFivgqjmGs2XcmT0TTz9C30AcSoq/LT6H82bznMHfUuV9jvl9Z6Lw/rMRke+fmCt 15 | jifZtwF02zqbYpprwDxT5Qt6anXJuY+EcBSucWR+AXmCiRNO1I6oC/kCgYEA0N9a 16 | oV+nBXmLPBk5CgOahjEC5t+wItRMw0d4RxxLHEQ+S+bbQ0vHSDmD03KATyFRzdH+ 17 | Kb6+mwndgqnridasX/s3BcOiHYUUUwfOops1nR563tmY/97gVYmNJgKoHirmrRey 18 | TjMSwA17fC0fFaXuysK9eRtl/p+TgV2m5EUqO3kCgYEA5gZME/TWF73rbudslZcn 19 | dFwZPwK32TEb3zFv48HgBTFPosKHqJ+uDG5N5Un6wMGWRsxMUJNxp2bLB+LMazTf 20 | pSNaASAGq2KZl2mXuhcfgjvmXUo8lT3l3eBLKG1DD3cMC8OuV6WHIJ/VYo5u/3H/ 21 | UvjsCRDGh/VXzIhbA50HZWECgYEAyanMTCAZFz0ZhEXp/1QpXkR1w8jWu1n0X1pi 22 | s22Ky1jMOOZEbn38gywcrgtP/JZz798+oLlsOC5XMu+x3NxqgyT+I8+VdyDk1aGl 23 | DPQD2JX0FbNLeEeMmql6oOdRwAa1vwsw+TVwdgLOZb7+0xzpJ9TLsOEWIbeaZ/2u 24 | w5hPxSkCgYEA7dpkaybr0rqSZuZb5Dtb4ooU5iGq0bnX3OgPH/rNfrEO5VyPnuBW 25 | 3Fw2LvFyOZ9D+IUkeZGzyXquVcyH/5n3gJZ8HDsKFQTERGdtFco5XZIUyxTPKTZO 26 | +yCl322lXymWYTHeteeGPWeaYzvEkSZIRtrbhjP3fvvyIt40NAiTQD8= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /utils/http/http_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package http 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "net/http" 21 | "testing" 22 | ) 23 | 24 | type AllocateIPRequest struct { 25 | Label Label 26 | } 27 | 28 | type Label struct { 29 | Network string 30 | ServiceType string 31 | ServiceName string 32 | Mobile bool 33 | Id string 34 | } 35 | 36 | func TestGetRequest(t *testing.T) { 37 | tr := &http.Transport{} 38 | data, err := GetRequest(context.Background(), tr, "http://www.mocky.io/v2/589bb17d100000701266e5e1") 39 | if err != nil { 40 | t.Error(err.Error()) 41 | } 42 | type AllocateIPReply struct { 43 | IpAddress string 44 | LinkLocalAddress string 45 | Error string 46 | XXX_unrecognized []byte `json:"-"` 47 | } 48 | var aip AllocateIPReply 49 | json.Unmarshal(data, &aip) 50 | if aip.IpAddress == "" || aip.LinkLocalAddress == "" { 51 | t.Fatal("expected IpAddress and LinkLocalAddress, but got empty") 52 | } 53 | } 54 | 55 | func TestGenBody(t *testing.T) { 56 | GenBody(AllocateIPRequest{ 57 | Label{ 58 | Network: "webtier", 59 | ServiceType: "t2", 60 | ServiceName: "s3", 61 | Mobile: true, 62 | Id: "id", 63 | }, 64 | }) 65 | } 66 | -------------------------------------------------------------------------------- /examples/vagrant/config/aurora-scheduler.conf: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | # 13 | description "Aurora scheduler" 14 | start on stopped rc RUNLEVEL=[2345] 15 | respawn 16 | post-stop exec sleep 5 17 | 18 | script 19 | . /etc/default/aurora-scheduler 20 | export JAVA_OPTS GLOG_v LIBPROCESS_PORT LIBPROCESS_IP 21 | exec start-stop-daemon --start -c aurora \ 22 | --exec /usr/share/aurora/bin/aurora-scheduler -- \ 23 | -cluster_name="$CLUSTER_NAME" \ 24 | -http_port="$HTTP_PORT" \ 25 | -native_log_quorum_size="$QUORUM_SIZE" \ 26 | -zk_endpoints="$ZK_ENDPOINTS" \ 27 | -mesos_master_address="$MESOS_MASTER" \ 28 | -serverset_path="$ZK_SERVERSET_PATH" \ 29 | -native_log_zk_group_path="$ZK_LOGDB_PATH" \ 30 | -native_log_file_path="$LOGDB_FILE_PATH" \ 31 | -backup_dir="$BACKUP_DIR" \ 32 | -thermos_executor_path="$THERMOS_EXECUTOR_PATH" \ 33 | -thermos_executor_resources="$THERMOS_EXECUTOR_RESOURCES" \ 34 | -thermos_executor_flags="$THERMOS_EXECUTOR_FLAGS" \ 35 | -allowed_container_types="$ALLOWED_CONTAINER_TYPES" \ 36 | -custom_executor_config="/home/vagrant/go/src/github.com/paypal/dce-go/examples/vagrant/config/docker-compose-executor.json" \ 37 | -enable_mesos_fetcher="true" \ 38 | -transient_task_state_timeout=10mins \ 39 | -min_offer_hold_time=1mins \ 40 | -offer_filter_duration=20secs \ 41 | -offer_reservation_duration=1mins \ 42 | -receive_revocable_resources=true 43 | $EXTRA_SCHEDULER_ARGS 44 | end script 45 | -------------------------------------------------------------------------------- /utils/http/transport.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "net" 5 | "net/http" 6 | "runtime" 7 | "time" 8 | ) 9 | 10 | // code source: https://github.com/hashicorp/go-cleanhttp/blob/master/cleanhttp.go 11 | // DefaultTransport returns a new http.Transport with similar default values to 12 | // http.DefaultTransport, but with idle connections and keepalives disabled. 13 | func DefaultTransport() *http.Transport { 14 | transport := DefaultPooledTransport() 15 | transport.DisableKeepAlives = true 16 | transport.MaxIdleConnsPerHost = -1 17 | return transport 18 | } 19 | 20 | // DefaultPooledTransport returns a new http.Transport with similar default 21 | // values to http.DefaultTransport. Do not use this for transient transports as 22 | // it can leak file descriptors over time. Only use this for transports that 23 | // will be re-used for the same host(s). 24 | func DefaultPooledTransport() *http.Transport { 25 | transport := &http.Transport{ 26 | Proxy: http.ProxyFromEnvironment, 27 | DialContext: (&net.Dialer{ 28 | Timeout: 30 * time.Second, 29 | KeepAlive: 30 * time.Second, 30 | DualStack: true, 31 | }).DialContext, 32 | MaxIdleConns: 100, 33 | IdleConnTimeout: 90 * time.Second, 34 | TLSHandshakeTimeout: 10 * time.Second, 35 | ExpectContinueTimeout: 1 * time.Second, 36 | MaxIdleConnsPerHost: runtime.GOMAXPROCS(0) + 1, 37 | } 38 | return transport 39 | } 40 | 41 | // DefaultClient returns a new http.Client with similar default values to 42 | // http.Client, but with a non-shared Transport, idle connections disabled, and 43 | // keepalives disabled. 44 | func DefaultClient() *http.Client { 45 | return &http.Client{ 46 | Transport: DefaultTransport(), 47 | } 48 | } 49 | 50 | // DefaultPooledClient returns a new http.Client with similar default values to 51 | // http.Client, but with a shared Transport. Do not use this function for 52 | // transient clients as it can leak file descriptors over time. Only use this 53 | // for clients that will be re-used for the same host(s). 54 | func DefaultPooledClient() *http.Client { 55 | return &http.Client{ 56 | Transport: DefaultPooledTransport(), 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /pluginimpl/general/impl_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package general 16 | 17 | import ( 18 | "context" 19 | "os/exec" 20 | "strconv" 21 | "testing" 22 | 23 | "github.com/mesos/mesos-go/api/v0/mesosproto" 24 | "github.com/paypal/dce-go/config" 25 | "github.com/paypal/dce-go/types" 26 | "github.com/stretchr/testify/assert" 27 | ) 28 | 29 | func TestCreateInfraContainer(t *testing.T) { 30 | config.GetConfig().SetDefault(types.NO_FOLDER, true) 31 | ctx := context.Background() 32 | _, err := CreateInfraContainer(ctx, "testdata/docker-infra-container.yml") 33 | if err != nil { 34 | t.Errorf("expected no error, but got %v", err) 35 | } 36 | } 37 | 38 | func TestGeneralExt_LaunchTaskPreImagePull(t *testing.T) { 39 | config.GetConfig().SetDefault(types.NO_FOLDER, true) 40 | g := new(generalExt) 41 | ctx := context.Background() 42 | begin, _ := strconv.ParseUint("1000", 10, 64) 43 | end, _ := strconv.ParseUint("1003", 10, 64) 44 | r := []*mesosproto.Value_Range{ 45 | {Begin: &begin, 46 | End: &end}, 47 | } 48 | ranges := &mesosproto.Value_Ranges{Range: r} 49 | name := types.PORTS 50 | taskInfo := &mesosproto.TaskInfo{ 51 | Resources: []*mesosproto.Resource{ 52 | { 53 | Name: &name, 54 | Ranges: ranges, 55 | }, 56 | }, 57 | } 58 | 59 | // No compose file 60 | err := g.LaunchTaskPreImagePull(ctx, &[]string{}, "exeutorid", taskInfo) 61 | assert.Equal(t, string(types.NoComposeFile), err.Error(), "Test if compose file list is empty") 62 | 63 | // One compose file 64 | compose := []string{"testdata/test.yml"} 65 | err = g.LaunchTaskPreImagePull(ctx, &compose, "exeutorid", taskInfo) 66 | assert.NoError(t, err) 67 | assert.Equal(t, 2, len(compose)) 68 | exec.Command("rm", "docker-infra-container.yml").Run() 69 | } 70 | -------------------------------------------------------------------------------- /utils/wait/wait_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package wait 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | 21 | "github.com/pkg/errors" 22 | ) 23 | 24 | func TestPollUntil(t *testing.T) { 25 | var done chan string 26 | // timeout 27 | _, err := PollUntil(1*time.Millisecond, done, 1*time.Millisecond, ConditionFunc(func() (string, error) { 28 | time.Sleep(20 * time.Second) 29 | return "hello", nil 30 | })) 31 | if err == nil || err != ErrTimeOut { 32 | t.Fatalf("expected timeout err, but got %v", err) 33 | } 34 | 35 | //conditionfunc finish 36 | res, _ := PollUntil(1*time.Second, done, 1*time.Second, ConditionFunc(func() (string, error) { 37 | return "hello", nil 38 | })) 39 | if res != "hello" { 40 | t.Fatalf("expected return message to be 'hello', but got %s", res) 41 | } 42 | } 43 | 44 | func TestWaitUntil(t *testing.T) { 45 | //timeout 46 | _, err := WaitUntil(1*time.Millisecond, ConditionCHFunc(func(reply chan string) { 47 | time.Sleep(2 * time.Second) 48 | reply <- "hello" 49 | })) 50 | if err == nil || err != ErrTimeOut { 51 | t.Fatalf("expected timeout err, but got %v", err) 52 | } 53 | 54 | //conditionfunc finish 55 | res, _ := WaitUntil(5*time.Second, ConditionCHFunc(func(reply chan string) { 56 | reply <- "hello" 57 | })) 58 | 59 | if res != "hello" { 60 | t.Fatalf("expected return message to be 'hello', but got %s", res) 61 | } 62 | } 63 | 64 | func TestPollRetry(t *testing.T) { 65 | //retry 3 times and return error 66 | err := PollRetry(3, 200*time.Millisecond, ConditionFunc(func() (string, error) { 67 | return "", errors.New("testerror") 68 | })) 69 | if err == nil || err != ErrTimeOut { 70 | t.Fatalf("expected timeout err, but got %v", err) 71 | } 72 | 73 | //success 74 | err = PollRetry(3, 200*time.Millisecond, ConditionFunc(func() (string, error) { 75 | return "", nil 76 | })) 77 | if err != nil { 78 | t.Fatalf("expected no err, but got %v", err) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /mesos-modules/README.rst: -------------------------------------------------------------------------------- 1 | ============================================ 2 | compose hook - cleanup dangling containers 3 | ============================================ 4 | 5 | 6 | Motivation 7 | ---------- 8 | 9 | We wish to cleanup after ourselves in case of executor failures. the leaked containers can lead to wasted resources on the slave node. 10 | when configured this hook gets called when excutor exits. The hook cleansup all the currently running containers associated with exited executor. 11 | This module is also responsible for cleaning up any cgroups that are left behind by mesos slave. 12 | 13 | The module does not remove containers but only stops them but this behavior can be changed easily. 14 | 15 | 16 | Goals 17 | ----- 18 | 19 | This is a `Mesos compose module`_ which can be loaded using the enclosed 20 | ``modules.json`` descriptor 21 | 22 | License & Allowed Use 23 | --------------------- 24 | 25 | Top level licencing for project follows. 26 | 27 | 28 | Build 29 | ----- 30 | 31 | Prerequisites 32 | ^^^^^^^^^^^^^ 33 | 34 | You obviously need `Apache Mesos`_ to build this 35 | project: in particular, you will need both the includes (``mesos``, ``stout`` 36 | and ``libprocess``) and the shared ``libmesos.so`` library. 37 | 38 | In addition, Mesos needs access to ``picojson.h`` and a subset of the ``boost`` 39 | header files: see the 40 | `3rdparty `_ 41 | folder in the mirrored github repository for Mesos, and in particular the 42 | `boost-1.53.0.tar.gz `_ 43 | archive. 44 | 45 | The "easiest" way to obtain all the prerequisites would probably be to clone the Mesos 46 | repository, build mesos and then install it in a local folder that you will then need to 47 | configure. 48 | 49 | In order to ease setting up prerequisites and build the module you can you the provided dockerfile to build the module from desired version of mesos. 50 | ``docker build -t mesos-build -f Dockerfile-mesos-modules --build-arg MESOS_VERSION=1.1.0 .`` 51 | The built binary module can be fetched at ``/output`` by running the generated docker image. 52 | 53 | Usage 54 | ----- 55 | 56 | In order to run a Mesos Agent with this module loaded, is a simple matter of 57 | adding the ``--modules`` flag, pointing it to the generated JSON 58 | ``modules.json`` file. and enable hook using ``--hook`` option 59 | 60 | $ ${MESOS_ROOT}/build/bin/mesos-slave.sh --work_dir=/var/lib/mesos \ 61 | --modules=/path/to/execute-module/gen/modules.json \ 62 | --hooks=org_apache_mesos_ComposePodCleanupHook \ 63 | --master=zk://zkclusteraddress:port 64 | 65 | See ``Configuration`` on the `Apache Mesos`_ documentation pages for more 66 | details on the various flags. 67 | 68 | Also, my `zk_mesos`_ github project provides an example `Vagrant`_ 69 | configuration showing how to deploy and run Mesos from the Mesosphere binary 70 | distributions. 71 | 72 | 73 | Tests 74 | ----- 75 | 76 | Not yet implemented 77 | 78 | -------- 79 | -------------------------------------------------------------------------------- /utils/http/http.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package http 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "encoding/json" 21 | "io" 22 | "io/ioutil" 23 | "net/http" 24 | 25 | "github.com/paypal/dce-go/config" 26 | log "github.com/sirupsen/logrus" 27 | ) 28 | 29 | // generate body for http request 30 | // parse interface{} to io.Reader 31 | func GenBody(t interface{}) io.Reader { 32 | tjson, err := json.Marshal(&t) 33 | if err != nil { 34 | log.Panic("Error marshalling : ", err.Error()) 35 | } 36 | log.Println("Request Body : ", string(tjson)) 37 | return bytes.NewReader(tjson) 38 | } 39 | 40 | // http post 41 | func PostRequest(ctx context.Context, transport http.RoundTripper, url string, body io.Reader) ([]byte, error) { 42 | if transport == nil { 43 | transport = http.DefaultTransport 44 | } 45 | client := &http.Client{ 46 | Transport: transport, 47 | Timeout: config.GetHttpTimeout(), 48 | } 49 | req, err := http.NewRequestWithContext(ctx, "POST", url, body) 50 | if err != nil { 51 | log.Println("Error creating http request : ", err.Error()) 52 | return nil, err 53 | } 54 | req.Header.Set("Content-Type", "application/json") 55 | resp, err := client.Do(req) 56 | if err != nil { 57 | log.Println("Error posting http request : ", err.Error()) 58 | return nil, err 59 | } 60 | respBody, err := ioutil.ReadAll(resp.Body) 61 | if err != nil { 62 | log.Println("Error reading http response : ", err.Error()) 63 | return nil, err 64 | } 65 | err = resp.Body.Close() 66 | if err != nil { 67 | log.Errorf("Failure to close response body :%v", err) 68 | return nil, err 69 | } 70 | return respBody, nil 71 | } 72 | 73 | // http get 74 | func GetRequest(ctx context.Context, transport http.RoundTripper, url string) ([]byte, error) { 75 | if transport == nil { 76 | transport = http.DefaultTransport 77 | } 78 | client := &http.Client{ 79 | Transport: transport, 80 | Timeout: config.GetHttpTimeout(), 81 | } 82 | req, err := http.NewRequestWithContext(ctx, "GET", url, nil) 83 | if err != nil { 84 | log.Println("Error creating http request : ", err.Error()) 85 | return nil, err 86 | } 87 | req.Header.Set("Accept", "application/json") 88 | resp, err := client.Do(req) 89 | if err != nil { 90 | log.Println("Error getting http request : ", err.Error()) 91 | return nil, err 92 | } 93 | respBody, err := ioutil.ReadAll(resp.Body) 94 | if err != nil { 95 | log.Println("Error reading http response : ", err.Error()) 96 | return nil, err 97 | } 98 | err = resp.Body.Close() 99 | if err != nil { 100 | log.Errorf("Failed to close response body :%v", err) 101 | return nil, err 102 | } 103 | return respBody, nil 104 | } 105 | -------------------------------------------------------------------------------- /plugin/type.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | //go:generate go-extpoints . ComposePlugin PodStatusHook Monitor 16 | package plugin 17 | 18 | import ( 19 | "context" 20 | 21 | "github.com/mesos/mesos-go/api/v0/executor" 22 | mesos "github.com/mesos/mesos-go/api/v0/mesosproto" 23 | "github.com/paypal/dce-go/types" 24 | ) 25 | 26 | type ComposePlugin interface { 27 | // Name gets the name of the plugin 28 | Name() string 29 | 30 | // execute some tasks before the Image is pulled 31 | LaunchTaskPreImagePull(ctx context.Context, composeFiles *[]string, executorId string, taskInfo *mesos.TaskInfo) error 32 | 33 | // execute some tasks after the Image is pulled 34 | LaunchTaskPostImagePull(ctx context.Context, composeFiles *[]string, executorId string, taskInfo *mesos.TaskInfo) error 35 | 36 | // execute the tasks after the pod is launched 37 | PostLaunchTask(ctx context.Context, composeFiles []string, taskInfo *mesos.TaskInfo) (string, error) 38 | 39 | // execute the task before we send a Kill to Mesos 40 | PreKillTask(ctx context.Context, taskInfo *mesos.TaskInfo) error 41 | 42 | // execute the task after we send a Kill to Mesos 43 | PostKillTask(ctx context.Context, taskInfo *mesos.TaskInfo) error 44 | 45 | // execute the task to shutdown the pod 46 | Shutdown(taskInfo *mesos.TaskInfo, ed executor.ExecutorDriver) error 47 | } 48 | 49 | // PodStatusHook allows custom implementations to be plugged when a Pod (mesos task) status changes. Currently this is 50 | // designed to be executed on task status changes during LaunchTask. 51 | type PodStatusHook interface { 52 | // TaskInfoInitializer is invoked to initialize the hook with TaskInfo 53 | TaskInfoInitializer(ctx context.Context, data interface{}) error 54 | // Execute is invoked when the pod.taskStatusCh channel has a new status. It returns an error on failure, 55 | // and also a flag "failExec" indicating if the error needs to fail the execution when a series of hooks are executed 56 | // This is to support cases where a few hooks can be executed in a best effort manner and need not fail the executor 57 | Execute(ctx context.Context, podStatus string, data interface{}) (failExec bool, err error) 58 | 59 | // This will be called from the pod.stopDriver. This will be used to end anything which needs clean-up at the end 60 | // of the executor 61 | Shutdown(ctx context.Context, podStatus string, data interface{}) 62 | } 63 | 64 | // Monitor inspects pods periodically until pod failed or terminated It also defines when to consider a pod as failed. 65 | // Move monitor as a plugin provides flexibility to replace default monitor logic. 66 | // Monitor name presents in config `monitorName` will be used, otherwise, default monitor will be used. 67 | type Monitor interface { 68 | Start(ctx context.Context) (types.PodStatus, error) 69 | } 70 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package config 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/mesos/mesos-go/api/v0/mesosproto" 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | func TestSetConfig(t *testing.T) { 25 | var val bool 26 | getConfigFromFile("config") 27 | SetConfig("test", true) 28 | val = GetConfig().GetBool("test") 29 | if !val { 30 | t.Errorf("expected val to be true, but got %v", val) 31 | } 32 | } 33 | 34 | func TestGetAppFolder(t *testing.T) { 35 | folder := GetAppFolder() 36 | if folder != "poddata" { 37 | t.Errorf("expected folder to be poddata, but got %s", folder) 38 | } 39 | } 40 | 41 | func TestGetStopTimeout(t *testing.T) { 42 | tests := []struct { 43 | name string 44 | input interface{} 45 | want int 46 | }{ 47 | // this default is picked from the config.yaml file 48 | {"check default value", "", 20}, 49 | {"incorrect integer value", 25, 0}, 50 | {"incorrect string value", "25", 0}, 51 | {"correct duration value", "25s", 25}, 52 | {"correct duration value", "25m", 1500}, 53 | {"incorrect value", "xyz", 0}, 54 | } 55 | 56 | for _, test := range tests { 57 | t.Run(test.name, func(t *testing.T) { 58 | // this is to use the default value 59 | if test.input != "" { 60 | GetConfig().Set(COMPOSE_STOP_TIMEOUT, test.input) 61 | } 62 | 63 | got := GetStopTimeout() 64 | if got != test.want { 65 | t.Errorf("expected cleanpod.timeout to be %d for input %v, but got %d", test.want, test.input, got) 66 | } 67 | }) 68 | } 69 | } 70 | 71 | func TestGetMaxRetry(t *testing.T) { 72 | max := GetMaxRetry() 73 | if max != 3 { 74 | t.Errorf("expected max retry to be 3, but got %d", max) 75 | } 76 | } 77 | 78 | func TestGetPullRetryCount(t *testing.T) { 79 | count := GetPullRetryCount() 80 | if count != 3 { 81 | t.Errorf("expected pull retry to be 3, but got %d", count) 82 | } 83 | } 84 | 85 | func TestOverrideConfig(t *testing.T) { 86 | type test struct { 87 | key string 88 | val string 89 | expectedKey string 90 | expectedVal string 91 | msg string 92 | } 93 | 94 | overrideTests := []test{ 95 | {"config.test1", "test1", "test1key", "", "shouldn't reset config if key isn't set"}, 96 | {"config.cleanpod.timeout", "1", "cleanpod.timeout", "1", "should reset config if key is set"}, 97 | {"config.launchtask.timeout", "1", "launchtask.timeout", "1", "should reset config if key is set"}, 98 | {"config1.launchtask.timeout", "2", "launchtask.timeout", "1", "shouldn't reset config with invalid prefix"}, 99 | } 100 | 101 | for _, ot := range overrideTests { 102 | var labels []*mesosproto.Label 103 | labels = append(labels, &mesosproto.Label{Key: &ot.key, Value: &ot.val}) 104 | taskInfo := &mesosproto.TaskInfo{Labels: &mesosproto.Labels{Labels: labels}} 105 | OverrideConfig(taskInfo) 106 | assert.Equal(t, ot.expectedVal, GetConfig().GetString(ot.expectedKey), ot.msg) 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /pluginimpl/example/impl.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package example 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/mesos/mesos-go/api/v0/executor" 21 | mesos "github.com/mesos/mesos-go/api/v0/mesosproto" 22 | "github.com/paypal/dce-go/config" 23 | "github.com/paypal/dce-go/plugin" 24 | "github.com/paypal/dce-go/types" 25 | "github.com/paypal/dce-go/utils/pod" 26 | log "github.com/sirupsen/logrus" 27 | ) 28 | 29 | var logger *log.Entry 30 | 31 | type exampleExt struct { 32 | } 33 | 34 | func init() { 35 | log.SetOutput(config.CreateFileAppendMode(types.DCE_OUT)) 36 | logger = log.WithFields(log.Fields{ 37 | "plugin": "example", 38 | }) 39 | 40 | logger.Println("Plugin Registering") 41 | 42 | plugin.ComposePlugins.Register(new(exampleExt), "example") 43 | } 44 | 45 | func (p *exampleExt) Name() string { 46 | return "example" 47 | } 48 | 49 | func (ex *exampleExt) LaunchTaskPreImagePull(ctx context.Context, composeFiles *[]string, executorId string, taskInfo *mesos.TaskInfo) error { 50 | logger.Println("LaunchTaskPreImagePull begin") 51 | // docker compose YML files are saved in context as type SERVICE_DETAIL which is map[interface{}]interface{}. 52 | // Massage YML files and save it in context. 53 | // Then pass to next plugin. 54 | 55 | // Get value from context 56 | filesMap := pod.GetServiceDetail() 57 | 58 | // Add label in each service, in each compose YML file 59 | for _, file := range *composeFiles { 60 | servMap := filesMap[file][types.SERVICES].(map[interface{}]interface{}) 61 | for serviceName := range servMap { 62 | containerDetails := filesMap[file][types.SERVICES].(map[interface{}]interface{})[serviceName].(map[interface{}]interface{}) 63 | if labels, ok := containerDetails[types.LABELS].(map[interface{}]interface{}); ok { 64 | labels["com.company.label"] = "awesome" 65 | containerDetails[types.LABELS] = labels 66 | } 67 | 68 | } 69 | 70 | } 71 | 72 | // Save the changes back to context 73 | pod.SetServiceDetail(filesMap) 74 | 75 | return nil 76 | } 77 | 78 | func (ex *exampleExt) LaunchTaskPostImagePull(ctx context.Context, composeFiles *[]string, executorId string, taskInfo *mesos.TaskInfo) error { 79 | logger.Println("LaunchTaskPostImagePull begin") 80 | return nil 81 | } 82 | 83 | func (ex *exampleExt) PostLaunchTask(ctx context.Context, composeFiles []string, taskInfo *mesos.TaskInfo) (string, error) { 84 | logger.Println("PostLaunchTask begin") 85 | return "", nil 86 | } 87 | 88 | func (ex *exampleExt) PreStopPod() error { 89 | logger.Println("PreStopPod Starting") 90 | return nil 91 | } 92 | 93 | func (ex *exampleExt) PreKillTask(ctx context.Context, taskInfo *mesos.TaskInfo) error { 94 | logger.Println("PreKillTask begin") 95 | return nil 96 | } 97 | 98 | func (ex *exampleExt) PostKillTask(ctx context.Context, taskInfo *mesos.TaskInfo) error { 99 | logger.Println("PostKillTask begin") 100 | return nil 101 | } 102 | 103 | func (ex *exampleExt) Shutdown(taskInfo *mesos.TaskInfo, ed executor.ExecutorDriver) error { 104 | logger.Println("Shutdown begin") 105 | return nil 106 | } 107 | -------------------------------------------------------------------------------- /dce/monitor/plugin/default/monitor.go: -------------------------------------------------------------------------------- 1 | package _default 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/paypal/dce-go/config" 7 | "github.com/paypal/dce-go/plugin" 8 | "github.com/paypal/dce-go/types" 9 | "github.com/paypal/dce-go/utils/pod" 10 | "github.com/paypal/dce-go/utils/wait" 11 | "github.com/pkg/errors" 12 | log "github.com/sirupsen/logrus" 13 | ) 14 | 15 | const name = "default" 16 | 17 | type monitor struct{} 18 | 19 | func init() { 20 | // Register default monitor plugin 21 | log.SetOutput(config.CreateFileAppendMode(types.DCE_OUT)) 22 | plugin.Monitors.Register(&monitor{}, name) 23 | log.Infof("Registered monitor plugin %s", name) 24 | } 25 | 26 | func (m *monitor) Start(ctx context.Context) (types.PodStatus, error) { 27 | logger := log.WithFields(log.Fields{ 28 | "monitor": name, 29 | }) 30 | // Get infra container ID 31 | var infraContainerId string 32 | var err error 33 | if !config.GetConfig().GetBool(types.RM_INFRA_CONTAINER) { 34 | infraContainerId, err = pod.GetContainerIdByService(pod.ComposeFiles, types.INFRA_CONTAINER) 35 | if err != nil { 36 | return types.POD_FAILED, errors.Wrap(err, "fail to get infra container ID") 37 | } 38 | logger.Debugf("Infra container ID: %s", infraContainerId) 39 | } 40 | 41 | run := func() (types.PodStatus, error) { 42 | for i := 0; i < len(pod.MonitorContainerList); i++ { 43 | hc, ok := pod.HealthCheckListId[pod.MonitorContainerList[i].ContainerId] 44 | healthy, running, exitCode, err := pod.CheckContainer(pod.MonitorContainerList[i].ContainerId, ok && hc) 45 | if err != nil { 46 | return types.POD_FAILED, err 47 | } 48 | logger.Debugf("container %s has health check, health status: %s, exitCode: %d, err : %v", 49 | pod.MonitorContainerList[i], healthy.String(), exitCode, err) 50 | 51 | if exitCode != 0 { 52 | return types.POD_FAILED, nil 53 | } 54 | 55 | if exitCode == 0 && !running { 56 | logger.Infof("Removed finished(exit with 0) container %s from monitor list", 57 | pod.MonitorContainerList[i]) 58 | pod.MonitorContainerList = append(pod.MonitorContainerList[:i], pod.MonitorContainerList[i+1:]...) 59 | i-- 60 | continue 61 | } 62 | 63 | if healthy == types.UNHEALTHY { 64 | err = pod.PrintInspectDetail(pod.MonitorContainerList[i].ContainerId) 65 | if err != nil { 66 | log.Warnf("failed to get container detail: %s ", err) 67 | } 68 | return types.POD_FAILED, nil 69 | } 70 | } 71 | 72 | // Send finished to mesos IF no container running or ONLY system proxy is running in the pod 73 | switch config.IsService() { 74 | case true: 75 | if len(pod.MonitorContainerList) == 0 { 76 | logger.Error("Task is SERVICE. All containers in the pod exit with code 0, sending FAILED") 77 | return types.POD_FAILED, nil 78 | } 79 | if len(pod.MonitorContainerList) == 1 && pod.MonitorContainerList[0].ContainerId == infraContainerId { 80 | logger.Error("Task is SERVICE. Only infra container is running in the pod, sending FAILED") 81 | return types.POD_FAILED, nil 82 | } 83 | case false: 84 | if len(pod.MonitorContainerList) == 0 { 85 | logger.Info("Task is ADHOC job. All containers in the pod exit with code 0, sending FINISHED") 86 | return types.POD_FINISHED, nil 87 | } 88 | if len(pod.MonitorContainerList) == 1 && pod.MonitorContainerList[0].ContainerId == infraContainerId { 89 | logger.Info("Task is ADHOC job. Only infra container is running in the pod, sending FINISHED") 90 | return types.POD_FINISHED, nil 91 | } 92 | } 93 | return types.POD_EMPTY, nil 94 | } 95 | 96 | res, err := wait.PollForever(config.GetPollInterval(), nil, func() (string, error) { 97 | status, err := run() 98 | if err != nil { 99 | // Error won't be considered as pod failure unless pod status is failed 100 | log.Warnf("error from monitor periodical check: %s", err) 101 | } 102 | return status.String(), nil 103 | }) 104 | 105 | return pod.ToPodStatus(res), err 106 | } 107 | -------------------------------------------------------------------------------- /utils/pod/util_test.go: -------------------------------------------------------------------------------- 1 | package pod 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/paypal/dce-go/types" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestPluginPanicHandler(t *testing.T) { 13 | _, err := PluginPanicHandler(ConditionFunc(func() (string, error) { 14 | panic("Test panic error") 15 | })) 16 | if err == nil { 17 | t.Error("Expected err not be nil, but got nil") 18 | } 19 | } 20 | 21 | func TestSetStepData(t *testing.T) { 22 | testErr := errors.New("unit-test") 23 | example := map[string][]*types.StepData{ 24 | "Image_Pull": { 25 | { 26 | RetryID: 0, 27 | StepName: "Image_Pull", 28 | Status: "Error", 29 | ErrorMsg: testErr, 30 | }, 31 | { 32 | RetryID: 1, 33 | StepName: "Image_Pull", 34 | Status: "Success", 35 | }, 36 | }, 37 | "HealthCheck": { 38 | { 39 | RetryID: 0, 40 | StepName: "HealthCheck", 41 | Status: "Success", 42 | }, 43 | }, 44 | } 45 | StartStep(StepMetrics, "Image_Pull") 46 | EndStep(StepMetrics, "Image_Pull", nil, testErr) 47 | 48 | StartStep(StepMetrics, "Image_Pull") 49 | EndStep(StepMetrics, "Image_Pull", nil, nil) 50 | 51 | StartStep(StepMetrics, "HealthCheck") 52 | EndStep(StepMetrics, "HealthCheck", nil, nil) 53 | 54 | for k, v1 := range example { 55 | v2, ok := StepMetrics[k] 56 | assert.True(t, ok) 57 | assert.Equal(t, len(v1), len(v2)) 58 | for i, s1 := range v1 { 59 | s2 := v2[i] 60 | assert.Equal(t, (s2.EndTime-s2.StartTime)*1000, s2.ExecTimeMS) 61 | assert.Equal(t, s1.RetryID, s2.RetryID) 62 | assert.Equal(t, s1.Status, s2.Status) 63 | assert.Equal(t, s1.StepName, s2.StepName) 64 | assert.Equal(t, s1.ErrorMsg, s2.ErrorMsg) 65 | assert.Equal(t, s1.Tags, s2.Tags) 66 | } 67 | } 68 | } 69 | 70 | func TestAddSvcContainers(t *testing.T) { 71 | t.Run("base container list is empty", func(t *testing.T) { 72 | var to []types.SvcContainer 73 | to = []types.SvcContainer{ 74 | { 75 | ServiceName: "test1", 76 | }, 77 | { 78 | ServiceName: "test2", 79 | }, 80 | } 81 | res := CopySvcContainers(to, []types.SvcContainer{}) 82 | assert.Equal(t, 0, len(res)) 83 | fmt.Println(res) 84 | }) 85 | t.Run("from container list is empty", func(t *testing.T) { 86 | var to []types.SvcContainer 87 | to = make([]types.SvcContainer, 3) 88 | res := CopySvcContainers(to, []types.SvcContainer{{ServiceName: "test1"}}) 89 | assert.Equal(t, 1, len(res)) 90 | fmt.Println(res) 91 | }) 92 | t.Run("copy duplicates to base container list", func(t *testing.T) { 93 | var to []types.SvcContainer 94 | to = []types.SvcContainer{ 95 | { 96 | ServiceName: "test1", 97 | }, 98 | { 99 | ServiceName: "test2", 100 | }, 101 | } 102 | res := CopySvcContainers(to, []types.SvcContainer{{ServiceName: "test2"}}) 103 | assert.Equal(t, 1, len(res)) 104 | assert.Equal(t, "test2", res[0].ServiceName) 105 | fmt.Println(res) 106 | }) 107 | t.Run("copy non-duplicates to base container list", func(t *testing.T) { 108 | var to []types.SvcContainer 109 | to = []types.SvcContainer{ 110 | { 111 | ServiceName: "test1", 112 | }, 113 | { 114 | ServiceName: "test2", 115 | }, 116 | } 117 | res := CopySvcContainers(to, []types.SvcContainer{{ServiceName: "test3"}}) 118 | assert.Equal(t, 1, len(res)) 119 | assert.Equal(t, "test3", res[0].ServiceName) 120 | fmt.Println(res) 121 | }) 122 | t.Run("copy duplicates & non-duplicates to base container list", func(t *testing.T) { 123 | var to []types.SvcContainer 124 | to = []types.SvcContainer{ 125 | { 126 | ServiceName: "test1", 127 | }, 128 | { 129 | ServiceName: "test2", 130 | }, 131 | } 132 | res := CopySvcContainers(to, []types.SvcContainer{{ServiceName: "test2"}, {ServiceName: "test3"}, {ServiceName: "test4"}}) 133 | assert.Equal(t, 3, len(res)) 134 | assert.Equal(t, "test2", res[0].ServiceName) 135 | assert.Equal(t, "test3", res[1].ServiceName) 136 | assert.Equal(t, "test4", res[2].ServiceName) 137 | fmt.Println(res) 138 | }) 139 | } 140 | -------------------------------------------------------------------------------- /utils/pod/util.go: -------------------------------------------------------------------------------- 1 | package pod 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "time" 7 | 8 | "github.com/paypal/dce-go/types" 9 | "github.com/pkg/errors" 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | type ConditionFunc func() (string, error) 14 | 15 | func PluginPanicHandler(condition ConditionFunc) (res string, err error) { 16 | defer func() { 17 | if r := recover(); r != nil { 18 | log.Printf("Recover : %v \n", r) 19 | err = errors.New(fmt.Sprintf("Recover : %v \n", r)) 20 | } 21 | }() 22 | 23 | if res, err = condition(); err != nil { 24 | log.Errorf("Error executing plugins: %v \n", err) 25 | return res, err 26 | } 27 | return res, err 28 | } 29 | 30 | func ToPodStatus(s string) types.PodStatus { 31 | switch s { 32 | case "POD_STAGING": 33 | return types.POD_STAGING 34 | case "POD_STARTING": 35 | return types.POD_STARTING 36 | case "POD_RUNNING": 37 | return types.POD_RUNNING 38 | case "POD_FAILED": 39 | return types.POD_FAILED 40 | case "POD_KILLED": 41 | return types.POD_KILLED 42 | case "POD_FINISHED": 43 | return types.POD_FINISHED 44 | case "POD_PULL_FAILED": 45 | return types.POD_PULL_FAILED 46 | case "POD_COMPOSE_CHECK_FAILED": 47 | return types.POD_COMPOSE_CHECK_FAILED 48 | } 49 | 50 | return types.POD_EMPTY 51 | } 52 | 53 | func ToHealthStatus(s string) types.HealthStatus { 54 | switch s { 55 | case "starting": 56 | return types.STARTING 57 | case "healthy": 58 | return types.HEALTHY 59 | case "unhealthy": 60 | return types.UNHEALTHY 61 | } 62 | 63 | return types.UNKNOWN_HEALTH_STATUS 64 | } 65 | 66 | // StartStep start a step of dce, and add a new item into the values, key is the stepName 67 | // In the map, key will be step, value will be each retry result, and duration. 68 | func StartStep(stepData map[string][]*types.StepData, stepName string) { 69 | if len(stepName) == 0 { 70 | log.Error("error while updating step data for Granular Metrics: step name can't be empty for stepData") 71 | } 72 | var ok bool 73 | 74 | stepValues, ok := stepData[stepName] 75 | if !ok { 76 | stepValues = []*types.StepData{} 77 | stepData[stepName] = stepValues 78 | } 79 | stepValue := &types.StepData{} 80 | 81 | stepValue.StepName = stepName 82 | stepValue.RetryID = len(stepValues) 83 | stepValue.StartTime = time.Now().Unix() 84 | stepValue.Status = "Starting" 85 | stepValues = append(stepValues, stepValue) 86 | stepData[stepName] = stepValues 87 | } 88 | 89 | // EndStep ends the current dce step, and update the result, duraiton. 90 | // current dce step can be fetch from stepData, key is the stepName, value is each retry results. Update the latest result 91 | func EndStep(stepData map[string][]*types.StepData, stepName string, tag map[string]string, err error) { 92 | if len(stepName) == 0 { 93 | log.Error("error while updating step data for Granular Metrics: step name can't be empty for stepData") 94 | return 95 | } 96 | var ok bool 97 | 98 | stepValues, ok := stepData[stepName] 99 | if !ok { 100 | log.Errorf("key %s not exist in stepData %+v", stepName, stepData) 101 | return 102 | } 103 | if len(stepValues) < 1 { 104 | log.Errorf("len of stepValues is %d, less than 1", len(stepValues)) 105 | return 106 | } 107 | 108 | step := stepValues[len(stepValues)-1] 109 | step.Tags = tag 110 | step.EndTime = time.Now().Unix() 111 | step.ErrorMsg = err 112 | step.ExecTimeMS = (step.EndTime - step.StartTime) * 1000 113 | if err != nil { 114 | step.Status = "Error" 115 | } else if healthStatus, ok := tag["healthStatus"]; ok && healthStatus != "healthy" { 116 | step.Status = "Error" 117 | } else { 118 | step.Status = "Success" 119 | } 120 | } 121 | 122 | func UpdateHealthCheckStatus(stepData map[string][]*types.StepData) { 123 | for stepName, stepVals := range stepData { 124 | if strings.HasPrefix(stepName, "HealthCheck-") && 125 | len(stepVals) > 0 && 126 | stepVals[len(stepVals)-1].Status == "Starting" { 127 | 128 | stepVals[len(stepVals)-1].Status = "Error" 129 | } 130 | } 131 | } 132 | 133 | func CopySvcContainers(to []types.SvcContainer, from []types.SvcContainer) []types.SvcContainer { 134 | to = make([]types.SvcContainer, len(from)) 135 | for i, c := range from { 136 | to[i] = c 137 | } 138 | return to 139 | } 140 | -------------------------------------------------------------------------------- /mesos-modules/hook/compose_pod_cleanup_hook.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | using namespace mesos; 35 | 36 | using process::Future; 37 | 38 | 39 | class ComposePodCleanupHook : public Hook 40 | { 41 | public: 42 | // This hook is called when the executor is being removed. 43 | virtual Try slaveRemoveExecutorHook( 44 | const FrameworkInfo& frameworkInfo, 45 | const ExecutorInfo& executorInfo) 46 | { 47 | LOG(INFO) << "Executing 'slaveRemoveExecutorHook'"; 48 | 49 | std::string executorId = executorInfo.executor_id().value(); 50 | std::string containerIdCommand = "docker ps -a --filter=\"label=executorId="+executorId+"\" -q 2>&1"; 51 | std::list result = exec(containerIdCommand); 52 | 53 | if(result.empty()) { 54 | LOG(INFO) << "ComposePodCleanupHook: nothing to clean"; 55 | return Nothing(); 56 | } 57 | 58 | std::string containers = join(result); 59 | stopContainer(containers); 60 | 61 | std::list::iterator it; 62 | std::string parentId; 63 | for(it = result.begin();it != result.end();++it) { 64 | std::string cmd = "docker inspect --format {{.HostConfig.CgroupParent}} " + *it; 65 | std::list cgParent = exec(cmd); 66 | if (!cgParent.empty()) { 67 | parentId = join(cgParent); 68 | break; 69 | } 70 | } 71 | 72 | // keep the containers for debugging purposes. 73 | //removeContainer(containers); 74 | 75 | if(!parentId.empty()){ 76 | deleteCgroup(parentId); 77 | } 78 | 79 | return Nothing(); 80 | } 81 | 82 | std::string join(std::list l, const char* delim=" ") { 83 | std::ostringstream imploded; 84 | std::copy(l.begin(), l.end(), 85 | std::ostream_iterator(imploded, delim)); 86 | return imploded.str(); 87 | } 88 | 89 | std::list stopContainer(std::string containerId) { 90 | std::string command = "docker stop "+containerId; 91 | return exec(command); 92 | } 93 | 94 | std::list removeContainer(std::string containerId) { 95 | std::string command = "docker rm -v "+containerId; 96 | return exec(command); 97 | } 98 | 99 | void deleteCgroup(std::string parentId) { 100 | std::string command = "lscgroup | grep " + parentId; 101 | std::list groupList = exec(command); 102 | if(!groupList.empty()) { 103 | command = "cgdelete "+ join(groupList); 104 | exec(command); 105 | } 106 | } 107 | 108 | std::list exec(std::string cmd) { 109 | char buffer[128]; 110 | std::string line; 111 | std::list result; 112 | FILE* pipe = popen(cmd.c_str(), "r"); 113 | if (!pipe) { 114 | LOG(INFO)<<"popen() failed! cmd:" << cmd; 115 | } 116 | try { 117 | while (!feof(pipe)) { 118 | if (fgets(buffer, 128, pipe) != NULL){ 119 | line.append(buffer); 120 | if(line.back() == '\n') { 121 | line.pop_back(); 122 | result.push_back(std::move(line)); 123 | line.clear(); 124 | } 125 | } 126 | } 127 | } catch (...) { 128 | LOG(INFO) <<"caught execption while processing shell cmd" << cmd; 129 | pclose(pipe); 130 | result.clear(); 131 | } 132 | 133 | if (pclose(pipe) < 0) { 134 | LOG(INFO) << "command failed with " << join(result); 135 | result.clear(); 136 | } 137 | return result; 138 | } 139 | 140 | }; 141 | 142 | 143 | static Hook* createHook(const Parameters& parameters) 144 | { 145 | return new ComposePodCleanupHook(); 146 | } 147 | 148 | 149 | // Declares a Hook module named 'org_apache_mesos_ComposePodCleanupHook'. 150 | mesos::modules::Module org_apache_mesos_ComposePodCleanupHook( 151 | MESOS_MODULE_API_VERSION, 152 | MESOS_VERSION, 153 | "Apache Mesos", 154 | "modules@mesos.apache.org", 155 | "Compose Pod Cleanup Hook module.", 156 | NULL, 157 | createHook); 158 | 159 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dce-go 2 | [![Go Reference](https://pkg.go.dev/badge/github.com/paypal/dce-go.svg)](https://pkg.go.dev/github.com/paypal/dce-go) ![CI Build Status](https://github.com/paypal/dce-go/actions/workflows/ci.yml/badge.svg) 3 | 4 | ## Overview 5 | 6 | dce-go project aims to enable Mesos frameworks to launch a pod of docker containers treating both Apache Mesos and docker as first class citizens. Kubernetes/K8 introduced the notion of a collection of docker containers that share namespaces and treat the collection as a single scaling unit. Brendan Burns talked about some design patterns/use cases for pods in [DockerCon'15](https://www.youtube.com/watch?v=Ph3t8jIt894). 7 | 8 | Docker Compose is a cherished tool used in docker community that helps us model a collection of docker containers. The specification is very flexible. Furthermore, you can also model a pod collapsing namespaces (net, IPC , pid). 9 | 10 | Composite containers representing an application is a common requirement for modular architecture. Composition requires co-location treating a set of containers as a single unit (aka pod) for scheduling. Sidecar, ambassador, adapter patterns use container pod. docker compose in docker community is an excellent way of defining a collection of containers and can be used to represent pod. Mesos on the other hand plays the critical role of a resource and cluster manager for large clusters. The native docker integration in Mesos can only launch a single container. In 1.1 Mesos has released Nested Container and Task Groups experimental feature to natively support a generic collection of tasks, not docker specific pods. However, frameworks need to change to support this and obviously the pod spec is separate than compose. universal containerizer with isolators is trying to model a different runtime than docker. docker swarm on the other hand as of 1.13/17.04 does not support local pods. compose-executor helps to immediately address the need of Mesos and docker users helping them launch a set of docker containers, aka pods as a single unit in Mesos without changing frameworks. 11 | 12 | ## Goal 13 | 14 | The project goal is to model a pod of containers with docker-compose and launch it with your favorite Mesos frameworks like Marathon, Apache Aurora, etc. One does not need to switch to Kubernetes on Mesos if all that they are looking to do is launch pods (model pod like workloads). With network and storage plugins supported directly in Docker, one can model advanced pods supported through compose. Furthermore, instead of using some different spec to define pods, we wanted to build around the compose spec that is well accepted in the docker community. A developer can now write the pod spec once, run it locally in their laptop using compose and later seamlessly move into Mesos without having to modify the pod spec. 15 | 16 | Running multiple pods on the same host may create many conflicts (containerId's , ports etc.). Executor takes care of resolving these conflicts. A new docker-compose file resolving all the conflicts is generated. Each container is tagged with specific taskId and executorId and this is used to clean up containers via mesos hooks if executor is terminated. Cgroup is introduced to limit, account for, and isolate the resource usage (cpu, memory at this point) of a Pod. 17 | 18 | dce-go is implemented in golang and provides a pluggable mechanism which gives developers more flexibilities to inject their custom logic. 19 | 20 | 21 | 22 | ### Plugins 23 | Pod is launched according to docker compose files provided by users. Docker compose files can be modified before pod is launched by dce-go. To allow developers implementing their own logic for customizing docker compose files based on specific requirements, pluggable structure is provided in dce-go. Please look into [How to develop](docs/how-to-develop.md) doc to understand how to implement plugins. 24 | 25 | ### Pod Modelling 26 | Pod containers essentially has to share namespace and containers in a pod need to share a common cgroup. 27 | 28 | ##### cgroup hierarchy 29 | dce-go mounts by default all the containers representing the pod under the parent mesos task cgroup. The memory subsystem use_hierarchy should be enabled for mesos cgroup. With this even if individual containers are not controlled, resources will be enforced as per the parent task limits. 30 | 31 | ##### Infrastructure Container 32 | This container is automatically added by the general plugin. Infrastructure container is the secret of how containers in a Pod can share the network namespace with it and the infra container gets the pod IP assigned to. We are not collapsing other namespaces like pid at this point in general plugin. 33 | 34 | ### Features 35 | - Implements mesos executor callbacks to maintain the lifecycle of a pod. 36 | - Massages compose file to add cgroup parent, mesos labels and edit certain sections to resolve any naming conflict etc. 37 | - Collapses network namespace by default. 38 | - Provides pod monitor to not only kill entire pod on unexpected container exit but also when a container becomes unhealthy as per docker healthchecks. 39 | - Supports running multiple compose files. 40 | - Mesos Module provided to prevent pod leaks in rare case of executor crashes. 41 | - Provides plugins. 42 | - Last but not the least any existing Mesos Frameworks like Aurora, Marathon etc can use DCE directly without making ANY framework changes. 43 | 44 | 45 | ### To start using dce-go 46 | 1. [Installing environment](docs/environment.md) 47 | 2. [How to use](docs/how-to-use.md) 48 | 49 | ### To start developing dce-go 50 | 1. [Installing environment](docs/environment.md) 51 | 2. [How to develop](docs/how-to-develop.md) 52 | 53 | ### Contributions 54 | Contributions are always welcome. Please raise an issue so that the contribution may be discussed before it's made. 55 | 56 | -------------------------------------------------------------------------------- /docs/how-to-use.md: -------------------------------------------------------------------------------- 1 | ## How to run dce-go with Mesos Frameworks 2 | 3 | In this document, we will cover using dce-go with mesos frameworks such as aurora and marathon. [Vagrantbox setup](https://github.com/paypal/dce-go/blob/master/docs/how-to-use.md) installs and configures mesos, marathon, apache aurora, dce-go etc. 4 | 5 | ### Running dce-go on Aurora 6 | #### Configuring Aurora Scheduler 7 | In order to use dce-go with Aurora, we must provide the scheduler a configuration file that contains information on how to run the executor. 8 | A sample config file for the docker-compose executor is shown below: 9 | ``` 10 | [ 11 | { 12 | "executor": { 13 | "command": { 14 | "value": "./executor", 15 | "shell": "true", 16 | "uris": [ 17 | { 18 | "cache": false, 19 | "executable": true, 20 | "extract": false, 21 | "value": "http://uri/executor" 22 | } 23 | ] 24 | }, 25 | "name": "docker-compose-executor", 26 | "resources": [ 27 | { 28 | "name": "cpus", 29 | "scalar": { 30 | "value": 0.25 31 | }, 32 | "type": "SCALAR" 33 | }, 34 | { 35 | "name": "mem", 36 | "scalar": { 37 | "value": 256 38 | }, 39 | "type": "SCALAR" 40 | } 41 | ] 42 | }, 43 | "task_prefix": "compose-" 44 | } 45 | ] 46 | ``` 47 | The example configuration is also available at dce-go/examples/vagrant/config/docker-compose-executor.json. Multiple executors can be provided in the configuration file. Vagrant setup takes care of setting up aurora scheduler configuration. 48 | More information on how an executor can be configured for consumption by Aurora can be found [here](https://github.com/apache/aurora/blob/master/docs/operations/configuration.md#custom-executors) 49 | under the custom executors section. 50 | 51 | 52 | [Gorealis](https://github.com/paypal/gorealis) is a go-library for communicating with apache aurora. The [Sample client](https://github.com/paypal/dce-go/blob/opensource/examples/client.go) in this project leverages Gorealis to launch job using aurora scheduler. This is a quick way to try out aurora scheduler apis using our vagrantbox setup. Follow [Installing environment](environment.md) for vagrant setup. 53 | 54 | **create** command launches a new aurora job. As indicated below, Environment, Role, and Name serve as JobKey, ExecutorName refers to the executor (as specified in scheduler configuration file) to be launched. Further, resources URIs are also specified for the job. 55 | 56 | ``` 57 | case "create": 58 | fmt.Println("Creating job") 59 | resp, err := r.CreateJob(job) 60 | if err != nil { 61 | fmt.Println(err) 62 | os.Exit(1) 63 | } 64 | fmt.Println(resp.String()) 65 | 66 | if resp.ResponseCode == aurora.ResponseCode_OK { 67 | // 500 will be the timeout for creating job 68 | if ok, err := monitor.Instances(job.JobKey(), job.GetInstanceCount(), 5, 500); !ok || err != nil { 69 | _, err := r.KillJob(job.JobKey()) 70 | if err != nil { 71 | fmt.Println(err) 72 | os.Exit(1) 73 | } 74 | } 75 | } 76 | ``` 77 | 78 | ``` 79 | job = realis.NewJob(). 80 | Environment("prod"). 81 | Role("vagrant"). 82 | Name("sampleapp"). 83 | ExecutorName("docker-compose-executor"). 84 | ExecutorData("{}"). 85 | CPU(0.5). 86 | RAM(64). 87 | Disk(100). 88 | IsService(true). 89 | InstanceCount(1). 90 | AddPorts(3). 91 | AddLabel("fileName", "sampleapp/docker-compose.yml,sampleapp/docker-compose-healthcheck.yml"). 92 | AddURIs(true, true, "http://192.168.33.8/app.tar.gz") 93 | ``` 94 | 95 | ##### Creating a Job 96 | ``` 97 | $ cd $GOPATH/src/github.com/paypal/dce-go/examples 98 | $ go run client.go -executor=compose -url=http://192.168.33.8:8081 -cmd=create 99 | ``` 100 | Mesos UI would show task details for the launched Job. App can be reached at https://192.168.33.8:${port}/index.html. 101 | 102 | File docker-infra-container.yml-generated.yml (shown below) in mesos sandbox for the task has information on exposed ports. 103 | We require to use host port-mapping for 443 -- ''31716 (in this case as indicated)'' in the app url: 104 | (https://192.168.33.8:31716/index.html) 105 | ``` 106 | networks: 107 | default: 108 | driver: bridge 109 | services: 110 | networkproxy: 111 | cgroup_parent: /mesos/011a235c-d49e-4085-9f0e-3e7862d24066 112 | container_name: vagrant-prod-sampleapp-0-a0694c60-85f5-48c7-90cb-1873dfb05004_networkproxy_0.1 113 | image: dcego/networkproxy:1.2 114 | labels: 115 | executorId: compose-vagrant-prod-sampleapp-0-a0694c60-85f5-48c7-90cb-1873dfb05004 116 | taskId: vagrant-prod-sampleapp-0-a0694c60-85f5-48c7-90cb-1873dfb05004 117 | networks: 118 | - default 119 | ports: 120 | - 31282:8081 121 | - 31645:80 122 | - 31716:443 123 | version: "2.1" 124 | ``` 125 | 126 | ##### Kill a Job 127 | ``` 128 | $ cd $GOPATH/src/github.com/paypal/dce-go/examples 129 | $ go run client.go -executor=compose -url=http://192.168.33.8:8081 -cmd=kill 130 | ``` 131 | 132 | 133 | ### Running dce-go on Marathon 134 | 135 | Marathon provides both REST APIs and UI to manage Jobs. 136 | An example of payload to create applications is provided as follows: 137 | ``` 138 | { 139 | "id": "docker-compose-demo", 140 | "cmd": " ", 141 | "cpus": 0.5, 142 | "mem": 64.0, 143 | "ports":[0,0,0], 144 | "instances": 1, 145 | "executor":"./executor", 146 | "labels": { 147 | "fileName": "sampleapp/docker-compose.yml,sampleapp/docker-compose-healthcheck.yml" 148 | }, 149 | "uris":["http://192.168.33.8/app.tar.gz","http://192.168.33.8/example/config.yaml","http://192.168.33.8/general.yaml","http://192.168.33.8/executor"] 150 | } 151 | ``` 152 | 153 | -------------------------------------------------------------------------------- /types/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package types 16 | 17 | import ( 18 | exec_cmd "os/exec" 19 | "strconv" 20 | 21 | "github.com/mesos/mesos-go/api/v0/mesosproto" 22 | ) 23 | 24 | type PodStatus int 25 | 26 | const ( 27 | POD_STAGING PodStatus = 1 + iota 28 | POD_STARTING 29 | POD_RUNNING 30 | POD_FAILED 31 | POD_KILLED 32 | POD_FINISHED 33 | POD_PULL_FAILED 34 | POD_COMPOSE_CHECK_FAILED 35 | POD_EMPTY 36 | ) 37 | 38 | func (status PodStatus) String() string { 39 | switch status { 40 | case POD_STAGING: 41 | return "POD_STAGING" 42 | case POD_STARTING: 43 | return "POD_STARTING" 44 | case POD_RUNNING: 45 | return "POD_RUNNING" 46 | case POD_FAILED: 47 | return "POD_FAILED" 48 | case POD_KILLED: 49 | return "POD_KILLED" 50 | case POD_FINISHED: 51 | return "POD_FINISHED" 52 | case POD_PULL_FAILED: 53 | return "POD_PULL_FAILED" 54 | case POD_COMPOSE_CHECK_FAILED: 55 | return "POD_COMPOSE_CHECK_FAILED" 56 | case POD_EMPTY: 57 | return "" 58 | } 59 | 60 | return "" 61 | } 62 | 63 | type HealthStatus int 64 | 65 | const ( 66 | STARTING HealthStatus = 1 + iota 67 | HEALTHY 68 | UNHEALTHY 69 | UNKNOWN_HEALTH_STATUS 70 | ) 71 | 72 | func (status HealthStatus) String() string { 73 | switch status { 74 | case STARTING: 75 | return "starting" 76 | case HEALTHY: 77 | return "healthy" 78 | case UNHEALTHY: 79 | return "unhealthy" 80 | case UNKNOWN_HEALTH_STATUS: 81 | return "unknown" 82 | } 83 | 84 | return "unknown" 85 | } 86 | 87 | func GetInstanceStatusTag(svcContainer SvcContainer, healthy HealthStatus, running bool, exitCode int) map[string]string { 88 | return map[string]string{ 89 | "serviceName": svcContainer.ServiceName, 90 | "containerId": svcContainer.ContainerId, 91 | "healthStatus": healthy.String(), 92 | "running": strconv.FormatBool(running), 93 | "exitCode": strconv.Itoa(exitCode), 94 | } 95 | } 96 | 97 | const ( 98 | LOGLEVEL = "loglevel" 99 | CONTAINER_NAME = "container_name" 100 | NETWORK_MODE = "network_mode" 101 | HEALTH_CHECK = "healthcheck" 102 | LINKS = "links" 103 | PORTS = "ports" 104 | LABELS = "labels" 105 | ENVIRONMENT = "environment" 106 | RESTART = "restart" 107 | APP_START_TIME = "appStartTime" 108 | SERVICES = "services" 109 | IMAGE = "image" 110 | VERSION = "version" 111 | NETWORKS = "networks" 112 | HOSTNAME = "hostname" 113 | VOLUMES = "volumes" 114 | DEPENDS_ON = "depends_on" 115 | EXTRA_HOSTS = "extra_hosts" 116 | CGROUP_PARENT = "cgroup_parent" 117 | HOST_MODE = "host" 118 | NONE_NETWORK_MODE = "none" 119 | NAME = "name" 120 | NETWORK_DRIVER = "driver" 121 | NETWORK_DEFAULT_DRIVER = "bridge" 122 | NETWORK_DEFAULT_NAME = "default" 123 | NETWORK_EXTERNAL = "external" 124 | PLUGIN_ORDER = "pluginorder" 125 | INFRA_CONTAINER_YML = "docker-infra-container.yml" 126 | INFRA_CONTAINER_GEN_YML = "docker-infra-container.yml-generated.yml" 127 | DEFAULT_FOLDER = "poddata" 128 | NO_FOLDER = "dontcreatefolder" 129 | RM_INFRA_CONTAINER = "rm_infra_container" 130 | COMPOSE_HTTP_TIMEOUT = "COMPOSE_HTTP_TIMEOUT" 131 | SERVICE_DETAIL = "serviceDetail" 132 | INFRA_CONTAINER = "networkproxy" 133 | IS_SERVICE = "isService" 134 | FOREVER = 1<<63 - 1 135 | DCE_OUT = "dce.out" 136 | DCE_ERR = "dce.err" 137 | ) 138 | 139 | // ServiceDetail key is filepath, value is map to store Unmarshal the docker-compose.yaml 140 | type ServiceDetail map[string]map[string]interface{} 141 | 142 | type CmdResult struct { 143 | Result error 144 | Command *exec_cmd.Cmd 145 | } 146 | 147 | type Network struct { 148 | PreExist bool 149 | Name string 150 | Driver string 151 | } 152 | 153 | type ContainerStatusDetails struct { 154 | ComposeTaskId *mesosproto.TaskID 155 | Pid int 156 | ContainerId string 157 | IsRunning bool 158 | ExitCode int 159 | HealthStatus string 160 | RestartCount int 161 | MaxRetryCount int 162 | Name string 163 | } 164 | 165 | func (c *ContainerStatusDetails) SetContainerId(containerId string) { 166 | c.ContainerId = containerId 167 | } 168 | 169 | func (c *ContainerStatusDetails) SetComposeTaskId(composeTaskId *mesosproto.TaskID) { 170 | c.ComposeTaskId = composeTaskId 171 | } 172 | 173 | type err string 174 | 175 | const NoComposeFile err = "no compose file specified" 176 | 177 | type StepData struct { 178 | RetryID int `json:"retryID,omitempty"` 179 | StepName string `json:"stepName,omitempty"` 180 | ErrorMsg error `json:"errorMsg,omitempty"` 181 | Status string `json:"status,omitempty"` 182 | Tags map[string]string `json:"tags,omitempty"` 183 | StartTime int64 `json:"startTime,omitempty"` 184 | EndTime int64 `json:"endTime,omitempty"` 185 | ExecTimeMS int64 `json:"execTimeMS,omitempty"` 186 | } 187 | 188 | type SvcContainer struct { 189 | ServiceName string 190 | ContainerId string 191 | Pid string 192 | } 193 | -------------------------------------------------------------------------------- /examples/client.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package main 16 | 17 | import ( 18 | "flag" 19 | "fmt" 20 | _ "io/ioutil" 21 | "os" 22 | 23 | realis "github.com/paypal/gorealis" 24 | "github.com/paypal/gorealis/gen-go/apache/aurora" 25 | _ "github.com/paypal/gorealis/response" 26 | ) 27 | 28 | func main() { 29 | cmd := flag.String("cmd", "", "Job request type to send to Aurora Scheduler") 30 | executor := flag.String("executor", "thermos", "Executor to use") 31 | url := flag.String("url", "", "URL at which the Aurora Scheduler exists as [url]:[port]") 32 | username := flag.String("username", "aurora", "Username to use for authorization") 33 | password := flag.String("password", "secret", "Password to use for authorization") 34 | flag.Parse() 35 | 36 | var job realis.Job 37 | var err error 38 | var monitor *realis.Monitor 39 | var r realis.Realis 40 | 41 | r, err = realis.NewRealisClient(realis.SchedulerUrl(*url), realis.BasicAuth(*username, *password), realis.ThriftJSON(), realis.TimeoutMS(20000)) 42 | if err != nil { 43 | fmt.Println(err) 44 | os.Exit(1) 45 | } 46 | 47 | monitor = &realis.Monitor{r} 48 | 49 | switch *executor { 50 | case "compose": 51 | job = realis.NewJob(). 52 | Environment("prod"). 53 | Role("vagrant"). 54 | Name("sampleapp"). 55 | ExecutorName("docker-compose-executor"). 56 | ExecutorData("{}"). 57 | CPU(0.25). 58 | RAM(256). 59 | Disk(100). 60 | IsService(true). 61 | InstanceCount(1). 62 | AddPorts(4). 63 | AddLabel("fileName", "sampleapp/docker-compose.yml,sampleapp/docker-compose-healthcheck.yml"). 64 | AddURIs(true, false, "http://192.168.33.8/app.tar.gz") 65 | break 66 | case "none": 67 | job = realis.NewJob(). 68 | Environment("prod"). 69 | Role("vagrant"). 70 | Name("docker_as_task"). 71 | CPU(1). 72 | RAM(64). 73 | Disk(100). 74 | IsService(true). 75 | InstanceCount(1). 76 | AddPorts(1) 77 | break 78 | default: 79 | fmt.Println("Only thermos, compose, and none are supported for now") 80 | os.Exit(1) 81 | } 82 | 83 | switch *cmd { 84 | case "create": 85 | fmt.Println("Creating job") 86 | resp, err := r.CreateJob(job) 87 | if err != nil { 88 | fmt.Println(err) 89 | os.Exit(1) 90 | } 91 | fmt.Println(resp.String()) 92 | 93 | if resp.ResponseCode == aurora.ResponseCode_OK { 94 | if ok, err := monitor.Instances(job.JobKey(), job.GetInstanceCount(), 5, 500); !ok || err != nil { 95 | _, err := r.KillJob(job.JobKey()) 96 | if err != nil { 97 | fmt.Println(err) 98 | os.Exit(1) 99 | } 100 | } 101 | } 102 | break 103 | case "createDocker": 104 | fmt.Println("Creating a docker based job") 105 | container := realis.NewDockerContainer().Image("python:2.7").AddParameter("network", "host") 106 | job.Container(container) 107 | resp, err := r.CreateJob(job) 108 | if err != nil { 109 | fmt.Println(err) 110 | os.Exit(1) 111 | } 112 | fmt.Println(resp.String()) 113 | 114 | if resp.ResponseCode == aurora.ResponseCode_OK { 115 | if ok, err := monitor.Instances(job.JobKey(), job.GetInstanceCount(), 10, 300); !ok || err != nil { 116 | _, err := r.KillJob(job.JobKey()) 117 | if err != nil { 118 | fmt.Println(err) 119 | os.Exit(1) 120 | } 121 | } 122 | } 123 | break 124 | case "kill": 125 | fmt.Println("Killing job") 126 | resp, err := r.KillJob(job.JobKey()) 127 | if err != nil { 128 | fmt.Println(err) 129 | os.Exit(1) 130 | } 131 | 132 | if resp.ResponseCode == aurora.ResponseCode_OK { 133 | if ok, err := monitor.Instances(job.JobKey(), 0, 5, 50); !ok || err != nil { 134 | fmt.Println("Unable to kill all instances of job") 135 | os.Exit(1) 136 | } 137 | } 138 | break 139 | case "restart": 140 | fmt.Println("Restarting job") 141 | resp, err := r.RestartJob(job.JobKey()) 142 | if err != nil { 143 | fmt.Println(err) 144 | os.Exit(1) 145 | } 146 | 147 | fmt.Println(resp.String()) 148 | break 149 | case "liveCount": 150 | fmt.Println("Getting instance count") 151 | 152 | live, err := r.GetInstanceIds(job.JobKey(), aurora.LIVE_STATES) 153 | if err != nil { 154 | fmt.Println(err) 155 | os.Exit(1) 156 | } 157 | 158 | fmt.Println("Number of live instances: ", len(live)) 159 | break 160 | case "activeCount": 161 | fmt.Println("Getting instance count") 162 | 163 | live, err := r.GetInstanceIds(job.JobKey(), aurora.ACTIVE_STATES) 164 | if err != nil { 165 | fmt.Println(err) 166 | os.Exit(1) 167 | } 168 | 169 | fmt.Println("Number of live instances: ", len(live)) 170 | break 171 | case "flexUp": 172 | fmt.Println("Flexing up job") 173 | 174 | numOfInstances := int32(2) 175 | resp, err := r.AddInstances(aurora.InstanceKey{job.JobKey(), 0}, numOfInstances) 176 | if err != nil { 177 | fmt.Println(err) 178 | os.Exit(1) 179 | } 180 | 181 | if ok, err := r.MonitorInstances(job.JobKey(), job.GetInstanceCount()+numOfInstances, 5, 500); !ok || err != nil { 182 | fmt.Println("Flexing up failed") 183 | } 184 | fmt.Println(resp.String()) 185 | break 186 | case "taskConfig": 187 | fmt.Println("Getting job info") 188 | config, err := r.FetchTaskConfig(aurora.InstanceKey{job.JobKey(), 0}) 189 | if err != nil { 190 | fmt.Println(err) 191 | os.Exit(1) 192 | } 193 | print(config.String()) 194 | break 195 | default: 196 | fmt.Println("Command not supported") 197 | os.Exit(1) 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /pluginimpl/general/editor_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package general 16 | 17 | import ( 18 | "container/list" 19 | "context" 20 | "strconv" 21 | "testing" 22 | 23 | "fmt" 24 | 25 | "github.com/paypal/dce-go/config" 26 | "github.com/paypal/dce-go/types" 27 | "github.com/paypal/dce-go/utils/file" 28 | "github.com/stretchr/testify/assert" 29 | ) 30 | 31 | func TestGenerateEditComposeFile(t *testing.T) { 32 | config.GetConfig().SetDefault(types.NO_FOLDER, true) 33 | var ports list.List 34 | u, _ := strconv.ParseUint("1000", 10, 64) 35 | ports.PushBack(u) 36 | u, _ = strconv.ParseUint("2000", 10, 64) 37 | ports.PushBack(u) 38 | u, _ = strconv.ParseUint("3000", 10, 64) 39 | ports.PushBack(u) 40 | 41 | var ctx context.Context 42 | ctx = context.Background() 43 | var servDetail types.ServiceDetail 44 | servDetail, err := file.ParseYamls(&[]string{"testdata/test.yml"}) 45 | if err != nil { 46 | t.Fatalf("Error to parse yaml files : %v", err) 47 | } 48 | 49 | var extraHosts = make(map[interface{}]bool) 50 | ctx = context.WithValue(ctx, types.SERVICE_DETAIL, servDetail) 51 | _, curPort, _ := editComposeFile("testdata/test.yml", "executorId", "taskId", ports.Front(), extraHosts) 52 | 53 | if curPort == nil || strconv.FormatUint(curPort.Value.(uint64), 10) != "3000" { 54 | t.Errorf("expected current port to be 3000 but got %v", curPort) 55 | } 56 | err = file.WriteChangeToFiles() 57 | if err != nil { 58 | t.Fatalf("Failed to write to files %v", err) 59 | } 60 | } 61 | 62 | func Test_editComposeFile(t *testing.T) { 63 | config.GetConfig().SetDefault(types.NO_FOLDER, true) 64 | var ports list.List 65 | u, _ := strconv.ParseUint("1000", 10, 64) 66 | ports.PushBack(u) 67 | u, _ = strconv.ParseUint("2000", 10, 64) 68 | ports.PushBack(u) 69 | u, _ = strconv.ParseUint("3000", 10, 64) 70 | ports.PushBack(u) 71 | 72 | var ctx context.Context 73 | ctx = context.Background() 74 | var servDetail types.ServiceDetail 75 | servDetail, err := file.ParseYamls(&[]string{"testdata/test.yml"}) 76 | if err != nil { 77 | t.Fatalf("Error to parse yaml files : %v", err) 78 | } 79 | ctx = context.WithValue(ctx, types.SERVICE_DETAIL, servDetail) 80 | 81 | // Before edit compose file 82 | containerDetails := servDetail["testdata/test.yml"][types.SERVICES].(map[interface{}]interface{})["mysql"].(map[interface{}]interface{}) 83 | assert.Equal(t, nil, containerDetails[types.LABELS], "Before editing compose file") 84 | 85 | var extraHosts = make(map[interface{}]bool) 86 | _, curPort, _ := editComposeFile("testdata/test.yml", "executorId", "taskId", ports.Front(), extraHosts) 87 | 88 | // After edit compose file 89 | if curPort == nil || strconv.FormatUint(curPort.Value.(uint64), 10) != "3000" { 90 | t.Errorf("expected current port to be 3000 but got %v", curPort) 91 | } 92 | servDetailAfter := ctx.Value(types.SERVICE_DETAIL).(types.ServiceDetail) 93 | containerDetailsAfter := servDetailAfter["testdata/test.yml-generated.yml"][types.SERVICES].(map[interface{}]interface{})["mysql"].(map[interface{}]interface{}) 94 | assert.Equal(t, 2, len(containerDetailsAfter[types.LABELS].(map[interface{}]interface{})), "After editing compose file") 95 | if _, ok := containerDetailsAfter[types.CGROUP_PARENT]; !ok { 96 | t.Error("Expected cgroup parent should be added") 97 | } 98 | if _, ok := containerDetailsAfter[types.RESTART]; ok { 99 | t.Error("Expected restart to be removed") 100 | } 101 | if _, ok := containerDetailsAfter[types.EXTRA_HOSTS]; ok { 102 | t.Error("Expected extra hosts should be removed") 103 | } 104 | assert.Equal(t, 1, len(extraHosts), "test extra host") 105 | } 106 | 107 | func Test_scanForExtraHostsSection(t *testing.T) { 108 | var extraHosts = make(map[interface{}]bool) 109 | containerDetail := make(map[interface{}]interface{}) 110 | containerDetail[types.EXTRA_HOSTS] = []interface{}{"redis1:0.0.0.0"} 111 | scanForExtraHostsSection(containerDetail, extraHosts) 112 | assert.Equal(t, 1, len(extraHosts), "Remove extra host from service") 113 | if _, ok := containerDetail[types.EXTRA_HOSTS]; ok { 114 | t.Errorf("Expected extra host is removed, but got %v\n", containerDetail[types.EXTRA_HOSTS]) 115 | } 116 | 117 | scanForExtraHostsSection(containerDetail, extraHosts) 118 | assert.Equal(t, 1, len(extraHosts), "Remove extra host from service if no extra host is defined") 119 | if _, ok := containerDetail[types.EXTRA_HOSTS]; ok { 120 | t.Errorf("Expected extra host is removed, but got %v\n", containerDetail[types.EXTRA_HOSTS]) 121 | } 122 | } 123 | 124 | func Test_addExtraHostsSection(t *testing.T) { 125 | config.GetConfig().SetDefault(types.NO_FOLDER, true) 126 | var servDetail types.ServiceDetail 127 | servDetail, err := file.ParseYamls(&[]string{"testdata/docker-extra-host.yml"}) 128 | if err != nil { 129 | t.Fatalf("Error to parse yaml files : %v", err) 130 | } 131 | fmt.Println(servDetail) 132 | var ctx context.Context 133 | var extraHosts = make(map[interface{}]bool) 134 | ctx = context.Background() 135 | ctx = context.WithValue(ctx, types.SERVICE_DETAIL, servDetail) 136 | containerDetail := make(map[interface{}]interface{}) 137 | containerDetail[types.EXTRA_HOSTS] = []interface{}{"redis2:0.0.0.2"} 138 | scanForExtraHostsSection(containerDetail, extraHosts) 139 | addExtraHostsSection(ctx, "testdata/docker-extra-host.yml", "redis", extraHosts) 140 | filesMap, ok := ctx.Value(types.SERVICE_DETAIL).(types.ServiceDetail) 141 | if !ok { 142 | t.Error("Couldn't get service detail") 143 | return 144 | } 145 | servMap, ok := filesMap["testdata/docker-extra-host.yml"][types.SERVICES].(map[interface{}]interface{}) 146 | if !ok { 147 | t.Error("Couldn't get content of compose file ") 148 | return 149 | } 150 | 151 | if containerDetail, ok := servMap["redis"].(map[interface{}]interface{}); !ok { 152 | t.Error("Couldn't convert service to map[interface{}]interface{}") 153 | } else { 154 | res := containerDetail[types.EXTRA_HOSTS].([]interface{}) 155 | assert.Equal(t, 3, len(res), "Adding extra host to service") 156 | fmt.Println("extra host:", res) 157 | } 158 | 159 | preCtx := ctx 160 | addExtraHostsSection(ctx, "testdata/docker-extra-host.yml", "fake", extraHosts) 161 | assert.Equal(t, preCtx, ctx, "Adding extra host to non exist service") 162 | } 163 | -------------------------------------------------------------------------------- /utils/file/testdata/yaml: -------------------------------------------------------------------------------- 1 | #---# 2 | #docker-compose-base.yml 3 | 4 | version: '2' 5 | services: 6 | 7 | ##Application pm2/node container 8 | paastestkrak13nodeweb: 9 | image: dockerhub.paypalcorp.com/altus-ci/paastestkrak13nodeweb:1.0.3_2017020216315929 10 | container_name: paastestkrak13nodeweb_1.0.3_2017020216315929 11 | stdin_open: true 12 | network_mode: "host" 13 | depends_on: 14 | - keymakeragent 15 | - appidentityfetcher 16 | user: cronusapp 17 | volumes: 18 | - /etc/nginx 19 | - /dependencies/paastestkrak13nodeweb/cronus/scripts 20 | - ./appdata:/appdata 21 | - ./log/applogs:/dependencies/paastestkrak13nodeweb/cronus/scripts/logs 22 | - ./log/deploylogs:/dependencies/paastestkrak13nodeweb/cronus/scripts/deploylogs 23 | - ./log/nginxlogs:/var/log/nginx 24 | - /etc/syshiera.yaml:/etc/syshiera.yaml 25 | - keymakeragent-identities:/x/web/keymaker/identities/ 26 | - paastestkrak13nodeweb:/data 27 | 28 | - keystore:/dependencies/paastestkrak13nodeweb_keystore 29 | 30 | environment: 31 | - CRONUSAPP_HOME=/dependencies 32 | - TERM=xterm 33 | - DISABLE_NGINX_AUTO_START=true 34 | - AUTHCODE 35 | - AUTHCODE2 36 | - METADATA_ENV_FILE 37 | - METADATA_JSON_FILE 38 | - APP_USER=altusapp 39 | 40 | ##App Nginx container 41 | nginx: 42 | image: dockerhub.paypalcorp.com/altus-ci/paastestkrak13nodeweb:1.0.3_2017020216315929 43 | container_name: nginx_1.0.3_2017020216315929 44 | network_mode: "host" 45 | entrypoint: /bin/bash -c "until [ -f /dependencies/paastestkrak13nodeweb/cronus/scripts/nginx.conf ]; do sleep 5; done; if ! grep -q 'daemon off' /dependencies/paastestkrak13nodeweb/cronus/scripts/nginx.conf ; then sed -i -e '1idaemon off;\' /dependencies/paastestkrak13nodeweb/cronus/scripts/nginx.conf; fi; /usr/sbin/nginx -c /dependencies/paastestkrak13nodeweb/cronus/scripts/nginx.conf || exit 1;" 46 | volumes_from: 47 | - paastestkrak13nodeweb 48 | environment: 49 | - TERM=xterm 50 | labels: 51 | primary.container: "true" 52 | healthcheck.container: "true" 53 | health.check.url: '/bin/bash -c "curl -kf https://\$HOSTNAME:443/ecv"' 54 | health.check.output: "in" 55 | 56 | #Keymaker Agent Container 57 | keymakeragent: 58 | image: dockerhub.paypalcorp.com/paas/keymakeragent:207.0-24264011 59 | container_name: keymakeragent_1.0.3_2017020216315929 60 | network_mode: "host" 61 | stdin_open: true 62 | restart: "on-failure:3" 63 | depends_on: 64 | - keymakeragentprotected 65 | volumes: 66 | - /etc/syshiera.yaml:/etc/syshiera.yaml 67 | - keymakeragent-protected:/x/web/LIVE/keymakeragent/protected 68 | - keymakeragent-identities:/x/web/keymaker/identities 69 | environment: 70 | - AUTHCODE 71 | - AUTHCODE2 72 | 73 | #Keymaker Protected Packages Container 74 | keymakeragentprotected: 75 | image: dockerhub.paypalcorp.com/paas/keymakeragent-protected 76 | container_name: keymakeragentprotected_1.0.3_2017020216315929 77 | volumes: 78 | - /etc/syshiera.yaml:/etc/syshiera.yaml 79 | - keymakeragent-protected:/x/web/LIVE/keymakeragent/protected 80 | 81 | #Keymaker Identity file Download Container 82 | appidentityfetcher: 83 | container_name: appidentityfetcher_1.0.3_2017020216315929 84 | image: dockerhub.paypalcorp.com/paas/appidentity-fetcher:1.1 85 | volumes: 86 | - /etc/syshiera.yaml:/etc/syshiera.yaml 87 | - keymakeragent-identities:/x/web/identities/ 88 | environment: 89 | - APPNAME 90 | 91 | 92 | #Application Protected Packages Container 93 | keystorecontainer: 94 | image: dockerhub.paypalcorp.com/keystore/paastestkrak13nodeweb_keystore:qa_1.0.50010 95 | container_name: keystorecontainer_1.0.3_2017020216315929 96 | volumes: 97 | - keystore:/keystorevolume 98 | 99 | 100 | volumes: 101 | #Application presistent data volume 102 | paastestkrak13nodeweb: 103 | driver: local 104 | #Keymaker Named Data Volumes 105 | keymakeragent-protected: 106 | driver: local 107 | keymakeragent-identities: 108 | driver: local 109 | 110 | keystore: 111 | driver: local 112 | 113 | 114 | #---# 115 | #docker-compose-qa.yml 116 | 117 | version: '2' 118 | services: 119 | 120 | ##Application container 121 | paastestkrak13nodeweb: 122 | environment: 123 | - SERVICE_HOST_FQDN=live.qa.paypal.com 124 | - SHARE_PWD=aardvark 125 | - HAPROXY_JSON 126 | 127 | #---# 128 | #docker-compose-production.yml 129 | 130 | version: '2' 131 | services: 132 | 133 | #Trace Logging Daemon 134 | svctracedaemon: 135 | image: dockerhub.paypalcorp.com/paas/svctracedaemon:201.0-18010465 136 | container_name: svctracedaemon_1.0.3_2017020216315929 137 | network_mode: "host" 138 | stdin_open: true 139 | restart: "on-failure:3" 140 | volumes: 141 | - /etc/syshiera.yaml:/etc/syshiera.yaml 142 | environment: 143 | - AUTHCODE 144 | - AUTHCODE2 145 | 146 | 147 | #Application Protected Packages Container Overrides 148 | keystorecontainer: 149 | image: dockerhub.paypalcorp.com/keystore/paastestkrak13nodeweb_keystore:production_1.0.50010 150 | 151 | 152 | #---# 153 | #docker-compose-sandbox.yml 154 | 155 | version: '2' 156 | services: 157 | 158 | #Trace Logging Daemon 159 | svctracedaemon: 160 | image: dockerhub.paypalcorp.com/paas/svctracedaemon:201.0-18010465 161 | container_name: svctracedaemon_1.0.3_2017020216315929 162 | network_mode: "host" 163 | stdin_open: true 164 | restart: "on-failure:3" 165 | volumes: 166 | - /etc/syshiera.yaml:/etc/syshiera.yaml 167 | environment: 168 | - AUTHCODE 169 | - AUTHCODE2 170 | 171 | 172 | #Application Protected Packages Container Overrides 173 | keystorecontainer: 174 | image: dockerhub.paypalcorp.com/keystore/paastestkrak13nodeweb_keystore:sandbox_1.0.50010 175 | 176 | 177 | #---# 178 | #docker-compose-debug.yml 179 | 180 | version: '2' 181 | services: 182 | 183 | paastestkrak13nodeweb: 184 | entrypoint: /bin/bash -c "sleep infinity" 185 | 186 | #---# 187 | #docker-compose-healthcheck.yml 188 | 189 | version: '2' 190 | services: 191 | 192 | ##Application pm2/node container 193 | paastestkrak13nodeweb: 194 | healthcheck: 195 | interval: 10s 196 | timeout: 5s 197 | retries: 100 198 | test: TEST_ECV=`grep '"ecvURL"' /dependencies/paastestkrak13nodeweb/cronus/scripts/manifest.json | awk 'match($$0, /".*"(.*)"/, a) {print a[1]}'` ; curl -kf $$TEST_ECV | grep -i 'in' || exit 1 199 | labels: 200 | healthcheck.container: "true" 201 | 202 | ##App Nginx container 203 | nginx: 204 | depends_on: 205 | paastestkrak13nodeweb: 206 | condition: service_healthy 207 | healthcheck: 208 | interval: 10s 209 | timeout: 5s 210 | retries: 100 211 | test: curl -kf https://`hostname -f`:443/ecv | grep -i 'in' || exit 1 212 | labels: 213 | healthcheck.container: "true" 214 | -------------------------------------------------------------------------------- /plugin/extpoints.go: -------------------------------------------------------------------------------- 1 | // generated by go-extpoints -- DO NOT EDIT 2 | package plugin 3 | 4 | import ( 5 | "reflect" 6 | "runtime" 7 | "strings" 8 | "sync" 9 | ) 10 | 11 | var extRegistry = ®istryType{m: make(map[string]*extensionPoint)} 12 | 13 | type registryType struct { 14 | sync.Mutex 15 | m map[string]*extensionPoint 16 | } 17 | 18 | // Top level registration 19 | 20 | func extensionTypes(extension interface{}) []string { 21 | var ifaces []string 22 | typ := reflect.TypeOf(extension) 23 | for name, ep := range extRegistry.m { 24 | if ep.iface.Kind() == reflect.Func && typ.AssignableTo(ep.iface) { 25 | ifaces = append(ifaces, name) 26 | } 27 | if ep.iface.Kind() != reflect.Func && typ.Implements(ep.iface) { 28 | ifaces = append(ifaces, name) 29 | } 30 | } 31 | return ifaces 32 | } 33 | 34 | func RegisterExtension(extension interface{}, name string) []string { 35 | extRegistry.Lock() 36 | defer extRegistry.Unlock() 37 | var ifaces []string 38 | for _, iface := range extensionTypes(extension) { 39 | if extRegistry.m[iface].register(extension, name) { 40 | ifaces = append(ifaces, iface) 41 | } 42 | } 43 | return ifaces 44 | } 45 | 46 | func UnregisterExtension(name string) []string { 47 | extRegistry.Lock() 48 | defer extRegistry.Unlock() 49 | var ifaces []string 50 | for iface, extpoint := range extRegistry.m { 51 | if extpoint.unregister(name) { 52 | ifaces = append(ifaces, iface) 53 | } 54 | } 55 | return ifaces 56 | } 57 | 58 | // Base extension point 59 | 60 | type extensionPoint struct { 61 | sync.Mutex 62 | iface reflect.Type 63 | extensions map[string]interface{} 64 | } 65 | 66 | func newExtensionPoint(iface interface{}) *extensionPoint { 67 | ep := &extensionPoint{ 68 | iface: reflect.TypeOf(iface).Elem(), 69 | extensions: make(map[string]interface{}), 70 | } 71 | extRegistry.Lock() 72 | extRegistry.m[ep.iface.Name()] = ep 73 | extRegistry.Unlock() 74 | return ep 75 | } 76 | 77 | func (ep *extensionPoint) lookup(name string) interface{} { 78 | ep.Lock() 79 | defer ep.Unlock() 80 | ext, ok := ep.extensions[name] 81 | if !ok { 82 | return nil 83 | } 84 | return ext 85 | } 86 | 87 | func (ep *extensionPoint) all() map[string]interface{} { 88 | ep.Lock() 89 | defer ep.Unlock() 90 | all := make(map[string]interface{}) 91 | for k, v := range ep.extensions { 92 | all[k] = v 93 | } 94 | return all 95 | } 96 | 97 | func (ep *extensionPoint) register(extension interface{}, name string) bool { 98 | ep.Lock() 99 | defer ep.Unlock() 100 | if name == "" { 101 | typ := reflect.TypeOf(extension) 102 | if typ.Kind() == reflect.Func { 103 | nameParts := strings.Split(runtime.FuncForPC( 104 | reflect.ValueOf(extension).Pointer()).Name(), ".") 105 | name = nameParts[len(nameParts)-1] 106 | } else { 107 | name = typ.Elem().Name() 108 | } 109 | } 110 | _, exists := ep.extensions[name] 111 | if exists { 112 | return false 113 | } 114 | ep.extensions[name] = extension 115 | return true 116 | } 117 | 118 | func (ep *extensionPoint) unregister(name string) bool { 119 | ep.Lock() 120 | defer ep.Unlock() 121 | _, exists := ep.extensions[name] 122 | if !exists { 123 | return false 124 | } 125 | delete(ep.extensions, name) 126 | return true 127 | } 128 | 129 | // ComposePlugin 130 | 131 | var ComposePlugins = &composePluginExt{ 132 | newExtensionPoint(new(ComposePlugin)), 133 | } 134 | 135 | type composePluginExt struct { 136 | *extensionPoint 137 | } 138 | 139 | func (ep *composePluginExt) Unregister(name string) bool { 140 | return ep.unregister(name) 141 | } 142 | 143 | func (ep *composePluginExt) Register(extension ComposePlugin, name string) bool { 144 | return ep.register(extension, name) 145 | } 146 | 147 | func (ep *composePluginExt) Lookup(name string) ComposePlugin { 148 | ext := ep.lookup(name) 149 | if ext == nil { 150 | return nil 151 | } 152 | return ext.(ComposePlugin) 153 | } 154 | 155 | func (ep *composePluginExt) Select(names []string) []ComposePlugin { 156 | var selected []ComposePlugin 157 | for _, name := range names { 158 | selected = append(selected, ep.Lookup(name)) 159 | } 160 | return selected 161 | } 162 | 163 | func (ep *composePluginExt) All() map[string]ComposePlugin { 164 | all := make(map[string]ComposePlugin) 165 | for k, v := range ep.all() { 166 | all[k] = v.(ComposePlugin) 167 | } 168 | return all 169 | } 170 | 171 | func (ep *composePluginExt) Names() []string { 172 | var names []string 173 | for k := range ep.all() { 174 | names = append(names, k) 175 | } 176 | return names 177 | } 178 | 179 | // PodStatusHook 180 | 181 | var PodStatusHooks = &podStatusHookExt{ 182 | newExtensionPoint(new(PodStatusHook)), 183 | } 184 | 185 | type podStatusHookExt struct { 186 | *extensionPoint 187 | } 188 | 189 | func (ep *podStatusHookExt) Unregister(name string) bool { 190 | return ep.unregister(name) 191 | } 192 | 193 | func (ep *podStatusHookExt) Register(extension PodStatusHook, name string) bool { 194 | return ep.register(extension, name) 195 | } 196 | 197 | func (ep *podStatusHookExt) Lookup(name string) PodStatusHook { 198 | ext := ep.lookup(name) 199 | if ext == nil { 200 | return nil 201 | } 202 | return ext.(PodStatusHook) 203 | } 204 | 205 | func (ep *podStatusHookExt) Select(names []string) []PodStatusHook { 206 | var selected []PodStatusHook 207 | for _, name := range names { 208 | selected = append(selected, ep.Lookup(name)) 209 | } 210 | return selected 211 | } 212 | 213 | func (ep *podStatusHookExt) All() map[string]PodStatusHook { 214 | all := make(map[string]PodStatusHook) 215 | for k, v := range ep.all() { 216 | all[k] = v.(PodStatusHook) 217 | } 218 | return all 219 | } 220 | 221 | func (ep *podStatusHookExt) Names() []string { 222 | var names []string 223 | for k := range ep.all() { 224 | names = append(names, k) 225 | } 226 | return names 227 | } 228 | 229 | // Monitor 230 | 231 | var Monitors = &monitorExt{ 232 | newExtensionPoint(new(Monitor)), 233 | } 234 | 235 | type monitorExt struct { 236 | *extensionPoint 237 | } 238 | 239 | func (ep *monitorExt) Unregister(name string) bool { 240 | return ep.unregister(name) 241 | } 242 | 243 | func (ep *monitorExt) Register(extension Monitor, name string) bool { 244 | return ep.register(extension, name) 245 | } 246 | 247 | func (ep *monitorExt) Lookup(name string) Monitor { 248 | ext := ep.lookup(name) 249 | if ext == nil { 250 | return nil 251 | } 252 | return ext.(Monitor) 253 | } 254 | 255 | func (ep *monitorExt) Select(names []string) []Monitor { 256 | var selected []Monitor 257 | for _, name := range names { 258 | selected = append(selected, ep.Lookup(name)) 259 | } 260 | return selected 261 | } 262 | 263 | func (ep *monitorExt) All() map[string]Monitor { 264 | all := make(map[string]Monitor) 265 | for k, v := range ep.all() { 266 | all[k] = v.(Monitor) 267 | } 268 | return all 269 | } 270 | 271 | func (ep *monitorExt) Names() []string { 272 | var names []string 273 | for k := range ep.all() { 274 | names = append(names, k) 275 | } 276 | return names 277 | } 278 | -------------------------------------------------------------------------------- /mesos-modules/m4/ax_cxx_compile_stdcxx_11.m4: -------------------------------------------------------------------------------- 1 | # ============================================================================ 2 | # http://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx_11.html 3 | # ============================================================================ 4 | # 5 | # SYNOPSIS 6 | # 7 | # AX_CXX_COMPILE_STDCXX_11([ext|noext],[mandatory|optional]) 8 | # 9 | # DESCRIPTION 10 | # 11 | # Check for baseline language coverage in the compiler for the C++11 12 | # standard; if necessary, add switches to CXXFLAGS to enable support. 13 | # 14 | # The first argument, if specified, indicates whether you insist on an 15 | # extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. 16 | # -std=c++11). If neither is specified, you get whatever works, with 17 | # preference for an extended mode. 18 | # 19 | # The second argument, if specified 'mandatory' or if left unspecified, 20 | # indicates that baseline C++11 support is required and that the macro 21 | # should error out if no mode with that support is found. If specified 22 | # 'optional', then configuration proceeds regardless, after defining 23 | # HAVE_CXX11 if and only if a supporting mode is found. 24 | # 25 | # LICENSE 26 | # 27 | # Copyright (c) 2008 Benjamin Kosnik 28 | # Copyright (c) 2012 Zack Weinberg 29 | # Copyright (c) 2013 Roy Stogner 30 | # 31 | # Copying and distribution of this file, with or without modification, are 32 | # permitted in any medium without royalty provided the copyright notice 33 | # and this notice are preserved. This file is offered as-is, without any 34 | # warranty. 35 | 36 | #serial 3 37 | 38 | m4_define([_AX_CXX_COMPILE_STDCXX_11_testbody], [ 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | 45 | template 46 | struct check 47 | { 48 | static_assert(sizeof(int) <= sizeof(T), "not big enough"); 49 | }; 50 | 51 | typedef check> right_angle_brackets; 52 | 53 | int a; 54 | decltype(a) b; 55 | 56 | typedef check check_type; 57 | check_type c; 58 | check_type&& cr = static_cast(c); 59 | 60 | auto d = a; 61 | 62 | struct Foo 63 | { 64 | void bar() const {} 65 | }; 66 | 67 | void f(const Foo& foo) { foo.bar(); } 68 | 69 | void baz() { 70 | std::unique_ptr p1(new Foo); // p1 owns Foo. 71 | p1->bar(); 72 | 73 | { 74 | std::unique_ptr p2(std::move(p1)); // Now p2 owns Foo. 75 | f(*p2); 76 | 77 | p1 = std::move(p2); // Ownership returns to p1. 78 | } 79 | 80 | p1->bar(); 81 | } 82 | 83 | std::shared_ptr k = std::make_shared(2); 84 | 85 | void mutexTest() 86 | { 87 | std::mutex _mutex; 88 | { 89 | // scope of lockGuard. 90 | std::lock_guard lockGuard(_mutex); 91 | // end scope of lockGuard. 92 | } 93 | 94 | { 95 | // scope of uniqueLock. 96 | std::unique_lock uniqueLock(_mutex); 97 | // end scope of uniqueLock. 98 | } 99 | } 100 | 101 | // check for std::enable_shared_from_this. 102 | struct SharedStruct : public std::enable_shared_from_this 103 | { 104 | std::shared_ptr get() 105 | { 106 | return shared_from_this(); 107 | } 108 | }; 109 | 110 | // construct a new shared_ptr using shared_from_this(). 111 | std::shared_ptr object = 112 | std::shared_ptr(new SharedStruct())->get(); 113 | 114 | // initializer lists. 115 | std::vector g = {"hello", "world"}; 116 | 117 | struct InitializerList 118 | { 119 | InitializerList(std::initializer_list) {} 120 | void doSomething(std::initializer_list) {} 121 | }; 122 | 123 | void initializerListClassTest() 124 | { 125 | InitializerList il{1,2,3,4}; 126 | il.doSomething({5,6,7,8}); 127 | } 128 | ]) 129 | 130 | AC_DEFUN([AX_CXX_COMPILE_STDCXX_11], [ 131 | m4_if([$1], [], [], 132 | [$1], [ext], [], 133 | [$1], [noext], [], 134 | [m4_fatal([invalid argument `$1' to AX_CXX_COMPILE_STDCXX_11])]) 135 | m4_if([$2], [], [ax_cxx_compile_cxx11_required=true], 136 | [$2], [mandatory], [ax_cxx_compile_cxx11_required=true], 137 | [$2], [optional], [ax_cxx_compile_cxx11_required=false], 138 | [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX_11])]) 139 | AC_LANG_PUSH([C++]) 140 | ac_success=no 141 | AC_CACHE_CHECK(whether $CXX supports C++11 features by default, 142 | ax_cv_cxx_compile_cxx11, 143 | [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_11_testbody])], 144 | [ax_cv_cxx_compile_cxx11=yes], 145 | [ax_cv_cxx_compile_cxx11=no])]) 146 | if test x$ax_cv_cxx_compile_cxx11 = xyes; then 147 | ac_success=yes 148 | fi 149 | 150 | m4_if([$1], [noext], [], [ 151 | if test x$ac_success = xno; then 152 | for switch in -std=gnu++11 -std=gnu++0x; do 153 | cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx11_$switch]) 154 | AC_CACHE_CHECK(whether $CXX supports C++11 features with $switch, 155 | $cachevar, 156 | [ac_save_CXXFLAGS="$CXXFLAGS" 157 | CXXFLAGS="$CXXFLAGS $switch" 158 | AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_11_testbody])], 159 | [eval $cachevar=yes], 160 | [eval $cachevar=no]) 161 | CXXFLAGS="$ac_save_CXXFLAGS"]) 162 | if eval test x\$$cachevar = xyes; then 163 | CXXFLAGS="$CXXFLAGS $switch" 164 | ac_success=yes 165 | break 166 | fi 167 | done 168 | fi]) 169 | 170 | m4_if([$1], [ext], [], [ 171 | if test x$ac_success = xno; then 172 | for switch in -std=c++11 -std=c++0x; do 173 | cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx11_$switch]) 174 | AC_CACHE_CHECK(whether $CXX supports C++11 features with $switch, 175 | $cachevar, 176 | [ac_save_CXXFLAGS="$CXXFLAGS" 177 | CXXFLAGS="$CXXFLAGS $switch" 178 | AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_11_testbody])], 179 | [eval $cachevar=yes], 180 | [eval $cachevar=no]) 181 | CXXFLAGS="$ac_save_CXXFLAGS"]) 182 | if eval test x\$$cachevar = xyes; then 183 | CXXFLAGS="$CXXFLAGS $switch" 184 | ac_success=yes 185 | break 186 | fi 187 | done 188 | fi]) 189 | AC_LANG_POP([C++]) 190 | if test x$ax_cxx_compile_cxx11_required = xtrue; then 191 | if test x$ac_success = xno; then 192 | AC_MSG_ERROR([*** A compiler with support for C++11 language features is required.]) 193 | fi 194 | else 195 | if test x$ac_success = xno; then 196 | HAVE_CXX11=0 197 | AC_MSG_NOTICE([No compiler with C++11 support was found]) 198 | else 199 | HAVE_CXX11=1 200 | AC_DEFINE(HAVE_CXX11,1, 201 | [define if the compiler supports basic C++11 syntax]) 202 | fi 203 | 204 | AC_SUBST(HAVE_CXX11) 205 | fi 206 | ]) 207 | -------------------------------------------------------------------------------- /utils/wait/wait.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package wait 16 | 17 | import ( 18 | "errors" 19 | "os" 20 | "os/exec" 21 | "strings" 22 | "time" 23 | 24 | log "github.com/sirupsen/logrus" 25 | 26 | "github.com/paypal/dce-go/config" 27 | "github.com/paypal/dce-go/types" 28 | ) 29 | 30 | type ConditionCHFunc func(done chan string) 31 | type ConditionFunc func() (string, error) 32 | 33 | var LogStatus = &LogCommandStatus{ 34 | IsRunning: false, 35 | } 36 | 37 | var ErrTimeOut = errors.New("timed out waiting for the condition") 38 | 39 | // Keep polling a condition func until it return a message or an error 40 | // PollForever always wait interval 41 | // PollForever will keep polling forever, no timeout 42 | func PollForever(interval time.Duration, done <-chan string, condition ConditionFunc) (string, error) { 43 | return PollUntil(interval, done, 0, condition) 44 | } 45 | 46 | // PollRetry repeats running condition functions with a backoff until it runs successfully 47 | // Or it already retried multiple times which is set in config 48 | // The backoff time will be retry * interval 49 | func PollRetry(retry int, interval time.Duration, condition ConditionFunc) error { 50 | log.Println("PullRetry : max pull retry is set as", retry) 51 | log.Println("PullRetry : interval:", interval) 52 | 53 | var err error 54 | factor := 1 55 | for i := 0; i < retry; i++ { 56 | if i != 0 { 57 | log.Println("Condition Func failed, Start Retrying : ", i) 58 | } 59 | _, err = condition() 60 | if err == nil { 61 | return nil 62 | } 63 | 64 | time.Sleep(time.Duration((i+1)*factor) * interval) 65 | } 66 | return ErrTimeOut 67 | } 68 | 69 | // Keep polling a condition func until timeout or a message/error is returned 70 | func PollUntil(interval time.Duration, done <-chan string, timeout time.Duration, condition ConditionFunc) (string, error) { 71 | ticker := time.NewTicker(interval) 72 | defer ticker.Stop() 73 | 74 | var after <-chan time.Time 75 | if timeout != 0 { 76 | timer := time.NewTimer(timeout) 77 | after = timer.C 78 | defer timer.Stop() 79 | } 80 | 81 | for { 82 | select { 83 | case <-ticker.C: 84 | res, err := WaitUntil(timeout, ConditionCHFunc(func(reply chan string) { 85 | condition_reply, _ := condition() 86 | reply <- condition_reply 87 | })) 88 | if err != nil { 89 | return res, err 90 | } 91 | if res != "" { 92 | return res, nil 93 | } 94 | case <-after: 95 | return "", ErrTimeOut 96 | case mesg := <-done: 97 | return mesg, nil 98 | } 99 | } 100 | } 101 | 102 | // Run condition in goroutine, wait for condition's return until timeout 103 | // If timeout, return ErrTimeOut 104 | // If message received from condition, return message 105 | func WaitUntil(timeout time.Duration, condition ConditionCHFunc) (string, error) { 106 | var after <-chan time.Time 107 | if timeout != 0 { 108 | timer := time.NewTimer(timeout) 109 | after = timer.C 110 | defer timer.Stop() 111 | } 112 | 113 | replyCH := make(chan string, 1) 114 | go condition(replyCH) 115 | 116 | select { 117 | case <-after: 118 | return "", ErrTimeOut 119 | case res := <-replyCH: 120 | return res, nil 121 | } 122 | } 123 | 124 | // wait on exec command finished or timeout 125 | func WaitCmd(timeout time.Duration, cmd_result *types.CmdResult) error { 126 | if timeout < time.Duration(0) { 127 | log.Println("TIMEOUT is less than zero") 128 | return ErrTimeOut 129 | } 130 | 131 | done := make(chan error, 1) 132 | go func() { 133 | done <- cmd_result.Command.Wait() 134 | }() 135 | 136 | select { 137 | case err := <-done: 138 | if err != nil { 139 | log.Println("cmd.wait() return error :", err.Error()) 140 | } 141 | 142 | return err 143 | 144 | case <-time.After(timeout): 145 | log.Println("cmd.wait() timeout :", ErrTimeOut.Error()) 146 | return ErrTimeOut 147 | } 148 | } 149 | 150 | // Retry command util reach the maximum try out count 151 | func RetryCmd(retry int, cmd *exec.Cmd) ([]byte, error) { 152 | var err error 153 | var out []byte 154 | //log.Debugf("RetryCmd: Run cmd: %s\n", cmd.Args) 155 | 156 | retryInterval := config.GetRetryInterval() 157 | factor := 1 158 | for i := 0; i < retry; i++ { 159 | _cmd := exec.Command(cmd.Args[0], cmd.Args[1:]...) 160 | 161 | if cmd.Stdout == nil { 162 | _cmd.Stderr = os.Stderr 163 | out, err = _cmd.Output() 164 | } else { 165 | _cmd.Stdout = cmd.Stdout 166 | _cmd.Stderr = cmd.Stderr 167 | err = _cmd.Run() 168 | } 169 | 170 | if err != nil { 171 | log.Warnf("Error to exec cmd %v with count %d : %v, retry after %v Millisecond", _cmd.Args, i, err, retryInterval) 172 | if strings.Contains(err.Error(), "already started") { 173 | return out, nil 174 | } 175 | time.Sleep(retryInterval * time.Duration((i+1)*factor)) 176 | continue 177 | } 178 | 179 | return out, nil 180 | } 181 | return nil, err 182 | } 183 | 184 | // Retry command forever 185 | func RetryCmdLogs(cmd *exec.Cmd, retry bool) ([]byte, error) { 186 | var err error 187 | var out []byte 188 | 189 | retryInterval := config.GetRetryInterval() 190 | 191 | for { 192 | _cmd := exec.Command(cmd.Args[0], cmd.Args[1:]...) 193 | log.Printf("Run cmd is: %s", _cmd.Args) 194 | 195 | if cmd.Stdout == nil { 196 | _cmd.Stdout = os.Stdout 197 | _cmd.Stderr = os.Stderr 198 | out, err = _cmd.Output() 199 | } else { 200 | 201 | _cmd.Stdout = cmd.Stdout 202 | _cmd.Stderr = cmd.Stderr 203 | 204 | //assuming that the log command will run successfully. 205 | SetLogStatus(true) 206 | err = _cmd.Run() 207 | 208 | //if this line executes, that means that either the log command returned which means something went wrong, 209 | //or there is an error. So either way, the logs are not getting logged. so reset it to false. 210 | SetLogStatus(false) 211 | log.Printf("Updated Log Status, Log command was Not successful") 212 | if err != nil { 213 | log.Printf("Error while running cmd: %v", err) 214 | } 215 | 216 | //So that we only retry and log all the container logs ones before fully killing/finishing the pod. 217 | //If we remove this, then this loop will print the container logs infinitely. 218 | //This is imp for small apps that complete before we start logging their logs. 219 | if !retry { 220 | return out, err 221 | } 222 | } 223 | log.Printf("cmd %s exits, retry...", _cmd.Args) 224 | time.Sleep(retryInterval) 225 | } 226 | return out, err 227 | } 228 | 229 | // Read log status 230 | func GetLogStatus() bool { 231 | LogStatus.RLock() 232 | defer LogStatus.RUnlock() 233 | return LogStatus.IsRunning 234 | } 235 | 236 | // Set log status 237 | func SetLogStatus(logCommandRunSuccess bool) { 238 | LogStatus.Lock() 239 | defer LogStatus.Unlock() 240 | LogStatus.IsRunning = logCommandRunSuccess 241 | } 242 | -------------------------------------------------------------------------------- /mesos-modules/m4/ax_compare_version.m4: -------------------------------------------------------------------------------- 1 | # =========================================================================== 2 | # http://www.gnu.org/software/autoconf-archive/ax_compare_version.html 3 | # =========================================================================== 4 | # SYNOPSIS 5 | # 6 | # AX_COMPARE_VERSION(VERSION_A, OP, VERSION_B, [ACTION-IF-TRUE], [ACTION-IF-FALSE]) 7 | # 8 | # DESCRIPTION 9 | # 10 | # This macro compares two version strings. Due to the various number of 11 | # minor-version numbers that can exist, and the fact that string 12 | # comparisons are not compatible with numeric comparisons, this is not 13 | # necessarily trivial to do in a autoconf script. This macro makes doing 14 | # these comparisons easy. 15 | # 16 | # The six basic comparisons are available, as well as checking equality 17 | # limited to a certain number of minor-version levels. 18 | # 19 | # The operator OP determines what type of comparison to do, and can be one 20 | # of: 21 | # 22 | # eq - equal (test A == B) 23 | # ne - not equal (test A != B) 24 | # le - less than or equal (test A <= B) 25 | # ge - greater than or equal (test A >= B) 26 | # lt - less than (test A < B) 27 | # gt - greater than (test A > B) 28 | # 29 | # Additionally, the eq and ne operator can have a number after it to limit 30 | # the test to that number of minor versions. 31 | # 32 | # eq0 - equal up to the length of the shorter version 33 | # ne0 - not equal up to the length of the shorter version 34 | # eqN - equal up to N sub-version levels 35 | # neN - not equal up to N sub-version levels 36 | # 37 | # When the condition is true, shell commands ACTION-IF-TRUE are run, 38 | # otherwise shell commands ACTION-IF-FALSE are run. The environment 39 | # variable 'ax_compare_version' is always set to either 'true' or 'false' 40 | # as well. 41 | # 42 | # Examples: 43 | # 44 | # AX_COMPARE_VERSION([3.15.7],[lt],[3.15.8]) 45 | # AX_COMPARE_VERSION([3.15],[lt],[3.15.8]) 46 | # 47 | # would both be true. 48 | # 49 | # AX_COMPARE_VERSION([3.15.7],[eq],[3.15.8]) 50 | # AX_COMPARE_VERSION([3.15],[gt],[3.15.8]) 51 | # 52 | # would both be false. 53 | # 54 | # AX_COMPARE_VERSION([3.15.7],[eq2],[3.15.8]) 55 | # 56 | # would be true because it is only comparing two minor versions. 57 | # 58 | # AX_COMPARE_VERSION([3.15.7],[eq0],[3.15]) 59 | # 60 | # would be true because it is only comparing the lesser number of minor 61 | # versions of the two values. 62 | # 63 | # Note: The characters that separate the version numbers do not matter. An 64 | # empty string is the same as version 0. OP is evaluated by autoconf, not 65 | # configure, so must be a string, not a variable. 66 | # 67 | # The author would like to acknowledge Guido Draheim whose advice about 68 | # the m4_case and m4_ifvaln functions make this macro only include the 69 | # portions necessary to perform the specific comparison specified by the 70 | # OP argument in the final configure script. 71 | # 72 | # LICENSE 73 | # 74 | # Copyright (c) 2008 Tim Toolan ; 75 | # 76 | # Copying and distribution of this file, with or without modification, are 77 | # permitted in any medium without royalty provided the copyright notice 78 | # and this notice are preserved. This file is offered as-is, without any 79 | # warranty. 80 | 81 | #serial 11 82 | 83 | dnl ######################################################################### 84 | AC_DEFUN([AX_COMPARE_VERSION], [ 85 | AC_REQUIRE([AC_PROG_AWK]) 86 | 87 | # Used to indicate true or false condition 88 | ax_compare_version=false 89 | 90 | # Convert the two version strings to be compared into a format that 91 | # allows a simple string comparison. The end result is that a version 92 | # string of the form 1.12.5-r617 will be converted to the form 93 | # 0001001200050617. In other words, each number is zero padded to four 94 | # digits, and non digits are removed. 95 | AS_VAR_PUSHDEF([A],[ax_compare_version_A]) 96 | A=`echo "$1" | sed -e 's/\([[0-9]]*\)/Z\1Z/g' \ 97 | -e 's/Z\([[0-9]]\)Z/Z0\1Z/g' \ 98 | -e 's/Z\([[0-9]][[0-9]]\)Z/Z0\1Z/g' \ 99 | -e 's/Z\([[0-9]][[0-9]][[0-9]]\)Z/Z0\1Z/g' \ 100 | -e 's/[[^0-9]]//g'` 101 | 102 | AS_VAR_PUSHDEF([B],[ax_compare_version_B]) 103 | B=`echo "$3" | sed -e 's/\([[0-9]]*\)/Z\1Z/g' \ 104 | -e 's/Z\([[0-9]]\)Z/Z0\1Z/g' \ 105 | -e 's/Z\([[0-9]][[0-9]]\)Z/Z0\1Z/g' \ 106 | -e 's/Z\([[0-9]][[0-9]][[0-9]]\)Z/Z0\1Z/g' \ 107 | -e 's/[[^0-9]]//g'` 108 | 109 | dnl # In the case of le, ge, lt, and gt, the strings are sorted as necessary 110 | dnl # then the first line is used to determine if the condition is true. 111 | dnl # The sed right after the echo is to remove any indented white space. 112 | m4_case(m4_tolower($2), 113 | [lt],[ 114 | ax_compare_version=`echo "x$A 115 | x$B" | sed 's/^ *//' | sort -r | sed "s/x${A}/false/;s/x${B}/true/;1q"` 116 | ], 117 | [gt],[ 118 | ax_compare_version=`echo "x$A 119 | x$B" | sed 's/^ *//' | sort | sed "s/x${A}/false/;s/x${B}/true/;1q"` 120 | ], 121 | [le],[ 122 | ax_compare_version=`echo "x$A 123 | x$B" | sed 's/^ *//' | sort | sed "s/x${A}/true/;s/x${B}/false/;1q"` 124 | ], 125 | [ge],[ 126 | ax_compare_version=`echo "x$A 127 | x$B" | sed 's/^ *//' | sort -r | sed "s/x${A}/true/;s/x${B}/false/;1q"` 128 | ],[ 129 | dnl Split the operator from the subversion count if present. 130 | m4_bmatch(m4_substr($2,2), 131 | [0],[ 132 | # A count of zero means use the length of the shorter version. 133 | # Determine the number of characters in A and B. 134 | ax_compare_version_len_A=`echo "$A" | $AWK '{print(length)}'` 135 | ax_compare_version_len_B=`echo "$B" | $AWK '{print(length)}'` 136 | 137 | # Set A to no more than B's length and B to no more than A's length. 138 | A=`echo "$A" | sed "s/\(.\{$ax_compare_version_len_B\}\).*/\1/"` 139 | B=`echo "$B" | sed "s/\(.\{$ax_compare_version_len_A\}\).*/\1/"` 140 | ], 141 | [[0-9]+],[ 142 | # A count greater than zero means use only that many subversions 143 | A=`echo "$A" | sed "s/\(\([[0-9]]\{4\}\)\{m4_substr($2,2)\}\).*/\1/"` 144 | B=`echo "$B" | sed "s/\(\([[0-9]]\{4\}\)\{m4_substr($2,2)\}\).*/\1/"` 145 | ], 146 | [.+],[ 147 | AC_WARNING( 148 | [illegal OP numeric parameter: $2]) 149 | ],[]) 150 | 151 | # Pad zeros at end of numbers to make same length. 152 | ax_compare_version_tmp_A="$A`echo $B | sed 's/./0/g'`" 153 | B="$B`echo $A | sed 's/./0/g'`" 154 | A="$ax_compare_version_tmp_A" 155 | 156 | # Check for equality or inequality as necessary. 157 | m4_case(m4_tolower(m4_substr($2,0,2)), 158 | [eq],[ 159 | test "x$A" = "x$B" && ax_compare_version=true 160 | ], 161 | [ne],[ 162 | test "x$A" != "x$B" && ax_compare_version=true 163 | ],[ 164 | AC_WARNING([illegal OP parameter: $2]) 165 | ]) 166 | ]) 167 | 168 | AS_VAR_POPDEF([A])dnl 169 | AS_VAR_POPDEF([B])dnl 170 | 171 | dnl # Execute ACTION-IF-TRUE / ACTION-IF-FALSE. 172 | if test "$ax_compare_version" = "true" ; then 173 | m4_ifvaln([$4],[$4],[:])dnl 174 | m4_ifvaln([$5],[else $5])dnl 175 | fi 176 | ]) dnl AX_COMPARE_VERSION 177 | -------------------------------------------------------------------------------- /utils/file/file_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package file 16 | 17 | import ( 18 | "fmt" 19 | "io/ioutil" 20 | "os" 21 | "reflect" 22 | "strings" 23 | "testing" 24 | 25 | "github.com/paypal/dce-go/config" 26 | "github.com/paypal/dce-go/types" 27 | "github.com/stretchr/testify/assert" 28 | ) 29 | 30 | func TestPrefixTaskId(t *testing.T) { 31 | config.GetConfig().SetDefault(types.NO_FOLDER, true) 32 | prefix := "taskid" 33 | session := "session" 34 | res := PrefixTaskId(prefix, session) 35 | a := strings.Split(res, "_") 36 | if len(a) != 2 { 37 | t.Fatalf("expected len to be 2, got %v", len(a)) 38 | } 39 | if a[0] != prefix { 40 | t.Fatalf("expected prefix to be 'taskid', got %s", a[0]) 41 | } 42 | if a[1] != session { 43 | t.Fatalf("expected session to be 'session', got %s", a[1]) 44 | } 45 | } 46 | 47 | func TestParseYamls(t *testing.T) { 48 | config.GetConfig().SetDefault(types.NO_FOLDER, true) 49 | yamls := []string{"testdata/docker-adhoc.yml", "testdata/docker-long.yml", "testdata/docker-empty.yml"} 50 | res, err := ParseYamls(&yamls) 51 | if err != nil { 52 | t.Fatalf("Got error to parseyamls %v", err) 53 | } 54 | //servs := res["testdata/docker-adhoc.yml"]["services"].(map[interface{}]interface{}) 55 | fmt.Println(res) 56 | } 57 | 58 | func TestWriteToGeneratedFile(t *testing.T) { 59 | config.GetConfig().SetDefault(types.NO_FOLDER, true) 60 | file, err := WriteToFile("wirtetofiletest.txt", []byte("hello,world")) 61 | if err != nil { 62 | t.Fatalf("Got error to write to file %v", err) 63 | } 64 | if file != "wirtetofiletest.txt" { 65 | t.Fatalf("expected file name to be 'wirtetofiletest.txt', got %s", file) 66 | } 67 | exist := CheckFileExist(file) 68 | if !exist { 69 | t.Fatal("file isn't generated") 70 | } 71 | data, err := ioutil.ReadFile(file) 72 | if err != nil { 73 | t.Errorf("Error to read file,%s", err.Error()) 74 | } 75 | if string(data) != "hello,world" { 76 | t.Fatalf("expected content to be 'hello,world', got %s", string(data)) 77 | } 78 | os.Remove(file) 79 | } 80 | 81 | func TestIndexArray(t *testing.T) { 82 | array := make([]string, 3) 83 | array[0] = "pen" 84 | array[1] = "apple" 85 | array[2] = "peach" 86 | i := IndexArray(array, "apple") 87 | assert.Equal(t, 1, i, "") 88 | i = IndexArray(array, "test") 89 | assert.Equal(t, -1, i, "") 90 | } 91 | 92 | func TestReplaceArrayElement(t *testing.T) { 93 | //Test array 94 | array := make([]interface{}, 3) 95 | array[0] = "pen" 96 | array[1] = "apple" 97 | array[2] = "peach" 98 | res := ReplaceElement(array, "pen", "pencil").([]interface{}) 99 | if len(res) != len(array) || res[0] != "pencil" { 100 | t.Fatalf("expected first element to be 'pencil', but got %s", res[0]) 101 | } 102 | 103 | //Replace element not exist in array 104 | res1 := ReplaceElement(array, "not_exist", "not_exist").([]interface{}) 105 | if len(res1) != len(res) { 106 | t.Fatalf("Expected array doesn't change, but got %v \n", res1) 107 | } 108 | 109 | array[0] = "fruit=banana" 110 | res2 := ReplaceElement(array, "^fruit=", "fruit=apple").([]interface{}) 111 | if res2[0] != "fruit=apple" { 112 | t.Fatalf("Expected fruit=apple replace fruit=banana, but got %v \n", res2) 113 | } 114 | 115 | array[0] = "fruit.banana" 116 | res5 := ReplaceElement(array, "^fruit$", "fruit=apple").([]interface{}) 117 | if res5[0] != "fruit.banana" { 118 | t.Fatalf("Expected no changes, but got %v \n", res5) 119 | } 120 | 121 | array[0] = "fruitbanana" 122 | res6 := ReplaceElement(array, "^fruit$", "fruit=apple").([]interface{}) 123 | if res6[0] != "fruitbanana" { 124 | t.Fatalf("Expected no changes, but got %v \n", res6) 125 | } 126 | 127 | //Test map 128 | m := make(map[interface{}]interface{}) 129 | m["key1"] = "val1" 130 | m["key2"] = "val2" 131 | res3 := ReplaceElement(m, "key2", "val3").(map[interface{}]interface{}) 132 | if res3["key2"].(string) != "val3" { 133 | t.Fatalf("expected first element to be 'val3', but got %s", res3["key3"]) 134 | } 135 | 136 | res4 := ReplaceElement(m, "key3", "val3").(map[interface{}]interface{}) 137 | _, ok := res4["key3"] 138 | if ok { 139 | t.Fatalf("Expected new element not added, but got %s", res4["key3"]) 140 | } 141 | } 142 | 143 | func TestAppendElement(t *testing.T) { 144 | //Test array 145 | array := make([]interface{}, 3) 146 | array[0] = "pen" 147 | array[1] = "apple" 148 | array[2] = "peach" 149 | res := AppendElement(array, "monkey", "monkey").([]interface{}) 150 | if len(res) != len(array)+1 || res[3] != "monkey" { 151 | t.Fatalf("expected first element to be 'monkey', but got %s", res[3]) 152 | } 153 | 154 | //Test duplicate element won't be appended 155 | res = AppendElement(array, "monkey", "monkey").([]interface{}) 156 | if len(res) != len(array)+1 || res[3] != "monkey" { 157 | t.Fatalf("expected first element to be 'monkey', but got %s", res[3]) 158 | } 159 | 160 | array[0] = "fruit=banana" 161 | res2 := AppendElement(array, "^fruit=", "fruit=apple").([]interface{}) 162 | if res2[0] != "fruit=apple" { 163 | t.Fatalf("Expected fruit=apple replace fruit=banana, but got %v \n", res2) 164 | } 165 | 166 | array[0] = "fruit.banana" 167 | res5 := AppendElement(array, "^fruit$", "fruit=apple1").([]interface{}) 168 | if res5[len(res5)-1] != "fruit=apple1" { 169 | t.Fatalf("Expected fruit=apple will be appended, but got %v \n", res5) 170 | } 171 | 172 | //Test map 173 | m := make(map[interface{}]interface{}) 174 | m["key1"] = "val1" 175 | m["key2"] = "val2" 176 | res1 := AppendElement(m, "key3", "val3").(map[interface{}]interface{}) 177 | if res1["key3"].(string) != "val3" { 178 | t.Fatalf("expected first element to be 'val3', but got %s", res1["key3"]) 179 | } 180 | } 181 | 182 | func TestSplitYAML(t *testing.T) { 183 | config.GetConfig().Set(types.NO_FOLDER, true) 184 | 185 | expectedFile1 := []string{"testdata/docker-compose-base.yml", "testdata/docker-compose-qa.yml", "testdata/docker-compose-production.yml", "testdata/docker-compose-sandbox.yml", "testdata/docker-compose-debug.yml", "testdata/docker-compose-healthcheck.yml"} 186 | files1, err := SplitYAML("testdata/yaml") 187 | if err != nil { 188 | t.Errorf(err.Error()) 189 | } 190 | 191 | if !reflect.DeepEqual(files1, expectedFile1) { 192 | t.Errorf("expected files to be %v, but got %v", expectedFile1, files1) 193 | } 194 | 195 | for _, file := range files1 { 196 | os.Remove(file) 197 | } 198 | 199 | expectedFile2 := []string{"testdata/docker-adhoc.yml"} 200 | files2, err := SplitYAML("testdata/docker-adhoc.yml") 201 | if err != nil { 202 | t.Errorf("Error split file %s\n", err.Error()) 203 | } 204 | if files2[0] != expectedFile2[0] { 205 | t.Errorf("expected files to be %s , but got %v", expectedFile2, files2) 206 | } 207 | 208 | for _, file := range files2 { 209 | os.Remove(file) 210 | } 211 | } 212 | 213 | func TestConvertArrayToMap(t *testing.T) { 214 | a := []interface{}{"a=b", "c", "d="} 215 | m := ConvertArrayToMap(a) 216 | assert.Equal(t, m["a"], "b", "a=b") 217 | assert.Equal(t, m["c"], "", "c") 218 | assert.Equal(t, m["d"], "", "d=") 219 | 220 | b := []interface{}{} 221 | m = ConvertArrayToMap(b) 222 | assert.Equal(t, len(m), 0, "empty map") 223 | } 224 | 225 | func TestGetDirFilesRecv(t *testing.T) { 226 | var files []string 227 | GetDirFilesRecv("testdata/docker-adhoc.yml", &files) 228 | assert.Equal(t, []string{"testdata/docker-adhoc.yml"}, files, "non archive format files should work") 229 | files = []string{} 230 | GetDirFilesRecv("testdata/config.zip", &files) 231 | assert.Equal(t, []string{"testdata/config/docker-adhoc.yml"}, files, "archive format files should work") 232 | } 233 | -------------------------------------------------------------------------------- /examples/vagrant/provision-dev-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D 17 | echo deb https://apt.dockerproject.org/repo ubuntu-trusty main > /etc/apt/sources.list.d/docker.list 18 | apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E56151BF 19 | echo deb http://repos.mesosphere.com/ubuntu trusty main > /etc/apt/sources.list.d/mesosphere.list 20 | 21 | apt-get update -q --fix-missing 22 | apt-get -qy install software-properties-common 23 | add-apt-repository ppa:george-edison55/cmake-3.x 24 | apt-get update -q 25 | apt-cache policy cmake 26 | 27 | add-apt-repository ppa:openjdk-r/ppa 28 | apt-get update 29 | apt-get -y install \ 30 | wget \ 31 | tar \ 32 | bison \ 33 | curl \ 34 | git \ 35 | openjdk-8-jdk \ 36 | docker-engine \ 37 | python-dev \ 38 | zookeeperd \ 39 | python-pip \ 40 | maven \ 41 | build-essential \ 42 | autoconf \ 43 | automake \ 44 | ca-certificates \ 45 | protobuf-compiler \ 46 | libprotobuf-dev \ 47 | make \ 48 | python \ 49 | python2.7 \ 50 | libpython-dev \ 51 | python-dev \ 52 | python-protobuf \ 53 | python-setuptools \ 54 | unzip \ 55 | --no-install-recommends 56 | 57 | sudo echo 1 > /sys/fs/cgroup/memory/memory.use_hierarchy 58 | 59 | update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java 60 | 61 | readonly IP_ADDRESS=192.168.33.8 62 | readonly MESOS_VERSION=1.0.0-2.0.89 63 | readonly MESOS_MODULE=v1.0.0 64 | readonly MARATHON_VERSION=1.1.2-1.0.482 65 | 66 | function install_go { 67 | 68 | wget -c "https://storage.googleapis.com/golang/go1.9.2.linux-amd64.tar.gz" "--no-check-certificate" 69 | tar -xvf go1.9.2.linux-amd64.tar.gz 70 | sudo mv go /usr/local 71 | 72 | export GOROOT=/usr/local/go 73 | export GOPATH=/home/vagrant/go 74 | export PATH=$PATH:$GOPATH/bin:$GOROOT/bin 75 | 76 | sed -e "\$aexport PATH=$PATH" /home/vagrant/.bashrc>/home/vagrant/.bashrc.new 77 | mv /home/vagrant/.bashrc.new /home/vagrant/.bashrc 78 | chmod 777 /home/vagrant/.bashrc 79 | 80 | sudo mkdir -p "$GOPATH/bin" "$GOPATH/src/github.com/paypal" 81 | 82 | curl https://glide.sh/get | sh 83 | 84 | } 85 | 86 | function install_mesos { 87 | apt-get -y install mesos=${MESOS_VERSION}.ubuntu1404 88 | } 89 | 90 | function prepare_extra { 91 | mkdir -p /etc/mesos-slave 92 | mkdir -p /etc/mesos-master 93 | mkdir -p /output/usr/local/lib/mesos 94 | mkdir -p /etc/docker || true 95 | cp /vagrant/examples/vagrant/config/daemon.json /etc/docker/ 96 | cp /vagrant/examples/vagrant/mesos_config/etc_mesos-slave/* /etc/mesos-slave 97 | cp /vagrant/examples/vagrant/mesos_config/etc_mesos-master/* /etc/mesos-master 98 | cp /vagrant/examples/vagrant/mesos-module/${MESOS_MODULE}/* /output/usr/local/lib/mesos/ 99 | cp /vagrant/examples/vagrant/marathon/marathon /etc/default/marathon 100 | } 101 | 102 | function install_aurora { 103 | wget -c https://apache.bintray.com/aurora/ubuntu-trusty/aurora-scheduler_0.16.0_amd64.deb 104 | sudo dpkg -i aurora-scheduler_0.16.0_amd64.deb 105 | sudo stop aurora-scheduler 106 | sudo -u aurora mkdir -p /var/lib/aurora/scheduler/db 107 | sudo -u aurora mesos-log initialize --path=/var/lib/aurora/scheduler/db 108 | sudo cp /vagrant/examples/vagrant/config/aurora-scheduler.conf /etc/init 109 | sudo start aurora-scheduler 110 | } 111 | 112 | function install_marathon { 113 | apt-get -y install marathon=${MARATHON_VERSION}.ubuntu1404 114 | } 115 | 116 | function install_docker_compose { 117 | pip install docker-compose 118 | } 119 | 120 | function build_docker_compose_executor { 121 | sudo ln -sf /vagrant $GOPATH/src/github.com/paypal/dce-go 122 | sudo chmod 777 $GOPATH/src/github.com/paypal/dce-go 123 | cd $GOPATH/src/github.com/paypal/dce-go 124 | glide install --strip-vendor 125 | go build -o executor $GOPATH/src/github.com/paypal/dce-go/dce/main.go 126 | mv executor /home/vagrant 127 | sed -i.bkp '$a export GOROOT=/usr/local/go;export GOPATH=/home/vagrant/go;export PATH=$PATH:$GOPATH/bin:$GOROOT/bin' /home/vagrant/.bashrc 128 | . /home/vagrant/.bashrc 129 | } 130 | 131 | function install_cluster_config { 132 | mkdir -p /etc/marathon 133 | ln -sf /vagrant/examples/vagrant/clusters.json /etc/marathon/clusters.json 134 | } 135 | 136 | function install_ssh_config { 137 | cat >> /etc/ssh/ssh_config < /home/vagrant/.netrc < /vagrant 183 | #ln -s /vagrant /var/www 184 | 185 | sudo cp /vagrant/examples/vagrant/config/config.yaml /usr/share/nginx/html 186 | sudo mkdir -p /usr/share/nginx/html/example 187 | sudo cp /vagrant/examples/vagrant/config/config_example.yaml /usr/share/nginx/html/example/config.yaml 188 | sudo cp /vagrant/examples/vagrant/config/general.yaml /usr/share/nginx/html 189 | sudo cp /home/vagrant/executor /usr/share/nginx/html 190 | sudo cp -r /vagrant/examples/sampleapp /usr/share/nginx/html/ 191 | cd /usr/share/nginx/html 192 | sudo tar -czvf app.tar.gz sampleapp 193 | } 194 | 195 | function install_mesos_modules { 196 | cd /vagrant 197 | sudo docker build -t mesos-build -f Dockerfile-mesos-modules --build-arg MESOS_VERSION=1.0.0 . 198 | sudo docker run --rm -v /output:/output mesos-build make install DESTDIR=/output 199 | } 200 | 201 | 202 | prepare_extra 203 | install_go 204 | #install_mesos_modules 205 | install_mesos 206 | install_aurora 207 | install_marathon 208 | install_docker_compose 209 | install_cluster_config 210 | install_ssh_config 211 | start_services 212 | configure_netrc 213 | sudoless_docker_setup 214 | build_docker_compose_executor 215 | configure_nginx 216 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | // Package config helps plugins extensions to easily set config as key/value by reading from configuration file 16 | // Package config also provides setter and getter functionality 17 | 18 | package config 19 | 20 | import ( 21 | "io/ioutil" 22 | "os" 23 | "path/filepath" 24 | "strconv" 25 | "strings" 26 | "time" 27 | 28 | mesos "github.com/mesos/mesos-go/api/v0/mesosproto" 29 | log "github.com/sirupsen/logrus" 30 | "github.com/spf13/viper" 31 | 32 | "github.com/paypal/dce-go/types" 33 | ) 34 | 35 | // Define default values 36 | const ( 37 | CONFIG_File = "config" 38 | FOLDER_NAME = "foldername" 39 | HEALTH_CHECK = "healthcheck" 40 | POD_MONITOR_INTERVAL = "launchtask.podmonitorinterval" 41 | TIMEOUT = "launchtask.timeout" 42 | INFRA_CONTAINER = "infracontainer" 43 | PULL_RETRY = "launchtask.pullretry" 44 | MAX_RETRY = "launchtask.maxretry" 45 | RETRY_INTERVAL = "launchtask.retryinterval" 46 | NETWORKS = "networks" 47 | PRE_EXIST = "pre_existing" 48 | NETWORK_NAME = "name" 49 | NETWORK_DRIVER = "driver" 50 | CLEANPOD = "cleanpod" 51 | CLEAN_CONTAINER_VOLUME_ON_MESOS_KILL = "cleanpod.cleanvolumeandcontaineronmesoskill" 52 | CLEAN_IMAGE_ON_MESOS_KILL = "cleanpod.cleanimageonmesoskill" 53 | CLEAN_FAIL_TASK = "cleanpod.cleanfailtask" 54 | DOCKER_COMPOSE_VERBOSE = "dockercomposeverbose" 55 | SKIP_PULL_IMAGES = "launchtask.skippull" 56 | COMPOSE_TRACE = "launchtask.composetrace" 57 | DEBUG_MODE = "launchtask.debug" 58 | COMPOSE_HTTP_TIMEOUT = "launchtask.composehttptimeout" 59 | HTTP_TIMEOUT = "launchtask.httptimeout" 60 | COMPOSE_STOP_TIMEOUT = "cleanpod.timeout" 61 | CONFIG_OVERRIDE_PREFIX = "config." 62 | monitorName = "podMonitor.monitorName" 63 | ) 64 | 65 | // Read from default configuration file and set config as key/values 66 | func init() { 67 | err := getConfigFromFile(CONFIG_File) 68 | 69 | if err != nil { 70 | log.Fatalf("Fail to retrieve data from file, err: %s\n", err.Error()) 71 | } 72 | } 73 | 74 | // Plugin extensions could merge configuration using ConfigInit with configuration file 75 | func ConfigInit(pluginConfig string) { 76 | log.Printf("Plugin Merge Config : %s", pluginConfig) 77 | 78 | file, err := os.Open(pluginConfig) 79 | if err != nil { 80 | log.Fatalf("Fail to open file, err: %s\n", err.Error()) 81 | } 82 | 83 | err = viper.MergeConfig(file) 84 | if err != nil { 85 | log.Fatalf("Fail to merge config, err: %s\n", err.Error()) 86 | } 87 | 88 | setDefaultConfig(GetConfig()) 89 | } 90 | 91 | func getConfigFromFile(cfgFile string) error { 92 | // Set config name 93 | viper.SetConfigName(cfgFile) 94 | 95 | // Set config type 96 | viper.SetConfigType("yaml") 97 | 98 | // Add path for searching config files including plugins' 99 | viper.AddConfigPath(".") 100 | abs_path, _ := filepath.Abs(".") 101 | viper.AddConfigPath(filepath.Join(filepath.Dir(filepath.Dir(abs_path)), "config")) 102 | dirs, _ := ioutil.ReadDir("./") 103 | for _, f := range dirs { 104 | if f.IsDir() { 105 | viper.AddConfigPath(f.Name()) 106 | } 107 | } 108 | 109 | err := viper.ReadInConfig() 110 | if err != nil { 111 | log.Errorf("No config file loaded, err: %s\n", err.Error()) 112 | return err 113 | } 114 | return nil 115 | } 116 | 117 | func SetConfig(key string, value interface{}) { 118 | log.Printf("Set config : %s = %v", key, value) 119 | GetConfig().Set(key, value) 120 | } 121 | 122 | func GetConfigSection(section string) map[string]string { 123 | return viper.GetStringMapString(section) 124 | } 125 | 126 | func GetConfig() *viper.Viper { 127 | return viper.GetViper() 128 | } 129 | 130 | func setDefaultConfig(conf *viper.Viper) { 131 | conf.SetDefault(POD_MONITOR_INTERVAL, "10s") 132 | conf.SetDefault(COMPOSE_HTTP_TIMEOUT, 300) 133 | conf.SetDefault(MAX_RETRY, 3) 134 | conf.SetDefault(PULL_RETRY, 3) 135 | conf.SetDefault(RETRY_INTERVAL, "10s") 136 | conf.SetDefault(TIMEOUT, "500s") 137 | conf.SetDefault(COMPOSE_STOP_TIMEOUT, "10s") 138 | conf.SetDefault(HTTP_TIMEOUT, "20s") 139 | conf.SetDefault(monitorName, "default") 140 | } 141 | 142 | func GetAppFolder() string { 143 | folder := GetConfig().GetString(FOLDER_NAME) 144 | if folder == "" { 145 | return types.DEFAULT_FOLDER 146 | } 147 | return folder 148 | } 149 | 150 | func GetPullRetryCount() int { 151 | return GetConfig().GetInt(PULL_RETRY) 152 | } 153 | 154 | // GetLaunchTimeout returns maximum time to wait until a pod becomes healthy. 155 | // Support value type as duration string. 156 | // A duration string is a possibly signed sequence of 157 | // decimal numbers, each with optional fraction and a unit suffix, 158 | // such as "300ms", "-1.5h" or "2h45m". 159 | // Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h" 160 | func GetLaunchTimeout() time.Duration { 161 | // Making backward compatible change to support config value of `launchtask.timeout` as duration (e.g. 500s), 162 | // as well as integer which is existing value type 163 | valStr := GetConfig().GetString(TIMEOUT) 164 | if valInt, err := strconv.Atoi(valStr); err == nil { 165 | return time.Duration(valInt) * time.Millisecond 166 | } 167 | duration, err := time.ParseDuration(valStr) 168 | if err != nil { 169 | log.Warningf("unable to parse launchtask.timeout from %s to duration, using 500s as default value", 170 | valStr) 171 | return 500 * time.Second 172 | } 173 | return duration 174 | } 175 | 176 | // GetStopTimeout returns the grace period time for a pod to die 177 | // Returns time in seconds as an integer 178 | func GetStopTimeout() int { 179 | // the timeout value is expected as duration. E.g: 10s 180 | duration := GetConfig().GetDuration(COMPOSE_STOP_TIMEOUT) 181 | return int(duration.Seconds()) 182 | } 183 | 184 | func GetRetryInterval() time.Duration { 185 | intervalStr := GetConfig().GetString(RETRY_INTERVAL) 186 | duration, err := time.ParseDuration(intervalStr) 187 | if err != nil { 188 | log.Warningf("unable to parse launchtask.retryinterval from %s to duration, using 10s as default value", 189 | intervalStr) 190 | return 10 * time.Second 191 | } 192 | return duration 193 | } 194 | 195 | func GetMaxRetry() int { 196 | return GetConfig().GetInt(MAX_RETRY) 197 | } 198 | 199 | func GetNetwork() (types.Network, bool) { 200 | nmap, ok := GetConfig().GetStringMap(INFRA_CONTAINER)[NETWORKS].(map[string]interface{}) 201 | if !ok { 202 | log.Println("networks section missing in config") 203 | return types.Network{}, false 204 | } 205 | 206 | if nmap[PRE_EXIST] == nil { 207 | nmap[PRE_EXIST] = false 208 | } 209 | 210 | if nmap[NETWORK_NAME] == nil { 211 | nmap[NETWORK_NAME] = "" 212 | } 213 | 214 | if nmap[NETWORK_DRIVER] == nil { 215 | nmap[NETWORK_DRIVER] = "" 216 | } 217 | 218 | network := types.Network{ 219 | PreExist: nmap[PRE_EXIST].(bool), 220 | Name: nmap[NETWORK_NAME].(string), 221 | Driver: nmap[NETWORK_DRIVER].(string), 222 | } 223 | return network, true 224 | } 225 | 226 | func EnableVerbose() bool { 227 | return GetConfig().GetBool(DOCKER_COMPOSE_VERBOSE) 228 | } 229 | 230 | func SkipPullImages() bool { 231 | return GetConfig().GetBool(SKIP_PULL_IMAGES) 232 | } 233 | 234 | func EnableComposeTrace() bool { 235 | return GetConfig().GetBool(COMPOSE_TRACE) 236 | } 237 | 238 | func GetPollInterval() time.Duration { 239 | intervalStr := GetConfig().GetString(POD_MONITOR_INTERVAL) 240 | duration, err := time.ParseDuration(intervalStr) 241 | if err != nil { 242 | log.Warningf("unable to parse launchtask.podmonitorinterval %s to duration, using 10s as default value", 243 | intervalStr) 244 | return 10 * time.Second 245 | } 246 | return duration 247 | } 248 | 249 | func GetHttpTimeout() time.Duration { 250 | timeoutStr := GetConfig().GetString(HTTP_TIMEOUT) 251 | duration, err := time.ParseDuration(timeoutStr) 252 | if err != nil { 253 | log.Warningf("unable to parse launchtask.httptimeout %s to duration, using 20s as default value", timeoutStr) 254 | return 20 * time.Second 255 | } 256 | return duration 257 | } 258 | 259 | func GetComposeHttpTimeout() int { 260 | return GetConfig().GetInt(COMPOSE_HTTP_TIMEOUT) 261 | } 262 | 263 | func EnableDebugMode() bool { 264 | return GetConfig().GetBool(DEBUG_MODE) 265 | } 266 | 267 | func IsService() bool { 268 | return GetConfig().GetBool(types.IS_SERVICE) 269 | } 270 | 271 | func CreateFileAppendMode(filename string) *os.File { 272 | 273 | File, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) 274 | if err != nil { 275 | log.Errorf("error in creating %v file, err: %v", filename, err) 276 | return os.Stdout 277 | } 278 | return File 279 | } 280 | 281 | // OverrideConfig gets labels with override prefix "config." and override configs with value of label 282 | // Checking labels contain key word "config." instead of prefix since different framework will add different prefix for labels 283 | func OverrideConfig(taskInfo *mesos.TaskInfo) { 284 | labelsList := taskInfo.GetLabels().GetLabels() 285 | 286 | for _, label := range labelsList { 287 | if strings.Contains(label.GetKey(), CONFIG_OVERRIDE_PREFIX) { 288 | parts := strings.SplitAfterN(label.GetKey(), CONFIG_OVERRIDE_PREFIX, 2) 289 | if len(parts) == 2 && GetConfig().IsSet(parts[1]) { 290 | GetConfig().Set(parts[1], label.GetValue()) 291 | log.Infof("override config %s with %s", parts[1], label.GetValue()) 292 | } 293 | } 294 | } 295 | } 296 | -------------------------------------------------------------------------------- /pluginimpl/general/impl.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package general 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "strconv" 21 | "strings" 22 | 23 | "github.com/mesos/mesos-go/api/v0/executor" 24 | mesos "github.com/mesos/mesos-go/api/v0/mesosproto" 25 | "github.com/paypal/dce-go/config" 26 | "github.com/paypal/dce-go/plugin" 27 | "github.com/paypal/dce-go/types" 28 | utils "github.com/paypal/dce-go/utils/file" 29 | "github.com/paypal/dce-go/utils/pod" 30 | "github.com/pkg/errors" 31 | log "github.com/sirupsen/logrus" 32 | "gopkg.in/yaml.v2" 33 | ) 34 | 35 | var logger *log.Entry 36 | 37 | type generalExt struct { 38 | } 39 | 40 | var infraYmlPath string 41 | 42 | func init() { 43 | log.SetOutput(config.CreateFileAppendMode(types.DCE_OUT)) 44 | 45 | logger = log.WithFields(log.Fields{ 46 | "plugin": "general", 47 | }) 48 | logger.Println("Plugin Registering") 49 | 50 | //Register plugin with name 51 | plugin.ComposePlugins.Register(new(generalExt), "general") 52 | 53 | //Merge plugin config file 54 | config.ConfigInit(utils.SearchFile(".", "general.yaml")) 55 | } 56 | 57 | func (p *generalExt) Name() string { 58 | return "general" 59 | } 60 | 61 | func (ge *generalExt) LaunchTaskPreImagePull(ctx context.Context, composeFiles *[]string, executorId string, taskInfo *mesos.TaskInfo) error { 62 | logger.Println("LaunchTaskPreImagePull begin") 63 | 64 | if composeFiles == nil || len(*composeFiles) == 0 { 65 | return fmt.Errorf(string(types.NoComposeFile)) 66 | } 67 | 68 | var editedFiles []string 69 | var err error 70 | 71 | logger.Println("====================context in====================") 72 | 73 | logger.Printf("Current compose files list: %v", *composeFiles) 74 | 75 | if len(pod.GetServiceDetail()) == 0 { 76 | var servDetail types.ServiceDetail 77 | servDetail, err = utils.ParseYamls(composeFiles) 78 | if err != nil { 79 | log.Errorf("Error parsing yaml files : %v", err) 80 | return err 81 | } 82 | 83 | pod.SetServiceDetail(servDetail) 84 | } 85 | 86 | currentPort := pod.GetPorts(taskInfo) 87 | 88 | // Create infra container yml file 89 | infrayml, err := CreateInfraContainer(ctx, types.INFRA_CONTAINER_YML) 90 | if err != nil { 91 | logger.Errorln("Error creating infra container : ", err.Error()) 92 | return err 93 | } 94 | *composeFiles = append(utils.FolderPath(*composeFiles), infrayml) 95 | 96 | var extraHosts = make(map[interface{}]bool) 97 | 98 | var indexInfra int 99 | for i, file := range *composeFiles { 100 | logger.Printf("Starting Edit compose file %s", file) 101 | var editedFile string 102 | editedFile, currentPort, err = editComposeFile(file, executorId, taskInfo.GetTaskId().GetValue(), currentPort, extraHosts) 103 | if err != nil { 104 | logger.Errorln("Error editing compose file : ", err.Error()) 105 | return err 106 | } 107 | 108 | if strings.Contains(editedFile, types.INFRA_CONTAINER_GEN_YML) { 109 | indexInfra = i 110 | infraYmlPath = editedFile 111 | } 112 | 113 | if editedFile != "" { 114 | editedFiles = append(editedFiles, editedFile) 115 | } 116 | } 117 | 118 | // Remove infra container yml file if network mode is host 119 | if config.GetConfig().GetBool(types.RM_INFRA_CONTAINER) { 120 | logger.Printf("Remove file: %s\n", types.INFRA_CONTAINER_GEN_YML) 121 | filesMap := pod.GetServiceDetail() 122 | delete(filesMap, editedFiles[indexInfra]) 123 | pod.SetServiceDetail(filesMap) 124 | editedFiles = append(editedFiles[:indexInfra], editedFiles[indexInfra+1:]...) 125 | err = utils.DeleteFile(types.INFRA_CONTAINER_YML) 126 | if err != nil { 127 | log.Errorf("Error deleting infra yml file %v", err) 128 | } 129 | } else { 130 | // Move extra_hosts from other compose files to infra container 131 | addExtraHostsSection(ctx, infraYmlPath, types.INFRA_CONTAINER, extraHosts) 132 | } 133 | 134 | logger.Println("====================context out====================") 135 | logger.Debugf("SERVICE_DETAIL: %+v", pod.GetServiceDetail()) 136 | 137 | *composeFiles = editedFiles 138 | 139 | logger.Println("Return massaged file list (Required Compose File List), ", *composeFiles) 140 | 141 | if currentPort != nil { 142 | logger.Println("Current port is ", strconv.FormatUint(currentPort.Value.(uint64), 10)) 143 | } 144 | 145 | return nil 146 | } 147 | 148 | func (gp *generalExt) LaunchTaskPostImagePull(ctx context.Context, composeFiles *[]string, executorId string, taskInfo *mesos.TaskInfo) error { 149 | logger.Println("LaunchTaskPostImagePull begin") 150 | return nil 151 | } 152 | 153 | func (gp *generalExt) PostLaunchTask(ctx context.Context, files []string, taskInfo *mesos.TaskInfo) (string, error) { 154 | logger.Println("PostLaunchTask begin") 155 | if pod.SinglePort { 156 | err := postEditComposeFile(infraYmlPath) 157 | if err != nil { 158 | log.Errorf("PostLaunchTask: Error editing compose file : %v", err) 159 | return types.POD_FAILED.String(), err 160 | } 161 | } 162 | return "", nil 163 | } 164 | 165 | func (gp *generalExt) PreKillTask(ctx context.Context, taskInfo *mesos.TaskInfo) error { 166 | logger.Println("PreKillTask begin") 167 | return nil 168 | } 169 | 170 | // PostKillTask cleans up containers, volumes, images if task is killed by mesos 171 | // Failed tasks will be cleaned up based on config cleanpod.cleanvolumeandcontaineronmesoskill and cleanpod.cleanimageonmesoskill 172 | // Non pre-existing networks will always be removed 173 | func (gp *generalExt) PostKillTask(ctx context.Context, taskInfo *mesos.TaskInfo) error { 174 | logger.Println("PostKillTask begin, pod status:", pod.GetPodStatus()) 175 | var err error 176 | if !pod.LaunchCmdAttempted { 177 | logger.Println("Pod hasn't started, no postKill work needed.") 178 | return nil 179 | } 180 | 181 | if pod.GetPodStatus() != types.POD_FAILED || config.GetConfig().GetBool(config.CLEAN_FAIL_TASK) { 182 | // clean pod volume and container if clean_container_volume_on_kill is true 183 | cleanVolumeAndContainer := config.GetConfig().GetBool(config.CLEAN_CONTAINER_VOLUME_ON_MESOS_KILL) 184 | if cleanVolumeAndContainer { 185 | err = pod.RemovePodVolume(pod.ComposeFiles) 186 | if err != nil { 187 | logger.Errorf("Error cleaning volumes: %v", err) 188 | } 189 | } 190 | logger.Println("Starting to clean image.") 191 | // clean pod images if clean_image_on_kill is true 192 | cleanImage := config.GetConfig().GetBool(config.CLEAN_IMAGE_ON_MESOS_KILL) 193 | if cleanImage { 194 | err = pod.RemovePodImage(pod.ComposeFiles) 195 | if err != nil { 196 | logger.Errorf("Error cleaning images: %v", err) 197 | } 198 | } 199 | } else { 200 | if network, ok := config.GetNetwork(); ok { 201 | if network.PreExist { 202 | return nil 203 | } 204 | } 205 | // skip removing network if network mode is host 206 | // RM_INFRA_CONTAINER is set as true if network mode is true during yml parsing 207 | if config.GetConfig().GetBool(types.RM_INFRA_CONTAINER) { 208 | return nil 209 | } 210 | 211 | // Get infra container id 212 | infraContainerId, err := pod.GetContainerIdByService(pod.ComposeFiles, types.INFRA_CONTAINER) 213 | if err != nil { 214 | logger.Errorf("Error getting container id of service %s: %v", types.INFRA_CONTAINER, err) 215 | return nil 216 | } 217 | 218 | networkName, err := pod.GetContainerNetwork(infraContainerId) 219 | if err != nil { 220 | logger.Errorf("Failed to clean up network :%v", err) 221 | } 222 | err = pod.RemoveNetwork(networkName) 223 | if err != nil { 224 | logger.Errorf("POD_CLEAN_NETWORK_FAIL -- %v", err) 225 | } 226 | } 227 | return err 228 | } 229 | 230 | func (gp *generalExt) Shutdown(taskInfo *mesos.TaskInfo, ed executor.ExecutorDriver) error { 231 | logger.Println("Shutdown begin") 232 | return nil 233 | } 234 | 235 | func CreateInfraContainer(ctx context.Context, path string) (string, error) { 236 | containerDetail := make(map[interface{}]interface{}) 237 | service := make(map[interface{}]interface{}) 238 | _yaml := make(map[string]interface{}) 239 | 240 | containerDetail[types.CONTAINER_NAME] = config.GetConfigSection(config.INFRA_CONTAINER)[types.CONTAINER_NAME] 241 | containerDetail[types.IMAGE] = config.GetConfigSection(config.INFRA_CONTAINER)[types.IMAGE] 242 | 243 | if network, ok := config.GetNetwork(); ok { 244 | if network.PreExist { 245 | serviceNetworks := make(map[string]interface{}) 246 | external := make(map[string]interface{}) 247 | name := make(map[string]string) 248 | if network.Name == "" { 249 | log.Warningln("Error in configuration file! Network Name is required if PreExist is true") 250 | return "", errors.New("NetworkName missing in general.yaml") 251 | } 252 | name[types.NAME] = network.Name 253 | external[types.NETWORK_EXTERNAL] = name 254 | serviceNetworks[types.NETWORK_DEFAULT_NAME] = external 255 | _yaml[types.NETWORKS] = serviceNetworks 256 | 257 | } else { 258 | serviceNetworks := make(map[string]interface{}) 259 | driver := make(map[string]string) 260 | if network.Driver == "" { 261 | network.Driver = types.NETWORK_DEFAULT_DRIVER 262 | } 263 | if network.Name == "" { 264 | network.Name = types.NETWORK_DEFAULT_NAME 265 | } 266 | driver[types.NETWORK_DRIVER] = network.Driver 267 | serviceNetworks[network.Name] = driver 268 | _yaml[types.NETWORKS] = serviceNetworks 269 | containerDetail[types.NETWORKS] = []string{network.Name} 270 | 271 | } 272 | } 273 | 274 | service[types.INFRA_CONTAINER] = containerDetail 275 | _yaml[types.SERVICES] = service 276 | _yaml[types.VERSION] = "2.1" 277 | log.Println(_yaml) 278 | 279 | content, _ := yaml.Marshal(_yaml) 280 | fileName, err := utils.WriteToFile(path, content) 281 | if err != nil { 282 | log.Errorf("Error writing infra container details into fils %v", err) 283 | return "", err 284 | } 285 | 286 | fileMap := pod.GetServiceDetail() 287 | 288 | fileMap[fileName] = _yaml 289 | 290 | pod.SetServiceDetail(fileMap) 291 | return fileName, nil 292 | } 293 | -------------------------------------------------------------------------------- /examples/sampleapp/app/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | DCE-GO 4 | 353 | 354 | 355 | 356 |

DCE-GO

357 |
358 |

Getting Started with DCE-GO

359 |

This project enables Mesos frameworks
to launch a pod of docker containers.

360 | 383 |
384 | 385 | 386 | 387 | 388 | -------------------------------------------------------------------------------- /mesos-modules/configure.ac: -------------------------------------------------------------------------------- 1 | AC_INIT([mesos-modules],[0.2]) 2 | AC_PREREQ([2.69]) 3 | 4 | AC_CANONICAL_HOST 5 | AC_CANONICAL_BUILD 6 | AC_CANONICAL_TARGET 7 | 8 | AC_CONFIG_FILES([Makefile]) 9 | AC_LANG([C++]) 10 | AC_PROG_CC 11 | 12 | # If the user did not provide any overrides of `CXXFLAGS` then assume 13 | # he actually means to have them empty. This may seem surprising to 14 | # experienced autotools users knowing that `AC_PROG_CXX` sets `CXXFLAGS` 15 | # to a default (commonly `-g -O2`) but note that debugging symbols and 16 | # optimizing are explicitely set further down using `MESOS_CXXFLAGS`. 17 | old_CXXFLAGS="$CXXFLAGS" 18 | AC_PROG_CXX 19 | if test "x$old_CXXFLAGS" = "x"; then 20 | CXXFLAGS= 21 | fi 22 | 23 | AC_PROG_CC_C_O 24 | AC_CONFIG_MACRO_DIR([m4]) 25 | 26 | AM_INIT_AUTOMAKE([foreign]) 27 | 28 | LT_PREREQ([2.2]) 29 | LT_INIT 30 | LT_LANG([C++]) 31 | LT_OUTPUT 32 | 33 | 34 | ############################################################################### 35 | # Optional features. 36 | ############################################################################### 37 | 38 | AC_ARG_ENABLE([debug], 39 | AS_HELP_STRING([--enable-debug], 40 | [Enable debug symbols @<:@default=yes@:>@]), 41 | [], [enable_debug=yes]) 42 | 43 | AC_ARG_ENABLE([optimize], 44 | AS_HELP_STRING([--enable-optimize], 45 | [Enable optimizing @<:@default=no@:>@]), 46 | [], [enable_optimize=yes]) 47 | 48 | 49 | ############################################################################### 50 | # Optional packages. 51 | ############################################################################### 52 | 53 | AC_ARG_WITH([mesos], 54 | [AS_HELP_STRING([--with-mesos=@<:@=DIR@:>@], 55 | [provide root dir for Mesos installation])], 56 | [mesos=$withval], []) 57 | 58 | AC_ARG_WITH([mesos_root], 59 | [AS_HELP_STRING([--with-mesos-root=@<:@=DIR@:>@], 60 | [provide root dir for Mesos sources])], 61 | [mesos_root=$withval], []) 62 | 63 | AC_ARG_WITH([mesos_build_dir], 64 | [AS_HELP_STRING([--with-mesos-build-dir=@<:@=DIR@:>@], 65 | [provide Mesos build dir])], 66 | [mesos_build_dir=$withval], []) 67 | 68 | AC_ARG_WITH([protobuf], 69 | [AS_HELP_STRING([--with-protobuf=@<:@=DIR@:>@], 70 | [provide install location of protobuf])], 71 | [with_protobuf=$withval], []) 72 | 73 | AC_ARG_WITH([glog], 74 | [AS_HELP_STRING([--with-glog=@<:@=DIR@:>@], 75 | [provide install location of glog])], 76 | [with_glog=$withval], []) 77 | 78 | AC_ARG_WITH([openssl], 79 | [AS_HELP_STRING([--with-openssl=@<:@=DIR@:>@], 80 | [provide install location of openssl])], 81 | [with_openssl=$withval], []) 82 | 83 | 84 | MESOS_CXXFLAGS= 85 | 86 | ############################################################################### 87 | # Debug/Optimization checks. 88 | ############################################################################### 89 | 90 | debug_flags="" 91 | if test "x$enable_debug" = "xyes"; then 92 | debug_flags="-g" 93 | fi 94 | 95 | optimize_flags="-O0" 96 | if test "x$enable_optimize" = "xyes"; then 97 | optimize_flags="-O2" 98 | fi 99 | 100 | MESOS_CXXFLAGS="${MESOS_CXXFLAGS} $debug_flags $optimize_flags" 101 | 102 | ############################################################################### 103 | ############################################################################### 104 | ############################################################################### 105 | 106 | MESOS_ROOT="$mesos_root" 107 | AC_SUBST([MESOS_ROOT]) 108 | 109 | MESOS_BUILD_DIR="$mesos_build_dir" 110 | AC_SUBST([MESOS_BUILD_DIR]) 111 | 112 | if test -d "$mesos"; then 113 | MESOS_CPPFLAGS="-I${mesos}/include" 114 | if test -d "${mesos}/lib64"; then 115 | MESOS_LDFLAGS="-L${mesos}/lib64 -lmesos" 116 | else 117 | MESOS_LDFLAGS="-L${mesos}/lib -lmesos" 118 | fi 119 | else 120 | if test -d "$mesos_root"; then 121 | MESOS_CPPFLAGS="${MESOS_CPPFLAGS} -I${mesos_root}/include" 122 | 123 | # Add src to include path. 124 | MESOS_CPPFLAGS="${MESOS_CPPFLAGS} -I${mesos_root}/src" 125 | 126 | # Add libprocess to include path. 127 | MESOS_CPPFLAGS+=" -I${mesos_root}/3rdparty/libprocess/include" 128 | 129 | # Add stout to include path. 130 | MESOS_CPPFLAGS="${MESOS_CPPFLAGS} -I${mesos_root}/3rdparty/stout/include" 131 | else 132 | AC_MSG_ERROR([Invalid mesos path; use --with-mesos-root= 133 | is the top-level directory of the Mesos distribution.]) 134 | fi 135 | 136 | if test -d "$mesos_build_dir"; then 137 | # Add built sources (i.e. protoc'd protobufs). 138 | MESOS_CPPFLAGS="${MESOS_CPPFLAGS} -I${mesos_build_dir}/include" 139 | 140 | # add picojson and boost headers 141 | MESOS_CPPFLAGS="${MESOS_CPPFLAGS} -I${mesos_build_dir}/3rdparty/boost-1.53.0" 142 | MESOS_CPPFLAGS="${MESOS_CPPFLAGS} -I${mesos_build_dir}/3rdparty/picojson-1.3.0" 143 | 144 | # Add src to include path. 145 | MESOS_CPPFLAGS="${MESOS_CPPFLAGS} -I${mesos_build_dir}/src" 146 | 147 | # Add the built libmesos. 148 | MESOS_LDFLAGS="${MESOS_LDFLAGS} -L${mesos_build_dir}/src -lmesos" 149 | else 150 | AC_MSG_ERROR([Invalid mesos build path; use --with-mesos-build-dir= 151 | is the Mesos build directory.]) 152 | fi 153 | fi 154 | 155 | if test -n "`echo $with_protobuf`"; then 156 | # Check for the protobuf compiler at user-provided location. 157 | AC_PATH_TOOL([PROTOCOMPILER], [protoc], [], [${with_protobuf}/bin]) 158 | MESOS_CPPFLAGS="${MESOS_CPPFLAGS} -I${with_protobuf}/include" 159 | MESOS_LDFLAGS="${MESOS_LDFLAGS} -L${with_protobuf}/lib -lprotobuf" 160 | else 161 | # Check for the protobuf compiler in default PATH. 162 | AC_PATH_TOOL([PROTOCOMPILER], [protoc], [], [$PATH]) 163 | fi 164 | 165 | if test -z "`echo $PROTOCOMPILER`"; then 166 | AC_MSG_ERROR([protoc not found in PATH]) 167 | fi 168 | 169 | if test -n "`echo $with_glog`"; then 170 | MESOS_CPPFLAGS="${MESOS_CPPFLAGS} -I${with_glog}/include" 171 | MESOS_LDFLAGS="${MESOS_LDFLAGS} -L${with_glog}/lib -lglog" 172 | fi 173 | 174 | if test -n "`echo $with_openssl`"; then 175 | MESOS_CPPFLAGS="${MESOS_CPPFLAGS} -I${with_openssl}/include" 176 | MESOS_LDFLAGS="${MESOS_LDFLAGS} -L${with_openssl}/lib" 177 | MESOS_LDFLAGS="${MESOS_LDFLAGS} -lssl -lcrypto" 178 | fi 179 | 180 | 181 | # Enable flag to check whether root and build dir were provided. 182 | AM_CONDITIONAL([SOURCE_PROVIDED], [test -d "$mesos_root"]) 183 | 184 | AC_SUBST(MESOS_CPPFLAGS) 185 | AC_SUBST(MESOS_CXXFLAGS) 186 | AC_SUBST(MESOS_LDFLAGS) 187 | 188 | AC_MSG_NOTICE([Setting up build environment for ${target_cpu} ${target_os}]) 189 | 190 | # Determine the current OS. 191 | case "${target_os}" in 192 | linux*) 193 | OS_NAME=linux 194 | LIB_EXT="so" 195 | ;; 196 | *) 197 | AC_MSG_ERROR("Modules are currently unsupported on your platform.") 198 | esac 199 | 200 | # Used for conditionally including source files. 201 | AM_CONDITIONAL([OS_LINUX], [test "x$OS_NAME" = "xlinux"]) 202 | 203 | AC_SUBST(LIB_EXT) 204 | 205 | # Check if we're using clang. 206 | AC_MSG_CHECKING([if compiling with clang]) 207 | 208 | AC_LANG_PUSH([C++]) 209 | AC_COMPILE_IFELSE( 210 | [AC_LANG_PROGRAM([], [[ 211 | #ifndef __clang__ 212 | not clang 213 | #endif 214 | ]])], 215 | [CLANG=yes], [CLANG=no]) 216 | AC_LANG_POP([C++]) 217 | 218 | AC_MSG_RESULT([$CLANG]) 219 | AC_SUBST([CLANG]) 220 | 221 | if test "x$CLANG" = "xno"; then 222 | # Check the version of gcc and add any flags as appropriate. Note 223 | # that '-dumpversion' works for clang as well but as of clang 3.3 it 224 | # reports version 4.2.1 (for gcc backwards compatibility). 225 | GCC_VERSION="`${CC} -dumpversion`" 226 | AC_MSG_NOTICE([GCC version: $GCC_VERSION]) 227 | test $? = 0 || AC_MSG_ERROR([failed to determine version of gcc]) 228 | 229 | # Check for GCC version 4.4. 230 | AX_COMPARE_VERSION([$GCC_VERSION], [eq2], [4.4], 231 | [is_gxx44=yes], [is_gxx44=no]) 232 | if test "x$is_gxx44" = "xyes"; then 233 | AC_MSG_NOTICE([Setting up CXXFLAGS for g++-4.4]) 234 | # We fail to build some protobuf generated code with gcc 4.4 235 | # without setting -fno-strict-aliasing. 236 | CFLAGS="$CFLAGS -fno-strict-aliasing" 237 | CXXFLAGS="$CXXFLAGS -fno-strict-aliasing" 238 | fi 239 | 240 | # Check for GCC version >= 4.8. 241 | AX_COMPARE_VERSION([$GCC_VERSION], [ge], [4.8], 242 | [is_ge_gxx48=yes], [is_ge_gxx48=no]) 243 | if test "x$is_ge_gxx48" = "xyes"; then 244 | AC_MSG_NOTICE([Setting up CXXFLAGS for g++ version >= 4.8]) 245 | # Boost 1.53.0 fails to compile with GCC 4.8 without 246 | # -Wno-unused-local-typedefs, and automake does not recognize the 247 | # flag. 248 | # TODO(brenden): Remove this when Boost has a resolution. 249 | CFLAGS="${CFLAGS} -Wno-unused-local-typedefs" 250 | CXXFLAGS="${CXXFLAGS} -Wno-unused-local-typedefs" 251 | fi 252 | 253 | # Check for GCC version == 4.7 and fail. Starting with 4.7 the '#if 254 | # __cplusplus >= 201103L' will evaluate to true which means the 255 | # C++11 code paths will be compiled but certain C++11 features that 256 | # we use are not supported by 4.7. Since we're requiring C++11 going 257 | # forward we can't support this compiler. 258 | AX_COMPARE_VERSION([$GCC_VERSION], [eq2], [4.7], 259 | [is_gxx47=yes], [is_gxx47=no]) 260 | if test "x$is_gxx47" = "xyes"; then 261 | AC_MSG_ERROR([Unable to build with g++-4.7 due to missing C++11 features]) 262 | fi 263 | 264 | # Suppress some erroneous warnings that break the build. 265 | MESOS_CPPFLAGS="${MESOS_CPPFLAGS} -Wno-maybe-uninitialized" 266 | fi 267 | 268 | # Ensure we can build the C++11 features we expect, and set the std 269 | # CXXFLAG as appropriate. 270 | AX_CXX_COMPILE_STDCXX_11([noext], [mandatory]) 271 | 272 | old_CPPFLAGS=${CPPFLAGS} 273 | CPPFLAGS="${CPPFLAGS} ${MESOS_CPPFLAGS}" 274 | 275 | old_LDFLAGS=${LDFLAGS} 276 | LDFLAGS="${LDFLAGS} ${MESOS_LDFLAGS}" 277 | 278 | # Check if headers and library were located. 279 | AC_CHECK_HEADER([glog/logging.h], 280 | [], 281 | [AC_MSG_ERROR([cannot find glog header.])]) 282 | 283 | AC_CHECK_HEADER([google/protobuf/message.h], 284 | [], 285 | [AC_MSG_ERROR([google protobuf is not installed.])]) 286 | 287 | AC_CHECK_HEADERS([openssl/ssl.h], 288 | [], 289 | [AC_MSG_ERROR([cannot find libssl headers])]) 290 | 291 | AC_CHECK_HEADERS([boost/lexical_cast.hpp boost/functional/hash.hpp], 292 | [], 293 | [AC_MSG_ERROR([boost is not installed.])]) 294 | 295 | AC_CHECK_HEADER([picojson.h], 296 | [MESOS_CPPFLAGS+=" -DPICOJSON_USE_INT64 -D__STDC_FORMAT_MACROS"], 297 | [AC_MSG_ERROR([picojson is not installed.])]) 298 | 299 | 300 | LDFLAGS=${old_LDFLAGS} 301 | CPPFLAGS=${old_CPPFLAGS} 302 | 303 | 304 | # Create Modules JSON blobs. 305 | AC_CONFIG_FILES([hook/modules.json], []) 306 | 307 | AC_OUTPUT 308 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. --------------------------------------------------------------------------------