├── .gitignore ├── LICENSE ├── README.md ├── docker-compose-faas-full.yml ├── docker-compose-faas.yml ├── docker-compose-web-prod.yml ├── docker-compose-web.yml ├── services ├── db │ ├── Dockerfile │ └── create.sql ├── functions │ ├── config │ │ ├── alert.rules.yml │ │ ├── alertmanager.yml │ │ └── prometheus.yml │ ├── eval │ │ ├── Dockerfile │ │ ├── Dockerfile01 │ │ ├── Dockerfile02 │ │ ├── handler.py │ │ ├── handler2.py │ │ └── requirements.txt │ ├── hook │ │ ├── Dockerfile │ │ ├── handler.py │ │ ├── handler2.py │ │ └── requirements.txt │ └── ping │ │ ├── Dockerfile │ │ ├── handler.py │ │ └── requirements.txt ├── nginx │ ├── Dockerfile │ └── dev.conf └── web │ ├── .dockerignore │ ├── Dockerfile │ ├── entrypoint.sh │ ├── manage.py │ ├── project │ ├── __init__.py │ ├── client │ │ ├── static │ │ │ ├── .jshintrc │ │ │ ├── main.css │ │ │ └── main.js │ │ └── templates │ │ │ ├── _base.html │ │ │ ├── errors │ │ │ ├── 401.html │ │ │ ├── 403.html │ │ │ ├── 404.html │ │ │ └── 500.html │ │ │ ├── footer.html │ │ │ ├── header.html │ │ │ ├── main │ │ │ ├── about.html │ │ │ ├── history.html │ │ │ └── home.html │ │ │ └── user │ │ │ ├── login.html │ │ │ ├── members.html │ │ │ └── register.html │ ├── config.py │ ├── server │ │ ├── __init__.py │ │ ├── main │ │ │ ├── __init__.py │ │ │ ├── tasks.py │ │ │ └── views.py │ │ ├── models.py │ │ └── user │ │ │ ├── __init__.py │ │ │ ├── forms.py │ │ │ └── views.py │ └── tests │ │ ├── __init__.py │ │ ├── base.py │ │ ├── helpers.py │ │ ├── test__config.py │ │ ├── test_main.py │ │ └── test_user.py │ └── requirements.txt └── template.yml /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | env 3 | venv 4 | __pycache__ 5 | *.pyc 6 | .DS_Store 7 | migrations/ 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) 2018 Michael Herman 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 5 | 6 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 7 | 8 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OpenCI 2 | 3 | A serverless continuous integration system powered by Python, Flask, and OpenFaaS. 4 | 5 | Built with Docker v18.03.0-ce. 6 | 7 | http://mherman.org/presentations/pycon-2018 8 | -------------------------------------------------------------------------------- /docker-compose-faas-full.yml: -------------------------------------------------------------------------------- 1 | version: '3.5' 2 | 3 | services: 4 | 5 | gateway: 6 | ports: 7 | - 8080:8080 8 | image: functions/gateway:0.7.9 9 | networks: 10 | - functions 11 | environment: 12 | functions_provider_url: 'http://faas-swarm:8080/' 13 | read_timeout: '25s' 14 | write_timeout: '25s' 15 | upstream_timeout: '20s' 16 | dnsrr: 'true' 17 | faas_nats_address: 'nats' 18 | faas_nats_port: 4222 19 | direct_functions: 'true' 20 | direct_functions_suffix: '' 21 | deploy: 22 | resources: 23 | reservations: 24 | memory: 100M 25 | restart_policy: 26 | condition: on-failure 27 | delay: 5s 28 | max_attempts: 20 29 | window: 380s 30 | placement: 31 | constraints: 32 | - 'node.platform.os == linux' 33 | 34 | faas-swarm: 35 | volumes: 36 | - '/var/run/docker.sock:/var/run/docker.sock' 37 | image: functions/faas-swarm:0.2.3 38 | networks: 39 | - functions 40 | environment: 41 | read_timeout: '25s' 42 | write_timeout: '25s' 43 | DOCKER_API_VERSION: '1.30' 44 | deploy: 45 | placement: 46 | constraints: 47 | - 'node.role == manager' 48 | - 'node.platform.os == linux' 49 | resources: 50 | reservations: 51 | memory: 100M 52 | restart_policy: 53 | condition: on-failure 54 | delay: 5s 55 | max_attempts: 20 56 | window: 380s 57 | 58 | nats: 59 | image: nats-streaming:0.6.0 60 | command: '--store memory --cluster_id faas-cluster' 61 | networks: 62 | - functions 63 | deploy: 64 | resources: 65 | limits: 66 | memory: 125M 67 | reservations: 68 | memory: 50M 69 | placement: 70 | constraints: 71 | - 'node.platform.os == linux' 72 | 73 | queue-worker: 74 | image: functions/queue-worker:0.4.3 75 | networks: 76 | - functions 77 | environment: 78 | max_inflight: '1' 79 | ack_timeout: '30s' 80 | deploy: 81 | resources: 82 | limits: 83 | memory: 50M 84 | reservations: 85 | memory: 20M 86 | restart_policy: 87 | condition: on-failure 88 | delay: 5s 89 | max_attempts: 20 90 | window: 380s 91 | placement: 92 | constraints: 93 | - 'node.platform.os == linux' 94 | 95 | prometheus: 96 | image: prom/prometheus:v2.2.0 97 | environment: 98 | no_proxy: "gateway" 99 | configs: 100 | - source: prometheus_config 101 | target: /etc/prometheus/prometheus.yml 102 | - source: prometheus_rules 103 | target: /etc/prometheus/alert.rules.yml 104 | command: 105 | - '--config.file=/etc/prometheus/prometheus.yml' 106 | # - '-storage.local.path=/prometheus' 107 | ports: 108 | - 9090:9090 109 | networks: 110 | - functions 111 | deploy: 112 | placement: 113 | constraints: 114 | - 'node.role == manager' 115 | - 'node.platform.os == linux' 116 | resources: 117 | limits: 118 | memory: 500M 119 | reservations: 120 | memory: 200M 121 | 122 | alertmanager: 123 | image: prom/alertmanager:v0.15.0-rc.0 124 | environment: 125 | no_proxy: "gateway" 126 | command: 127 | - '--config.file=/alertmanager.yml' 128 | - '--storage.path=/alertmanager' 129 | networks: 130 | - functions 131 | # Uncomment the following port mapping if you wish to expose the Prometheus 132 | # Alertmanager UI. 133 | # ports: 134 | # - 9093:9093 135 | deploy: 136 | resources: 137 | limits: 138 | memory: 50M 139 | reservations: 140 | memory: 20M 141 | placement: 142 | constraints: 143 | - 'node.role == manager' 144 | - 'node.platform.os == linux' 145 | configs: 146 | - source: alertmanager_config 147 | target: /alertmanager.yml 148 | 149 | ping: 150 | image: python-ping:latest 151 | labels: 152 | function: 'true' 153 | com.openfaas.scale.min: 5 154 | networks: 155 | - functions 156 | environment: 157 | no_proxy: 'gateway' 158 | https_proxy: $https_proxy 159 | deploy: 160 | placement: 161 | constraints: 162 | - 'node.platform.os == linux' 163 | 164 | eval: 165 | image: python-eval:latest 166 | labels: 167 | function: 'true' 168 | networks: 169 | - functions 170 | environment: 171 | no_proxy: 'gateway' 172 | https_proxy: $https_proxy 173 | deploy: 174 | placement: 175 | constraints: 176 | - 'node.platform.os == linux' 177 | 178 | hook: 179 | image: python-hook:latest 180 | labels: 181 | function: 'true' 182 | networks: 183 | - functions 184 | environment: 185 | no_proxy: 'gateway' 186 | https_proxy: $https_proxy 187 | deploy: 188 | placement: 189 | constraints: 190 | - 'node.platform.os == linux' 191 | 192 | configs: 193 | prometheus_config: 194 | file: ./functions/services/config/prometheus.yml 195 | prometheus_rules: 196 | file: ./functions/services/config/alert.rules.yml 197 | alertmanager_config: 198 | file: ./functions/services/config/alertmanager.yml 199 | 200 | networks: 201 | functions: 202 | driver: overlay 203 | attachable: true 204 | labels: 205 | - 'openfaas=true' 206 | -------------------------------------------------------------------------------- /docker-compose-faas.yml: -------------------------------------------------------------------------------- 1 | version: '3.5' 2 | 3 | services: 4 | 5 | gateway: 6 | ports: 7 | - 8080:8080 8 | image: functions/gateway:0.7.9 9 | networks: 10 | - functions 11 | environment: 12 | functions_provider_url: 'http://faas-swarm:8080/' 13 | read_timeout: '25s' 14 | write_timeout: '25s' 15 | upstream_timeout: '20s' 16 | dnsrr: 'true' 17 | direct_functions: 'true' 18 | direct_functions_suffix: '' 19 | deploy: 20 | resources: 21 | reservations: 22 | memory: 100M 23 | restart_policy: 24 | condition: on-failure 25 | delay: 5s 26 | max_attempts: 20 27 | window: 380s 28 | placement: 29 | constraints: 30 | - 'node.platform.os == linux' 31 | 32 | faas-swarm: 33 | volumes: 34 | - '/var/run/docker.sock:/var/run/docker.sock' 35 | image: functions/faas-swarm:0.2.3 36 | networks: 37 | - functions 38 | environment: 39 | read_timeout: '25s' 40 | write_timeout: '25s' 41 | DOCKER_API_VERSION: '1.30' 42 | deploy: 43 | placement: 44 | constraints: 45 | - 'node.role == manager' 46 | - 'node.platform.os == linux' 47 | resources: 48 | reservations: 49 | memory: 100M 50 | restart_policy: 51 | condition: on-failure 52 | delay: 5s 53 | max_attempts: 20 54 | window: 380s 55 | 56 | eval: 57 | image: python-eval:latest 58 | labels: 59 | function: 'true' 60 | networks: 61 | - functions 62 | environment: 63 | no_proxy: 'gateway' 64 | https_proxy: $https_proxy 65 | deploy: 66 | placement: 67 | constraints: 68 | - 'node.platform.os == linux' 69 | 70 | ping: 71 | image: python-ping:latest 72 | labels: 73 | function: 'true' 74 | networks: 75 | - functions 76 | environment: 77 | no_proxy: 'gateway' 78 | https_proxy: $https_proxy 79 | deploy: 80 | placement: 81 | constraints: 82 | - 'node.platform.os == linux' 83 | 84 | hook: 85 | image: python-hook:latest 86 | labels: 87 | function: 'true' 88 | networks: 89 | - functions 90 | environment: 91 | no_proxy: 'gateway' 92 | https_proxy: $https_proxy 93 | deploy: 94 | placement: 95 | constraints: 96 | - 'node.platform.os == linux' 97 | 98 | networks: 99 | functions: 100 | driver: overlay 101 | attachable: true 102 | labels: 103 | - 'openfaas=true' 104 | -------------------------------------------------------------------------------- /docker-compose-web-prod.yml: -------------------------------------------------------------------------------- 1 | version: '3.5' 2 | 3 | services: 4 | 5 | web: 6 | image: web 7 | build: 8 | context: ./services/web 9 | dockerfile: Dockerfile 10 | ports: 11 | - 5002:5000 12 | environment: 13 | - APP_NAME=OpenCI 14 | - FLASK_DEBUG=1 15 | - PYTHONUNBUFFERED=0 16 | - APP_SETTINGS=project.config.ProductionConfig 17 | - DATABASE_URL=postgres://postgres:postgres@web-db:5432/web_dev 18 | - DATABASE_TEST_URL=postgres://postgres:postgres@web-db:5432/web_test 19 | - SECRET_KEY=change_me_in_prod 20 | - OPENFAAS_URL=http://104.131.75.55 21 | depends_on: 22 | - web-db 23 | 24 | web-db: 25 | image: web-db 26 | build: 27 | context: ./services/db 28 | dockerfile: Dockerfile 29 | ports: 30 | - 5435:5432 31 | environment: 32 | - POSTGRES_USER=postgres 33 | - POSTGRES_PASSWORD=postgres 34 | 35 | web-nginx: 36 | image: web-nginx 37 | build: 38 | context: ./services/nginx 39 | dockerfile: Dockerfile 40 | restart: always 41 | ports: 42 | - 80:80 43 | depends_on: 44 | - web 45 | 46 | web-worker: 47 | image: web 48 | environment: 49 | - APP_SETTINGS=project.config.DevelopmentConfig 50 | depends_on: 51 | - web-redis 52 | - web 53 | command: python manage.py run_worker 54 | 55 | web-redis: 56 | image: redis:3.2.11 57 | -------------------------------------------------------------------------------- /docker-compose-web.yml: -------------------------------------------------------------------------------- 1 | version: '3.5' 2 | 3 | services: 4 | 5 | web: 6 | image: web 7 | build: 8 | context: ./services/web 9 | dockerfile: Dockerfile 10 | volumes: 11 | - './services/web:/usr/src/app' 12 | ports: 13 | - 5002:5000 14 | environment: 15 | - APP_NAME=OpenCI 16 | - FLASK_DEBUG=1 17 | - PYTHONUNBUFFERED=0 18 | - APP_SETTINGS=project.config.DevelopmentConfig 19 | - DATABASE_URL=postgres://postgres:postgres@web-db:5432/web_dev 20 | - DATABASE_TEST_URL=postgres://postgres:postgres@web-db:5432/web_test 21 | - SECRET_KEY=change_me_in_prod 22 | - OPENFAAS_URL=http://104.131.75.55 23 | depends_on: 24 | - web-db 25 | 26 | web-db: 27 | image: web-db 28 | build: 29 | context: ./services/db 30 | dockerfile: Dockerfile 31 | ports: 32 | - 5435:5432 33 | environment: 34 | - POSTGRES_USER=postgres 35 | - POSTGRES_PASSWORD=postgres 36 | 37 | web-nginx: 38 | image: web-nginx 39 | build: 40 | context: ./services/nginx 41 | dockerfile: Dockerfile 42 | restart: always 43 | ports: 44 | - 80:80 45 | depends_on: 46 | - web 47 | 48 | web-worker: 49 | image: web 50 | volumes: 51 | - ./services/web:/usr/src/app 52 | environment: 53 | - APP_SETTINGS=project.config.DevelopmentConfig 54 | depends_on: 55 | - web-redis 56 | - web 57 | command: python manage.py run_worker 58 | 59 | web-redis: 60 | image: redis:3.2.11 61 | -------------------------------------------------------------------------------- /services/db/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM postgres 2 | 3 | # run create.sql on init 4 | ADD create.sql /docker-entrypoint-initdb.d 5 | -------------------------------------------------------------------------------- /services/db/create.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE web_prod; 2 | CREATE DATABASE web_stage; 3 | CREATE DATABASE web_dev; 4 | CREATE DATABASE web_test; 5 | -------------------------------------------------------------------------------- /services/functions/config/alert.rules.yml: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: prometheus/alert.rules 3 | rules: 4 | - alert: service_down 5 | expr: up == 0 6 | - alert: APIHighInvocationRate 7 | expr: sum(rate(gateway_function_invocation_total{code="200"}[10s])) BY (function_name) > 5 8 | for: 5s 9 | labels: 10 | service: gateway 11 | severity: major 12 | value: '{{$value}}' 13 | annotations: 14 | description: High invocation total on {{ $labels.instance }} 15 | summary: High invocation total on {{ $labels.instance }} 16 | -------------------------------------------------------------------------------- /services/functions/config/alertmanager.yml: -------------------------------------------------------------------------------- 1 | route: 2 | group_by: ['alertname', 'cluster', 'service'] 3 | group_wait: 5s 4 | group_interval: 10s 5 | repeat_interval: 30s 6 | receiver: scale-up 7 | routes: 8 | - match: 9 | service: gateway 10 | receiver: scale-up 11 | severity: major 12 | inhibit_rules: 13 | - source_match: 14 | severity: 'critical' 15 | target_match: 16 | severity: 'warning' 17 | equal: ['alertname', 'cluster', 'service'] 18 | receivers: 19 | - name: 'scale-up' 20 | webhook_configs: 21 | - url: http://gateway:8080/system/alert 22 | send_resolved: true 23 | -------------------------------------------------------------------------------- /services/functions/config/prometheus.yml: -------------------------------------------------------------------------------- 1 | # my global config 2 | global: 3 | scrape_interval: 15s # By default, scrape targets every 15 seconds. 4 | evaluation_interval: 15s # By default, scrape targets every 15 seconds. 5 | # scrape_timeout is set to the global default (10s). 6 | 7 | # Attach these labels to any time series or alerts when communicating with 8 | # external systems (federation, remote storage, Alertmanager). 9 | external_labels: 10 | monitor: 'faas-monitor' 11 | 12 | # Load rules once and periodically evaluate them according to the global 'evaluation_interval'. 13 | rule_files: 14 | - 'alert.rules.yml' 15 | 16 | 17 | # A scrape configuration containing exactly one endpoint to scrape: 18 | # Here it's Prometheus itself. 19 | scrape_configs: 20 | # The job name is added as a label `job=` to any timeseries scraped from this config. 21 | - job_name: 'prometheus' 22 | 23 | # Override the global default and scrape targets from this job every 5 seconds. 24 | scrape_interval: 5s 25 | 26 | # metrics_path defaults to '/metrics' 27 | # scheme defaults to 'http'. 28 | static_configs: 29 | - targets: ['localhost:9090'] 30 | 31 | - job_name: "gateway" 32 | scrape_interval: 5s 33 | dns_sd_configs: 34 | - names: ['tasks.gateway'] 35 | port: 8080 36 | type: A 37 | refresh_interval: 5s 38 | 39 | alerting: 40 | alertmanagers: 41 | - static_configs: 42 | - targets: 43 | - alertmanager:9093 44 | -------------------------------------------------------------------------------- /services/functions/eval/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | 3 | RUN apk add -Uuv --no-cache python3 \ 4 | && apk upgrade -v --available --no-cache \ 5 | && apk add ca-certificates git && pip3 install --no-cache-dir --upgrade pip setuptools wheel \ 6 | && pip3 install requests certifi 7 | 8 | ADD https://github.com/openfaas/faas/releases/download/0.7.9/fwatchdog /usr/bin 9 | RUN chmod +x /usr/bin/fwatchdog 10 | 11 | WORKDIR /root/ 12 | RUN mkdir -p /root/tmp 13 | 14 | COPY requirements.txt . 15 | 16 | RUN pip install -r requirements.txt 17 | COPY handler2.py . 18 | 19 | ENV fprocess="python3 handler2.py" 20 | 21 | HEALTHCHECK --interval=1s CMD [ -e /tmp/.lock ] || exit 1 22 | 23 | CMD ["fwatchdog"] 24 | -------------------------------------------------------------------------------- /services/functions/eval/Dockerfile01: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | 3 | RUN apk add -Uuv --no-cache python3 \ 4 | && apk upgrade -v --available --no-cache \ 5 | && apk add ca-certificates git && pip3 install --no-cache-dir --upgrade pip setuptools wheel \ 6 | && pip3 install requests certifi 7 | 8 | ADD https://github.com/openfaas/faas/releases/download/0.7.9/fwatchdog /usr/bin 9 | RUN chmod +x /usr/bin/fwatchdog 10 | 11 | WORKDIR /root/ 12 | RUN mkdir -p /root/tmp 13 | 14 | COPY requirements.txt . 15 | 16 | RUN pip install -r requirements.txt 17 | COPY handler.py . 18 | 19 | ENV fprocess="python3 handler.py" 20 | 21 | HEALTHCHECK --interval=1s CMD [ -e /tmp/.lock ] || exit 1 22 | 23 | CMD ["fwatchdog"] 24 | -------------------------------------------------------------------------------- /services/functions/eval/Dockerfile02: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | 3 | RUN apk add -Uuv --no-cache python3 \ 4 | && apk upgrade -v --available --no-cache \ 5 | && apk add ca-certificates git && pip3 install --no-cache-dir --upgrade pip setuptools wheel \ 6 | && pip3 install requests certifi 7 | 8 | ADD https://github.com/openfaas/faas/releases/download/0.7.9/fwatchdog /usr/bin 9 | RUN chmod +x /usr/bin/fwatchdog 10 | 11 | WORKDIR /root/ 12 | RUN mkdir -p /root/tmp 13 | 14 | COPY requirements.txt . 15 | 16 | RUN pip install -r requirements.txt 17 | COPY handler2.py . 18 | 19 | ENV fprocess="python3 handler2.py" 20 | 21 | HEALTHCHECK --interval=1s CMD [ -e /tmp/.lock ] || exit 1 22 | 23 | CMD ["fwatchdog"] 24 | -------------------------------------------------------------------------------- /services/functions/eval/handler.py: -------------------------------------------------------------------------------- 1 | import io 2 | import os 3 | import sys 4 | import contextlib 5 | import subprocess 6 | import shutil 7 | 8 | from git import Repo 9 | 10 | 11 | @contextlib.contextmanager 12 | def stdoutIO(stdout=None): 13 | old = sys.stdout 14 | if stdout is None: 15 | stdout = io.StringIO() 16 | sys.stdout = stdout 17 | yield stdout 18 | sys.stdout = old 19 | 20 | 21 | def main(): 22 | try: 23 | shutil.rmtree('./tmp', ignore_errors=True) 24 | Repo.clone_from( 25 | 'https://github.com/testdrivenio/pycon-sample', './tmp') 26 | except Exception as e: 27 | print(e) 28 | raise 29 | with stdoutIO() as std_out: 30 | try: 31 | p1 = subprocess.Popen( 32 | ['python3', 'tmp/test.py'], 33 | stdout=subprocess.PIPE, 34 | stderr=subprocess.PIPE 35 | ) 36 | p2 = p1.stdout.read().decode('utf-8') 37 | p3 = p1.stderr.read().decode('utf-8') 38 | if len(p2) > 0: 39 | print(p2) 40 | else: 41 | print(p3) 42 | except Exception as e: 43 | print(e) 44 | print(std_out.getvalue()) 45 | 46 | 47 | if __name__ == '__main__': 48 | main() 49 | -------------------------------------------------------------------------------- /services/functions/eval/handler2.py: -------------------------------------------------------------------------------- 1 | import io 2 | import os 3 | import sys 4 | import contextlib 5 | import subprocess 6 | import shutil 7 | import json 8 | 9 | from git import Repo 10 | 11 | 12 | ERROR_MSG = 'Please provide a namespace and a repo name in the payload!' 13 | 14 | 15 | @contextlib.contextmanager 16 | def stdoutIO(stdout=None): 17 | old = sys.stdout 18 | if stdout is None: 19 | stdout = io.StringIO() 20 | sys.stdout = stdout 21 | yield stdout 22 | sys.stdout = old 23 | 24 | 25 | def get_stdin(): 26 | buf = '' 27 | for line in sys.stdin: 28 | buf = buf + line 29 | return buf 30 | 31 | 32 | def main(): 33 | try: 34 | payload = json.loads(get_stdin()) 35 | namespace = payload['namespace'] 36 | repo_name = payload['repo_name'] 37 | url = f'https://github.com/{namespace}/{repo_name}' 38 | except Exception as e: 39 | print(ERROR_MSG) 40 | raise 41 | try: 42 | shutil.rmtree(f'./tmp/{repo_name}', ignore_errors=True) 43 | Repo.clone_from(url, f'./tmp/{repo_name}') 44 | except Exception as e: 45 | print(e) 46 | raise 47 | with stdoutIO() as std_out: 48 | try: 49 | p1 = subprocess.Popen( 50 | ['python3', f'tmp/{repo_name}/test.py'], 51 | stdout=subprocess.PIPE, 52 | stderr=subprocess.PIPE 53 | ) 54 | p2 = p1.stdout.read().decode('utf-8') 55 | p3 = p1.stderr.read().decode('utf-8') 56 | status = False 57 | if len(p2) > 0: 58 | print(p2) 59 | else: 60 | print(p3) 61 | if 'failures' not in p3: 62 | status = True 63 | except Exception as e: 64 | print(e) 65 | response_object = { 66 | 'status': status, 67 | 'data': std_out.getvalue() 68 | } 69 | print(json.dumps(response_object)) 70 | 71 | 72 | if __name__ == '__main__': 73 | main() 74 | -------------------------------------------------------------------------------- /services/functions/eval/requirements.txt: -------------------------------------------------------------------------------- 1 | gitpython==2.1.9 2 | -------------------------------------------------------------------------------- /services/functions/hook/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | 3 | RUN apk add -Uuv --no-cache python3 \ 4 | && apk upgrade -v --available --no-cache \ 5 | && apk add ca-certificates git && pip3 install --no-cache-dir --upgrade pip setuptools wheel \ 6 | && pip3 install requests certifi 7 | 8 | ADD https://github.com/openfaas/faas/releases/download/0.7.9/fwatchdog /usr/bin 9 | RUN chmod +x /usr/bin/fwatchdog 10 | 11 | WORKDIR /root/ 12 | RUN mkdir -p /root/tmp 13 | 14 | COPY requirements.txt . 15 | 16 | RUN pip install -r requirements.txt 17 | COPY handler2.py . 18 | 19 | ENV fprocess="python3 handler2.py" 20 | 21 | HEALTHCHECK --interval=1s CMD [ -e /tmp/.lock ] || exit 1 22 | 23 | CMD ["fwatchdog"] 24 | -------------------------------------------------------------------------------- /services/functions/hook/handler.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | 4 | def get_stdin(): 5 | buf = '' 6 | for line in sys.stdin: 7 | buf = buf + line 8 | return buf 9 | 10 | 11 | def main(): 12 | print(get_stdin()) 13 | 14 | 15 | if __name__ == '__main__': 16 | main() 17 | -------------------------------------------------------------------------------- /services/functions/hook/handler2.py: -------------------------------------------------------------------------------- 1 | import io 2 | import os 3 | import sys 4 | import contextlib 5 | import subprocess 6 | import shutil 7 | import json 8 | 9 | import requests 10 | from git import Repo 11 | 12 | 13 | ERROR_MSG = 'Please provide a namespace and a repo name in the payload!' 14 | WEB_URL = 'http://165.227.188.115' 15 | 16 | 17 | @contextlib.contextmanager 18 | def stdoutIO(stdout=None): 19 | old = sys.stdout 20 | if stdout is None: 21 | stdout = io.StringIO() 22 | sys.stdout = stdout 23 | yield stdout 24 | sys.stdout = old 25 | 26 | 27 | def get_stdin(): 28 | buf = '' 29 | for line in sys.stdin: 30 | buf = buf + line 31 | return buf 32 | 33 | 34 | def main(): 35 | try: 36 | payload = json.loads(get_stdin()) 37 | namespace = payload['repository']['owner']['login'] 38 | repo_name = payload['repository']['name'] 39 | url = f'https://github.com/{namespace}/{repo_name}' 40 | except Exception as e: 41 | print(ERROR_MSG) 42 | raise 43 | try: 44 | shutil.rmtree(f'./tmp/{repo_name}', ignore_errors=True) 45 | Repo.clone_from(url, f'./tmp/{repo_name}') 46 | except Exception as e: 47 | print(e) 48 | raise 49 | with stdoutIO() as std_out: 50 | try: 51 | p1 = subprocess.Popen( 52 | ['python3', f'tmp/{repo_name}/test.py'], 53 | stdout=subprocess.PIPE, 54 | stderr=subprocess.PIPE 55 | ) 56 | p2 = p1.stdout.read().decode('utf-8') 57 | p3 = p1.stderr.read().decode('utf-8') 58 | status = False 59 | if len(p2) > 0: 60 | print(p2) 61 | else: 62 | print(p3) 63 | if 'failures' not in p3: 64 | status = True 65 | payload = { 66 | 'repo_name': repo_name, 67 | 'status': status 68 | 69 | } 70 | r = requests.put(f'{WEB_URL}/projects/update', json=payload) 71 | print(json.dumps(r.json())) 72 | except Exception as e: 73 | print(e) 74 | 75 | 76 | if __name__ == '__main__': 77 | main() 78 | -------------------------------------------------------------------------------- /services/functions/hook/requirements.txt: -------------------------------------------------------------------------------- 1 | gitpython==2.1.9 2 | requests==2.18.4 3 | -------------------------------------------------------------------------------- /services/functions/ping/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | 3 | RUN apk add -Uuv --no-cache python3 \ 4 | && apk upgrade -v --available --no-cache \ 5 | && apk add ca-certificates && pip3 install --no-cache-dir --upgrade pip setuptools wheel \ 6 | && pip3 install requests certifi 7 | 8 | ADD https://github.com/openfaas/faas/releases/download/0.7.9/fwatchdog /usr/bin 9 | RUN chmod +x /usr/bin/fwatchdog 10 | 11 | WORKDIR /root/ 12 | 13 | COPY requirements.txt . 14 | 15 | RUN pip install -r requirements.txt 16 | COPY handler.py . 17 | 18 | ENV fprocess="python3 handler.py" 19 | 20 | HEALTHCHECK --interval=1s CMD [ -e /tmp/.lock ] || exit 1 21 | 22 | CMD ["fwatchdog"] 23 | -------------------------------------------------------------------------------- /services/functions/ping/handler.py: -------------------------------------------------------------------------------- 1 | print('pong') 2 | -------------------------------------------------------------------------------- /services/functions/ping/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/testdrivenio/openci/2ca5b5c2554d36272fca53d6cf23e2c10a6976a8/services/functions/ping/requirements.txt -------------------------------------------------------------------------------- /services/nginx/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nginx:1.13.12 2 | 3 | RUN rm /etc/nginx/conf.d/default.conf 4 | COPY /dev.conf /etc/nginx/conf.d 5 | -------------------------------------------------------------------------------- /services/nginx/dev.conf: -------------------------------------------------------------------------------- 1 | server { 2 | 3 | listen 80; 4 | 5 | location / { 6 | proxy_pass http://web:5000; 7 | proxy_redirect default; 8 | proxy_set_header Host $host; 9 | proxy_set_header X-Real-IP $remote_addr; 10 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 11 | proxy_set_header X-Forwarded-Host $server_name; 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /services/web/.dockerignore: -------------------------------------------------------------------------------- 1 | migrations/ 2 | -------------------------------------------------------------------------------- /services/web/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6.5 2 | 3 | # install environment dependencies 4 | RUN apt-get update -yqq \ 5 | && apt-get install -yqq --no-install-recommends \ 6 | netcat \ 7 | && apt-get -q clean 8 | 9 | # set working directory 10 | RUN mkdir -p /usr/src/app 11 | WORKDIR /usr/src/app 12 | 13 | # add requirements 14 | COPY ./requirements.txt /usr/src/app/requirements.txt 15 | 16 | # install requirements 17 | RUN pip install -r requirements.txt 18 | 19 | # add entrypoint.sh 20 | COPY ./entrypoint.sh /usr/src/app/entrypoint.sh 21 | 22 | # add app 23 | COPY . /usr/src/app 24 | 25 | # run server 26 | CMD ["./entrypoint.sh"] 27 | -------------------------------------------------------------------------------- /services/web/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "Waiting for postgres..." 4 | 5 | while ! nc -z web-db 5432; do 6 | sleep 0.1 7 | done 8 | 9 | echo "PostgreSQL started" 10 | 11 | python manage.py run -h 0.0.0.0 12 | -------------------------------------------------------------------------------- /services/web/manage.py: -------------------------------------------------------------------------------- 1 | # manage.py 2 | 3 | 4 | import sys 5 | import subprocess 6 | import unittest 7 | from datetime import datetime 8 | 9 | import redis 10 | from rq import Connection, Worker 11 | import coverage 12 | from flask.cli import FlaskGroup 13 | 14 | from project.server import create_app, db 15 | from project.server.models import User, Project, Build 16 | 17 | 18 | app = create_app() 19 | cli = FlaskGroup(create_app=create_app) 20 | 21 | # code coverage 22 | COV = coverage.coverage( 23 | branch=True, 24 | include='project/*', 25 | omit=[ 26 | 'project/tests/*', 27 | 'project/server/config.py', 28 | 'project/server/*/__init__.py' 29 | ] 30 | ) 31 | COV.start() 32 | 33 | 34 | @cli.command() 35 | def create_db(): 36 | db.drop_all() 37 | db.create_all() 38 | db.session.commit() 39 | 40 | 41 | @cli.command() 42 | def drop_db(): 43 | """Drops the db tables.""" 44 | db.drop_all() 45 | 46 | 47 | @cli.command() 48 | def create_admin(): 49 | """Creates the admin user.""" 50 | db.session.add(User(email='ad@min.com', password='admin', admin=True)) 51 | db.session.commit() 52 | 53 | 54 | @cli.command() 55 | def create_data(): 56 | """Creates a user, project, and build.""" 57 | user = User(email='michael@mherman.org', password='herman') 58 | db.session.add(user) 59 | db.session.commit() 60 | project = Project( 61 | user_id=user.id, 62 | name='pycon-sample', 63 | url='https://github.com/testdrivenio/pycon-sample' 64 | ) 65 | db.session.add(project) 66 | db.session.commit() 67 | db.session.add( 68 | Build( 69 | project_id=project.id, 70 | status=False, 71 | datetime=datetime.today().strftime('%d-%m-%Y %H:%M:%S') 72 | ) 73 | ) 74 | db.session.commit() 75 | 76 | 77 | @cli.command() 78 | def test(): 79 | """Runs the unit tests without test coverage.""" 80 | tests = unittest.TestLoader().discover('project/tests', pattern='test*.py') 81 | result = unittest.TextTestRunner(verbosity=2).run(tests) 82 | if result.wasSuccessful(): 83 | sys.exit(0) 84 | else: 85 | sys.exit(1) 86 | 87 | 88 | @cli.command() 89 | def cov(): 90 | """Runs the unit tests with coverage.""" 91 | tests = unittest.TestLoader().discover('project/tests') 92 | result = unittest.TextTestRunner(verbosity=2).run(tests) 93 | if result.wasSuccessful(): 94 | COV.stop() 95 | COV.save() 96 | print('Coverage Summary:') 97 | COV.report() 98 | COV.html_report() 99 | COV.erase() 100 | sys.exit(0) 101 | else: 102 | sys.exit(1) 103 | 104 | 105 | @cli.command() 106 | def flake(): 107 | """Runs flake8 on the project.""" 108 | subprocess.run(['flake8', 'project']) 109 | 110 | 111 | @cli.command() 112 | def run_worker(): 113 | redis_url = app.config['REDIS_URL'] 114 | redis_connection = redis.from_url(redis_url) 115 | with Connection(redis_connection): 116 | worker = Worker(app.config['QUEUES']) 117 | worker.work() 118 | 119 | 120 | if __name__ == '__main__': 121 | cli() 122 | -------------------------------------------------------------------------------- /services/web/project/__init__.py: -------------------------------------------------------------------------------- 1 | # project/__init__.py 2 | -------------------------------------------------------------------------------- /services/web/project/client/static/.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "globals": { 3 | "esnext": true, 4 | "jasmine": false, 5 | "spyOn": false, 6 | "it": false, 7 | "console": false, 8 | "describe": false, 9 | "expect": false, 10 | "beforeEach": false, 11 | "afterEach": false, 12 | "waits": false, 13 | "waitsFor": false, 14 | "runs": false, 15 | "$": false, 16 | "confirm": false 17 | }, 18 | "esnext": true, 19 | "node" : true, 20 | "browser" : true, 21 | "boss" : false, 22 | "curly": false, 23 | "debug": false, 24 | "devel": false, 25 | "eqeqeq": true, 26 | "evil": true, 27 | "forin": false, 28 | "immed": true, 29 | "laxbreak": false, 30 | "newcap": true, 31 | "noarg": true, 32 | "noempty": false, 33 | "nonew": false, 34 | "nomen": false, 35 | "onevar": true, 36 | "plusplus": false, 37 | "regexp": false, 38 | "undef": true, 39 | "sub": true, 40 | "strict": false, 41 | "white": true, 42 | "unused": false 43 | } 44 | -------------------------------------------------------------------------------- /services/web/project/client/static/main.css: -------------------------------------------------------------------------------- 1 | /* 2 | global 3 | */ 4 | 5 | .site-content { 6 | padding-top: 6rem; 7 | } 8 | 9 | /* 10 | table 11 | */ 12 | 13 | .build-fail { 14 | color: red; 15 | } 16 | 17 | .build-pass { 18 | color: green; 19 | } 20 | -------------------------------------------------------------------------------- /services/web/project/client/static/main.js: -------------------------------------------------------------------------------- 1 | const term = new Terminal({cursorBlink: false, rows: 20}); 2 | 3 | $(() => { 4 | console.log('Sanity Check!'); 5 | term.open(document.getElementById('terminal')); 6 | }); 7 | 8 | $('.delete-project').on('click', (event) => { 9 | const result = confirm('Are you sure?'); 10 | if (!result) event.preventDefault(); 11 | }); 12 | 13 | $('body').on('click', '.grade-project', function() { 14 | term.clear(); 15 | term.writeln('Running...'); 16 | const projectID = $(this).data('id'); 17 | $.ajax({ 18 | url: `/projects/grade/${projectID}`, 19 | type: 'GET', 20 | }) 21 | .done((res) => { 22 | getStatus(res.data.task_id, projectID); 23 | }) 24 | .fail((err) => { console.log(err); }); 25 | 26 | }); 27 | 28 | function getStatus(taskID, projectID) { 29 | $.ajax({ 30 | url: `/tasks/${projectID}/${taskID}`, 31 | method: 'GET' 32 | }) 33 | .done((res) => { 34 | const taskStatus = res.data.task_status; 35 | if (taskStatus === 'finished') { 36 | if (res.data.task_result.status) { 37 | $(`.status-${projectID}`).html( 38 | '' 39 | ); 40 | } else { 41 | $(`.status-${projectID}`).html( 42 | '' 43 | ); 44 | } 45 | const termData = (res.data.task_result.data).replace(/\r?\n/g, "\r\n"); 46 | term.clear(); 47 | term.writeln(termData); 48 | term.scrollToBottom(); 49 | return false; 50 | } 51 | if (taskStatus === 'failed') { 52 | $(`.status-${projectID}`).html( 53 | '' 54 | ); 55 | return false; 56 | } 57 | setTimeout(function() { 58 | getStatus(res.data.task_id, projectID); 59 | }, 1000); 60 | }) 61 | .fail((err) => { 62 | console.log(err); 63 | }); 64 | } 65 | -------------------------------------------------------------------------------- /services/web/project/client/templates/_base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | {{ config.APP_NAME }}{% block title %}{% endblock %} 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | {% block css %}{% endblock %} 17 | 18 | 19 | 20 | 21 | {% include 'header.html' %} 22 | 23 |
24 |
25 | 26 | 27 | {% with messages = get_flashed_messages(with_categories=true) %} 28 | {% if messages %} 29 |
30 |
31 |
32 | {% for category, message in messages | sort(reverse=true) %} 33 | {% if loop.index == 1 %} 34 |
35 | × 36 | {{message}} 37 |
38 | {% endif %} 39 | {% endfor %} 40 |
41 |
42 | {% endif %} 43 | {% endwith %} 44 | 45 | 46 | {% block content %}{% endblock %} 47 | 48 |
49 | 50 | 51 | {% if error %} 52 |

Error: {{ error }}

53 | {% endif %} 54 | 55 |
56 |
57 | 58 |

59 | 60 | {% include 'footer.html' %} 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | {% block js %}{% endblock %} 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /services/web/project/client/templates/errors/401.html: -------------------------------------------------------------------------------- 1 | {% extends "_base.html" %} 2 | {% block page_title %}- Unauthorized{% endblock %} 3 | {% block content %} 4 | 5 |
6 |
7 |

401

8 |

You are not authorized to view this page. Please log in.

9 |
10 |
11 | 12 | {% endblock %} 13 | -------------------------------------------------------------------------------- /services/web/project/client/templates/errors/403.html: -------------------------------------------------------------------------------- 1 | {% extends "_base.html" %} 2 | {% block page_title %}- Forbidden{% endblock %} 3 | {% block content %} 4 | 5 |
6 |
7 |

403

8 |

You are forbidden from viewing this page. Please log in.

9 |
10 |
11 | 12 | {% endblock %} 13 | -------------------------------------------------------------------------------- /services/web/project/client/templates/errors/404.html: -------------------------------------------------------------------------------- 1 | {% extends "_base.html" %} 2 | {% block page_title %}- Page Not Found{% endblock %} 3 | {% block content %} 4 | 5 |
6 |
7 |

404

8 |

Sorry. The requested page doesn't exist. Go home.

9 |
10 |
11 | 12 | {% endblock %} 13 | -------------------------------------------------------------------------------- /services/web/project/client/templates/errors/500.html: -------------------------------------------------------------------------------- 1 | {% extends "_base.html" %} 2 | {% block page_title %}- Server Error{% endblock %} 3 | {% block content %} 4 | 5 |
6 |
7 |

500

8 |

Sorry. Something went terribly wrong. Go home.

9 |
10 |
11 | 12 | {% endblock %} 13 | -------------------------------------------------------------------------------- /services/web/project/client/templates/footer.html: -------------------------------------------------------------------------------- 1 | 12 | -------------------------------------------------------------------------------- /services/web/project/client/templates/header.html: -------------------------------------------------------------------------------- 1 | 2 | 23 | -------------------------------------------------------------------------------- /services/web/project/client/templates/main/about.html: -------------------------------------------------------------------------------- 1 | {% extends "_base.html" %} 2 | {% block content %} 3 | 4 |
5 |
6 |

About

7 |

8 |

Add some content here!

9 |
10 |
11 | 12 | {% endblock %} 13 | -------------------------------------------------------------------------------- /services/web/project/client/templates/main/history.html: -------------------------------------------------------------------------------- 1 | {% extends "_base.html" %} 2 | 3 | {% block content %} 4 | 5 |
6 | 7 |

Build History for {{name}}

8 |


9 | 10 |
11 |
12 | 13 |

Home

14 | 15 | 16 | {% if build_history %} 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | {% for history in build_history|sort(attribute='datetime',reverse = True) %} 26 | 27 | 34 | 35 | 36 | {% endfor %} 37 | 38 |
StatusDatetime
28 | {% if history.status %} 29 | 30 | {% else %} 31 | 32 | {% endif %} 33 | {{ history.datetime }}
39 | {% else %} 40 |

No builds yet!

41 | {% endif %} 42 | 43 |
44 |
45 | 46 |
47 | 48 | 49 | {% endblock %} 50 | -------------------------------------------------------------------------------- /services/web/project/client/templates/main/home.html: -------------------------------------------------------------------------------- 1 | {% extends "_base.html" %} 2 | 3 | {% block content %} 4 | 5 |
6 | 7 |

Projects

8 |


9 | 10 |
11 |
12 | 13 |

14 | 15 | 16 | {% if projects %} 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | {% for project in projects %} 27 | 28 | 29 | 36 | 46 | 47 | {% endfor %} 48 | 49 |
Project NameStatusActions
{{ project.name }} 30 | {% if project.status %} 31 | 32 | {% else %} 33 | 34 | {% endif %} 35 | 37 | 43 | View history 44 | Delete Project 45 |
50 | {% else %} 51 |

No projects! Add one.

52 | {% endif %} 53 | 54 |
55 | 56 | {% if current_user.is_authenticated %} 57 |
58 |
59 |
60 | {% endif %} 61 | 62 |
63 | 64 |
65 | 66 | 67 | 95 | 96 | {% endblock %} 97 | 98 | {% block js %} 99 | 100 | {% if current_user.is_authenticated %} 101 | 102 | {% endif %} 103 | 104 | {% endblock %} 105 | -------------------------------------------------------------------------------- /services/web/project/client/templates/user/login.html: -------------------------------------------------------------------------------- 1 | {% extends '_base.html' %} 2 | {% import "bootstrap/wtf.html" as wtf %} 3 | {% block content %} 4 | 5 |
6 |

Login

7 |

8 |
9 | 10 |
11 | 12 |
13 | {{ form.csrf_token }} 14 | {{ form.hidden_tag() }} 15 | {{ wtf.form_errors(form, hiddens="only") }} 16 | 17 |
18 | {{ wtf.form_field(form.email) }} 19 | {{ wtf.form_field(form.password) }} 20 | 21 | 22 |

23 |

Need to Register?

24 |
25 |
26 | 27 | {% endblock %} 28 | -------------------------------------------------------------------------------- /services/web/project/client/templates/user/members.html: -------------------------------------------------------------------------------- 1 | {% extends "_base.html" %} 2 | {% block content %} 3 | 4 |

Welcome, {{ current_user.email }}!

5 |

This is the members-only page.

6 | 7 | {% endblock %} 8 | -------------------------------------------------------------------------------- /services/web/project/client/templates/user/register.html: -------------------------------------------------------------------------------- 1 | {% extends '_base.html' %} 2 | {% import "bootstrap/wtf.html" as wtf %} 3 | {% block content %} 4 | 5 |
6 |

Register

7 |

8 |
9 | 10 |
11 | 12 |
13 | {{ form.csrf_token }} 14 | {{ form.hidden_tag() }} 15 | {{ wtf.form_errors(form, hiddens="only") }} 16 | 17 |
18 | {{ wtf.form_field(form.email) }} 19 | {{ wtf.form_field(form.password) }} 20 | {{ wtf.form_field(form.confirm) }} 21 | 22 | 23 |

24 |

Already have an account? Sign in.

25 |
26 |
27 | 28 | {% endblock %} 29 | -------------------------------------------------------------------------------- /services/web/project/config.py: -------------------------------------------------------------------------------- 1 | # project/server/config.py 2 | 3 | import os 4 | 5 | basedir = os.path.abspath(os.path.dirname(__file__)) 6 | 7 | 8 | class BaseConfig(object): 9 | """Base configuration.""" 10 | APP_NAME = os.getenv('APP_NAME') 11 | BCRYPT_LOG_ROUNDS = 4 12 | DEBUG_TB_ENABLED = False 13 | SECRET_KEY = os.getenv('SECRET_KEY', default='my_precious') 14 | SQLALCHEMY_TRACK_MODIFICATIONS = False 15 | WTF_CSRF_ENABLED = False 16 | OPENFAAS_URL = os.environ.get('OPENFAAS_URL') 17 | REDIS_URL = 'redis://web-redis:6379/0' 18 | QUEUES = ['default'] 19 | 20 | 21 | class DevelopmentConfig(BaseConfig): 22 | """Development configuration.""" 23 | DEBUG_TB_ENABLED = True 24 | DEBUG_TB_INTERCEPT_REDIRECTS = False 25 | SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL') 26 | 27 | 28 | class TestingConfig(BaseConfig): 29 | """Testing configuration.""" 30 | PRESERVE_CONTEXT_ON_EXCEPTION = False 31 | SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_TEST_URL') 32 | TESTING = True 33 | 34 | 35 | class ProductionConfig(BaseConfig): 36 | """Production configuration.""" 37 | BCRYPT_LOG_ROUNDS = 13 38 | SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL') 39 | WTF_CSRF_ENABLED = True 40 | -------------------------------------------------------------------------------- /services/web/project/server/__init__.py: -------------------------------------------------------------------------------- 1 | # project/server/__init__.py 2 | 3 | 4 | import os 5 | 6 | from flask import Flask, render_template 7 | from flask_login import LoginManager 8 | from flask_bcrypt import Bcrypt 9 | from flask_debugtoolbar import DebugToolbarExtension 10 | from flask_bootstrap import Bootstrap 11 | from flask_sqlalchemy import SQLAlchemy 12 | from flask_migrate import Migrate 13 | 14 | 15 | # instantiate the extensions 16 | login_manager = LoginManager() 17 | bcrypt = Bcrypt() 18 | toolbar = DebugToolbarExtension() 19 | bootstrap = Bootstrap() 20 | db = SQLAlchemy() 21 | migrate = Migrate() 22 | 23 | 24 | def create_app(script_info=None): 25 | 26 | # instantiate the app 27 | app = Flask( 28 | __name__, 29 | template_folder='../client/templates', 30 | static_folder='../client/static' 31 | ) 32 | 33 | # set config 34 | app_settings = os.getenv( 35 | 'APP_SETTINGS', 'project.server.config.DevelopmentConfig') 36 | app.config.from_object(app_settings) 37 | 38 | # set up extensions 39 | login_manager.init_app(app) 40 | bcrypt.init_app(app) 41 | toolbar.init_app(app) 42 | bootstrap.init_app(app) 43 | db.init_app(app) 44 | migrate.init_app(app, db) 45 | 46 | # register blueprints 47 | from project.server.user.views import user_blueprint 48 | from project.server.main.views import main_blueprint 49 | app.register_blueprint(user_blueprint) 50 | app.register_blueprint(main_blueprint) 51 | 52 | # flask login 53 | from project.server.models import User 54 | login_manager.login_view = 'user.login' 55 | login_manager.login_message_category = 'danger' 56 | 57 | @login_manager.user_loader 58 | def load_user(user_id): 59 | return User.query.filter(User.id == int(user_id)).first() 60 | 61 | # error handlers 62 | @app.errorhandler(401) 63 | def unauthorized_page(error): 64 | return render_template('errors/401.html'), 401 65 | 66 | @app.errorhandler(403) 67 | def forbidden_page(error): 68 | return render_template('errors/403.html'), 403 69 | 70 | @app.errorhandler(404) 71 | def page_not_found(error): 72 | return render_template('errors/404.html'), 404 73 | 74 | @app.errorhandler(500) 75 | def server_error_page(error): 76 | return render_template('errors/500.html'), 500 77 | 78 | # shell context for flask cli 79 | @app.shell_context_processor 80 | def ctx(): 81 | return {'app': app, 'db': db} 82 | 83 | return app 84 | -------------------------------------------------------------------------------- /services/web/project/server/main/__init__.py: -------------------------------------------------------------------------------- 1 | # project/server/main/__init__.py 2 | -------------------------------------------------------------------------------- /services/web/project/server/main/tasks.py: -------------------------------------------------------------------------------- 1 | # project/server/main/tasks.py 2 | 3 | 4 | import json 5 | 6 | import requests 7 | 8 | 9 | def create_task(project_url, openfass_url): 10 | url_list = project_url.split('/') 11 | payload = { 12 | 'namespace': url_list[3], 13 | 'repo_name': url_list[4] 14 | } 15 | url = f'{openfass_url}:8080/function/func_eval' 16 | headers = {'Content-Type': 'text/plain'} 17 | r = requests.post(url, data=json.dumps(payload), headers=headers) 18 | data = r.json() 19 | return data 20 | -------------------------------------------------------------------------------- /services/web/project/server/main/views.py: -------------------------------------------------------------------------------- 1 | # project/server/main/views.py 2 | 3 | 4 | import json 5 | from datetime import datetime 6 | 7 | import redis 8 | from rq import Queue, Connection 9 | from flask import render_template, Blueprint, request, redirect, \ 10 | url_for, flash, jsonify, current_app 11 | from flask_login import login_required, current_user 12 | 13 | from project.server import db 14 | from project.server.models import Project, Build 15 | from project.server.main.tasks import create_task 16 | 17 | 18 | main_blueprint = Blueprint('main', __name__,) 19 | 20 | 21 | @main_blueprint.route('/', methods=['GET']) 22 | @login_required 23 | def home(): 24 | projects = Project.query.all() 25 | return render_template('main/home.html', projects=projects) 26 | 27 | 28 | @main_blueprint.route('/projects', methods=['POST']) 29 | @login_required 30 | def add_project(): 31 | user_id = current_user.id 32 | name = request.form['name'] 33 | url = request.form['url'] 34 | db.session.add(Project(user_id=user_id, name=name, url=url)) 35 | db.session.commit() 36 | flash('Project added!', 'success') 37 | return redirect(url_for('main.home')) 38 | 39 | 40 | @main_blueprint.route('/projects/delete/', methods=['GET']) 41 | @login_required 42 | def remove_project(project_id): 43 | project = Project.query.filter_by(id=project_id).first_or_404() 44 | db.session.delete(project) 45 | db.session.commit() 46 | flash('Project removed!', 'success') 47 | return redirect(url_for('main.home')) 48 | 49 | 50 | @main_blueprint.route('/projects/grade/', methods=['GET']) 51 | @login_required 52 | def grade_project(project_id): 53 | project = Project.query.filter_by(id=project_id).first_or_404() 54 | with Connection(redis.from_url(current_app.config['REDIS_URL'])): 55 | q = Queue() 56 | task = q.enqueue( 57 | create_task, 58 | project.url, 59 | current_app.config["OPENFAAS_URL"] 60 | ) 61 | response_object = { 62 | 'status': 'success', 63 | 'data': { 64 | 'task_id': task.get_id() 65 | } 66 | } 67 | return jsonify(response_object), 202 68 | 69 | 70 | @main_blueprint.route('/tasks//', methods=['GET']) 71 | def get_status(project_id, task_id): 72 | with Connection(redis.from_url(current_app.config['REDIS_URL'])): 73 | q = Queue() 74 | task = q.fetch_job(task_id) 75 | if task: 76 | response_object = { 77 | 'status': 'success', 78 | 'data': { 79 | 'task_id': task.get_id(), 80 | 'task_status': task.get_status(), 81 | 'task_result': task.result 82 | } 83 | } 84 | if task.get_status() == 'finished': 85 | project = Project.query.filter_by(id=project_id).first() 86 | project.status = False 87 | if bool(task.result['status']): 88 | project.status = True 89 | db.session.commit() 90 | db.session.add( 91 | Build( 92 | project_id=project.id, 93 | status=project.status, 94 | datetime=datetime.today().strftime('%d-%m-%Y %H:%M:%S') 95 | ) 96 | ) 97 | db.session.commit() 98 | else: 99 | response_object = {'status': 'error'} 100 | return jsonify(response_object) 101 | 102 | 103 | @main_blueprint.route('/projects/update', methods=['PUT']) 104 | def update_project(): 105 | data = request.get_json() 106 | project = Project.query.filter_by(name=data['repo_name']).first() 107 | project.status = False 108 | if bool(data['status']): 109 | project.status = True 110 | db.session.commit() 111 | db.session.add( 112 | Build( 113 | project_id=project.id, 114 | status=project.status, 115 | datetime=datetime.today().strftime('%d-%m-%Y %H:%M:%S') 116 | ) 117 | ) 118 | db.session.commit() 119 | response_object = { 120 | 'status': 'success', 121 | 'message': 'Project updated!' 122 | } 123 | return jsonify(response_object) 124 | 125 | 126 | @main_blueprint.route('/projects/history/', methods=['GET']) 127 | @login_required 128 | def get_history(project_id): 129 | project = Project.query.filter_by(id=project_id).first_or_404() 130 | build_history = Build.query.filter_by(project_id=project_id).all() 131 | return render_template( 132 | 'main/history.html', name=project.name, build_history=build_history) 133 | 134 | 135 | @main_blueprint.route('/about/') 136 | def about(): 137 | return render_template('main/about.html') 138 | -------------------------------------------------------------------------------- /services/web/project/server/models.py: -------------------------------------------------------------------------------- 1 | # project/server/models.py 2 | 3 | 4 | from flask import current_app 5 | 6 | from project.server import db, bcrypt 7 | 8 | 9 | class User(db.Model): 10 | 11 | __tablename__ = 'users' 12 | 13 | id = db.Column(db.Integer, primary_key=True, autoincrement=True) 14 | email = db.Column(db.String(255), unique=True, nullable=False) 15 | password = db.Column(db.String(255), nullable=False) 16 | admin = db.Column(db.Boolean, nullable=False, default=False) 17 | projects = db.relationship('Project', backref='users', lazy=True) 18 | 19 | def __init__(self, email, password, admin=False): 20 | self.email = email 21 | self.password = bcrypt.generate_password_hash( 22 | password, current_app.config.get('BCRYPT_LOG_ROUNDS') 23 | ).decode('utf-8') 24 | self.admin = admin 25 | 26 | def is_authenticated(self): 27 | return True 28 | 29 | def is_active(self): 30 | return True 31 | 32 | def is_anonymous(self): 33 | return False 34 | 35 | def get_id(self): 36 | return self.id 37 | 38 | def __repr__(self): 39 | return ''.format(self.email) 40 | 41 | 42 | class Project(db.Model): 43 | 44 | __tablename__ = 'projects' 45 | 46 | id = db.Column(db.Integer, primary_key=True, autoincrement=True) 47 | name = db.Column(db.String(120), nullable=False) 48 | url = db.Column(db.String, nullable=False) 49 | status = db.Column(db.Boolean, nullable=False, default=False) 50 | user_id = db.Column( 51 | db.Integer, 52 | db.ForeignKey('users.id'), 53 | nullable=False 54 | ) 55 | builds = db.relationship('Build', backref='builds', lazy=True) 56 | 57 | def __init__(self, user_id, name, url, status=False): 58 | self.user_id = user_id 59 | self.name = name 60 | self.url = url 61 | self.status = status 62 | 63 | 64 | class Build(db.Model): 65 | 66 | __tablename__ = 'builds' 67 | 68 | id = db.Column(db.Integer, primary_key=True, autoincrement=True) 69 | status = db.Column(db.Boolean, nullable=False) 70 | datetime = db.Column(db.DateTime, nullable=False) 71 | project_id = db.Column( 72 | db.Integer, 73 | db.ForeignKey('projects.id'), 74 | nullable=False 75 | ) 76 | 77 | def __init__(self, project_id, status, datetime): 78 | self.project_id = project_id 79 | self.status = status 80 | self.datetime = datetime 81 | 82 | def to_json(self): 83 | return { 84 | 'id': self.id, 85 | 'project_id': self.project_id, 86 | 'status': self.status, 87 | 'datetime': self.datetime 88 | } 89 | -------------------------------------------------------------------------------- /services/web/project/server/user/__init__.py: -------------------------------------------------------------------------------- 1 | # project/server/user/__init__.py 2 | -------------------------------------------------------------------------------- /services/web/project/server/user/forms.py: -------------------------------------------------------------------------------- 1 | # project/server/user/forms.py 2 | 3 | 4 | from flask_wtf import FlaskForm 5 | from wtforms import StringField, PasswordField 6 | from wtforms.validators import DataRequired, Email, Length, EqualTo 7 | 8 | 9 | class LoginForm(FlaskForm): 10 | email = StringField('Email Address', [DataRequired(), Email()]) 11 | password = PasswordField('Password', [DataRequired()]) 12 | 13 | 14 | class RegisterForm(FlaskForm): 15 | email = StringField( 16 | 'Email Address', 17 | validators=[ 18 | DataRequired(), 19 | Email(message=None), 20 | Length(min=6, max=40) 21 | ] 22 | ) 23 | password = PasswordField( 24 | 'Password', 25 | validators=[DataRequired(), Length(min=6, max=25)] 26 | ) 27 | confirm = PasswordField( 28 | 'Confirm password', 29 | validators=[ 30 | DataRequired(), 31 | EqualTo('password', message='Passwords must match.') 32 | ] 33 | ) 34 | -------------------------------------------------------------------------------- /services/web/project/server/user/views.py: -------------------------------------------------------------------------------- 1 | # project/server/user/views.py 2 | 3 | 4 | from flask import render_template, Blueprint, url_for, \ 5 | redirect, flash, request 6 | from flask_login import login_user, logout_user, login_required 7 | 8 | from project.server import bcrypt, db 9 | from project.server.models import User 10 | from project.server.user.forms import LoginForm, RegisterForm 11 | 12 | 13 | user_blueprint = Blueprint('user', __name__,) 14 | 15 | 16 | @user_blueprint.route('/register', methods=['GET', 'POST']) 17 | def register(): 18 | form = RegisterForm(request.form) 19 | if form.validate_on_submit(): 20 | user = User( 21 | email=form.email.data, 22 | password=form.password.data 23 | ) 24 | db.session.add(user) 25 | db.session.commit() 26 | login_user(user) 27 | flash('Thank you for registering.', 'success') 28 | return redirect(url_for('main.home')) 29 | 30 | return render_template('user/register.html', form=form) 31 | 32 | 33 | @user_blueprint.route('/login', methods=['GET', 'POST']) 34 | def login(): 35 | form = LoginForm(request.form) 36 | if form.validate_on_submit(): 37 | user = User.query.filter_by(email=form.email.data).first() 38 | if user and bcrypt.check_password_hash( 39 | user.password, request.form['password']): 40 | login_user(user) 41 | flash('You are logged in. Welcome!', 'success') 42 | return redirect(url_for('main.home')) 43 | else: 44 | flash('Invalid email and/or password.', 'danger') 45 | return render_template('user/login.html', form=form) 46 | return render_template('user/login.html', title='Please Login', form=form) 47 | 48 | 49 | @user_blueprint.route('/logout') 50 | @login_required 51 | def logout(): 52 | logout_user() 53 | flash('You were logged out. Bye!', 'success') 54 | return redirect(url_for('main.home')) 55 | 56 | 57 | @user_blueprint.route('/members') 58 | @login_required 59 | def members(): 60 | return render_template('user/members.html') 61 | -------------------------------------------------------------------------------- /services/web/project/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # project/server/tests/__init__.py 2 | -------------------------------------------------------------------------------- /services/web/project/tests/base.py: -------------------------------------------------------------------------------- 1 | # project/server/tests/base.py 2 | 3 | 4 | from flask_testing import TestCase 5 | 6 | from project.server import db, create_app 7 | from project.server.models import User 8 | 9 | app = create_app() 10 | 11 | 12 | class BaseTestCase(TestCase): 13 | 14 | def create_app(self): 15 | app.config.from_object('project.config.TestingConfig') 16 | return app 17 | 18 | def setUp(self): 19 | db.create_all() 20 | user = User(email="ad@min.com", password="admin_user") 21 | db.session.add(user) 22 | db.session.commit() 23 | 24 | def tearDown(self): 25 | db.session.remove() 26 | db.drop_all() 27 | -------------------------------------------------------------------------------- /services/web/project/tests/helpers.py: -------------------------------------------------------------------------------- 1 | # tests/helpers.py 2 | -------------------------------------------------------------------------------- /services/web/project/tests/test__config.py: -------------------------------------------------------------------------------- 1 | # project/server/tests/test_config.py 2 | 3 | 4 | import unittest 5 | import os 6 | 7 | from flask import current_app 8 | from flask_testing import TestCase 9 | 10 | from project.server import create_app 11 | 12 | app = create_app() 13 | 14 | 15 | class TestDevelopmentConfig(TestCase): 16 | 17 | def create_app(self): 18 | app.config.from_object('project.config.DevelopmentConfig') 19 | return app 20 | 21 | def test_app_is_development(self): 22 | self.assertFalse(current_app.config['TESTING']) 23 | self.assertTrue(app.config['WTF_CSRF_ENABLED'] is False) 24 | self.assertTrue(app.config['DEBUG_TB_ENABLED'] is True) 25 | self.assertFalse(current_app is None) 26 | 27 | 28 | class TestTestingConfig(TestCase): 29 | 30 | def create_app(self): 31 | app.config.from_object('project.config.TestingConfig') 32 | return app 33 | 34 | def test_app_is_testing(self): 35 | self.assertTrue(current_app.config['TESTING']) 36 | self.assertTrue(app.config['BCRYPT_LOG_ROUNDS'] == 4) 37 | self.assertTrue(app.config['WTF_CSRF_ENABLED'] is False) 38 | 39 | 40 | class TestProductionConfig(TestCase): 41 | 42 | def create_app(self): 43 | app.config.from_object('project.config.ProductionConfig') 44 | return app 45 | 46 | def test_app_is_production(self): 47 | self.assertFalse(current_app.config['TESTING']) 48 | self.assertTrue(app.config['DEBUG_TB_ENABLED'] is False) 49 | self.assertTrue(app.config['WTF_CSRF_ENABLED'] is True) 50 | self.assertTrue(app.config['BCRYPT_LOG_ROUNDS'] == 13) 51 | 52 | def test_secret_key_has_been_set(self): 53 | self.assertTrue(app.secret_key == os.getenv( 54 | 'SECRET_KEY', default='my_precious')) 55 | 56 | 57 | if __name__ == '__main__': 58 | unittest.main() 59 | -------------------------------------------------------------------------------- /services/web/project/tests/test_main.py: -------------------------------------------------------------------------------- 1 | # project/server/tests/test_main.py 2 | 3 | 4 | import unittest 5 | 6 | from base import BaseTestCase 7 | 8 | 9 | class TestMainBlueprint(BaseTestCase): 10 | 11 | def test_index(self): 12 | # Ensure Flask is setup. 13 | response = self.client.get('/', follow_redirects=True) 14 | self.assertEqual(response.status_code, 200) 15 | self.assertNotIn(b'Projects', response.data) 16 | self.assertIn(b'Login', response.data) 17 | 18 | def test_about(self): 19 | # Ensure about route behaves correctly. 20 | response = self.client.get('/about', follow_redirects=True) 21 | self.assertEqual(response.status_code, 200) 22 | self.assertIn(b'About', response.data) 23 | 24 | def test_404(self): 25 | # Ensure 404 error is handled. 26 | response = self.client.get('/404') 27 | self.assert404(response) 28 | self.assertTemplateUsed('errors/404.html') 29 | 30 | 31 | if __name__ == '__main__': 32 | unittest.main() 33 | -------------------------------------------------------------------------------- /services/web/project/tests/test_user.py: -------------------------------------------------------------------------------- 1 | # project/server/tests/test_user.py 2 | 3 | 4 | import unittest 5 | 6 | from flask_login import current_user 7 | 8 | from base import BaseTestCase 9 | from project.server import bcrypt 10 | from project.server.models import User 11 | from project.server.user.forms import LoginForm 12 | 13 | 14 | class TestUserBlueprint(BaseTestCase): 15 | 16 | def test_correct_login(self): 17 | # Ensure login behaves correctly with correct credentials. 18 | with self.client: 19 | response = self.client.post( 20 | '/login', 21 | data=dict(email="ad@min.com", password="admin_user"), 22 | follow_redirects=True 23 | ) 24 | self.assertIn(b'Welcome', response.data) 25 | self.assertIn(b'Logout', response.data) 26 | self.assertIn(b'Projects', response.data) 27 | self.assertTrue(current_user.email == "ad@min.com") 28 | self.assertTrue(current_user.is_active()) 29 | self.assertEqual(response.status_code, 200) 30 | 31 | def test_logout_behaves_correctly(self): 32 | # Ensure logout behaves correctly - regarding the session. 33 | with self.client: 34 | self.client.post( 35 | '/login', 36 | data=dict(email="ad@min.com", password="admin_user"), 37 | follow_redirects=True 38 | ) 39 | response = self.client.get('/logout', follow_redirects=True) 40 | self.assertIn(b'You were logged out. Bye!', response.data) 41 | self.assertFalse(current_user.is_active) 42 | 43 | def test_logout_route_requires_login(self): 44 | # Ensure logout route requres logged in user. 45 | response = self.client.get('/logout', follow_redirects=True) 46 | self.assertIn(b'Please log in to access this page', response.data) 47 | 48 | def test_member_route_requires_login(self): 49 | # Ensure member route requres logged in user. 50 | response = self.client.get('/members', follow_redirects=True) 51 | self.assertIn(b'Please log in to access this page', response.data) 52 | 53 | def test_validate_success_login_form(self): 54 | # Ensure correct data validates. 55 | form = LoginForm(email='ad@min.com', password='admin_user') 56 | self.assertTrue(form.validate()) 57 | 58 | def test_validate_invalid_email_format(self): 59 | # Ensure invalid email format throws error. 60 | form = LoginForm(email='unknown', password='example') 61 | self.assertFalse(form.validate()) 62 | 63 | def test_get_by_id(self): 64 | # Ensure id is correct for the current/logged in user. 65 | with self.client: 66 | self.client.post('/login', data=dict( 67 | email='ad@min.com', password='admin_user' 68 | ), follow_redirects=True) 69 | self.assertTrue(current_user.id == 1) 70 | 71 | def test_check_password(self): 72 | # Ensure given password is correct after unhashing. 73 | user = User.query.filter_by(email='ad@min.com').first() 74 | self.assertTrue( 75 | bcrypt.check_password_hash(user.password, 'admin_user')) 76 | self.assertFalse(bcrypt.check_password_hash(user.password, 'foobar')) 77 | 78 | def test_validate_invalid_password(self): 79 | # Ensure user can't login when the pasword is incorrect. 80 | with self.client: 81 | response = self.client.post('/login', data=dict( 82 | email='ad@min.com', password='foo_bar' 83 | ), follow_redirects=True) 84 | self.assertIn(b'Invalid email and/or password.', response.data) 85 | 86 | def test_register_route(self): 87 | # Ensure about route behaves correctly. 88 | response = self.client.get('/register', follow_redirects=True) 89 | self.assertIn(b'

Register

\n', response.data) 90 | 91 | def test_user_registration(self): 92 | # Ensure registration behaves correctlys. 93 | with self.client: 94 | response = self.client.post( 95 | '/register', 96 | data=dict(email="test@tester.com", password="testing", 97 | confirm="testing"), 98 | follow_redirects=True 99 | ) 100 | self.assertIn(b'Projects', response.data) 101 | self.assertTrue(current_user.email == "test@tester.com") 102 | self.assertTrue(current_user.is_active()) 103 | self.assertEqual(response.status_code, 200) 104 | 105 | 106 | if __name__ == '__main__': 107 | unittest.main() 108 | -------------------------------------------------------------------------------- /services/web/requirements.txt: -------------------------------------------------------------------------------- 1 | coverage==4.5.1 2 | flake8==3.5.0 3 | Flask==0.12.2 4 | Flask-Bcrypt==0.7.1 5 | Flask-Bootstrap==3.3.7.1 6 | Flask-DebugToolbar==0.10.1 7 | Flask-Login==0.4.1 8 | Flask-Migrate==2.1.1 9 | Flask-SQLAlchemy==2.3.2 10 | Flask-Testing==0.7.1 11 | Flask-WTF==0.14.2 12 | psycopg2==2.7.3.2 13 | redis==2.10.6 14 | requests==2.18.4 15 | rq==0.10.0 16 | -------------------------------------------------------------------------------- /template.yml: -------------------------------------------------------------------------------- 1 | provider: 2 | name: faas 3 | gateway: http://localhost:8080 4 | 5 | functions: 6 | ping: 7 | lang: dockerfile 8 | handler: ./services/functions/ping 9 | image: functions/openfaas-ping:latest 10 | eval: 11 | lang: dockerfile 12 | handler: ./services/functions/eval 13 | image: functions/openfaas-eval:latest 14 | --------------------------------------------------------------------------------