├── one_click ├── __init__.py ├── resources │ └── app │ │ ├── uwsgi.ini │ │ ├── docker-compose.yml │ │ └── Dockerfile ├── terraform │ ├── output.tf │ ├── provision_project │ │ ├── variables.tf │ │ └── main.tf │ ├── variables.tf │ └── main.tf ├── utils.py └── cli.py ├── requirements.txt ├── pyproject.toml ├── .pre-commit-config.yaml ├── setup.py ├── tests ├── utils_test.go ├── test_utils.py └── integration_test.go ├── Gopkg.toml ├── .circleci └── config.yml ├── .gitignore ├── Gopkg.lock └── README.md /one_click/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /one_click/resources/app/uwsgi.ini: -------------------------------------------------------------------------------- 1 | [uwsgi] 2 | module = run 3 | callable = app -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # application 2 | click 3 | python-terraform 4 | 5 | # dev 6 | black 7 | ipdb 8 | pytest -------------------------------------------------------------------------------- /one_click/terraform/output.tf: -------------------------------------------------------------------------------- 1 | output "public_dns" { 2 | value = "${aws_instance.flask_server.public_dns}" 3 | } -------------------------------------------------------------------------------- /one_click/resources/app/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | app: 4 | build: ./ 5 | ports: 6 | - 80:80 -------------------------------------------------------------------------------- /one_click/resources/app/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG IMAGE=tiangolo/uwsgi-nginx-flask:python3.7 2 | 3 | FROM $IMAGE 4 | 5 | COPY ./app /app 6 | 7 | RUN chmod -R 707 $STATIC_PATH 8 | 9 | RUN pip install -r /app/requirements.txt -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | skip-string-normalization = true 3 | line-length = 79 4 | include = '\.pyi?$' 5 | exclude = ''' 6 | /( 7 | \.git 8 | | \.hg 9 | | \.mypy_cache 10 | | \.tox 11 | | \.venv 12 | | _build 13 | | buck-out 14 | | build 15 | | dist 16 | )/ 17 | ''' -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/ambv/black 3 | rev: stable 4 | hooks: 5 | - id: black 6 | language_version: python3.7 7 | - repo: https://github.com/pre-commit/pre-commit-hooks 8 | rev: v1.2.3 9 | hooks: 10 | - id: flake8 11 | args: [--max-line-length=88] -------------------------------------------------------------------------------- /one_click/terraform/provision_project/variables.tf: -------------------------------------------------------------------------------- 1 | variable "host" {} 2 | 3 | variable "path_to_private_key" {} 4 | 5 | variable "base_directory" {} 6 | 7 | variable "project_link_or_path" {} 8 | 9 | variable "image_version" {} 10 | 11 | variable "use_github" {} 12 | 13 | variable "use_local" {} 14 | 15 | variable "public_ip" {} -------------------------------------------------------------------------------- /one_click/terraform/variables.tf: -------------------------------------------------------------------------------- 1 | variable "instance_name_base" { 2 | default = "flask-server" 3 | } 4 | 5 | variable "base_directory" {} 6 | 7 | variable "instance_type" {} 8 | 9 | variable "image_version" {} 10 | 11 | variable "project_link_or_path" {} 12 | 13 | variable "path_to_public_key" {} 14 | 15 | variable "path_to_private_key" {} 16 | 17 | variable "use_github" {} 18 | 19 | variable "use_local" {} -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | 4 | with open('requirements.txt', 'r') as f: 5 | install_requires = f.read().splitlines() 6 | 7 | setup( 8 | name='One-Click', 9 | version='0.1', 10 | install_requires=install_requires, 11 | entry_points=''' 12 | [console_scripts] 13 | one-click=one_click.cli:main 14 | ''', 15 | packages=find_packages(), 16 | ) 17 | -------------------------------------------------------------------------------- /tests/utils_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "time" 7 | ) 8 | 9 | // GetWithRetryE tests if url responds with 200 after n_retries 10 | func GetWithRetryE(url string, nRetries int) (response *http.Response, e error) { 11 | for i := 0; i < nRetries; i++ { 12 | response, e := http.Get(url) 13 | if e == nil { 14 | return response, e 15 | } 16 | time.Sleep(time.Second) 17 | } 18 | 19 | return nil, fmt.Errorf("Failed to receive a response after %d retries", nRetries) 20 | } 21 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | # Gopkg.toml example 2 | # 3 | # Refer to https://golang.github.io/dep/docs/Gopkg.toml.html 4 | # for detailed Gopkg.toml documentation. 5 | # 6 | # required = ["github.com/user/thing/cmd/thing"] 7 | # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] 8 | # 9 | # [[constraint]] 10 | # name = "github.com/user/project" 11 | # version = "1.0.0" 12 | # 13 | # [[constraint]] 14 | # name = "github.com/user/project2" 15 | # branch = "dev" 16 | # source = "github.com/myfork/project2" 17 | # 18 | # [[override]] 19 | # name = "github.com/x/y" 20 | # version = "2.4.0" 21 | # 22 | # [prune] 23 | # non-go = false 24 | # go-tests = true 25 | # unused-packages = true 26 | 27 | 28 | [prune] 29 | go-tests = true 30 | unused-packages = true 31 | 32 | [[constraint]] 33 | name = "github.com/gruntwork-io/terratest" 34 | version = "0.13.23" 35 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import click 3 | from pathlib import Path 4 | 5 | from one_click import utils 6 | 7 | 8 | def test_dict_to_tfvars(): 9 | sample = { 10 | "project_link_or_path": "~/MyName/projects/project", 11 | "use_github": "1", 12 | "use_local": "0", 13 | } 14 | utils.dict_to_tfvars(sample) 15 | 16 | 17 | def test_py_version_to_image(): 18 | with pytest.raises(click.BadParameter): 19 | utils.py_version_to_image("3.8") 20 | utils.py_version_to_image("3.6.3") 21 | utils.py_version_to_image("best_version") 22 | 23 | assert ( 24 | utils.py_version_to_image("3.7") 25 | == "tiangolo/uwsgi-nginx-flask:python3.7" 26 | ) 27 | 28 | 29 | def test_pre_destroy_check(tmpdir): 30 | tmpdir = Path(tmpdir) 31 | required_state_files = ( 32 | ".terraform", 33 | "main.tf", 34 | "terraform.tfstate", 35 | "terraform.tfvars", 36 | ) 37 | 38 | for f in required_state_files: 39 | (tmpdir / f).open('w').close() 40 | 41 | utils.pre_destroy_check(tmpdir) 42 | 43 | for f in required_state_files: 44 | (tmpdir / f).unlink() # delete required files one by one 45 | with pytest.raises(click.UsageError): 46 | utils.pre_destroy_check(tmpdir) 47 | -------------------------------------------------------------------------------- /one_click/terraform/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "us-west-2" 3 | } 4 | 5 | resource "random_string" "deployment_id" { 6 | length = 6 7 | special = false 8 | } 9 | 10 | resource aws_key_pair "one_click" { 11 | key_name = "one-click-key - ${random_string.deployment_id.result}" 12 | public_key = "${file("${var.path_to_public_key}")}" 13 | } 14 | 15 | resource "aws_instance" "flask_server" { 16 | ami = "ami-70e90210", 17 | instance_type = "${var.instance_type}" 18 | key_name = "${aws_key_pair.one_click.key_name}" 19 | 20 | vpc_security_group_ids = ["${aws_security_group.allow_flask_and_ssh.id}"] 21 | 22 | tags { 23 | Name = "${var.instance_name_base} - ${random_string.deployment_id.result}" 24 | } 25 | } 26 | 27 | module "provision_project" { 28 | source = "./provision_project" 29 | 30 | host = "${aws_instance.flask_server.public_ip}" 31 | path_to_private_key = "${var.path_to_private_key}" 32 | base_directory = "${var.base_directory}" 33 | project_link_or_path = "${var.project_link_or_path}" 34 | image_version = "${var.image_version}" 35 | use_github = "${var.use_github}" 36 | use_local = "${var.use_local}" 37 | public_ip = "${aws_instance.flask_server.public_ip}" 38 | } 39 | 40 | resource "aws_security_group" "allow_flask_and_ssh" { 41 | name = "allow_flask_and_ssh - ${random_string.deployment_id.result}" 42 | 43 | ingress { 44 | protocol = "tcp" 45 | from_port = 80 46 | to_port = 80 47 | cidr_blocks = ["0.0.0.0/0"] 48 | } 49 | 50 | ingress { 51 | protocol = "tcp" 52 | from_port = 22 53 | to_port = 22 54 | cidr_blocks = ["0.0.0.0/0"] 55 | } 56 | 57 | egress { 58 | from_port = 0 59 | to_port = 0 60 | protocol = "-1" 61 | cidr_blocks = ["0.0.0.0/0"] 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | docker: 5 | - image: circleci/python:3.6.1 6 | 7 | steps: 8 | - checkout 9 | 10 | # Download and cache dependencies 11 | - restore_cache: 12 | keys: 13 | - v1-dependencies-{{ checksum "requirements.txt" }} 14 | # fallback to using the latest cache if no exact match is found 15 | - v1-dependencies- 16 | 17 | - run: 18 | name: install dependencies 19 | command: | 20 | python3 -m venv venv 21 | . venv/bin/activate 22 | pip install -r requirements.txt 23 | pip install . 24 | 25 | - save_cache: 26 | paths: 27 | - ./venv 28 | key: v1-dependencies-{{ checksum "requirements.txt" }} 29 | 30 | # run tests! 31 | - run: 32 | name: run tests 33 | command: | 34 | . venv/bin/activate 35 | pytest 36 | 37 | build_go: 38 | docker: 39 | - image: circleci/golang:1.11 40 | 41 | working_directory: /go/src/github.com/gusostow/one-click 42 | 43 | environment: 44 | TEST_RESULTS_DIR: /tmp/test-results 45 | 46 | steps: 47 | - checkout 48 | 49 | - run: mkdir -p $TEST_RESULTS_DIR 50 | 51 | - run: 52 | name: install dependencies 53 | command: | 54 | wget https://releases.hashicorp.com/terraform/0.11.11/terraform_0.11.11_linux_amd64.zip 55 | sudo unzip terraform_0.11.11_linux_amd64.zip -d /usr/local/bin 56 | go get github.com/tools/godep 57 | dep ensure 58 | 59 | - run: 60 | name: run infrastructure tests 61 | command: | 62 | go test -v ./... 63 | 64 | 65 | workflows: 66 | version: 2 67 | build_and_test: 68 | jobs: 69 | - build 70 | - build_go -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | pip-wheel-metadata/ 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # IPython 79 | profile_default/ 80 | ipython_config.py 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # celery beat schedule file 86 | celerybeat-schedule 87 | 88 | # SageMath parsed files 89 | *.sage.py 90 | 91 | # Environments 92 | .env 93 | .venv 94 | env/ 95 | venv/ 96 | ENV/ 97 | env.bak/ 98 | venv.bak/ 99 | 100 | # Spyder project settings 101 | .spyderproject 102 | .spyproject 103 | 104 | # Rope project settings 105 | .ropeproject 106 | 107 | # mkdocs documentation 108 | /site 109 | 110 | # mypy 111 | .mypy_cache/ 112 | .dmypy.json 113 | dmypy.json 114 | 115 | # Pyre type checker 116 | .pyre/ 117 | 118 | # terraform 119 | .terraform 120 | *.tfstate 121 | *.tfstate.backup 122 | *.tfvars 123 | 124 | # golang 125 | *.test 126 | *.out 127 | vendor/ 128 | .test-data 129 | -------------------------------------------------------------------------------- /one_click/terraform/provision_project/main.tf: -------------------------------------------------------------------------------- 1 | resource "null_resource" "remote_exec_from_github" { 2 | count = "${var.use_github}" 3 | 4 | connection { 5 | host = "${var.host}" 6 | type = "ssh" 7 | user = "ubuntu" 8 | private_key = "${file("${var.path_to_private_key}")}" 9 | } 10 | 11 | provisioner "file" { 12 | source = "${var.base_directory}/resources/app" 13 | destination = "/home/ubuntu/app/" 14 | } 15 | 16 | provisioner "remote-exec" { 17 | inline = [ 18 | "mkdir ./app/app", 19 | "git clone ${var.project_link_or_path} ./app/app", 20 | "mv app/uwsgi.ini ./app/app/", 21 | "sudo apt-get update && sudo apt-get install -y docker.io", 22 | "sudo curl -L https://github.com/docker/compose/releases/download/1.18.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose", 23 | "sudo chmod +x /usr/local/bin/docker-compose", 24 | "sudo usermod -aG docker ubuntu", 25 | "cd ./app", 26 | "sudo docker-compose build --build-arg IMAGE=${var.image_version} app", 27 | "sudo docker-compose up -d" 28 | ] 29 | } 30 | } 31 | 32 | resource "null_resource" "remote_exec_from_local" { 33 | count = "${var.use_local}" 34 | 35 | connection { 36 | host = "${var.host}" 37 | type = "ssh" 38 | user = "ubuntu" 39 | private_key = "${file("${var.path_to_private_key}")}" 40 | } 41 | 42 | provisioner "file" { 43 | source = "${var.base_directory}/resources/app" 44 | destination = "/home/ubuntu/app/" 45 | } 46 | 47 | provisioner "local-exec" { 48 | command = "rsync -avz --progress -e \"ssh -o StrictHostKeyChecking=no\" ${var.project_link_or_path}/ ubuntu@${var.public_ip}:/home/ubuntu/app/app" 49 | } 50 | 51 | provisioner "remote-exec" { 52 | inline = [ 53 | "mv app/uwsgi.ini ./app/app/", 54 | "sudo apt-get update && sudo apt-get install -y docker.io", 55 | "sudo curl -L https://github.com/docker/compose/releases/download/1.18.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose", 56 | "sudo chmod +x /usr/local/bin/docker-compose", 57 | "sudo usermod -aG docker ubuntu", 58 | "cd ./app", 59 | "sudo docker-compose build --build-arg IMAGE=${var.image_version} app", 60 | "sudo docker-compose up -d" 61 | ] 62 | } 63 | } -------------------------------------------------------------------------------- /one_click/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Optional 2 | from pathlib import Path 3 | 4 | import click 5 | 6 | BASE_DIR = str(Path(__file__).parent) 7 | 8 | 9 | def dict_to_tfvars(vars: Dict[str, str]) -> str: 10 | """ 11 | Convert a dictionary of strings to a string suitable for a tfvars file 12 | (or a bash enviornment variable file for that matter) 13 | """ 14 | return "\n".join(f"{key} = \"{val}\"" for key, val in vars.items()) 15 | 16 | 17 | def py_version_to_image(py_version: str) -> Optional[str]: 18 | if py_version not in {"3.7", "3.6", "3.5", "2.7"}: 19 | raise click.BadParameter( 20 | 'Invalid version of Python', 21 | param=py_version, 22 | param_hint=["3.7", "3.6", "3.5", "2.7"], 23 | ) 24 | return f"tiangolo/uwsgi-nginx-flask:python{py_version}" 25 | 26 | 27 | def build_and_validate_tfvars( 28 | project_link_or_path, 29 | public_key_path, 30 | private_key_path, 31 | py, 32 | instance_type, 33 | deployment_source="github", 34 | ): 35 | # Drop trailing slash from local path so it doesn't interfere with rsync 36 | if (deployment_source == "local") and (project_link_or_path[-1] == "/"): 37 | project_link_or_path = project_link_or_path[:-1] 38 | 39 | github_local_switches = { 40 | "github": {"use_github": 1, "use_local": 0}, 41 | "local": {"use_github": 0, "use_local": 1}, 42 | } 43 | 44 | image_tag = py_version_to_image(py) 45 | 46 | var = { 47 | "base_directory": str(BASE_DIR), 48 | "path_to_public_key": public_key_path, 49 | "path_to_private_key": private_key_path, 50 | "project_link_or_path": project_link_or_path, 51 | "image_version": image_tag, 52 | "instance_type": instance_type, 53 | **github_local_switches[deployment_source], 54 | } 55 | 56 | return dict_to_tfvars(var) 57 | 58 | 59 | def pre_destroy_check(deployment_directory): 60 | required_state_files = ( 61 | ".terraform", 62 | "terraform.tfstate", 63 | "terraform.tfvars", 64 | ) 65 | has_all_required_files = all( 66 | map( 67 | lambda path: any(deployment_directory.glob(path)), 68 | required_state_files, 69 | ) 70 | ) 71 | if not has_all_required_files: 72 | raise click.UsageError( 73 | f""" 74 | Deployment directory is missing some or all of the required state 75 | files: {required_state_files}. Make sure that you actually have a 76 | project deployed and that you are in its correct directory.""" 77 | ) 78 | -------------------------------------------------------------------------------- /one_click/cli.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import click 4 | import python_terraform as pt 5 | 6 | from one_click import utils 7 | 8 | 9 | DEPLOYMENT_DIR = Path.cwd() 10 | BASE_DIR = Path(__file__).parent 11 | TERRAFORM_DIR = str(BASE_DIR / "terraform") 12 | 13 | 14 | def deploy( 15 | project_link_or_path, 16 | public_key_path, 17 | private_key_path, 18 | py, 19 | instance_type, 20 | deployment_source="github", 21 | ): 22 | tfvars = utils.build_and_validate_tfvars( 23 | project_link_or_path, 24 | public_key_path, 25 | private_key_path, 26 | py, 27 | instance_type, 28 | deployment_source=deployment_source, 29 | ) 30 | 31 | with open(DEPLOYMENT_DIR / "terraform.tfvars", "w") as f: 32 | f.write(tfvars) 33 | 34 | tf = pt.Terraform() 35 | tf.init( 36 | dir_or_plan=str(DEPLOYMENT_DIR), 37 | from_module=TERRAFORM_DIR, 38 | capture_output=False, 39 | ) 40 | return_code, _, _ = tf.apply(capture_output=False) 41 | 42 | 43 | @click.group() 44 | def main(): 45 | pass 46 | 47 | 48 | def deployment_options(deployment_function): 49 | deployment_function = click.option( 50 | "--py", 51 | default="3.7", 52 | help='Python version. Options are 3.7, 3.6, 3.5, 2.7.', 53 | )(deployment_function) 54 | deployment_function = click.option( 55 | "--instance_type", 56 | default="t2.medium", 57 | help="See what's available here https://aws.amazon.com/ec2/instance-types/", 58 | )(deployment_function) 59 | deployment_function = click.option( 60 | "--private_key_path", default="~/.ssh/id_rsa" 61 | )(deployment_function) 62 | deployment_function = click.option( 63 | "--public_key_path", default="~/.ssh/id_rsa.pub" 64 | )(deployment_function) 65 | deployment_function = main.command()(deployment_function) 66 | 67 | return deployment_function 68 | 69 | 70 | @deployment_options 71 | @click.argument("git_path") 72 | def deploy_github( 73 | git_path, 74 | public_key_path=None, 75 | private_key_path=None, 76 | py=None, 77 | instance_type=None, 78 | ): 79 | deploy( 80 | git_path, 81 | public_key_path, 82 | private_key_path, 83 | py, 84 | instance_type, 85 | deployment_source="github", 86 | ) 87 | 88 | 89 | @deployment_options 90 | @click.argument("local_path") 91 | def deploy_local( 92 | local_path, 93 | public_key_path=None, 94 | private_key_path=None, 95 | py=None, 96 | instance_type=None, 97 | ): 98 | deploy( 99 | local_path, 100 | public_key_path, 101 | private_key_path, 102 | py, 103 | instance_type, 104 | deployment_source="local", 105 | ) 106 | 107 | 108 | @main.command() 109 | def destroy(): 110 | # Ensure that the proper backend files are in the deployment directory 111 | utils.pre_destroy_check(DEPLOYMENT_DIR) 112 | tf = pt.Terraform() 113 | tf.destroy(capture_output=False) 114 | -------------------------------------------------------------------------------- /tests/integration_test.go: -------------------------------------------------------------------------------- 1 | // Integration test that deploys github flask project to real infrastructure 2 | package test 3 | 4 | import ( 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "path/filepath" 9 | "runtime" 10 | "testing" 11 | 12 | "github.com/gruntwork-io/terratest/modules/ssh" 13 | "github.com/gruntwork-io/terratest/modules/terraform" 14 | test_structure "github.com/gruntwork-io/terratest/modules/test-structure" 15 | ) 16 | 17 | func TestGithub(t *testing.T) { 18 | terraformDirectory := "../one_click/terraform" 19 | tempdirName := "deployment_directory" 20 | dnsFileName := "public_dns" 21 | 22 | // Destroy infrastructure and temporary files at the end of the test 23 | defer test_structure.RunTestStage(t, "teardown", func() { 24 | terraformOptions := test_structure.LoadTerraformOptions(t, terraformDirectory) 25 | terraform.Destroy(t, terraformOptions) 26 | 27 | dir := filepath.Join(os.TempDir(), tempdirName) 28 | os.Remove(dir) 29 | 30 | // remove dns file if it exists 31 | dnsFilePath := filepath.Join(terraformDirectory, dnsFileName) + ".json" 32 | if _, e := os.Stat(dnsFilePath); e == nil { 33 | os.Remove(dnsFilePath) 34 | } 35 | }) 36 | 37 | test_structure.RunTestStage(t, "setup", func() { 38 | // Give distinct "namespace" for test resources 39 | instanceNameBase := "terratest - flask-server" 40 | 41 | dir, e := ioutil.TempDir("", tempdirName) 42 | if e != nil { 43 | log.Fatal(e) 44 | } 45 | 46 | privatePath := filepath.Join(dir, "id_rsa") 47 | publicPath := filepath.Join(dir, "id_rsa.pub") 48 | privateF, _ := os.Create(privatePath) 49 | publicF, _ := os.Create(publicPath) 50 | 51 | // Create temporary keys 52 | keyPair := ssh.GenerateRSAKeyPair(t, 1096) 53 | privateF.WriteString(keyPair.PrivateKey) 54 | publicF.WriteString(keyPair.PublicKey) 55 | 56 | _, callerPath, _, _ := runtime.Caller(0) 57 | baseDirectory := filepath.Join(callerPath, "../../one_click") 58 | 59 | terraformOptions := &terraform.Options{ 60 | TerraformDir: terraformDirectory, 61 | 62 | // Variables to pass to our Terraform code using -var options 63 | Vars: map[string]interface{}{ 64 | "instance_name_base": instanceNameBase, 65 | "base_directory": baseDirectory, 66 | "instance_type": "t2.micro", 67 | "image_version": "tiangolo/uwsgi-nginx-flask:python3.6", 68 | "project_link_or_path": "https://github.com/gusostow/EXAMPLE-hosteldirt.git", 69 | "path_to_public_key": publicPath, 70 | "path_to_private_key": privatePath, 71 | "use_github": "1", 72 | "use_local": "0", 73 | }, 74 | } 75 | 76 | test_structure.SaveTerraformOptions(t, terraformDirectory, terraformOptions) 77 | 78 | terraform.InitAndApply(t, terraformOptions) 79 | 80 | // Save this information for the validation stage 81 | publicDNS := terraform.Output(t, terraformOptions, "public_dns") 82 | test_structure.SaveString(t, terraformDirectory, dnsFileName, publicDNS) 83 | }) 84 | 85 | test_structure.RunTestStage(t, "validate", func() { 86 | // Load address of test webserver 87 | publicDNS := test_structure.LoadString(t, terraformDirectory, dnsFileName) 88 | 89 | // Check if webserver is responding to requests 90 | publicDNSFullURL := "http://" + publicDNS 91 | response, e := GetWithRetryE(publicDNSFullURL, 30) 92 | 93 | if e != nil { 94 | t.Error(e) 95 | } else if response.StatusCode != 200 { 96 | t.Error("Test failed: Got status code", response.StatusCode) 97 | } else { 98 | log.Print("Successfully received response from", publicDNS) 99 | } 100 | }) 101 | } 102 | -------------------------------------------------------------------------------- /Gopkg.lock: -------------------------------------------------------------------------------- 1 | # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. 2 | 3 | 4 | [[projects]] 5 | digest = "1:4fe4da54f9556db12625848681a944b28b7041229e9e2675a566a69bbbf87482" 6 | name = "github.com/aws/aws-sdk-go" 7 | packages = [ 8 | "aws", 9 | "aws/awserr", 10 | "aws/awsutil", 11 | "aws/client", 12 | "aws/client/metadata", 13 | "aws/corehandlers", 14 | "aws/credentials", 15 | "aws/credentials/ec2rolecreds", 16 | "aws/credentials/endpointcreds", 17 | "aws/credentials/processcreds", 18 | "aws/credentials/stscreds", 19 | "aws/csm", 20 | "aws/defaults", 21 | "aws/ec2metadata", 22 | "aws/endpoints", 23 | "aws/request", 24 | "aws/session", 25 | "aws/signer/v4", 26 | "internal/ini", 27 | "internal/s3err", 28 | "internal/sdkio", 29 | "internal/sdkrand", 30 | "internal/sdkuri", 31 | "internal/shareddefaults", 32 | "private/protocol", 33 | "private/protocol/ec2query", 34 | "private/protocol/eventstream", 35 | "private/protocol/eventstream/eventstreamapi", 36 | "private/protocol/json/jsonutil", 37 | "private/protocol/jsonrpc", 38 | "private/protocol/query", 39 | "private/protocol/query/queryutil", 40 | "private/protocol/rest", 41 | "private/protocol/restxml", 42 | "private/protocol/xml/xmlutil", 43 | "service/acm", 44 | "service/autoscaling", 45 | "service/cloudwatchlogs", 46 | "service/ec2", 47 | "service/iam", 48 | "service/kms", 49 | "service/rds", 50 | "service/s3", 51 | "service/s3/s3iface", 52 | "service/s3/s3manager", 53 | "service/sns", 54 | "service/sqs", 55 | "service/sts", 56 | ] 57 | pruneopts = "UT" 58 | revision = "81f3829f5a9d041041bdf56e55926691309d7699" 59 | version = "v1.16.26" 60 | 61 | [[projects]] 62 | digest = "1:7b94d37d65c0445053c6f3e73090e3966c1c29127035492c349e14f25c440359" 63 | name = "github.com/boombuler/barcode" 64 | packages = [ 65 | ".", 66 | "qr", 67 | "utils", 68 | ] 69 | pruneopts = "UT" 70 | revision = "3cfea5ab600ae37946be2b763b8ec2c1cf2d272d" 71 | version = "v1.0.0" 72 | 73 | [[projects]] 74 | digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" 75 | name = "github.com/davecgh/go-spew" 76 | packages = ["spew"] 77 | pruneopts = "UT" 78 | revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" 79 | version = "v1.1.1" 80 | 81 | [[projects]] 82 | digest = "1:ec6f9bf5e274c833c911923c9193867f3f18788c461f76f05f62bb1510e0ae65" 83 | name = "github.com/go-sql-driver/mysql" 84 | packages = ["."] 85 | pruneopts = "UT" 86 | revision = "72cd26f257d44c1114970e19afddcd812016007e" 87 | version = "v1.4.1" 88 | 89 | [[projects]] 90 | digest = "1:8f8811f9be822914c3a25c6a071e93beb4c805d7b026cbf298bc577bc1cc945b" 91 | name = "github.com/google/uuid" 92 | packages = ["."] 93 | pruneopts = "UT" 94 | revision = "064e2069ce9c359c118179501254f67d7d37ba24" 95 | version = "0.2" 96 | 97 | [[projects]] 98 | digest = "1:88cf7b3f31faf3082741e1fa3056d67e257eb45bb2f535374a5ba9685a405bf3" 99 | name = "github.com/gruntwork-io/terratest" 100 | packages = [ 101 | "modules/aws", 102 | "modules/collections", 103 | "modules/customerrors", 104 | "modules/files", 105 | "modules/logger", 106 | "modules/packer", 107 | "modules/random", 108 | "modules/retry", 109 | "modules/shell", 110 | "modules/ssh", 111 | "modules/terraform", 112 | "modules/test-structure", 113 | ] 114 | pruneopts = "UT" 115 | revision = "913f60214f225bac27ef03ee2b93a92931bc9786" 116 | version = "v0.13.23" 117 | 118 | [[projects]] 119 | digest = "1:bb81097a5b62634f3e9fec1014657855610c82d19b9a40c17612e32651e35dca" 120 | name = "github.com/jmespath/go-jmespath" 121 | packages = ["."] 122 | pruneopts = "UT" 123 | revision = "c2b33e84" 124 | 125 | [[projects]] 126 | digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" 127 | name = "github.com/pmezard/go-difflib" 128 | packages = ["difflib"] 129 | pruneopts = "UT" 130 | revision = "792786c7400a136282c1664665ae0a8db921c6c2" 131 | version = "v1.0.0" 132 | 133 | [[projects]] 134 | digest = "1:dc56e7178a82f8e84960ed9efa7d4ebd0a6d7864894bfc46f8a35ee5eae7153a" 135 | name = "github.com/pquerna/otp" 136 | packages = [ 137 | ".", 138 | "hotp", 139 | "totp", 140 | ] 141 | pruneopts = "UT" 142 | revision = "be78767b3e392ce45ea73444451022a6fc32ad0d" 143 | version = "v1.1.0" 144 | 145 | [[projects]] 146 | digest = "1:5da8ce674952566deae4dbc23d07c85caafc6cfa815b0b3e03e41979cedb8750" 147 | name = "github.com/stretchr/testify" 148 | packages = [ 149 | "assert", 150 | "require", 151 | ] 152 | pruneopts = "UT" 153 | revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053" 154 | version = "v1.3.0" 155 | 156 | [[projects]] 157 | branch = "master" 158 | digest = "1:a741998d8633b7f444053abdf218de3d8f66dc02f832e182b983b80f8c7ba852" 159 | name = "golang.org/x/crypto" 160 | packages = [ 161 | "curve25519", 162 | "ed25519", 163 | "ed25519/internal/edwards25519", 164 | "internal/chacha20", 165 | "internal/subtle", 166 | "poly1305", 167 | "ssh", 168 | "ssh/agent", 169 | ] 170 | pruneopts = "UT" 171 | revision = "b01c7a72566457eb1420261cdafef86638fc3861" 172 | 173 | [[projects]] 174 | branch = "master" 175 | digest = "1:76ee51c3f468493aff39dbacc401e8831fbb765104cbf613b89bef01cf4bad70" 176 | name = "golang.org/x/net" 177 | packages = ["context"] 178 | pruneopts = "UT" 179 | revision = "d26f9f9a57f3fab6a695bec0d84433c2c50f8bbf" 180 | 181 | [[projects]] 182 | digest = "1:c25289f43ac4a68d88b02245742347c94f1e108c534dda442188015ff80669b3" 183 | name = "google.golang.org/appengine" 184 | packages = ["cloudsql"] 185 | pruneopts = "UT" 186 | revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1" 187 | version = "v1.4.0" 188 | 189 | [solve-meta] 190 | analyzer-name = "dep" 191 | analyzer-version = 1 192 | input-imports = [ 193 | "github.com/gruntwork-io/terratest/modules/ssh", 194 | "github.com/gruntwork-io/terratest/modules/terraform", 195 | "github.com/gruntwork-io/terratest/modules/test-structure", 196 | ] 197 | solver-name = "gps-cdcl" 198 | solver-version = 1 199 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # one-click 2 | [![CircleCI](https://circleci.com/gh/InsightDataCommunity/one-click/tree/master.svg?style=svg)](https://circleci.com/gh/InsightDataCommunity/one-click/tree/master) 3 | 4 | One-click deployment for Machine Learning Flask apps 5 | 6 | ## Before you Can Deploy 7 | 8 | The deploy might be one click ... installing dependencies, making your AWS account, and ensuring your project is compatible with one-click is not. If you've already setup your machine and your project skip to the [quick-start guide](#quick-start-guide). 9 | 10 | Windows is not directly supported at this time. The [Ubuntu subsystem](https://helloacm.com/the-ubuntu-sub-system-new-bash-shell-in-windows-10/) for Windows 10 is recommended. 11 | 12 | ### AWS Setup 13 | 14 | 1. Create a [AWS account](https://aws.amazon.com/) (or use an existing one). 15 | 2. Create an [IAM admin user and group](https://docs.aws.amazon.com/IAM/latest/UserGuide/getting-started_create-admin-group.html). AWS credentialling is confusing. This instruction creates a new sub-user that will be safer to use than the root account you just created in step 1. 16 | 3. Get the access key and secret access key from the IAM administrator user you just created. 17 | - Go to the [IAM console](https://console.aws.amazon.com/iam/home?#home) 18 | - Choose **Users** and then the administrator user you just created. 19 | - Select the **Security Credentials** tab and then hit **Create Access Key** 20 | - Choose **Show** 21 | - We need to export these as enviornment variables in your `~/.bash_profile`. You should add something that looks like this to the bottom of your profile using your favorite text editor, where the keys are your own of course: 22 | ```bash 23 | export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE 24 | export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY 25 | ``` 26 | Then source your profile `souce ~/.bash_profile` and now your laptop will be fully authorized to create resources on AWS! 27 | 28 | ### Create RSA Key Pair 29 | 30 | 1. Check to see if you already have keys with default names: `ls ~/.ssh` (It's fine if it says the directory `~/.ssh` doesn't exist move on to step 2). If you have two files with names `id_rsa` and `id_rsa.pub` then you are all set to skip this section, if not then continue on to creating the key pair. 31 | 2. `ssh-keygen` 32 | 3. Continue by pressing enter repeatedly (you don't need to enter anything in the text boxes) until you see something like this 33 | ``` 34 | +--[ RSA 2048]----+ 35 | | o=. | 36 | | o o++E | 37 | | + . Ooo. | 38 | | + O B.. | 39 | | = *S. | 40 | | o | 41 | | | 42 | | | 43 | | | 44 | +-----------------+ 45 | ``` 46 | 47 | ### Software Requirements 48 | 49 | - You need terraform installed. 50 | - MacOs: `brew install terraform`. If you don't have homebrew, install it with this command: `/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"`. 51 | . 52 | - linux: https://www.terraform.io/downloads.html 53 | 54 | ### App Compatibility 55 | 56 | One-click has several strict requirements for apps it can deploy. Rigid specifications keeps the tool easy to use. Check out some example [one-click compatible projects](#example-apps) that are compliant. 57 | 58 | #### Directory Structure 59 | 60 | - It is strongly recommended that your directory structure be flat. Having your app defined, your templates, and your static folder in a nested subfolder e.g. in `yourapp/flaskexample/` might cause problems. 61 | - There must be a python file called `run.py` in the root of your project directory that will run your app. _**The name and the location are non-negotiable.**_ The file might looks something like: 62 | ```python 63 | from views import app 64 | 65 | if __name__ == '__main__': 66 | app.run(host='0.0.0.0', port=80) 67 | ``` 68 | - As of now, run your app in `run.py` on `host='0.0.0.0'` and `port=80` 69 | 70 | #### Requirements File 71 | 72 | One-click builds a fresh python environment in ubuntu for every deployment. You need to clearly specify which python requirements your app depends on. 73 | 74 | - Put the name (and potentially the version number) of every requirement in a file `requirements.txt` in the root of your project. Once again, _**The name and the location of `requirements.txt` are non-negotiable.**_ 75 | 76 | - If you haven't been keeping track of your requirements you could: 77 | - Use a tool like [pigar](https://github.com/damnever/pigar) to automatically generate it based on searching your project. 78 | - If you've been using a conda environment or a virtualenv for the project you can run `pip freeze > requirements.txt` 79 | 80 | - **HINT:** A good way to test if your `requirements.txt` file is comprehensive is to create a fresh conda or virtual enviornment and try to run your app after installing from the file. 81 | ```bash 82 | conda create -n test_env python=3.6 83 | source activate test_env 84 | pip install -r requirements.txt 85 | python run.py 86 | ``` 87 | 88 | ## Quick-start Guide 89 | 90 | Consult the [app compatibility guidelines](#app-compatibility) before deploying for the first time. You may have to restructure your project before it will work with one-click. 91 | 92 | ### Deploy Instructions 93 | 94 | 1. Clone the repo 95 | 2. Install the one-click package (from inside the cloned repo) `pip install -e .` 96 | 3. Make a new directory to track the state of your deployment. It can be anywhere. This new *deployment directory* has nothing to do with your project directory that has your code. It will hold the backend state files for the deployment. Any time you want to reference this specific deployment you must be using one-click from its deployment directory. 97 | 4. Deploy your project! Inside the deployment directory you just created, run for github deployment (**NOTE:** if you didn't use the default names when you generated your RSA keys, or if you're on windows, then you will have to specify the paths with the `--private_key_path` and `--public_key_path`command line options) 98 | ``` 99 | one-click deploy-github https://github.com/gusostow/EXAMPLE-localtype_site 100 | ``` 101 | or for local deployment 102 | ``` 103 | one-click deploy-local ~/path/to/your/flask/project/folder 104 | ``` 105 | 106 | Your app should now be publicly available from the `public_dns` output in your console. If you want to ssh into the instance this can be done with `ssh ubuntu@` 107 | 108 | ### Destroy Instructions 109 | 110 | 1. Navigate to your deployment directory, which is where the terraform state is located. 111 | 2. Run `one-click destroy` 112 | 113 | ### Updating your App 114 | 115 | As of now one-click does not provision automatic CI/CD support to keep your deployment up to date with pushes to your app's repo. To make updates: 116 | 1. Push your changes to github 117 | 2. Make sure you are inside the directory used for deployment, then destroy and re-deploy your project: 118 | ``` 119 | one-click destroy 120 | one-click deploy-github 121 | ``` 122 | 123 | ## Troubleshooting your Deployment 124 | 125 | A lot can go wrong with a one size fits all automatic deployment. Most issues will be visible with some detective work. 126 | 127 | ### Problems with Provisioning the Server and Building your App Environment 128 | 129 | Build logs for installations on the server and building the docker environment are piped to console. Here you can see if there's an issue with making the ssh connection to remotely execute commands, cloning your repo to the server, or installing your requirements. If your url is completely unaccessable, then the error can likely be diagnosed here. 130 | 131 | ### Problems with Running your Code 132 | 133 | However, once the environment is set up, the server logs won't be directly visible in your console. If you get a 403 error when you visit your webpage url, then that means there is an error in your code, which probably has something to do with porting it a docker environment. 134 | 135 | You need to ssh into the server to view get visibility in those logs: 136 | 1. Get shell access to the server. `ssh ubuntu@`. You don't need to specify the path to a key file because you already did that in the deploy phase. 137 | 2. `cd app` 138 | 3. View the logs. `docker-compose logs` 139 | 140 | Here you will find the python errors you are accustomed to diagnosing when developing your app. 141 | 142 | ### Other Fixes to Common Problems 143 | 144 | #### Broken Paths 145 | - **Problem:** Absolute paths to files like datasets or models will break. The path `~/gusostow/python/myproject/data/data.csv` might work fine on your laptop, it won't in the docker container built for your app, which has a different directory structure. 146 | - **Solution:** Switch to referencing paths relatively to the python file that uses them, so they will be invariant to where the script is run. The `__file__` variable has that information. 147 | 148 | ## Example Apps 149 | 150 | - [Localtype](https://github.com/gusostow/EXAMPLE-localtype_site) 151 | - [Hosteldirt](https://github.com/gusostow/EXAMPLE-hosteldirt) 152 | 153 | --------------------------------------------------------------------------------