├── .editorconfig ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── lambda_function.py └── tf ├── .gitignore ├── .terraform.lock.hcl ├── apigw.tf ├── dynamo.tf ├── lambda.tf ├── main.tf ├── role.tf ├── s3.tf ├── secrets.tf └── vars.tf /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | # A special property that should be specified at the top of the file outside of 4 | # any sections. Set to true to stop .editor config file search on current file 5 | root = true 6 | 7 | [*] 8 | # Indentation style 9 | # Possible values - tab, space 10 | indent_style = space 11 | 12 | # Indentation size in single-spaced characters 13 | # Possible values - an integer, tab 14 | indent_size = 4 15 | 16 | # Line ending file format 17 | # Possible values - lf, crlf, cr 18 | end_of_line = lf 19 | 20 | # File character encoding 21 | # Possible values - latin1, utf-8, utf-16be, utf-16le 22 | charset = utf-8 23 | 24 | # Denotes whether to trim whitespace at the end of lines 25 | # Possible values - true, false 26 | trim_trailing_whitespace = true 27 | 28 | # Denotes whether file should end with a newline 29 | # Possible values - true, false 30 | insert_final_newline = true 31 | 32 | [Makefile] 33 | indent_style = tab 34 | 35 | [*.tf] 36 | indent_size = 2 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | out 2 | *.auto.tfvars 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | 2 | FROM amazonlinux:latest AS build-stage 3 | 4 | RUN yum upgrade -y 5 | RUN yum install -y gcc gcc-c++ make freetype-devel yum-utils findutils openssl-devel git zip 6 | 7 | ARG PYTHON_VERSION_WITH_DOT=3.8 8 | ARG PYTHON_VERSION_WITHOUT_DOT=38 9 | 10 | RUN amazon-linux-extras install -y python${PYTHON_VERSION_WITH_DOT} && \ 11 | yum install -y python${PYTHON_VERSION_WITHOUT_DOT}-devel 12 | 13 | ARG INSTBASE=/var/task 14 | 15 | WORKDIR ${INSTBASE} 16 | RUN python${PYTHON_VERSION_WITH_DOT} -m venv venv 17 | RUN venv/bin/pip install \ 18 | pypicloud[dynamo] \ 19 | typing_extensions \ 20 | apig-wsgi 21 | 22 | # Create lambda_venv_path.py 23 | RUN INSTBASE=${INSTBASE} venv/bin/python -c \ 24 | 'import os; import sys; instbase = os.environ["INSTBASE"]; print("import sys; sys.path[:0] = %s" % [p for p in sys.path if p.startswith(instbase)])' \ 25 | > ${INSTBASE}/lambda_venv_path.py 26 | 27 | # Remove artifacts that won't be used. 28 | # If lib64 is a symlink, remove it. 29 | RUN rm -rf venv/bin venv/share venv/include && \ 30 | (if test -h venv/lib64 ; then rm -f venv/lib64 ; fi) 31 | 32 | COPY lambda_function.py . 33 | 34 | RUN zip -r9q /tmp/lambda_pypicloud.zip * 35 | 36 | # Generate a filesystem image with just the zip file as the output. 37 | # See: https://docs.docker.com/engine/reference/commandline/build/#custom-build-outputs 38 | FROM scratch AS export-stage 39 | COPY --from=build-stage /tmp/lambda_pypicloud.zip / 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021 Shane Hathaway 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | default: out/lambda_pypicloud.zip 3 | 4 | out/lambda_pypicloud.zip: Dockerfile lambda_function.py 5 | mkdir -p out && \ 6 | DOCKER_BUILDKIT=1 docker build -o out . 7 | 8 | .PHONY: default 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyPICloud + AWS Lambda + Terraform 2 | 3 | This project brings together a few pieces so you can easily run a small 4 | PyPICloud instance as an AWS Lambda function with the help of Terraform. 5 | PyPICloud lets you publish Python packages privately, AWS Lambda lets you run 6 | a small service for free, and Terraform ensures the service is deployed and 7 | maintained correctly. 8 | 9 | ## Prerequisites 10 | 11 | This project was tested in Ubuntu 20.04. It may work in other environments. 12 | Feel free to submit issues. 13 | 14 | The following software should be installed before you start: 15 | 16 | - Docker (command line) 17 | - The AWS CLI - https://aws.amazon.com/cli/ 18 | - Terraform 19 | - Make (optional) 20 | 21 | Note that this project only uses Docker for building a ZIP file and does not 22 | use Docker in production. Some of the Python libraries contain native code 23 | that must be compiled in the same type of environment where the code will run. 24 | Docker makes it possible to ensure the build environment matches the AWS 25 | Lambda environment. 26 | 27 | ## Build 28 | 29 | `git clone` this project and `cd` to it. If you have `make` installed, type 30 | `make`. If not, type: 31 | 32 | ```sh 33 | mkdir -p out && DOCKER_BUILDKIT=1 docker build -o out . 34 | ``` 35 | 36 | The file `out/lambda_pypicloud.zip` will be generated. The zip file contains 37 | all the Python code and libraries needed for the service. See `Dockerfile` and 38 | `lambda_function.py` if you're interested in a simple way to run a Python app 39 | on AWS Lambda. 40 | 41 | ## Authenticate to AWS 42 | 43 | See: 44 | https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html 45 | 46 | If you have logged in before, use `aws sts get-caller-identity` to find out 47 | who you're currently authenticated as. If you manage mutiple AWS profiles, see 48 | https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html . 49 | 50 | ## Configure Deployment 51 | 52 | Terraform needs some variable settings. In the `tf` folder, create a text file 53 | with a name ending with `.auto.tfvars`. An example: 54 | 55 | ```sh 56 | vim tf/my.auto.tfvars 57 | ``` 58 | 59 | Your `.auto.tfvars` file needs to set the `region`, `package_bucket`, and 60 | `log_group` variables. The following template is a start, but replace the 61 | bucket name `MY-PYPICLOUD-BUCKET` with a bucket name of your choosing. 62 | 63 | ``` 64 | region = "us-east-1" 65 | package_bucket = "MY-PYPICLOUD-BUCKET" 66 | log_group = "pypicloud" 67 | ``` 68 | 69 | The bucket name must be globally unique and must not already exist (unless 70 | you're already skilled with Terraform and you prefer to `terraform import` 71 | the bucket instead.) 72 | 73 | ## Deploy 74 | 75 | Once you have built the ZIP file, authenticated to AWS, and set the variables, 76 | run Terraform to deploy. 77 | 78 | ```sh 79 | cd tf 80 | terraform init && terraform apply 81 | ``` 82 | 83 | When Terraform completes successfully, you'll have a lambda function and an 84 | API gateway URL connected to it. Find the URL of the service by typing: 85 | 86 | ``` 87 | terraform show -no-color | grep invoke_url 88 | ``` 89 | 90 | Visit that URL in your browser. Add yourself as an administrator. Read the 91 | PyPICloud documentation to learn how to use your private package index when 92 | installing or publishing packages: 93 | 94 | https://pypicloud.readthedocs.io/en/latest/topics/getting_started.html#installing-packages 95 | 96 | At this point, you have a serverless PyPICloud instance. Unless you use the 97 | service a lot, it will probably stay entirely within the AWS free tier, but 98 | remember to set up AWS billing alerts to notify you of usage spikes. 99 | 100 | ## Custom Domain Name 101 | 102 | It is simple to use your own domain name. 103 | 104 | - Use AWS Certificate Manager to create a free certificate for the domain or 105 | subdomain where you want to host your package index. 106 | 107 | - Visit the AWS console and find the API Gateway object called `pypicloud`. 108 | Visit the *Custom domain names* link. Add your domain name and use the 109 | certificate you created. Enable TLS 1.2, but don't enable 110 | *Mutual TLS Authentication* unless you know how to create and use 111 | client certificates. 112 | 113 | - Configure a CNAME in your DNS to map your domain or subdomain to the *API 114 | Gateway domain name* shown in the custom domain name's *Endpoint 115 | configuration* box. The API Gateway domain name is not the same as the 116 | DNS name used in the Invoke URL. 117 | 118 | Terraform is not aware of the custom domain name setting unless you tell it 119 | otherwise, so your domain name setting won't conflict with Terraform. 120 | 121 | ## Store Terraform State Remotely 122 | 123 | You should store the Terraform state in an S3 bucket to ensure the state 124 | doesn't get lost. Remote state storage is also important for working with a 125 | team. 126 | 127 | Create a new S3 storage bucket (or reuse one where you've already stored 128 | Terraform state.) In the `tf` folder, create a file called `remote-state.tf`. 129 | Use the following template, replacing `MY-TFSTATE-BUCKET` with the name of your 130 | bucket: 131 | 132 | ``` 133 | terraform { 134 | backend "s3" { 135 | bucket = "MY-TFSTATE-BUCKET" 136 | key = "tfstate/pypicloud" 137 | region = "us-east-1" 138 | } 139 | } 140 | ``` 141 | 142 | Run `terraform init` to migrate the local state to the storage bucket, then 143 | use `terraform plan` to ensure Terraform is still working as intended. 144 | 145 | ## Security 146 | 147 | This project takes the following security precautions. 148 | 149 | - The session keys are auto-generated and stored in a secret. 150 | 151 | - The server configuration file, which contains the session keys, is generated 152 | on the fly to minimize the possibility of exposure. 153 | 154 | - The lambda function runs with a role limited to accessing only what it 155 | needs. 156 | 157 | - PyPICloud is configured to display no packages to unauthenticated users. 158 | 159 | Please submit an issue if you discover that these precautions are not being 160 | applied correctly or if you are aware of other security precautions that 161 | should be implemented. 162 | 163 | ## Scope 164 | 165 | This project is not intended to be a framework. If the project stops working 166 | due to changes in AWS, Terraform, Python libraries, etc., then let's fix it, 167 | but if you want to enable LDAP, add a plugin, or customize in a way that 168 | doesn't apply generally, you probably ought to fork the project instead. 169 | 170 | ## Troubleshooting 171 | 172 | - 10 MB limit: AWS API Gateway has a fixed 10 MB limit. If the limit keeps you 173 | from uploading a package, one way around it is to temporarily run PyPICloud 174 | on your own box, using almost the same .ini file as the Lambda function uses. 175 | There is an "upload" button in the PyPICloud UI. 176 | -------------------------------------------------------------------------------- /lambda_function.py: -------------------------------------------------------------------------------- 1 | 2 | import lambda_venv_path # noqa 3 | 4 | import boto3 5 | import json 6 | import os 7 | from paste.deploy import loadapp 8 | from apig_wsgi import make_lambda_handler 9 | 10 | 11 | ini_template = """ 12 | [app:main] 13 | use = egg:pypicloud 14 | pyramid.reload_templates = False 15 | pyramid.debug_authorization = false 16 | pyramid.debug_notfound = false 17 | pyramid.debug_routematch = false 18 | pyramid.default_locale_name = en 19 | pypi.default_read = authenticated 20 | pypi.storage = s3 21 | storage.bucket = {BUCKET} 22 | storage.region_name = {BUCKET_REGION} 23 | pypi.db = dynamo 24 | db.region_name = {DYNAMO_REGION} 25 | pypi.auth = pypicloud.access.aws_secrets_manager.AWSSecretsManagerAccessBackend 26 | auth.secret_id = {AUTH_SECRET_ID} 27 | session.encrypt_key = {SESSION_ENCRYPT_KEY} 28 | session.validate_key = {SESSION_VALIDATE_KEY} 29 | session.secure = True 30 | session.invalidate_corrupt = true 31 | pypi.fallback = redirect 32 | 33 | [loggers] 34 | keys = root 35 | 36 | [handlers] 37 | keys = stdout 38 | 39 | [formatters] 40 | keys = generic 41 | 42 | [logger_root] 43 | level = INFO 44 | handlers = stdout 45 | 46 | [handler_stdout] 47 | class = StreamHandler 48 | args = (sys.stdout,) 49 | level = NOTSET 50 | formatter = generic 51 | 52 | [formatter_generic] 53 | format = %(levelname)s %(asctime)s [%(name)s] %(message)s 54 | """ 55 | 56 | 57 | def generate_secret(): 58 | import base64 59 | import secrets 60 | return base64.encodebytes(secrets.token_bytes(32)).decode('ascii').strip() 61 | 62 | 63 | def get_config_fn(): 64 | """Get the environment overlay file from Secrets Manager""" 65 | env_overlay = {} 66 | 67 | secret_id = os.environ.get('ENV_SECRET_ID') 68 | if secret_id: 69 | # Get or create the session encrypt and validate secrets. 70 | session = boto3.session.Session() 71 | client = session.client('secretsmanager') 72 | 73 | try: 74 | response = client.get_secret_value(SecretId=secret_id) 75 | env_overlay = json.loads(response['SecretString']) 76 | except client.exceptions.ResourceNotFoundException: 77 | env_overlay = { 78 | 'SESSION_ENCRYPT_KEY': generate_secret(), 79 | 'SESSION_VALIDATE_KEY': generate_secret(), 80 | } 81 | client.put_secret_value( 82 | SecretId=secret_id, 83 | SecretString=json.dumps(env_overlay), 84 | ) 85 | 86 | env = {} 87 | env.update(os.environ) 88 | env.update({ 89 | 'SESSION_ENCRYPT_KEY': env_overlay['SESSION_ENCRYPT_KEY'], 90 | 'SESSION_VALIDATE_KEY': env_overlay['SESSION_VALIDATE_KEY'], 91 | }) 92 | ini_content = ini_template.format(**env) 93 | 94 | fn = '/tmp/server.ini' 95 | with open(fn, 'w') as f: 96 | f.write(ini_content) 97 | 98 | return fn 99 | 100 | 101 | config_fn = get_config_fn() 102 | app = loadapp(f'config:{config_fn}') 103 | lambda_handler = make_lambda_handler(app) 104 | -------------------------------------------------------------------------------- /tf/.gitignore: -------------------------------------------------------------------------------- 1 | .terraform 2 | remote-state.tf 3 | *.auto.tfvars 4 | -------------------------------------------------------------------------------- /tf/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/aws" { 5 | version = "3.22.0" 6 | hashes = [ 7 | "h1:8aWXjFcmEi64P0TMHOCQXWws+/SmvJQrNvHlzdktKOM=", 8 | "zh:4a9a66caf1964cdd3b61fb3ebb0da417195a5529cb8e496f266b0778335d11c8", 9 | "zh:514f2f006ae68db715d86781673faf9483292deab235c7402ff306e0e92ea11a", 10 | "zh:5277b61109fddb9011728f6650ef01a639a0590aeffe34ed7de7ba10d0c31803", 11 | "zh:67784dc8c8375ab37103eea1258c3334ee92be6de033c2b37e3a2a65d0005142", 12 | "zh:76d4c8be2ca4a3294fb51fb58de1fe03361d3bc403820270cc8e71a04c5fa806", 13 | "zh:8f90b1cfdcf6e8fb1a9d0382ecaa5056a3a84c94e313fbf9e92c89de271cdede", 14 | "zh:d0ac346519d0df124df89be2d803eb53f373434890f6ee3fb37112802f9eac59", 15 | "zh:d6256feedada82cbfb3b1dd6dd9ad02048f23120ab50e6146a541cb11a108cc1", 16 | "zh:db2fe0d2e77c02e9a74e1ed694aa352295a50283f9a1cf896e5be252af14e9f4", 17 | "zh:eda61e889b579bd90046939a5b40cf5dc9031fb5a819fc3e4667a78bd432bdb2", 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /tf/apigw.tf: -------------------------------------------------------------------------------- 1 | resource "aws_apigatewayv2_api" "pypicloud" { 2 | name = "pypicloud" 3 | protocol_type = "HTTP" 4 | } 5 | 6 | resource "aws_apigatewayv2_integration" "pypicloud" { 7 | api_id = aws_apigatewayv2_api.pypicloud.id 8 | integration_type = "AWS_PROXY" 9 | 10 | connection_type = "INTERNET" 11 | // content_handling_strategy = "CONVERT_TO_TEXT" 12 | // description = "Lambda example" 13 | integration_method = "POST" 14 | integration_uri = aws_lambda_function.pypicloud.arn 15 | passthrough_behavior = "WHEN_NO_MATCH" 16 | payload_format_version = "2.0" 17 | } 18 | 19 | resource "aws_apigatewayv2_route" "pypicloud" { 20 | api_id = aws_apigatewayv2_api.pypicloud.id 21 | route_key = "$default" 22 | target = "integrations/${aws_apigatewayv2_integration.pypicloud.id}" 23 | } 24 | 25 | resource "aws_apigatewayv2_stage" "default" { 26 | api_id = aws_apigatewayv2_api.pypicloud.id 27 | name = "$default" 28 | auto_deploy = true 29 | 30 | access_log_settings { 31 | destination_arn = aws_cloudwatch_log_group.pypicloud.arn 32 | format = "$context.identity.sourceIp - - [$context.requestTime] \"$context.httpMethod $context.routeKey $context.protocol\" $context.status $context.responseLength $context.requestId $context.integrationErrorMessage" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /tf/dynamo.tf: -------------------------------------------------------------------------------- 1 | resource "aws_dynamodb_table" "DynamoPackage" { 2 | name = "DynamoPackage" 3 | hash_key = "filename" 4 | read_capacity = 5 5 | write_capacity = 5 6 | 7 | attribute { 8 | name = "filename" 9 | type = "S" 10 | } 11 | 12 | attribute { 13 | name = "name" 14 | type = "S" 15 | } 16 | 17 | global_secondary_index { 18 | hash_key = "name" 19 | name = "name-index" 20 | non_key_attributes = [] 21 | projection_type = "ALL" 22 | read_capacity = 5 23 | write_capacity = 5 24 | } 25 | } 26 | 27 | resource "aws_dynamodb_table" "PackageSummary" { 28 | name = "PackageSummary" 29 | hash_key = "name" 30 | read_capacity = 5 31 | write_capacity = 5 32 | 33 | attribute { 34 | name = "name" 35 | type = "S" 36 | } 37 | } 38 | 39 | resource "aws_iam_policy" "pypicloud_dynamo" { 40 | name = "pypicloud_dynamo" 41 | policy = <