├── MANIFEST.in ├── torchlambda ├── _version.py ├── arguments │ ├── __init__.py │ ├── parser.py │ └── subparsers.py ├── implementation │ ├── utils │ │ ├── __init__.py │ │ └── template │ │ │ ├── __init__.py │ │ │ ├── macro.py │ │ │ ├── validator.py │ │ │ ├── header.py │ │ │ └── imputation.py │ ├── __init__.py │ ├── layer.py │ ├── build.py │ ├── general.py │ ├── docker.py │ └── template.py ├── __init__.py ├── subcommands │ ├── __init__.py │ ├── build.py │ ├── template.py │ ├── settings.py │ └── layer.py ├── .dockerignore ├── main.py ├── templates │ ├── settings │ │ ├── torchlambda.yaml │ │ └── main.cpp │ └── custom │ │ └── main.cpp ├── build.sh ├── dependencies │ ├── aws-lambda.sh │ ├── aws-sdk.sh │ └── torch.sh ├── Dockerfile └── CMakeLists.txt ├── assets ├── banner.png └── logos │ └── torchlambda.png ├── scripts ├── ci │ ├── dependencies.sh │ └── build.sh ├── release │ └── update_version.sh └── analysis │ ├── header.md │ └── performance.py ├── tests └── settings │ └── automated │ ├── src │ ├── utils.py │ ├── process.py │ ├── model.py │ ├── payload.py │ ├── validate.py │ ├── gather.py │ └── setup.py │ └── run.sh ├── LICENSE ├── setup.py ├── .gitignore ├── .github └── workflows │ └── update.yml ├── CODE_OF_CONDUCT.md ├── README.md └── BENCHMARKS.md /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include torchlambda * 2 | -------------------------------------------------------------------------------- /torchlambda/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "NON-CI" 2 | -------------------------------------------------------------------------------- /torchlambda/arguments/__init__.py: -------------------------------------------------------------------------------- 1 | from . import parser 2 | -------------------------------------------------------------------------------- /torchlambda/implementation/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from . import template 2 | -------------------------------------------------------------------------------- /torchlambda/__init__.py: -------------------------------------------------------------------------------- 1 | from . import arguments, implementation, subcommands 2 | -------------------------------------------------------------------------------- /torchlambda/subcommands/__init__.py: -------------------------------------------------------------------------------- 1 | from . import build, layer, settings, template 2 | -------------------------------------------------------------------------------- /assets/banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szymonmaszke/torchlambda/HEAD/assets/banner.png -------------------------------------------------------------------------------- /torchlambda/implementation/__init__.py: -------------------------------------------------------------------------------- 1 | from . import build, docker, general, layer, template 2 | -------------------------------------------------------------------------------- /torchlambda/.dockerignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | templates 3 | arguments 4 | commands 5 | utils 6 | 7 | *.py 8 | -------------------------------------------------------------------------------- /torchlambda/implementation/utils/template/__init__.py: -------------------------------------------------------------------------------- 1 | from . import header, imputation, macro, validator 2 | -------------------------------------------------------------------------------- /assets/logos/torchlambda.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szymonmaszke/torchlambda/HEAD/assets/logos/torchlambda.png -------------------------------------------------------------------------------- /scripts/ci/dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | pip install -e . && pip install numpy torch torchvision 4 | docker pull lambci/lambda:provided 5 | -------------------------------------------------------------------------------- /scripts/ci/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | TORCH_VERSION=${1:-"latest"} 4 | torchlambda build . --no-run --pytorch-version "$TORCH_VERSION" --image "szymonmaszke/torchlambda:$TORCH_VERSION" 5 | -------------------------------------------------------------------------------- /torchlambda/subcommands/build.py: -------------------------------------------------------------------------------- 1 | from .. import implementation 2 | 3 | 4 | def run(args): 5 | """Entrypoint for `torchlambda build` subcommand""" 6 | implementation.docker.check() 7 | 8 | with implementation.general.message("deployment."): 9 | image = implementation.build.get_image(args) 10 | if not args.no_run: 11 | implementation.build.get_package(args, image) 12 | -------------------------------------------------------------------------------- /scripts/release/update_version.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | repository="$(basename "$(git rev-parse --show-toplevel)")" 6 | # Version equal to current timestamp 7 | echo "Current working directory: $(pwd)" 8 | echo "OLD VERSION CHECK:" 9 | cat "$repository"/_version.py 10 | 11 | echo "__version__ = \"$(date +%s)\"" >"$repository"/_version.py 12 | 13 | echo "NEW VERSION:" 14 | cat "$repository"/_version.py 15 | -------------------------------------------------------------------------------- /tests/settings/automated/src/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pathlib 3 | import sys 4 | import typing 5 | 6 | import yaml 7 | 8 | import torchlambda 9 | 10 | 11 | def load_settings() -> typing.Dict: 12 | with open(pathlib.Path(os.environ["SETTINGS"]).absolute(), "r") as file: 13 | try: 14 | return torchlambda.implementation.utils.template.validator.get().normalized( 15 | yaml.safe_load(file) 16 | ) 17 | except yaml.YAMLError as error: 18 | print("Test error:: Error during user settings loading.") 19 | print(error) 20 | sys.exit(1) 21 | -------------------------------------------------------------------------------- /torchlambda/main.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | 3 | from . import arguments, subcommands 4 | from ._version import __version__ 5 | 6 | 7 | def main() -> None: 8 | """Entrypoint run after torchlambda command. 9 | 10 | Responsibilities for functionalities are fully transferred to specific 11 | subcommand. 12 | 13 | See subcommands module for details. 14 | 15 | """ 16 | 17 | args = arguments.parser.get() 18 | module = importlib.import_module( 19 | ".subcommands." + args.subcommand, package=__package__ 20 | ) 21 | module.run(args) 22 | 23 | 24 | if __name__ == "__main__": 25 | main() 26 | -------------------------------------------------------------------------------- /tests/settings/automated/src/process.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | 4 | import yaml 5 | 6 | import utils 7 | 8 | 9 | def process(settings): 10 | pattern = re.compile("Duration: (\d+\.?\d*)") 11 | with open(os.environ["DATA"], "r") as f: 12 | init, duration, billed = [float(value) for value in pattern.findall(f.read())] 13 | settings["init"] = init 14 | settings["duration"] = duration 15 | settings["billed"] = billed 16 | with open(os.environ["SETTINGS"], "w") as file: 17 | yaml.dump(settings, file, default_flow_style=False) 18 | 19 | 20 | if __name__ == "__main__": 21 | process(utils.load_settings()) 22 | -------------------------------------------------------------------------------- /torchlambda/subcommands/template.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | 3 | from .. import implementation 4 | 5 | 6 | def run(args): 7 | """Entrypoint for `torchlambda template` subcommand""" 8 | with implementation.general.message( 9 | "creating C++ scheme at {}.".format(args.destination) 10 | ): 11 | if args.yaml is None: 12 | implementation.general.run( 13 | "cp -r ./templates/custom {}".format( 14 | pathlib.Path(args.destination).absolute() 15 | ), 16 | operation="copying CPP sources", 17 | silent=args.silent, 18 | ) 19 | else: 20 | implementation.template.create_template(args) 21 | -------------------------------------------------------------------------------- /torchlambda/templates/settings/torchlambda.yaml: -------------------------------------------------------------------------------- 1 | # Reference file for fields: 2 | # https://github.com/szymonmaszke/torchlambda/wiki/YAML-settings-file-reference 3 | --- 4 | grad: false 5 | optimize: false 6 | validate_json: true 7 | model: /opt/model.ptc 8 | input: 9 | name: data 10 | type: base64 11 | validate: true 12 | shape: [1, 3, width, height] 13 | validate_shape: true 14 | cast: float 15 | divide: 255 16 | normalize: 17 | means: [0.485, 0.456, 0.406] 18 | stddevs: [0.229, 0.224, 0.225] 19 | return: 20 | output: 21 | type: double 22 | name: output 23 | item: false 24 | result: 25 | type: int 26 | name: result 27 | operations: argmax 28 | arguments: 1 29 | item: true 30 | -------------------------------------------------------------------------------- /torchlambda/subcommands/settings.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | 3 | from .. import implementation 4 | 5 | 6 | def run(args): 7 | """Entrypoint for `torchlambda settings` subcommand""" 8 | destination = pathlib.Path(args.destination).absolute() 9 | with implementation.general.message( 10 | "creating YAML settings at {}.".format(destination) 11 | ): 12 | template_path = ( 13 | pathlib.Path(__file__).resolve().parent.parent 14 | / "templates/settings/torchlambda.yaml" 15 | ) 16 | implementation.general.run( 17 | "cp -r {} {}".format(template_path, destination), 18 | operation="copying YAML source code", 19 | silent=args.silent, 20 | ) 21 | -------------------------------------------------------------------------------- /tests/settings/automated/src/model.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | 4 | import torch 5 | import torchvision 6 | import yaml 7 | 8 | import utils 9 | 10 | 11 | def create_model(): 12 | possible_models = [ 13 | "shufflenet_v2_x1_0", 14 | "resnet18", 15 | "mobilenet_v2", 16 | "mnasnet1_0", 17 | "mnasnet1_3", 18 | ] 19 | 20 | settings = utils.load_settings() 21 | model_name = random.choice(possible_models) 22 | print("Test:: Model: {}".format(model_name)) 23 | model = getattr(torchvision.models, model_name)() 24 | 25 | script_model = torch.jit.script(model) 26 | script_model.save(os.environ["MODEL"]) 27 | 28 | settings["model_name"] = model_name 29 | with open(os.environ["SETTINGS"], "w") as file: 30 | yaml.dump(settings, file, default_flow_style=False) 31 | 32 | 33 | if __name__ == "__main__": 34 | create_model() 35 | -------------------------------------------------------------------------------- /torchlambda/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copy necessary dependencies from build 4 | 5 | AWS_COMPONENTS="$1" 6 | COMPILATION="$2" 7 | 8 | CMAKE_ARGS=() 9 | CMAKE_ARGS+=("-DBUILD_SHARED_LIBS=OFF") 10 | CMAKE_ARGS+=("-DAWS_COMPONENTS=${AWS_COMPONENTS}") 11 | 12 | echo "torchlambda:: Building AWS Lambda .zip package." 13 | echo "torchlambda:: Compilation flags: ${COMPILATION}" 14 | echo "torchlambda:: Final build arguments: ${CMAKE_ARGS[*]}" 15 | 16 | mkdir build && 17 | cd build && 18 | cmake3 -E env CXXFLAGS="${COMPILATION}" cmake3 "${CMAKE_ARGS[@]}" .. && 19 | cmake3 --build . --config Release && 20 | make aws-lambda-package-torchlambda 21 | 22 | printf "\ntorchlambda:: App size:\n\n" 23 | du -sh /usr/local/build/torchlambda 24 | 25 | printf "\ntorchlambda:: Zipped app size:\n\n" 26 | du -sh /usr/local/build/torchlambda.zip 27 | printf "\ntorchlambda:: Deployment finished successfully." 28 | -------------------------------------------------------------------------------- /torchlambda/arguments/parser.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from .subparsers import build, layer, settings, template 4 | 5 | 6 | def get(): 7 | """Get user provided arguments.""" 8 | parser = argparse.ArgumentParser( 9 | formatter_class=argparse.RawTextHelpFormatter, 10 | description="Lightweight tool to deploy PyTorch models on AWS Lambda.\n" 11 | "For more information see documentation: https://github.com/szymonmaszke/torchlambda/wiki", 12 | ) 13 | 14 | parser.add_argument( 15 | "--silent", 16 | required=False, 17 | action="store_true", 18 | help="Will only output torchlambda information (e.g. building PyTorch output won't be displayed).\n" 19 | "Default: False", 20 | ) 21 | 22 | subparsers = parser.add_subparsers(help="Subcommands:", dest="subcommand") 23 | settings(subparsers) 24 | template(subparsers) 25 | build(subparsers) 26 | layer(subparsers) 27 | 28 | return parser.parse_args() 29 | -------------------------------------------------------------------------------- /torchlambda/subcommands/layer.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import zipfile 3 | 4 | from .. import implementation 5 | 6 | 7 | def run(args): 8 | """Entrypoint for `torchlambda layer` subcommand""" 9 | destination = pathlib.Path(args.destination).absolute() 10 | with implementation.general.message( 11 | "packing Torchscript model to {}.".format(destination) 12 | ): 13 | with zipfile.ZipFile( 14 | destination, 15 | "w", 16 | compression=getattr(zipfile, "ZIP_{}".format(args.compression)), 17 | compresslevel=implementation.layer.compression_level( 18 | args.compression, args.compression_level 19 | ), 20 | ) as file: 21 | implementation.layer.validate(args) 22 | file.write( 23 | pathlib.Path(args.source), 24 | implementation.layer.path(args) 25 | if args.directory is not None 26 | else args.source, 27 | ) 28 | -------------------------------------------------------------------------------- /torchlambda/dependencies/aws-lambda.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Original source: 4 | # https://github.com/pytorch/pytorch/blob/master/scripts/build_mobile.sh#L63 5 | 6 | if [ -z "$MAX_JOBS" ]; then 7 | if [ "$(uname)" == 'Darwin' ]; then 8 | MAX_JOBS=$(sysctl -n hw.ncpu) 9 | else 10 | MAX_JOBS=$(nproc) 11 | fi 12 | else 13 | MAX_JOBS=1 14 | fi 15 | 16 | CMAKE_ARGS=() 17 | 18 | if [ -x "$(command -v ninja)" ]; then 19 | CMAKE_ARGS+=("-GNinja") 20 | fi 21 | 22 | CMAKE_ARGS+=("-DBUILD_SHARED_LIBS=OFF") 23 | 24 | echo "torchlambda:: AWS Lambda C++ Runtime build arguments:" 25 | echo "${CMAKE_ARGS[@]}" 26 | 27 | echo "torchlambda:: Cloning and building AWS Lambda C++ Runtime..." 28 | git clone https://github.com/awslabs/aws-lambda-cpp.git && 29 | cd aws-lambda-cpp && 30 | mkdir -p build && 31 | cd build && 32 | cmake3 .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local "${CMAKE_ARGS[@]}" && 33 | cmake3 --build . --target install -- "-j${MAX_JOBS}" 34 | 35 | echo "torchlambda:: AWS Lambda C++ Runtime built successfully." 36 | -------------------------------------------------------------------------------- /torchlambda/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazonlinux:latest AS builder 2 | 3 | ARG AWS=" " 4 | ARG PYTORCH=" " 5 | ARG PYTORCH_VERSION="latest" 6 | 7 | RUN yum -y group install "Development Tools" && \ 8 | yum -y install unzip git wget rh-python37 ninja-build curl-devel \ 9 | libcurl-devel libuuid-devel openssl-devel && \ 10 | pip3 install --no-cache-dir pyyaml==5.3 && \ 11 | pip3 install cmake && \ 12 | ln -s /usr/bin/ninja-build /usr/bin/ninja 13 | 14 | WORKDIR /home/app 15 | COPY . /home/app 16 | 17 | RUN cd dependencies && \ 18 | ./aws-lambda.sh && \ 19 | ./aws-sdk.sh ${AWS} && \ 20 | ./torch.sh ${PYTORCH_VERSION} ${PYTORCH} && \ 21 | cp -r pytorch/build_mobile/install/* /usr/local/ && \ 22 | cp ../CMakeLists.txt ../build.sh /usr/local/ 23 | 24 | # Final image with copied install dependencies 25 | FROM amazonlinux:latest 26 | COPY --from=builder /usr/local /usr/local 27 | 28 | RUN yum -y install libcurl-devel libuuid-devel openssl-devel gcc-c++ make cmake3 zip 29 | 30 | LABEL maintainer="szymon.maszke@protonmail.com" 31 | 32 | WORKDIR /usr/local 33 | 34 | ENTRYPOINT ["./build.sh"] 35 | CMD [] 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Szymon Maszke 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /scripts/analysis/header.md: -------------------------------------------------------------------------------- 1 | # Performance matrix 2 | 3 | __You shouldn't base your settings on information provided in this section. Perform your 4 | own measurements, this section acts more as a rule of thumb__. 5 | 6 | 7 | Exact number of tests (on which this comparison is based) can be seen as the footer of 8 | this page 9 | Below matrices show latest performance measurements gathered from automated tests from last 10 | week. 11 | 12 | __Each entry has the following format:__ 13 | 14 | ``` 15 | DurationType: Trait1 x Trait2 16 | ``` 17 | 18 | - __`DurationType`__: 19 | - `init` - how long it took to initialize Lambda function 20 | - `duration` - how long the request took 21 | - `billed` - same as `duration but rounded` to the next `100ms`. How much you would 22 | be charged for running this function. See how it translates to money on 23 | [AWS Lambda pricing](https://aws.amazon.com/lambda/pricing/) documentation 24 | - __`Trait1`__ - setting value by which analysis was taken. For `grad` it could 25 | be `True` and `False`. It could be `model_name` (one of five tested `torchvision` neural networks). 26 | - __`Trait2`__ - same as `Trait1` but different axis 27 | 28 | _See concrete comparisons below_ 29 | -------------------------------------------------------------------------------- /torchlambda/dependencies/aws-sdk.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Original source: 4 | # https://github.com/pytorch/pytorch/blob/master/scripts/build_mobile.sh#L63 5 | 6 | if [ -z "$MAX_JOBS" ]; then 7 | if [ "$(uname)" == 'Darwin' ]; then 8 | MAX_JOBS=$(sysctl -n hw.ncpu) 9 | else 10 | MAX_JOBS=$(nproc) 11 | fi 12 | else 13 | MAX_JOBS=1 14 | fi 15 | 16 | # Build arguments and custom user defined args 17 | 18 | CMAKE_ARGS=() 19 | 20 | CMAKE_ARGS+=("-DENABLE_TESTING=OFF") 21 | CMAKE_ARGS+=("-DBUILD_ONLY=core") 22 | 23 | CMAKE_ARGS+=("-DENABLE_TESTING=OFF") 24 | 25 | CMAKE_ARGS+=("-DMINIMIZE_SIZE=ON") 26 | 27 | CMAKE_ARGS+=("-DCUSTOM_MEMORY_MANAGEMENT=OFF") 28 | CMAKE_ARGS+=("-DCPP_STANDARD=17") 29 | 30 | CMAKE_ARGS+=("$@") 31 | 32 | if [ -x "$(command -v ninja)" ]; then 33 | CMAKE_ARGS+=("-GNinja") 34 | fi 35 | 36 | CMAKE_ARGS+=("-DBUILD_SHARED_LIBS=OFF") 37 | 38 | echo "torchlambda:: AWS C++ SDK build arguments:" 39 | echo "${CMAKE_ARGS[@]}" 40 | 41 | echo "torchlambda:: Cloning and building AWS C++ SDK..." 42 | git clone https://github.com/aws/aws-sdk-cpp.git && 43 | cd aws-sdk-cpp && 44 | mkdir -p build && 45 | cd build && 46 | cmake3 .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local "${CMAKE_ARGS[@]}" && 47 | cmake3 --build . --target install -- "-j${MAX_JOBS}" 48 | 49 | echo "torchlambda:: AWS C++ SDK built successfully." 50 | -------------------------------------------------------------------------------- /torchlambda/dependencies/torch.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | TORCH_VERSION=$1 4 | shift 5 | 6 | OP_LIST="/home/app/model.yaml" 7 | 8 | # Build args targeting AWS Lambda capabilities 9 | BUILD_ARGS=() 10 | BUILD_ARGS+=("-DBUILD_PYTHON=OFF") 11 | 12 | BUILD_ARGS+=("-DUSE_MPI=OFF") 13 | BUILD_ARGS+=("-DUSE_NUMPY=OFF") 14 | BUILD_ARGS+=("-DUSE_ROCM=OFF") 15 | BUILD_ARGS+=("-DUSE_NCCL=OFF") 16 | BUILD_ARGS+=("-DUSE_NUMA=OFF") 17 | BUILD_ARGS+=("-DUSE_MKLDNN=ON") 18 | BUILD_ARGS+=("-DUSE_GLOO=OFF") 19 | BUILD_ARGS+=("-DUSE_OPENMP=OFF") 20 | 21 | BUILD_ARGS+=("$@") 22 | 23 | # Display gathered arguments 24 | printf "\n\ntorchlambda:: Libtorch build arguments:\n\n" 25 | echo "${BUILD_ARGS[@]}" 26 | 27 | # Export selected operations if provided 28 | if [ -f "${OP_LIST}" ]; then 29 | export SELECTED_OP_LIST="${OP_LIST}" 30 | printf "torchlambda:: Building with following customized operations:\n\n" 31 | cat "$OP_LIST" 32 | fi 33 | 34 | echo "torchlambda:: Cloning and building Libtorch..." 35 | git clone --recursive https://github.com/pytorch/pytorch.git 36 | 37 | if [ "$TORCH_VERSION" != "latest" ]; then 38 | echo "torchlambda:: Resetting PyTorch to commit/tag: $TORCH_VERSION" 39 | cd pytorch && git reset --hard "${TORCH_VERSION}" && cd - || exit 1 40 | else 41 | echo "torchlambda:: PyTorch master head on GitHub will be used" 42 | fi 43 | 44 | # We are using cmake3 and python3 so replace all occurrences of it in the script 45 | # Only python with space as it's used in two more places. No difference for cmake, consistency's sake 46 | sed -i 's/cmake\ /cmake3\ /g' /home/app/dependencies/pytorch/scripts/build_mobile.sh 47 | sed -i 's/python\ /python3\ /g' /home/app/dependencies/pytorch/scripts/build_mobile.sh 48 | 49 | ./pytorch/scripts/build_mobile.sh "${BUILD_ARGS[@]}" 50 | 51 | echo "torchlambda:: Libtorch built successfully." 52 | -------------------------------------------------------------------------------- /torchlambda/implementation/utils/template/macro.py: -------------------------------------------------------------------------------- 1 | import typing 2 | 3 | 4 | def define(name: str, value: str = "") -> str: 5 | """ 6 | Return C++ macro-string #define name value 7 | 8 | Used for header defines. 9 | 10 | Parameters 11 | ---------- 12 | name : str 13 | Name of macro to be defined 14 | value : str, optional 15 | Value of defined macro (if any), 16 | 17 | Returns 18 | ------- 19 | str: 20 | "#define name" or "#define name value" 21 | """ 22 | return "#define {} {}".format(name, value) 23 | 24 | 25 | def key(dictionary, key: str, mapping: typing.Dict = None) -> str: 26 | """ 27 | Return C++ macro-string "#define name value" based on provided key. 28 | 29 | Used for header defines based on user-provided setting. 30 | 31 | Parameters 32 | ---------- 33 | name : str 34 | Name of macro to be defined 35 | value : str, optional 36 | Value of defined macro (if any), 37 | 38 | Returns 39 | ------- 40 | str: 41 | "#define name value" or "#define name value" 42 | """ 43 | value = dictionary.get(key, None) 44 | if value is not None: 45 | if mapping is not None: 46 | value = mapping[value] 47 | return define(key.upper(), value) 48 | return "" 49 | 50 | 51 | def conditional(condition: bool, name: str, value: typing.Any = "") -> str: 52 | """ 53 | Return C++ macro-string "#define name value" if condition is True. 54 | 55 | Used for header defines based on user-provided setting. 56 | 57 | Parameters 58 | ---------- 59 | name : str 60 | Name of macro to be defined 61 | value : str, optional 62 | Value of defined macro (if any), 63 | 64 | Returns 65 | ------- 66 | str: 67 | "#define name" or "#define name value" 68 | """ 69 | if condition: 70 | return define(name, value) 71 | return "" 72 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | 3 | import setuptools 4 | 5 | 6 | def read(filepath, variable): 7 | namespace = {} 8 | exec(open(filepath).read(), namespace) 9 | return namespace[variable] 10 | 11 | 12 | setuptools.setup( 13 | name="torchlambda", 14 | version=read("torchlambda/_version.py", "__version__"), 15 | license="MIT", 16 | author="Szymon Maszke", 17 | author_email="szymon.maszke@protonmail.com", 18 | description="Minimalistic & easy deployment of PyTorch models on AWS Lambda with C++", 19 | long_description=pathlib.Path("README.md").read_text(), 20 | long_description_content_type="text/markdown", 21 | url="https://github.com/pypa/torchlambda", 22 | packages=setuptools.find_packages(), 23 | include_package_data=True, 24 | python_requires=">=3.6", 25 | install_requires=["PyYAML>=5.3", "Cerberus>=1.3.2"], 26 | entry_points={"console_scripts": ["torchlambda=torchlambda.main:main"]}, 27 | classifiers=[ 28 | "Development Status :: 2 - Pre-Alpha", 29 | "Programming Language :: Python", 30 | "Programming Language :: Python :: 3", 31 | "Programming Language :: Python :: 3.6", 32 | "Programming Language :: Python :: 3.7", 33 | "Programming Language :: Python :: 3.8", 34 | "License :: OSI Approved :: MIT License", 35 | "Intended Audience :: Developers", 36 | "Operating System :: OS Independent", 37 | "Topic :: Scientific/Engineering", 38 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 39 | ], 40 | project_urls={ 41 | "Website": "https://szymonmaszke.github.io/torchlambda", 42 | "Documentation": "https://szymonmaszke.github.io/torchlambda/#torchlambda", 43 | "Issues": "https://github.com/szymonmaszke/torchlambda/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc", 44 | }, 45 | keywords="aws lambda pytorch deployment minimal c++ neural network model", 46 | ) 47 | -------------------------------------------------------------------------------- /torchlambda/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.5...3.16) 2 | 3 | if(${CMAKE_VERSION} VERSION_LESS 3.12) 4 | cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) 5 | endif() 6 | 7 | project(torchlambda VERSION 0.1.0 8 | DESCRIPTION "PyTorch AWS Lambda model inference deployment" 9 | LANGUAGES CXX 10 | ) 11 | 12 | set(AWS_COMPONENTS "core" CACHE STRING "AWS-SDK Components used by user") 13 | 14 | # All dependencies are in /usr/local already 15 | 16 | # Find torch package 17 | find_package(Torch REQUIRED) 18 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}") 19 | 20 | # Find AWS Lambda 21 | find_package(aws-lambda-runtime REQUIRED) 22 | 23 | # AWS_COMPONENTS is already set during image build 24 | find_package(AWSSDK REQUIRED COMPONENTS ${AWS_COMPONENTS}) 25 | 26 | # Add all C/C++ files 27 | file(GLOB_RECURSE DEPLOYMENT_SRC 28 | "${CMAKE_CURRENT_SOURCE_DIR}/user_code/*.hpp" 29 | "${CMAKE_CURRENT_SOURCE_DIR}/user_code/*.h" 30 | "${CMAKE_CURRENT_SOURCE_DIR}/user_code/*.cpp" 31 | "${CMAKE_CURRENT_SOURCE_DIR}/user_code/*.c" 32 | ) 33 | 34 | add_executable(${PROJECT_NAME} ${DEPLOYMENT_SRC}) 35 | 36 | set_target_properties(${PROJECT_NAME} PROPERTIES 37 | CXX_STANDARD 17 38 | CXX_STANDARD_REQUIRED YES 39 | CXX_EXTENSIONS NO 40 | ) 41 | 42 | include(CheckIPOSupported) 43 | check_ipo_supported(RESULT result) 44 | if(result) 45 | set_target_properties(${PROJECT_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION TRUE) 46 | endif() 47 | 48 | target_compile_options(${PROJECT_NAME} PRIVATE ${COMPILATION_OPTIONS}) 49 | 50 | # Linking and options (--whole-archive is a current workaround) 51 | target_link_libraries(${PROJECT_NAME} PRIVATE -lm 52 | AWS::aws-lambda-runtime 53 | ${AWSSDK_LINK_LIBRARIES} 54 | -Wl,--whole-archive "${TORCH_LIBRARIES}" 55 | -Wl,--no-whole-archive 56 | -lpthread 57 | ${CMAKE_DL_LIBS}) 58 | 59 | 60 | # This line creates a target that packages your binary and zips it up 61 | aws_lambda_package_target(${PROJECT_NAME}) 62 | -------------------------------------------------------------------------------- /torchlambda/implementation/layer.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import typing 3 | 4 | 5 | def validate(args) -> None: 6 | """ 7 | Validate torchscript model is a file. Exit with 1 otherwise 8 | 9 | Parameters 10 | ---------- 11 | args : dict-like 12 | User provided arguments parsed by argparse.ArgumentParser instance. 13 | 14 | """ 15 | source = pathlib.Path(args.source) 16 | if not source.is_file: 17 | print("torchlambda:: Error: provided path to torchscript model is not a file!") 18 | exit(1) 19 | 20 | 21 | def path(args) -> pathlib.Path: 22 | """ 23 | Return path where model will be placed upon `.zip` unpacking. 24 | 25 | Parameters 26 | ---------- 27 | args : dict-like 28 | User provided arguments parsed by argparse.ArgumentParser instance. 29 | 30 | Returns 31 | ------- 32 | pathlib.Path 33 | 34 | """ 35 | if args.directory is not None: 36 | return pathlib.Path(args.directory) / args.source 37 | return pathlib.Path(args.source) 38 | 39 | 40 | def compression_level(compression, level): 41 | """ 42 | Validate compression and compression's level user arguments. 43 | 44 | This function return appropriate compression level for any `zipfile.Zipfile` 45 | supported values. 46 | 47 | Check: https://docs.python.org/3/library/zipfile.html#zipfile-objects 48 | to see possible values for each type. 49 | 50 | Parameters 51 | ---------- 52 | args : dict-like 53 | User provided arguments parsed by argparse.ArgumentParser instance. 54 | 55 | Returns 56 | ------- 57 | pathlib.Path 58 | 59 | """ 60 | 61 | def _wrong_parameters(minimum, maximum): 62 | print( 63 | "--level should be in range [{}, {}] for compression type {}".format( 64 | minimum, maximum, level 65 | ) 66 | ) 67 | exit(1) 68 | 69 | if compression == "DEFLATED": 70 | if not 0 <= level <= 9: 71 | _wrong_parameters(0, 9) 72 | return level 73 | if compression == "BZIP2": 74 | if not 1 <= level <= 9: 75 | _wrong_parameters(1, 9) 76 | return level 77 | 78 | return level 79 | -------------------------------------------------------------------------------- /tests/settings/automated/src/payload.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import json 3 | import os 4 | import pathlib 5 | import random 6 | import struct 7 | import typing 8 | 9 | import numpy as np 10 | import yaml 11 | 12 | import utils 13 | 14 | 15 | def create_payload(data, batch, channels, width, height) -> pathlib.Path: 16 | request = { 17 | "batch": batch, 18 | "channels": channels, 19 | "width": width, 20 | "height": height, 21 | "data": data, 22 | } 23 | 24 | payload_file = pathlib.Path(os.environ["OUTPUT"]).absolute() 25 | with open(payload_file, "w") as file: 26 | json.dump(request, file) 27 | return payload_file 28 | 29 | 30 | def create_data( 31 | settings, 32 | type_mapping: typing.Dict = { 33 | "byte": np.uint8, 34 | "char": np.int8, 35 | "short": np.int16, 36 | "int": np.int32, 37 | "long": np.int64, 38 | "float": np.float32, 39 | "double": np.float64, 40 | }, 41 | ): 42 | def _get_shape(settings): 43 | return [1, 3] + random.choice( 44 | [[128, 128], [256, 256], [512, 512], [1024, 1024]] 45 | ) 46 | 47 | batch, channels, width, height = _get_shape(settings) 48 | settings["payload"] = [batch, channels, width, height] 49 | print( 50 | "Test:: Request shape: [{}, {}, {}, {}]".format(batch, channels, width, height) 51 | ) 52 | data = np.random.randint( 53 | low=0, high=255, size=(batch, channels, width, height) 54 | ).flatten() 55 | if settings["input"]["type"] == "base64": 56 | data = base64.b64encode( 57 | struct.pack("<{}B".format(len(data)), *(data.tolist())) 58 | ).decode() 59 | else: 60 | data = data.astype(type_mapping[settings["input"]["type"]]).tolist() 61 | with open(os.environ["SETTINGS"], "w") as file: 62 | yaml.dump(settings, file, default_flow_style=False) 63 | return data, (batch, channels, width, height) 64 | 65 | 66 | if __name__ == "__main__": 67 | print("Test:: Creating payload") 68 | settings = utils.load_settings() 69 | data, (batch, channels, width, height) = create_data(settings) 70 | create_payload(data, batch, channels, width, height) 71 | -------------------------------------------------------------------------------- /tests/settings/automated/src/validate.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import sys 4 | 5 | import torchlambda 6 | import utils 7 | 8 | 9 | def load_response(): 10 | try: 11 | with open(os.environ["RESPONSE"], "r") as file: 12 | return json.load(file) 13 | except json.JSONDecodeError as e: 14 | print("Test:: Failed during error response loading! Error: \n {}".format(e)) 15 | sys.exit(1) 16 | 17 | 18 | def validate_response(response, settings): 19 | def _get_type(name: str): 20 | mapping = { 21 | "int": int, 22 | "long": int, 23 | "float": float, 24 | "double": float, 25 | } 26 | if settings["return"][name] is not None: 27 | return ( 28 | settings["return"][name]["item"], 29 | mapping[settings["return"][name]["type"]], 30 | ) 31 | return (None, None) 32 | 33 | def _get_value(name: str): 34 | if settings["return"][name] is None: 35 | return None 36 | return response[name] 37 | 38 | def _validate(name, value, is_item, value_type): 39 | def _check_type(item): 40 | if not isinstance(item, value_type): 41 | print("Test:: {}'s item is not of type {}".format(name, value_type)) 42 | sys.exit(1) 43 | 44 | if not any(value is None for value in (value, is_item, value_type)): 45 | print("Test:: Validating {} correctness...".format(name)) 46 | if is_item: 47 | _check_type(value) 48 | else: 49 | if not isinstance(value, list): 50 | print("Test:: {} is not a list!".format(name)) 51 | sys.exit(1) 52 | _check_type(value[0]) 53 | 54 | print("Test:: Getting desired output types to assert...") 55 | (output_is_item, output_type), (result_is_item, result_type) = ( 56 | _get_type("output"), 57 | _get_type("result"), 58 | ) 59 | 60 | output_value, result_value = _get_value("output"), _get_value("result") 61 | 62 | print("Test:: Validating response correctness...") 63 | _validate("output", output_value, output_is_item, output_type) 64 | _validate("result", result_value, result_is_item, result_type) 65 | 66 | 67 | if __name__ == "__main__": 68 | response = load_response() 69 | settings = utils.load_settings() 70 | validate_response(response, settings) 71 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore vim tags 2 | tags 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | pip-wheel-metadata/ 27 | share/python-wheels/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | MANIFEST 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .nox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *.cover 53 | *.py,cover 54 | .hypothesis/ 55 | .pytest_cache/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | db.sqlite3 65 | db.sqlite3-journal 66 | 67 | # Flask stuff: 68 | instance/ 69 | .webassets-cache 70 | 71 | # Scrapy stuff: 72 | .scrapy 73 | 74 | # Sphinx documentation 75 | docs/_build/ 76 | 77 | # PyBuilder 78 | target/ 79 | 80 | # Jupyter Notebook 81 | .ipynb_checkpoints 82 | 83 | # IPython 84 | profile_default/ 85 | ipython_config.py 86 | 87 | # pyenv 88 | .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 98 | __pypackages__/ 99 | 100 | # Celery stuff 101 | celerybeat-schedule 102 | celerybeat.pid 103 | 104 | # SageMath parsed files 105 | *.sage.py 106 | 107 | # Environments 108 | .env 109 | .venv 110 | env/ 111 | venv/ 112 | ENV/ 113 | env.bak/ 114 | venv.bak/ 115 | 116 | # Spyder project settings 117 | .spyderproject 118 | .spyproject 119 | 120 | # Rope project settings 121 | .ropeproject 122 | 123 | # mkdocs documentation 124 | /site 125 | 126 | # mypy 127 | .mypy_cache/ 128 | .dmypy.json 129 | dmypy.json 130 | 131 | # Pyre type checker 132 | .pyre/ 133 | -------------------------------------------------------------------------------- /tests/settings/automated/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e # Crash if anything returns non-zero code 4 | 5 | TORCH_VERSION=${1:-"latest"} 6 | 7 | # Global test run settings 8 | DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" 9 | 10 | # Where performance analysis will be saved 11 | DATA="$DIR/data" 12 | mkdir -p "$DATA" 13 | 14 | IMAGE="szymonmaszke/torchlambda:$TORCH_VERSION" 15 | 16 | TEST_CPP_SOURCE_FOLDER="$DIR/test_cpp_source_folder" 17 | TEST_PACKAGE="$DIR/deployment.zip" 18 | TEST_CODE="$DIR/test_code" 19 | MODEL="$DIR/model.ptc" 20 | 21 | RESPONSE="output.json" 22 | PAYLOAD="payload.json" 23 | 24 | # Run for each test case 25 | FINAL_DATA="analysis" 26 | 27 | SECS=7200 28 | ENDTIME=$(($(date +%s) + SECS)) 29 | START=$(date +%s) 30 | 31 | while [ $(date +%s) -lt $ENDTIME ]; do 32 | # Insert test case specific values into settings 33 | OUTPUT="$DATA/$i.yaml" python "$DIR"/src/setup.py 34 | echo "Test $i :: Tested settings:" 35 | cat "$DATA/$i.yaml" 36 | 37 | # # Use test settings to create C++ code template 38 | echo "Test $i :: Creating source code from settings" 39 | torchlambda template --yaml "$DATA/$i.yaml" --destination "$TEST_CPP_SOURCE_FOLDER" 40 | 41 | # # Build code template into deployment package 42 | echo "Test $i :: Building source code" 43 | torchlambda build "$TEST_CPP_SOURCE_FOLDER" --destination "$TEST_PACKAGE" --image "$IMAGE" 44 | unzip -qq "$TEST_PACKAGE" -d "$TEST_CODE" 45 | 46 | # # Create example model 47 | MODEL="$MODEL" SETTINGS="$DATA/$i.yaml" python "$DIR"/src/model.py 48 | echo "Test $i :: Model size:" 49 | du -sh "$MODEL" 50 | 51 | SETTINGS="$DATA/$i.yaml" OUTPUT="$PAYLOAD" python "$DIR"/src/payload.py 52 | echo "Test $i :: Payload size:" 53 | du -sh "$PAYLOAD" 54 | 55 | echo "Test $i :: Setting up server" 56 | 57 | docker run --rm -v \ 58 | "$TEST_CODE":/var/task:ro,delegated -v \ 59 | "$MODEL":/opt/model.ptc:ro,delegated \ 60 | -i -e DOCKER_LAMBDA_USE_STDIN=1 \ 61 | lambci/lambda:provided \ 62 | torchlambda <"$PAYLOAD" >"$RESPONSE" 2>"temp" 63 | 64 | # Remove colored output from lambci 65 | sed 's/\x1b\[[0-9;]*m//g' "temp" | tail -n 1 >"temp2" 66 | 67 | echo "Test $i :: Validating received response" 68 | SETTINGS="$DATA/$i.yaml" RESPONSE="$RESPONSE" python "$DIR"/src/validate.py 69 | 70 | echo "Test $i :: Adding time measurements" 71 | SETTINGS="$DATA/$i.yaml" DATA="temp2" python "$DIR"/src/process.py 72 | 73 | # Clean up 74 | echo "Test $i :: Cleaning up" 75 | rm -rf "temp" "temp2" "$TEST_CPP_SOURCE_FOLDER" "$TEST_PACKAGE" "$TEST_CODE" "$MODEL" "$PAYLOAD" "$RESPONSE" 76 | i=$((i + 1)) 77 | done 78 | 79 | echo "Test :: Creating statistics: $FINAL_DATA" 80 | OUTPUT="$FINAL_DATA" DATA="$DATA" python "$DIR"/src/gather.py 81 | rm -rf "$DATA" 82 | -------------------------------------------------------------------------------- /torchlambda/implementation/build.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import shutil 3 | import sys 4 | import uuid 5 | 6 | from . import docker, general 7 | 8 | 9 | def get_image(args) -> str: 10 | """ 11 | Get torchlambda deployment image. 12 | 13 | If image specified by --image exists locally it's name will be returned. 14 | Else if image should be build from source it is built and it's name is returned 15 | as specified by --image. 16 | Otherwise pre-built torchlambda:latest image will be used. 17 | 18 | Parameters 19 | ---------- 20 | args : dict-like 21 | User provided arguments parsed by argparse.ArgumentParser instance. 22 | 23 | Returns 24 | ------- 25 | str: 26 | Name of obtained image 27 | """ 28 | 29 | def _custom_build(args): 30 | """True if any of the flags specified as non-default.""" 31 | flags = ( 32 | "pytorch", 33 | "aws", 34 | "operations", 35 | "aws_components", 36 | "docker_build", 37 | "pytorch_version", 38 | ) 39 | # If any is not None or empty list 40 | return any(map(lambda flag: getattr(args, flag), flags)) 41 | 42 | image_exists: bool = docker.image_exists(args.image) 43 | if image_exists: 44 | print( 45 | "torchlambda:: Image {} was found locally and will be used.".format( 46 | args.image 47 | ) 48 | ) 49 | return args.image 50 | print("torchlambda:: Image {} was not found locally.".format(args.image)) 51 | if _custom_build(args): 52 | general.copy_operations(args) 53 | return docker.build(args) 54 | 55 | print("torchlambda:: Default szymonmaszke/torchlambda:latest image will be used.") 56 | return "szymonmaszke/torchlambda:latest" 57 | 58 | 59 | def get_package(args, image): 60 | """Generate deployment package by running provided image. 61 | 62 | Deployment package will be .zip file containing compiled source code 63 | and statically linked AWS and Libtorch (with optional user-specified flags). 64 | 65 | Parameters 66 | ---------- 67 | args : dict-like 68 | User provided arguments parsed by argparse.ArgumentParser instance. 69 | image : str 70 | Name of image used to create container. 71 | """ 72 | container: str = docker.run(args, image) 73 | with docker.rm(container): 74 | destination = pathlib.Path(args.destination).absolute() 75 | if not destination.is_file(): 76 | docker.cp( 77 | args, container, "/usr/local/build/torchlambda.zip", destination, 78 | ) 79 | else: 80 | print( 81 | "torchlambda:: Error: path {} exists. Please run the script again with the same --image argument.".format( 82 | destination 83 | ) 84 | ) 85 | -------------------------------------------------------------------------------- /.github/workflows/update.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: update 3 | on: 4 | push: 5 | 6 | jobs: 7 | tests: 8 | name: ${{ matrix.os }}-py${{ matrix.python }}-torch${{ matrix.pytorch }} 9 | runs-on: ${{ matrix.os }} 10 | strategy: 11 | matrix: 12 | pytorch: 13 | - "v1.6.0" 14 | - "latest" 15 | python: 16 | - 3.6 17 | - 3.7 18 | - 3.8 19 | os: 20 | - ubuntu-latest 21 | - ubuntu-16.04 22 | steps: 23 | - uses: actions/checkout@v1 24 | - name: Set up Python ${{ matrix.python }} 25 | uses: actions/setup-python@v1 26 | with: 27 | python-version: ${{ matrix.python }} 28 | - name: Update torchlambda version 29 | run: ./scripts/release/update_version.sh 30 | - name: Install dependencies 31 | run: ./scripts/ci/dependencies.sh 32 | - name: Build docker image locally 33 | run: ./scripts/ci/build.sh ${{ matrix.pytorch }} 34 | - name: Perform tests 35 | run: ./tests/settings/automated/run.sh ${{ matrix.pytorch }} 36 | - name: Upload test results 37 | uses: actions/upload-artifact@v1 38 | with: 39 | name: ${{ matrix.os }}-py${{ matrix.python }}-torch${{ matrix.pytorch }}.npz 40 | path: analysis.npz 41 | 42 | docker: 43 | needs: tests 44 | name: Deployment image szymonmaszke/torchlambda:${{ matrix.pytorch}} 45 | runs-on: ubuntu-latest 46 | strategy: 47 | matrix: 48 | pytorch: 49 | - "v1.6.0" 50 | - "latest" 51 | steps: 52 | - uses: actions/checkout@v1 53 | - name: Set up Python 54 | uses: actions/setup-python@v1 55 | with: 56 | python-version: 3.7 57 | - name: Update torchlambda version 58 | run: ./scripts/release/update_version.sh 59 | - name: Install dependencies 60 | run: ./scripts/ci/dependencies.sh 61 | - name: Build docker image locally 62 | run: ./scripts/ci/build.sh ${{ matrix.pytorch }} 63 | - name: Login to Docker 64 | run: > 65 | docker login 66 | -u ${{ secrets.DOCKER_USERNAME }} 67 | -p ${{ secrets.DOCKER_PASSWORD }} 68 | - name: Deploy image szymonmaszke/torchlambda:${{ matrix.pytorch }} 69 | run: > 70 | docker push szymonmaszke/torchlambda:${{ matrix.pytorch }} 71 | 72 | pip: 73 | needs: tests 74 | name: Create and publish package to PyPI with current timestamp 75 | runs-on: ubuntu-latest 76 | steps: 77 | - uses: actions/checkout@master 78 | - name: Update torchlambda version 79 | run: ./scripts/release/update_version.sh 80 | - name: Set up Python 81 | uses: actions/setup-python@v1 82 | with: 83 | python-version: "3.7" 84 | - name: Install dependencies 85 | run: | 86 | python -m pip install --upgrade pip 87 | pip install setuptools wheel 88 | - name: Build package 89 | run: python setup.py sdist bdist_wheel 90 | - name: Publish package to PyPI 91 | uses: pypa/gh-action-pypi-publish@master 92 | with: 93 | password: ${{ secrets.PYPI_PASSWORD }} 94 | -------------------------------------------------------------------------------- /torchlambda/implementation/general.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import pathlib 3 | import shlex 4 | import shutil 5 | import subprocess 6 | import sys 7 | import typing 8 | 9 | 10 | def parse_none(*args): 11 | """Return tuple of arguments excluding the ones being None.""" 12 | return tuple(arg if arg is not None else "" for arg in args) 13 | 14 | 15 | @contextlib.contextmanager 16 | def message(operation: str): 17 | """ 18 | Verbose logging of operation to be performed. 19 | 20 | Used only by `utils.run_command` 21 | 22 | Parameters 23 | ---------- 24 | operation : str 25 | String describing operation to be run. 26 | 27 | """ 28 | print("torchlambda:: Started {}".format(operation), file=sys.stderr) 29 | yield 30 | print("torchlambda:: Finished {}".format(operation), file=sys.stderr) 31 | 32 | 33 | def run( 34 | command: str, 35 | operation: str, 36 | silent: bool, 37 | exit_on_failure: bool = True, 38 | no_stdout: bool = False, 39 | no_stderr: bool = False, 40 | ) -> int: 41 | """ 42 | Run specific cli command, parse and return it's results. 43 | 44 | Parameters 45 | ---------- 46 | command : str 47 | CLI Linux command to be run. 48 | operation : str 49 | Name of operation for verbose logging of output. 50 | exit_on_failure : bool, optional 51 | Whether program should exit with error if the command didn't return 0. 52 | Default: `True` 53 | 54 | Returns 55 | ------- 56 | int 57 | Value returned by command 58 | 59 | """ 60 | 61 | def _set_streams() -> typing.Dict: 62 | kwargs = {} 63 | if no_stdout or silent: 64 | kwargs["stdout"] = subprocess.DEVNULL 65 | if no_stderr or silent: 66 | kwargs["stderr"] = subprocess.DEVNULL 67 | return kwargs 68 | 69 | with message(operation): 70 | return_value = subprocess.call( 71 | shlex.split(command), 72 | cwd=pathlib.Path(__file__).absolute().parent.parent, 73 | **_set_streams() 74 | ) 75 | if return_value != 0 and exit_on_failure: 76 | print( 77 | "torchlambda:: Error: Failed during {}".format(operation), 78 | file=sys.stderr, 79 | ) 80 | exit(1) 81 | 82 | return return_value 83 | 84 | 85 | @message("copying provided model operations.") 86 | def copy_operations(args) -> None: 87 | """ 88 | Copy custom operations.yaml of PyTorch model if provided by user. 89 | 90 | Providing custom operations.yaml will trigger docker image build from scratch. 91 | 92 | Parameters 93 | ---------- 94 | args : dict-like 95 | User provided arguments parsed by argparse.ArgumentParser instance. 96 | 97 | """ 98 | if args.operations is not None: 99 | operations_path = pathlib.Path(args.operations).absolute() 100 | if operations_path.is_file(): 101 | print("torchlambda:: Copying custom model operations...") 102 | shutil.copyfile( 103 | pathlib.Path(args.operations).absolute(), 104 | pathlib.Path(__file__).absolute().parent.parent / "model.yaml", 105 | ) 106 | else: 107 | print("torchlambda:: Error, provided operations are not a file!") 108 | exit(1) 109 | -------------------------------------------------------------------------------- /tests/settings/automated/src/gather.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import os 3 | import pathlib 4 | import re 5 | import sys 6 | import typing 7 | 8 | import numpy as np 9 | import yaml 10 | 11 | import utils 12 | 13 | 14 | def load(result): 15 | with open(result, "r") as f: 16 | try: 17 | return yaml.safe_load(f) 18 | except yaml.YAMLError as error: 19 | print("Test error:: Error during {} loading.".format(result)) 20 | print(error) 21 | sys.exit(1) 22 | 23 | 24 | def gather(): 25 | # init, duration, billed 3 26 | # grad [True, False] 2 27 | # type [] 8 28 | # size [[256, 256], [256, 512], [512, 256], [512, 512], [64, 64], [128, 64], [64, 128], [128, 128]] 3 29 | # normalize [True, False] 30 | # return [output, result, output+result] 31 | mapping = { 32 | "grad": {True: 0, False: 1}, 33 | "type": { 34 | "base64": 0, 35 | "byte": 1, 36 | "char": 2, 37 | "short": 3, 38 | "int": 4, 39 | "long": 5, 40 | "float": 6, 41 | "double": 7, 42 | }, 43 | "payload": {(128, 128): 0, (256, 256): 1, (512, 512): 2, (1024, 1024): 3}, 44 | "model_name": { 45 | "shufflenet_v2_x1_0": 0, 46 | "resnet18": 1, 47 | "mobilenet_v2": 2, 48 | "mnasnet1_0": 3, 49 | "mnasnet1_3": 4, 50 | } 51 | # normalize: None, Smth 52 | # return: output, result, output+result, 53 | # duration, init, billed 54 | } 55 | 56 | def index(key: str, result, input_field: bool = False): 57 | def _return(): 58 | output_exists = "output" in result["return"] 59 | result_exists = "result" in result["return"] 60 | if output_exists and result_exists: 61 | return 0 62 | if output_exists: 63 | return 1 64 | return 2 65 | 66 | def _normalize(): 67 | return 0 if result["normalize"] is None else 1 68 | 69 | if key == "return": 70 | return _return() 71 | if key == "normalize": 72 | return _normalize() 73 | 74 | if input_field: 75 | return mapping[key][result["input"][key]] 76 | if key == "payload": 77 | _, _, width, height = result["payload"] 78 | return mapping[key][(width, height)] 79 | return mapping[key][result[key]] 80 | 81 | # + normalize, return, duration 82 | data = np.zeros([len(value) for value in mapping.values()] + [2, 3, 3]) 83 | occurrences = np.zeros_like(data).astype(int) 84 | for file in pathlib.Path(os.environ["DATA"]).iterdir(): 85 | result = load(file) 86 | grad, types, payload, model_name, normalize, output = [ 87 | index("grad", result), 88 | index("type", result, True), 89 | index("payload", result), 90 | index("model_name", result), 91 | index("normalize", result), 92 | index("return", result), 93 | ] 94 | data[grad, types, payload, model_name, normalize, output, ...] = [ 95 | result["init"], 96 | result["duration"], 97 | result["billed"], 98 | ] 99 | occurrences[grad, types, payload, model_name, normalize, output, ...] += 1 100 | 101 | np.savez(os.environ["OUTPUT"], data=data, occurrences=occurrences) 102 | 103 | 104 | if __name__ == "__main__": 105 | gather() 106 | -------------------------------------------------------------------------------- /scripts/analysis/performance.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import itertools 3 | import pathlib 4 | from datetime import datetime 5 | 6 | import numpy as np 7 | import pandas as pd 8 | 9 | 10 | def parse(): 11 | parser = argparse.ArgumentParser() 12 | 13 | parser.add_argument( 14 | "directory", help="Directory containing performance file created by analysis", 15 | ) 16 | 17 | parser.add_argument( 18 | "output", help="Markdown file to output performance analysis combinations.", 19 | ) 20 | 21 | return parser.parse_args() 22 | 23 | 24 | def duration_mapping(name: str): 25 | return {"init": 0, "duration": 1, "billed": 2}[name] 26 | 27 | 28 | def matrix_mapping(name: str): 29 | mapping = { 30 | key: index 31 | for index, key in enumerate( 32 | ("grad", "type", "payload", "model_name", "normalize", "return") 33 | ) 34 | } 35 | return mapping[name] 36 | 37 | 38 | def sum_of_trials(args): 39 | def gather_all(key: str): 40 | return np.stack( 41 | [np.load(path)[key] for path in pathlib.Path(args.directory).glob("*.npz")] 42 | ).sum(axis=0) 43 | 44 | trials, occurrences = gather_all("data"), gather_all("occurrences") 45 | return trials, occurrences 46 | 47 | 48 | def axis_description(name: str): 49 | mapping = { 50 | "grad": [True, False], 51 | "type": ["base64", "byte", "char", "short", "int", "long", "float", "double"], 52 | "payload": ["128x128", "256x256", "512x512", "1024x1024"], 53 | "model_name": [ 54 | "shufflenet_v2_x1_0", 55 | "resnet18", 56 | "mobilenet_v2", 57 | "mnasnet1_0", 58 | "mnasnet1_3", 59 | ], 60 | "normalize": [True, False], 61 | "return": ["output", "result", "result+output"], 62 | } 63 | 64 | return mapping[name] 65 | 66 | 67 | def get_results(trials, occurrences, rows, columns, duration): 68 | def _reduce(matrix): 69 | return np.sum( 70 | matrix, 71 | axis=tuple( 72 | i 73 | for i in range(len(matrix.shape) - 1) 74 | if i not in (matrix_mapping(rows), matrix_mapping(columns)) 75 | ), 76 | )[:, :, duration_mapping(duration)] 77 | 78 | all_tests = _reduce(occurrences) 79 | return np.sum(all_tests), _reduce(trials) / all_tests 80 | 81 | 82 | def header() -> str: 83 | with open("header.md", "r") as f: 84 | return f.read() + "\n\n" 85 | 86 | 87 | if __name__ == "__main__": 88 | args = parse() 89 | trials, occurrences = sum_of_trials(args) 90 | fields = ("grad", "type", "payload", "model_name", "normalize", "return") 91 | times = ("init", "duration", "billed") 92 | 93 | markdown = header() 94 | for time, (row, column) in itertools.product( 95 | times, itertools.combinations(fields, 2) 96 | ): 97 | 98 | _, result_matrix = get_results(trials, occurrences, row, column, time) 99 | df = pd.DataFrame( 100 | data=result_matrix, 101 | index=axis_description(row), 102 | columns=axis_description(column), 103 | ) 104 | if not df.isnull().any().any(): 105 | markdown += "\n\n## {}: {} x {}\n\n".format(time, row, column) 106 | markdown += df.to_markdown() 107 | 108 | all_tests, _ = get_results(trials, occurrences, row, column, time) 109 | markdown += "\n\n _This file was auto-generated on {} based on {} tests_".format( 110 | datetime.today().strftime("%Y-%m-%d-%H:%M:%S"), all_tests 111 | ) 112 | 113 | with open(args.output, "w") as f: 114 | f.write(markdown) 115 | -------------------------------------------------------------------------------- /tests/settings/automated/src/setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | import typing 4 | 5 | import yaml 6 | 7 | import torchlambda 8 | 9 | 10 | class OptionGenerator: 11 | def __init__(self, options: int): 12 | self.options = options 13 | 14 | def __call__(self): 15 | return random.choice(self.options) 16 | 17 | 18 | def generate(possibilities: typing.Dict) -> typing.Dict: 19 | return { 20 | key: value() if isinstance(value, OptionGenerator) else generate(value) 21 | for key, value in possibilities.items() 22 | } 23 | 24 | 25 | def post_process(settings: typing.Dict) -> typing.Dict: 26 | def _remove_nones(dictionary): 27 | return { 28 | key: value if not isinstance(value, dict) else _remove_nones(value) 29 | for key, value in dictionary.items() 30 | if value is not None 31 | } 32 | 33 | def _possibly_nullify(dictionary): 34 | def _randomly_nullify_field(field, key): 35 | choice = random.choice([True, False]) 36 | if choice: 37 | field.pop(key) 38 | return choice 39 | 40 | # cast cannot be nullified as model hangs on other input than float 41 | _randomly_nullify_field(dictionary["input"], "divide") 42 | _randomly_nullify_field(dictionary, "normalize") 43 | no_output = _randomly_nullify_field(dictionary["return"], "output") 44 | if not no_output: 45 | _randomly_nullify_field(dictionary["return"], "result") 46 | 47 | _possibly_nullify(settings) 48 | return _remove_nones(settings) 49 | 50 | 51 | def create_settings() -> None: 52 | validator = torchlambda.implementation.utils.template.validator.get() 53 | possibilities = { 54 | "grad": OptionGenerator([False, True]), 55 | # Do not optimize first pass 56 | "optimize": OptionGenerator([False]), 57 | "validate_json": OptionGenerator([True, False]), 58 | "input": { 59 | "validate": OptionGenerator([True, False]), 60 | "type": OptionGenerator( 61 | ["base64", "byte", "char", "short", "int", "long", "float", "double"] 62 | ), 63 | "shape": OptionGenerator( 64 | [["batch", "channels", "width", "height"], [1, 3, "width", "height"]] 65 | ), 66 | "validate_shape": OptionGenerator([True, False]), 67 | # cast cannot be different as model hangs on different input than float 68 | "cast": OptionGenerator(["float"]), 69 | "divide": OptionGenerator([255]), 70 | }, 71 | "normalize": { 72 | "means": OptionGenerator([[0.485, 0.456, 0.406], [0.485], [0.444, 0.444]]), 73 | "stddevs": OptionGenerator([[0.44], [0.229, 0.224, 0.225]]), 74 | }, 75 | "return": { 76 | "output": { 77 | "type": OptionGenerator(["double"]), 78 | "item": OptionGenerator([False]), 79 | }, 80 | "result": { 81 | "type": OptionGenerator(["int", "long"]), 82 | "item": OptionGenerator([True, False]), 83 | "operations": OptionGenerator(["argmax", ["sigmoid", "argmax"]]), 84 | "arguments": OptionGenerator([None, [[], 1]]), 85 | }, 86 | }, 87 | } 88 | 89 | settings = generate(possibilities) 90 | settings = post_process(settings) 91 | if not validator.validate(settings): 92 | create_settings() 93 | else: 94 | with open(os.environ["OUTPUT"], "w") as file: 95 | yaml.dump(settings, file, default_flow_style=False) 96 | 97 | 98 | if __name__ == "__main__": 99 | create_settings() 100 | -------------------------------------------------------------------------------- /torchlambda/templates/custom/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | /*! 12 | * 13 | * HANDLE REQUEST 14 | * 15 | */ 16 | 17 | static aws::lambda_runtime::invocation_response 18 | handler(torch::jit::script::Module &module, 19 | const Aws::Utils::Base64::Base64 &transformer, 20 | const aws::lambda_runtime::invocation_request &request) { 21 | 22 | const Aws::String data_field{"data"}; 23 | 24 | /*! 25 | * 26 | * PARSE AND VALIDATE REQUEST 27 | * 28 | */ 29 | 30 | const auto json = Aws::Utils::Json::JsonValue{request.payload}; 31 | if (!json.WasParseSuccessful()) 32 | return aws::lambda_runtime::invocation_response::failure( 33 | "Failed to parse input JSON file.", "InvalidJSON"); 34 | 35 | const auto json_view = json.View(); 36 | if (!json_view.KeyExists(data_field)) 37 | return aws::lambda_runtime::invocation_response::failure( 38 | "Required data was not provided.", "InvalidJSON"); 39 | 40 | /*! 41 | * 42 | * LOAD DATA, TRANSFORM TO TENSOR, NORMALIZE 43 | * 44 | */ 45 | 46 | const auto base64_data = json_view.GetString(data_field); 47 | Aws::Utils::ByteBuffer decoded = transformer.Decode(base64_data); 48 | 49 | torch::Tensor tensor = 50 | torch::from_blob(decoded.GetUnderlyingData(), 51 | { 52 | static_cast(decoded.GetLength()), 53 | }, 54 | torch::kUInt8) 55 | .reshape({1, 3, 64, 64}) 56 | .toType(torch::kFloat32) / 57 | 255.0; 58 | 59 | torch::Tensor normalized_tensor = torch::data::transforms::Normalize<>{ 60 | {0.485, 0.456, 0.406}, {0.229, 0.224, 0.225}}(tensor); 61 | 62 | /*! 63 | * 64 | * MAKE INFERENCE 65 | * 66 | */ 67 | 68 | auto output = module.forward({normalized_tensor}).toTensor(); 69 | const int label = torch::argmax(output).item(); 70 | 71 | /*! 72 | * 73 | * RETURN JSON 74 | * 75 | */ 76 | 77 | return aws::lambda_runtime::invocation_response::success( 78 | Aws::Utils::Json::JsonValue{} 79 | .WithInteger("label", label) 80 | .View() 81 | .WriteCompact(), 82 | "application/json"); 83 | } 84 | 85 | int main() { 86 | /*! 87 | * 88 | * LOAD MODEL ON CPU 89 | * & SET IT TO EVALUATION MODE 90 | * 91 | */ 92 | 93 | /* Turn off gradient */ 94 | torch::NoGradGuard no_grad_guard{}; 95 | /* No optimization during first pass as it might slow down inference by 30s */ 96 | torch::jit::setGraphExecutorOptimize(false); 97 | 98 | constexpr auto model_path = "/opt/model.ptc"; 99 | 100 | torch::jit::script::Module module = torch::jit::load(model_path, torch::kCPU); 101 | module.eval(); 102 | 103 | /*! 104 | * 105 | * INITIALIZE AWS SDK 106 | * & REGISTER REQUEST HANDLER 107 | * 108 | */ 109 | 110 | Aws::SDKOptions options; 111 | Aws::InitAPI(options); 112 | { 113 | const Aws::Utils::Base64::Base64 transformer{}; 114 | const auto handler_fn = 115 | [&module, 116 | &transformer](const aws::lambda_runtime::invocation_request &request) { 117 | return handler(module, transformer, request); 118 | }; 119 | aws::lambda_runtime::run_handler(handler_fn); 120 | } 121 | Aws::ShutdownAPI(options); 122 | return 0; 123 | } 124 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | [szymon.maszke@protonmail.com](mailto:szymon.maszke@protonmail.com). 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /torchlambda/implementation/docker.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import pathlib 3 | import shutil 4 | import sys 5 | import typing 6 | import uuid 7 | 8 | from . import general 9 | 10 | 11 | def check() -> None: 12 | """Check whether docker is available for usage by torchlambda.""" 13 | if shutil.which("docker") is None: 14 | print( 15 | "Docker is required but was not found on $PATH.\n" 16 | "Please install docker and make it available for torchlambda (see https://docs.docker.com/install/).", 17 | file=sys.stderr, 18 | ) 19 | exit(1) 20 | 21 | 22 | def image_exists(name: str) -> bool: 23 | """ 24 | Return true if specified image exist locally on host. 25 | 26 | Parameters 27 | ---------- 28 | name : str 29 | Name of image to be searched for 30 | 31 | Returns 32 | ------- 33 | bool 34 | Whether image exists on localhost 35 | 36 | """ 37 | return not bool( 38 | general.run( 39 | "docker inspect --type=image {}".format(name), 40 | operation="image existence check.", 41 | silent=True, 42 | exit_on_failure=False, 43 | ) 44 | ) 45 | 46 | 47 | def build(args) -> str: 48 | """ 49 | Build docker image from scratch. 50 | 51 | User can provide various flags to docker and build commands except `-t` which 52 | is specified by `image` argument and parsed separately. 53 | 54 | Parameters 55 | ---------- 56 | args : dict-like 57 | User provided arguments parsed by argparse.ArgumentParser instance. 58 | 59 | Returns 60 | ------- 61 | str: 62 | Name of created image 63 | """ 64 | 65 | def _cmake_environment_variables(name: str, values: typing.List[str]) -> str: 66 | return ( 67 | '--build-arg {}="{}" '.format( 68 | name, " ".join(["-D{}".format(value) for value in values]), 69 | ) 70 | if values 71 | else " " 72 | ) 73 | 74 | def _create_aws_components(components): 75 | return ["BUILD_ONLY='{}'".format(";".join(components))] if components else [] 76 | 77 | def _pytorch_version(version: str): 78 | return ( 79 | '--build-arg PYTORCH_VERSION="{}" '.format(version) 80 | if version is not None 81 | else " " 82 | ) 83 | 84 | command = "docker {} build {} -t {} ".format( 85 | *general.parse_none(args.docker, args.docker_build, args.image) 86 | ) 87 | command += _cmake_environment_variables("PYTORCH", args.pytorch) 88 | command += _pytorch_version(args.pytorch_version) 89 | command += _cmake_environment_variables( 90 | "AWS", _create_aws_components(args.aws_components) + args.aws 91 | ) 92 | command += "." 93 | general.run(command, operation="custom PyTorch build.", silent=args.silent) 94 | 95 | return args.image 96 | 97 | 98 | def run(args, image: str) -> str: 99 | """ 100 | Run docker image and mount user-provided folder with C++ files. 101 | 102 | Parameters 103 | ---------- 104 | args : dict-like 105 | User provided arguments parsed by argparse.ArgumentParser instance. 106 | image : str 107 | Name of image from which container is run 108 | 109 | Returns 110 | ------- 111 | str: 112 | Name of created container. Consist of torchlambda prefix and random string 113 | """ 114 | 115 | def _add_components(args): 116 | return ( 117 | '"' + ";".join(args.aws_components) + '"' 118 | if args.aws_components 119 | else '"core"' 120 | ) 121 | 122 | def _compilation(args): 123 | return '"' + args.compilation + '"' if args.compilation else "" 124 | 125 | container_name = "torchlambda-" + str(uuid.uuid4()) 126 | source_directory = pathlib.Path(args.source).absolute() 127 | if source_directory.is_dir(): 128 | command = "docker {} run {} -v {}:/usr/local/user_code --name {} {} {} ".format( 129 | *general.parse_none( 130 | args.docker, 131 | args.docker_run, 132 | source_directory, 133 | container_name, 134 | image, 135 | _add_components(args), 136 | ) 137 | ) 138 | 139 | command += _compilation(args) 140 | 141 | general.run( 142 | command, 143 | operation="building inference AWS Lambda package.", 144 | silent=args.silent, 145 | ) 146 | return container_name 147 | 148 | print("torchlambda:: Provided source files are not directory, exiting.") 149 | exit(1) 150 | 151 | 152 | def cp(args, name: str, source: str, destination: str) -> None: 153 | """ 154 | Copy source to destination from named container. 155 | 156 | Will verbosely copy file from container, yet no output of `docker` command 157 | will be displayed. 158 | 159 | Will crash the program if anything goes wrong. 160 | 161 | Parameters 162 | ---------- 163 | args : dict-like 164 | User provided arguments parsed by argparse.ArgumentParser instance. 165 | args : dict-like 166 | User provided arguments 167 | name : str 168 | Name of container 169 | source : str 170 | Path to source to be copied 171 | destination : str 172 | Destination where source will be copied 173 | 174 | """ 175 | general.run( 176 | "docker container cp {}:{} {}".format( 177 | name, source, pathlib.Path(destination).absolute() 178 | ), 179 | operation="copying {} from docker container {} to {}.".format( 180 | source, name, destination 181 | ), 182 | silent=args.silent, 183 | ) 184 | 185 | 186 | @contextlib.contextmanager 187 | def rm(name: str) -> None: 188 | """ 189 | Remove container specified by name 190 | 191 | Container to be removed after deployment was copied from it to localhost. 192 | 193 | Parameters 194 | ---------- 195 | name : str 196 | Name of the container to be removed 197 | 198 | """ 199 | try: 200 | yield 201 | finally: 202 | general.run( 203 | "docker rm {}".format(name), 204 | operation="removing created docker container named {}.".format(name), 205 | silent=True, 206 | ) 207 | -------------------------------------------------------------------------------- /torchlambda/implementation/utils/template/validator.py: -------------------------------------------------------------------------------- 1 | import collections 2 | 3 | import cerberus 4 | 5 | 6 | class Validator(cerberus.Validator): 7 | def _validate_is_shorter(self, other: str, field, value): 8 | """Test whether one field (arguments) is shorter than other (operations). 9 | 10 | The rule's arguments are validated against this schema: 11 | {"type": "string"} 12 | """ 13 | arguments_length = ( 14 | 1 if not isinstance(value, collections.Iterable) else len(value) 15 | ) 16 | operations_length = ( 17 | 1 if isinstance(self.document[other], str) else len(self.document[other]) 18 | ) 19 | if arguments_length > operations_length: 20 | self._error( 21 | field, 22 | "More arguments provided in field: {} than in operations!".format( 23 | field, 24 | ), 25 | ) 26 | 27 | def _validate_broadcastable(self, shape_field: str, field, value): 28 | """Test whether one field (arguments) is shorter than other (operations). 29 | 30 | The rule's arguments are validated against this schema: 31 | {"type": "string"} 32 | """ 33 | if isinstance(value, (list, tuple)): 34 | shapes = self.root_document["input"][shape_field] 35 | if len(value) != 1 and len(value) != shapes[1]: 36 | self._error( 37 | field, 38 | "{} field shape ({}) is not broadcastable to provided input shape: {}".format( 39 | field, value, shapes 40 | ), 41 | ) 42 | 43 | 44 | def _is_not_dict(field, value, error): 45 | if isinstance(value, dict): 46 | error(field, "field cannot be an instance of dict") 47 | 48 | 49 | # Fix validator 50 | def get(): 51 | return Validator( 52 | { 53 | "grad": {"type": "boolean", "default": False}, 54 | "optimize": {"type": "boolean", "default": False}, 55 | "validate_json": {"type": "boolean", "default": True}, 56 | "model": {"type": "string", "default": "/opt/model.ptc", "empty": False}, 57 | "input": { 58 | "type": "dict", 59 | "schema": { 60 | "name": {"type": "string", "default": "data", "empty": False}, 61 | "validate": {"type": "boolean", "default": True}, 62 | "type": { 63 | "type": "string", 64 | "allowed": [ 65 | "base64", 66 | "byte", 67 | "char", 68 | "short", 69 | "int", 70 | "long", 71 | "float", 72 | "double", 73 | ], 74 | "required": True, 75 | }, 76 | "shape": { 77 | "type": "list", 78 | "schema": {"type": ["string", "integer"]}, 79 | "required": True, 80 | "minlength": 2, 81 | "empty": False, 82 | }, 83 | "validate_shape": {"type": "boolean", "default": True}, 84 | "cast": { 85 | "type": "string", 86 | "allowed": [ 87 | "byte", 88 | "char", 89 | "short", 90 | "int", 91 | "long", 92 | "half", 93 | "float", 94 | "double", 95 | ], 96 | "nullable": True, 97 | "default": None, 98 | }, 99 | "divide": {"type": "number", "nullable": True, "default": None}, 100 | }, 101 | }, 102 | "normalize": { 103 | "type": "dict", 104 | "nullable": True, 105 | "schema": { 106 | "means": { 107 | "required": True, 108 | "broadcastable": "shape", 109 | "anyof_type": ["list", "number"], 110 | "schema": {"type": "number"}, 111 | }, 112 | "stddevs": { 113 | "required": True, 114 | "broadcastable": "shape", 115 | "anyof_type": ["list", "number"], 116 | "schema": {"type": "number"}, 117 | }, 118 | }, 119 | "default": None, 120 | }, 121 | # Return can be either result, output or both 122 | "return": { 123 | "type": "dict", 124 | "required": True, 125 | "schema": { 126 | "output": { 127 | "type": "dict", 128 | "nullable": True, 129 | "schema": { 130 | # Return only single item, not array 131 | "name": { 132 | "type": "string", 133 | "default": "output", 134 | "empty": False, 135 | }, 136 | "type": { 137 | "type": "string", 138 | "allowed": ["int", "long", "double", "bool"], 139 | "required": True, 140 | }, 141 | "item": {"type": "boolean", "default": False}, 142 | }, 143 | "default": None, 144 | }, 145 | "result": { 146 | "type": "dict", 147 | "nullable": True, 148 | "schema": { 149 | # Return only single item, not array 150 | "name": {"type": "string", "default": "result"}, 151 | "type": { 152 | "type": "string", 153 | "allowed": ["int", "long", "double", "bool"], 154 | "required": True, 155 | }, 156 | "item": {"type": "boolean", "default": False}, 157 | "operations": { 158 | "oneof": [ 159 | { 160 | "type": "list", 161 | "schema": {"type": "string", "empty": False}, 162 | "empty": False, 163 | }, 164 | {"type": "string", "empty": False}, 165 | ], 166 | "required": True, 167 | }, 168 | "arguments": { 169 | "is_shorter": "operations", 170 | "check_with": _is_not_dict, 171 | "empty": False, 172 | "dependencies": "operations", 173 | "nullable": True, 174 | "default": None, 175 | }, 176 | }, 177 | "default": None, 178 | }, 179 | }, 180 | }, 181 | } 182 | ) 183 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | [__torchlambda__](https://github.com/szymonmaszke/torchlambda/wiki) is a tool to deploy [PyTorch](https://pytorch.org/) models 4 | on [Amazon's AWS Lambda](https://aws.amazon.com/lambda/) using [AWS SDK for C++](https://aws.amazon.com/sdk-for-cpp/) 5 | and [custom C++ runtime](https://github.com/awslabs/aws-lambda-cpp). 6 | 7 | Using statically compiled dependencies __whole package is shrunk to only `30MB`__. 8 | 9 | Due to small size of compiled source code users can pass their models as [AWS Lambda layers](https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html). 10 | __Services like [Amazon S3](https://aws.amazon.com/s3/) are no longer necessary to load your model__. 11 | 12 | [__torchlambda__](https://github.com/szymonmaszke/torchlambda/wiki) has it's PyTorch & AWS dependencies always tested & up to date because of daily [continuous deployment](https://en.wikipedia.org/wiki/Continuous_deployment) runs. 13 | 14 | 15 | | Docs | Deployment | Package | Python | PyTorch | Docker | CodeBeat | Images | 16 | |------|------------|---------|--------|---------|--------|----------|--------| 17 | |[![Documentation](https://img.shields.io/static/v1?label=&message=Wiki&color=EE4C2C&style=for-the-badge)](https://github.com/szymonmaszke/torchlambda/wiki) | ![CD](https://img.shields.io/github/workflow/status/szymonmaszke/torchlambda/update?label=%20&style=for-the-badge) | [![PyPI](https://img.shields.io/static/v1?label=&message=PyPI&color=377EF0&style=for-the-badge)](https://pypi.org/project/torchlambda/) | [![Python](https://img.shields.io/static/v1?label=&message=3.6&color=377EF0&style=for-the-badge&logo=python&logoColor=F8C63D)](https://www.python.org/) | [![PyTorch](https://img.shields.io/static/v1?label=&message=1.4.0&color=EE4C2C&style=for-the-badge)](https://pytorch.org/) | [![Docker](https://img.shields.io/static/v1?label=&message=17.05&color=309cef&style=for-the-badge)](https://cloud.docker.com/u/szymonmaszke/repository/docker/szymonmaszke/torchlambda) | [![codebeat badge](https://codebeat.co/badges/ca6f19c8-29ad-4ddb-beb3-4d4e2fb3aba2)](https://codebeat.co/projects/github-com-szymonmaszke-torchlambda-master) | [![Images](https://img.shields.io/static/v1?label=&message=Tags&color=309cef&style=for-the-badge)](https://hub.docker.com/r/szymonmaszke/torchlambda/tags)| 18 | 19 | 20 | ## :heavy_check_mark: Why should I use `torchlambda`? 21 | 22 | - __Lightweight & latest dependencies__ - compiled source code weights only `30MB`. Previous approach to PyTorch network deployment on AWS Lambda ([fastai](https://course.fast.ai/deployment_aws_lambda.html)) uses outdated PyTorch (`1.1.0`) as dependency layer and requires AWS S3 to host your model. Now you can only use AWS Lambda and host your model as layer and PyTorch `master` and latest stable release are supported on a daily basis. 23 | - __Cheaper and less resource hungry__ - available solutions run server hosting incoming requests all the time. AWS Lambda (and torchlambda) runs only when the request comes. 24 | - __Easy automated scaling__ usually autoscaling is done with [Kubernetes](https://kubernetes.io/) or similar tools (see [KubeFlow](https://www.kubeflow.org/docs/gke/deploy/)). This approach requires knowledge of another tool, setting up appropriate services (e.g. [Amazon EKS](https://aws.amazon.com/eks/)). In AWS Lambda case you just push your neural network inference code and you are done. 25 | - __Easy to use__ - no need to learn new tool. `torchlambda` has at most 26 | `4` commands and deployment is done via [YAML](https://yaml.org/) settings. No need to modify your PyTorch code. 27 | - __Do one thing and do it well__ - most deployment tools are complex solutions 28 | including multiple frameworks and multiple services. `torchlambda` focuses 29 | solely on inference of PyTorch models on AWS Lambda. 30 | - __Write programs to work together__ - This tool does not repeat PyTorch & AWS's functionalities (like `aws-cli`). You can also use your favorite third party tools (say [saws](https://github.com/donnemartin/saws), [Terraform](https://www.terraform.io/) with AWS and [MLFlow](https://www.mlflow.org/docs/latest/index.html), [PyTorch-Lightning](https://github.com/PyTorchLightning/pytorch-lightning) to train your model). 31 | - __Test locally, run in the cloud__ - `torchlambda` uses [Amazon Linux 2](https://aws.amazon.com/amazon-linux-2/) Docker [images](https://hub.docker.com/_/amazonlinux) under the hood & allows you to use [lambci/docker-lambda](https://github.com/lambci/docker-lambda) to test your deployment on `localhost` before pushing deployment to the cloud (see [Test Lambda deployment locally](https://github.com/szymonmaszke/torchlambda/wiki/Test-Lambda-deployment-locally) tutorial). 32 | - __Extensible when you need it__ - All you usually need are a few lines of YAML settings, but if you wish to fine-tune your deployment you can use `torchlambda build` `--flags` (changing various properties of PyTorch and AWS dependencies themselves). You can also write your own C++ deployment code (generate template via `torchlambda template` command). 33 | - __Small is beautiful__ - `3000` LOC (most being convenience wrapper creating this tool) 34 | make it easy to jump into source code and check what's going on under the hood. 35 | 36 | 37 | # :house: Table Of Contents 38 | 39 | - [Installation](https://github.com/szymonmaszke/torchlambda/wiki/Installation) 40 | - [Tutorials](https://github.com/szymonmaszke/torchlambda/wiki/Tutorials) 41 | - [ResNet18 deployment on AWS Lambda](https://github.com/szymonmaszke/torchlambda/wiki/ResNet18-deployment-on-AWS-Lambda) 42 | - [Test Lambda deployment locally](https://github.com/szymonmaszke/torchlambda/wiki/Test-Lambda-deployment-locally) 43 | - [`base64` image encoding](https://github.com/szymonmaszke/torchlambda/wiki/base64-image-encoding) 44 | - [Commands](https://github.com/szymonmaszke/torchlambda/wiki/Commands) 45 | - [settings](https://github.com/szymonmaszke/torchlambda/wiki/Commands#torchlambda-settings) 46 | - [template](https://github.com/szymonmaszke/torchlambda/wiki/Commands#torchlambda-template) 47 | - [build](https://github.com/szymonmaszke/torchlambda/wiki/Commands#torchlambda-build) 48 | - [layer](https://github.com/szymonmaszke/torchlambda/wiki/Commands#torchlambda-layer) 49 | - [YAML settings file reference](https://github.com/szymonmaszke/torchlambda/wiki/YAML-settings-file-reference) 50 | - [C++ code](https://github.com/szymonmaszke/torchlambda/wiki/CPP---code) 51 | 52 | ## :page_with_curl: Benchmarks 53 | 54 | Benchmarks can be seen in [`BENCHMARKS.md`](https://github.com/szymonmaszke/torchlambda/blob/master/BENCHMARKS.md) file and are comprised of around ~30000 test cases. 55 | 56 | Results are divided based on settings used, model type, payload, AWS Lambda timing etc. Below is an example of how inference performance changes due to higher resolution images and type of encoding: 57 | 58 | | | 128x128 | 256x256 | 512x512 | 1024x1024 | 59 | |:-------|----------:|----------:|----------:|------------:| 60 | | base64 | 120.622 | 165.184 | 311.129 | 995.249 | 61 | | byte | 133.315 | 203.628 | 498.391 | 1738.97 | 62 | | char | 128.331 | 209.306 | 517.482 | 1822.56 | 63 | | short | 135.859 | 207.389 | 497.818 | 1740.91 | 64 | | int | 133.42 | 216.163 | 519.502 | 1783.02 | 65 | | long | 126.979 | 228.497 | 516.98 | 1760.93 | 66 | | float | 135.825 | 223.045 | 515.245 | 1802.25 | 67 | | double | 137.281 | 209.267 | 536.959 | 1811.83 | 68 | 69 | Clearly the bigger image, the more important it is to use `base64` encoding. For all results and description [click here](https://github.com/szymonmaszke/torchlambda/blob/master/BENCHMARKS.md). 70 | 71 | ## :question: Contributing 72 | 73 | If you find an issue or you think some functionality may be useful to you, please [open new Issue](https://help.github.com/en/articles/creating-an-issue) or [create Pull Request](https://help.github.com/en/articles/creating-a-pull-request-from-a-fork) with your changes, thanks! 74 | -------------------------------------------------------------------------------- /torchlambda/implementation/utils/template/header.py: -------------------------------------------------------------------------------- 1 | from . import macro 2 | 3 | 4 | def static(settings) -> str: 5 | """ 6 | Return #define STATIC if all inputs fields are integer. 7 | 8 | If specified, no field checks will be performed during request to 9 | Lambda function. 10 | 11 | Parameters 12 | ---------- 13 | settings : typing.Dict 14 | YAML parsed to dict 15 | 16 | Returns 17 | ------- 18 | str: 19 | Either "" or "#define STATIC" 20 | """ 21 | return macro.conditional( 22 | all(isinstance(x, int) for x in settings["input"]["shape"]), "STATIC" 23 | ) 24 | 25 | 26 | def grad(settings) -> str: 27 | """ 28 | Return #define GRAD if grad: True specified. 29 | 30 | If specified, libtorch's grad addition to ATen will be turned on. 31 | 32 | Parameters 33 | ---------- 34 | settings : typing.Dict 35 | YAML parsed to dict 36 | 37 | Returns 38 | ------- 39 | str: 40 | Either "" or "#define GRAD" 41 | """ 42 | return macro.conditional(settings["grad"], "GRAD") 43 | 44 | 45 | def optimize(settings) -> str: 46 | """ 47 | Return #define GRAD if grad: True specified. 48 | 49 | If specified, libtorch's grad addition to ATen will be turned on. 50 | 51 | Parameters 52 | ---------- 53 | settings : typing.Dict 54 | YAML parsed to dict 55 | 56 | Returns 57 | ------- 58 | str: 59 | Either "" or "#define GRAD" 60 | """ 61 | return macro.conditional(settings["optimize"], "OPTIMIZE") 62 | 63 | 64 | def validate_json(settings) -> str: 65 | """ 66 | Return #define VALIDATE_JSON if validate_json: True specified. 67 | 68 | If specified, it will be validated if JSON was parsed successfully. 69 | If not, InvalidJson with appropriate text will be returned. 70 | 71 | Parameters 72 | ---------- 73 | settings : typing.Dict 74 | YAML parsed to dict 75 | 76 | Returns 77 | ------- 78 | str: 79 | Either "" or "#define VALIDATE_JSON" 80 | """ 81 | return macro.conditional(settings["validate_json"], "VALIDATE_JSON") 82 | 83 | 84 | def base64(settings) -> str: 85 | """ 86 | Return #define BASE64 if input->type is equal to base64. 87 | 88 | If specified, it is assumed data is in base64 string format and will be decoded 89 | in handler. In this case input_type IS NOT USED and data type will be 90 | unsigned int 8 upon creation (which can be optionally casted). 91 | 92 | If not, it is assumed data will be a flat array of specified by user via 93 | input_type. 94 | 95 | Parameters 96 | ---------- 97 | settings : typing.Dict 98 | YAML parsed to dict 99 | 100 | Returns 101 | ------- 102 | str: 103 | Either "" or "#define BASE64" 104 | """ 105 | return macro.conditional(settings["input"]["type"] == "base64", "BASE64") 106 | 107 | 108 | def validate_field(settings) -> str: 109 | """ 110 | Return #define VALIDATE_FIELD if validate: True specified. 111 | 112 | If specified, data will be checked for correctness 113 | (whether `data` field exists and whether it's type is string or array). 114 | 115 | If not, InvalidJson with appropriate text will be returned. 116 | 117 | Parameters 118 | ---------- 119 | settings : typing.Dict 120 | YAML parsed to dict 121 | 122 | Returns 123 | ------- 124 | str: 125 | Either "" or "#define VALIDATE_FIELD" 126 | """ 127 | return macro.conditional(settings["input"]["validate"], "VALIDATE_FIELD") 128 | 129 | 130 | def validate_shape(settings) -> str: 131 | """ 132 | Return #define VALIDATE_SHAPE if validate_inputs: True specified. 133 | 134 | If specified, input fields will be checked (if any exist). 135 | All of them will be checked for existence and whether their type 136 | is integer. 137 | 138 | If not, InvalidJson with appropriate text will be returned. 139 | 140 | Parameters 141 | ---------- 142 | settings : typing.Dict 143 | YAML parsed to dict 144 | 145 | Returns 146 | ------- 147 | str: 148 | Either "" or "#define VALIDATE_SHAPE" 149 | """ 150 | return macro.conditional(settings["input"]["validate_shape"], "VALIDATE_SHAPE") 151 | 152 | 153 | def normalize(settings) -> str: 154 | """ 155 | Return #define NORMALIZE if normalize: True specified. 156 | 157 | Will normalize image-like tensor across channels using 158 | torch::data::transforms::Normalize<>. 159 | 160 | Parameters 161 | ---------- 162 | settings : typing.Dict 163 | YAML parsed to dict 164 | 165 | Returns 166 | ------- 167 | str: 168 | Either "" or "#define NORMALIZE" 169 | """ 170 | return macro.conditional(settings["normalize"], "NORMALIZE") 171 | 172 | 173 | def cast(settings) -> str: 174 | """ 175 | Impute libtorch specific type from user provided "human-readable" form. 176 | 177 | See `type_mapping` in source code for exact mapping. 178 | 179 | Parameters 180 | ---------- 181 | settings : typing.Dict 182 | YAML parsed to dict 183 | 184 | Returns 185 | ------- 186 | str: 187 | String specifying type, e.g. "torch::kFloat16" 188 | """ 189 | type_mapping = { 190 | "byte": "torch::kUInt8", 191 | "char": "torch::kInt8", 192 | "short": "torch::kInt16", 193 | "int": "torch::kInt32", 194 | "long": "torch::kInt64", 195 | "half": "torch::kFloat16", 196 | "float": "torch::kFloat32", 197 | "double": "torch::kFloat64", 198 | } 199 | 200 | return macro.key(settings["input"], key="cast", mapping=type_mapping,) 201 | 202 | 203 | def divide(settings) -> str: 204 | """ 205 | Impute value by which casted tensor will be divided. 206 | 207 | If user doesn't want to cast tensor, he should simply specify `1`, 208 | though it won't be used too often. 209 | 210 | Parameters 211 | ---------- 212 | settings : typing.Dict 213 | YAML parsed to dict 214 | 215 | Returns 216 | ------- 217 | str: 218 | string representation of number, e.g. "255.0" 219 | 220 | """ 221 | return macro.key(settings["input"], key="divide") 222 | 223 | 224 | def return_output(settings): 225 | """ 226 | Return #define RETURN_OUTPUT if field return->output is specified. 227 | 228 | Will return output from neural network as JSON array of specified 229 | OUTPUT_TYPE if macro defined. 230 | 231 | Parameters 232 | ---------- 233 | settings : typing.Dict 234 | YAML parsed to dict 235 | 236 | Returns 237 | ------- 238 | str: 239 | Either "" or "#define RETURN_OUTPUT" 240 | """ 241 | return macro.conditional( 242 | settings["return"]["output"] and not settings["return"]["output"]["item"], 243 | "RETURN_OUTPUT", 244 | ) 245 | 246 | 247 | def return_output_item(settings): 248 | """ 249 | Return #define RETURN_OUTPUT if field return->output->item is specified. 250 | 251 | Will return output from neural network as single element of specified 252 | OUTPUT_TYPE if macro defined. 253 | 254 | Parameters 255 | ---------- 256 | settings : typing.Dict 257 | YAML parsed to dict 258 | 259 | Returns 260 | ------- 261 | str: 262 | Either "" or "#define RETURN_OUTPUT_ITEM" 263 | """ 264 | return macro.conditional( 265 | settings["return"]["output"] and settings["return"]["output"]["item"], 266 | "RETURN_OUTPUT_ITEM", 267 | ) 268 | 269 | 270 | def return_result(settings): 271 | """ 272 | Return #define RETURN_RESULT if field return->result is specified. 273 | 274 | Will return modified by RESULT_OPERATION output from neural network 275 | as JSON array of specified RESULT_TYPE if macro defined. 276 | 277 | Parameters 278 | ---------- 279 | settings : typing.Dict 280 | YAML parsed to dict 281 | 282 | Returns 283 | ------- 284 | str: 285 | Either "" or "#define RETURN_RESULT" 286 | """ 287 | return macro.conditional( 288 | settings["return"]["result"] and not settings["return"]["result"]["item"], 289 | "RETURN_RESULT", 290 | ) 291 | 292 | 293 | def return_result_item(settings): 294 | """ 295 | Return #define RETURN_RESULT_ITEM if field return->result->item is specified. 296 | 297 | Will return modified by RESULT_OPERATION output from neural network 298 | as single item of specified RESULT_TYPE if macro defined 299 | 300 | Parameters 301 | ---------- 302 | settings : typing.Dict 303 | YAML parsed to dict 304 | 305 | Returns 306 | ------- 307 | str: 308 | Either "" or "#define RETURN_RESULT_ITEM" 309 | """ 310 | return macro.conditional( 311 | settings["return"]["result"] and settings["return"]["result"]["item"], 312 | "RETURN_RESULT_ITEM", 313 | ) 314 | -------------------------------------------------------------------------------- /torchlambda/templates/settings/main.cpp: -------------------------------------------------------------------------------- 1 | {STATIC} 2 | 3 | {GRAD} 4 | 5 | {OPTIMIZE} 6 | 7 | {VALIDATE_JSON} 8 | 9 | {BASE64} 10 | 11 | {VALIDATE_FIELD} 12 | 13 | {VALIDATE_SHAPE} 14 | 15 | {NORMALIZE} 16 | 17 | {CAST} 18 | 19 | {DIVIDE} 20 | 21 | {RETURN_OUTPUT} 22 | 23 | {RETURN_OUTPUT_ITEM} 24 | 25 | {RETURN_RESULT} 26 | 27 | {RETURN_RESULT_ITEM} 28 | 29 | #include /* To use for InvalidJson return if any of shape fields not provided */ 30 | 31 | #include 32 | #include 33 | 34 | #include 35 | #include 36 | #include 37 | #include 38 | 39 | #include 40 | 41 | #include 42 | #include 43 | 44 | /*! 45 | * 46 | * UTILITY MACROS FOR OUTPUT & RESULT PROCESSING 47 | * 48 | */ 49 | 50 | #define CREATE_JSON_ARRAY(decoded, data, ptr_name, func, torch_type, cpp_type) \ 51 | const auto *ptr_name = data.data_ptr(); \ 52 | for (int64_t i = 0; i < data.numel(); ++i) \ 53 | decoded[i] = Aws::Utils::Json::JsonValue{{}}.func( \ 54 | (static_cast(*(ptr_name + i)))); 55 | 56 | #define ADD_ITEM(value, func, name, torch_type, cpp_type) \ 57 | func(name, static_cast(value.flatten().item())) 58 | 59 | /*! 60 | * 61 | * REQUEST HANDLER 62 | * 63 | */ 64 | 65 | static aws::lambda_runtime::invocation_response 66 | handler(std::shared_ptr &module, 67 | const aws::lambda_runtime::invocation_request &request 68 | #ifdef BASE64 69 | , 70 | const Aws::Utils::Base64::Base64 &transformer 71 | #endif 72 | ) {{ 73 | const Aws::String data_field{{ {DATA} }}; 74 | 75 | /*! 76 | * 77 | * PARSE AND VALIDATE REQUEST 78 | * 79 | */ 80 | 81 | const auto json = Aws::Utils::Json::JsonValue{{request.payload}}; 82 | 83 | #ifdef VALIDATE_JSON 84 | if (!json.WasParseSuccessful()) 85 | return aws::lambda_runtime::invocation_response::failure( 86 | "Failed to parse request JSON file.", "InvalidJSON"); 87 | #endif 88 | 89 | const auto json_view = json.View(); 90 | 91 | #ifdef VALIDATE_FIELD 92 | if (!json_view.KeyExists(data_field)) 93 | return aws::lambda_runtime::invocation_response::failure( 94 | "Required field: \"" {DATA} "\" was not provided.", "InvalidJSON"); 95 | #ifdef BASE64 96 | if (!json_view.GetObject(data_field).IsString()) 97 | return aws::lambda_runtime::invocation_response::failure( 98 | "Required field: \"" {DATA} "\" is not string.", "InvalidJSON"); 99 | #else 100 | if (!json_view.GetObject(data_field).IsListType()) 101 | return aws::lambda_runtime::invocation_response::failure( 102 | "Required field: \"" {DATA} "\" is not list type.", "InvalidJSON"); 103 | #endif 104 | #endif 105 | 106 | #if not defined(STATIC) && defined(VALIDATE_SHAPE) 107 | /* Check whether all necessary fields are passed */ 108 | 109 | const Aws::String fields[]{{{FIELDS}}}; 110 | for (const auto &field : fields) {{ 111 | if (!json_view.KeyExists(field)) 112 | return aws::lambda_runtime::invocation_response::failure( 113 | "Required input shape field: '" + 114 | std::string{{field.c_str(), field.size()}} + 115 | "' was not provided.", 116 | "InvalidJSON"); 117 | 118 | if (!json_view.GetObject(field).IsIntegerType()) 119 | return aws::lambda_runtime::invocation_response::failure( 120 | "Required shape field: '" + 121 | std::string{{field.c_str(), field.size()}} + 122 | "' is not of integer type.", 123 | "InvalidJSON"); 124 | }} 125 | 126 | #endif 127 | 128 | /*! 129 | * 130 | * LOAD DATA, TRANSFORM TO TENSOR, NORMALIZE 131 | * 132 | */ 133 | 134 | #ifdef BASE64 135 | const auto base64_string = json_view.GetString(data_field); 136 | auto data = transformer.Decode(base64_string); 137 | auto *data_pointer = data.GetUnderlyingData(); 138 | const std::size_t data_length = data.GetLength(); 139 | #else 140 | const auto nested_json = json_view.GetArray(data_field); 141 | Aws::Vector<{DATA_TYPE}> data; 142 | data.reserve(nested_json.GetLength()); 143 | 144 | for (size_t i = 0; i < nested_json.GetLength(); ++i) {{ 145 | data.push_back(static_cast<{DATA_TYPE}>(nested_json[i].{DATA_FUNC}())); 146 | }} 147 | 148 | auto *data_pointer = data.data(); 149 | const std::size_t data_length = nested_json.GetLength(); 150 | #endif 151 | const torch::Tensor tensor = 152 | #ifdef NORMALIZE 153 | torch::data::transforms::Normalize<>{{ 154 | {{{NORMALIZE_MEANS}}}, {{{NORMALIZE_STDDEVS}}} 155 | }}( 156 | #endif 157 | torch::from_blob( 158 | data_pointer, 159 | {{ 160 | /* Explicit cast as PyTorch has long int for some reason */ 161 | static_cast(data_length), 162 | }}, 163 | {TORCH_DATA_TYPE}) 164 | .reshape({{{INPUTS}}}) 165 | #ifdef CAST 166 | .toType(CAST) 167 | #endif 168 | #ifdef DIVIDE 169 | / DIVIDE 170 | #endif 171 | #ifdef NORMALIZE 172 | ) 173 | #endif 174 | ; 175 | 176 | /*! 177 | * 178 | * MAKE INFERENCE AND RETURN JSON RESPONSE 179 | * 180 | */ 181 | 182 | /* Support for multi-output/multi-input? */ 183 | 184 | const auto output = module->forward({{tensor}}) 185 | .toTensor() 186 | #ifdef RETURN_OUTPUT 187 | .toType({OUTPUT_CAST}) 188 | #endif 189 | ; 190 | 191 | /* Perform operation to create result */ 192 | #if defined(RETURN_RESULT) || defined(RETURN_RESULT_ITEM) 193 | const auto result = ({OPERATIONS_AND_ARGUMENTS}).toType({RESULT_CAST}); 194 | #endif 195 | 196 | /* If array of outputs to be returned gather values as JSON */ 197 | #ifdef RETURN_OUTPUT 198 | Aws::Utils::Array output_array{{ 199 | static_cast(output.numel()) 200 | }}; 201 | CREATE_JSON_ARRAY(output_array, output, output_ptr, {AWS_OUTPUT_FUNCTION}, 202 | {TORCH_OUTPUT_TYPE}, {OUTPUT_TYPE}) 203 | #endif 204 | 205 | /* If array of results to be returned gather values as JSON */ 206 | #ifdef RETURN_RESULT 207 | Aws::Utils::Array result_array{{ 208 | static_cast(result.numel()) 209 | }}; 210 | CREATE_JSON_ARRAY(result_array, result, result_ptr, {AWS_RESULT_FUNCTION}, 211 | {TORCH_RESULT_TYPE}, {RESULT_TYPE}) 212 | #endif 213 | 214 | /* Return JSON with response */ 215 | return aws::lambda_runtime::invocation_response::success( 216 | Aws::Utils::Json::JsonValue{{}} 217 | #ifdef RETURN_OUTPUT 218 | .WithArray("{OUTPUT_NAME}", output_array) 219 | #elif defined(RETURN_OUTPUT_ITEM) 220 | .ADD_ITEM(output, {AWS_OUTPUT_ITEM_FUNCTION}, "{OUTPUT_NAME}", 221 | {TORCH_OUTPUT_TYPE}, {OUTPUT_TYPE}) 222 | #endif 223 | #ifdef RETURN_RESULT 224 | .WithArray("{RESULT_NAME}", result_array) 225 | #elif defined(RETURN_RESULT_ITEM) 226 | .ADD_ITEM(result, {AWS_RESULT_ITEM_FUNCTION}, "{RESULT_NAME}", 227 | {TORCH_RESULT_TYPE}, {RESULT_TYPE}) 228 | #endif 229 | .View() 230 | .WriteCompact(), 231 | "application/json" 232 | ); 233 | }} 234 | 235 | int main() {{ 236 | /*! 237 | * 238 | * INITIALIZE AWS SDK 239 | * 240 | */ 241 | 242 | Aws::SDKOptions options; 243 | Aws::InitAPI(options); 244 | {{ 245 | #ifndef GRAD 246 | torch::NoGradGuard no_grad_guard{{}}; 247 | #endif 248 | #ifndef OPTIMIZE 249 | torch::jit::setGraphExecutorOptimize(false); 250 | #endif 251 | 252 | /* Change name/path to your model if you so desire */ 253 | /* Layers are unpacked to /opt, so you are better off keeping it */ 254 | constexpr auto model_path = {MODEL_PATH}; 255 | 256 | /* You could add some checks whether the module is loaded correctly */ 257 | auto module = Aws::MakeShared( 258 | "TORCHSCRIPT_MODEL", torch::jit::load(model_path, torch::kCPU)); 259 | if (module == nullptr) 260 | return -1; 261 | #ifndef GRAD 262 | module->eval(); 263 | #endif 264 | 265 | const Aws::Utils::Base64::Base64 transformer{{}}; 266 | const auto handler_fn = [&module 267 | #ifdef BASE64 268 | , 269 | &transformer 270 | #endif 271 | ](const aws::lambda_runtime::invocation_request &request){{ 272 | return handler(module, request 273 | #ifdef BASE64 274 | , 275 | transformer 276 | #endif 277 | ); 278 | }}; 279 | aws::lambda_runtime::run_handler(handler_fn); 280 | }} 281 | 282 | Aws::ShutdownAPI(options); 283 | return 0; 284 | }} 285 | -------------------------------------------------------------------------------- /torchlambda/implementation/utils/template/imputation.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import itertools 3 | 4 | 5 | def data(settings) -> str: 6 | """ 7 | Impute name of field containing base64 encoded data. 8 | 9 | Parameters 10 | ---------- 11 | settings : typing.Dict 12 | YAML parsed to dict 13 | 14 | Returns 15 | ------- 16 | str: 17 | "name_of_data_field" 18 | """ 19 | return '"' + settings["input"]["name"] + '"' 20 | 21 | 22 | def fields(settings) -> str: 23 | """ 24 | Impute name of fields (if any) specifying tensor shape during request. 25 | 26 | Fields can be empty if tensor shape is known beforehand 27 | (STATIC macro defined, see `header.static` function). 28 | 29 | Parameters 30 | ---------- 31 | settings : typing.Dict 32 | YAML parsed to dict 33 | 34 | Returns 35 | ------- 36 | str: 37 | "field1", "field2", ..., "fieldN" 38 | """ 39 | return ", ".join( 40 | [ 41 | '"' + field + '"' 42 | for field in settings["input"]["shape"] 43 | if isinstance(field, str) 44 | ] 45 | ) 46 | 47 | 48 | def data_type(settings) -> str: 49 | type_mapping = { 50 | "base64": "", 51 | "byte": "uint8_t", 52 | "char": "int8_t", 53 | "short": "int16_t", 54 | "int": "int32_t", 55 | "long": "int64_t", 56 | "float": "float", 57 | "double": "double", 58 | } 59 | return type_mapping[settings["input"]["type"]] 60 | 61 | 62 | def data_func(settings) -> str: 63 | type_mapping = { 64 | "base64": "", 65 | "byte": "Integer", 66 | "char": "Integer", 67 | "short": "Integer", 68 | "int": "Integer", 69 | "long": "Int64", 70 | "float": "Double", 71 | "double": "Double", 72 | } 73 | return "As" + type_mapping[settings["input"]["type"]] 74 | 75 | 76 | def normalize(settings, key: str) -> str: 77 | """ 78 | Impute normalization values if any. 79 | 80 | Parameters 81 | ---------- 82 | settings : typing.Dict 83 | YAML parsed to dict 84 | key : str 85 | Name of YAML settings field (either means or stddevs) to be imputed 86 | 87 | Returns 88 | ------- 89 | str: 90 | "" or "value1, value2, value3" 91 | """ 92 | if settings["normalize"] is None: 93 | return "" 94 | return ", ".join(map(str, settings["normalize"][key])) 95 | 96 | 97 | def torch_data_type(settings): 98 | type_mapping = { 99 | "base64": "torch::kUInt8", 100 | "byte": "torch::kUInt8", 101 | "char": "torch::kInt8", 102 | "short": "torch::kInt16", 103 | "int": "torch::kInt32", 104 | "long": "torch::kInt64", 105 | "float": "torch::kFloat32", 106 | "double": "torch::kFloat64", 107 | } 108 | return type_mapping[settings["input"]["type"]] 109 | 110 | 111 | def inputs(settings) -> str: 112 | """ 113 | Impute input shapes. 114 | 115 | Shapes may be name of fields passed during request (dynamic input shape) 116 | or integers (static input shape) or mix of both. 117 | 118 | If field is a name (string), it will be transformed to `json_view.GetInteger(name)`. 119 | 120 | Parameters 121 | ---------- 122 | settings : typing.Dict 123 | YAML parsed to dict 124 | 125 | Returns 126 | ------- 127 | str: 128 | String like "1, 3, json_view.GetInteger("width"), json_view.GetInteger("height")" 129 | """ 130 | return ", ".join( 131 | str(elem) 132 | if isinstance(elem, int) 133 | else 'json_view.GetInteger("{}")'.format(elem) 134 | for elem in settings["input"]["shape"] 135 | ) 136 | 137 | 138 | def aws_to_torch(settings, key: str) -> str: 139 | """ 140 | Impute libtorch specific type from user provided "human-readable" form. 141 | 142 | See `type_mapping` in source code for exact mapping. 143 | 144 | Parameters 145 | ---------- 146 | settings : typing.Dict 147 | YAML parsed to dict 148 | 149 | Returns 150 | ------- 151 | str: 152 | String specifying type, e.g. "torch::kFloat16" 153 | """ 154 | type_mapping = { 155 | "bool": "torch::kInt8", 156 | "int": "torch::kInt32", 157 | "long": "torch::kInt64", 158 | "double": "torch::kFloat64", 159 | } 160 | 161 | if settings["return"][key] is None: 162 | return "" 163 | 164 | return type_mapping[settings["return"][key]["type"].lower()] 165 | 166 | 167 | def torch_approximation(settings, key: str) -> str: 168 | """ 169 | PyTorch has no `bool` type in libtorch hence we approximate one. 170 | 171 | Each item will be later static_casted to appropriate type if needed, 172 | which is essentially a no-op for for already correct types. 173 | 174 | Usually only `bool` will be casted (eventually other "hard-types" if 175 | the architecture is specific). 176 | 177 | Parameters 178 | ---------- 179 | settings : typing.Dict 180 | YAML parsed to dict 181 | 182 | Returns 183 | ------- 184 | str: 185 | String specifying type, e.g. "int8_t" 186 | """ 187 | type_mapping = { 188 | "bool": "int8_t", 189 | "int": "int32_t", 190 | "long": "int64_t", 191 | "double": "double", 192 | } 193 | 194 | if settings["return"][key] is None: 195 | return "" 196 | 197 | return type_mapping[settings["return"][key]["type"].lower()] 198 | 199 | 200 | def operations_and_arguments(settings): 201 | """ 202 | If return->result specified get names of operations to apply on output tensor. 203 | 204 | Merges return->result->operations and return->result->arguments into 205 | single string to input. 206 | 207 | return ->result->operations is required. 208 | 209 | Names of operations or arguments isn't verified and it may result in compilation 210 | error if user specifies unavailable tensor function. 211 | 212 | Current weak point in design, check if it can be improved and "safer". 213 | 214 | Parameters 215 | ---------- 216 | settings : typing.Dict 217 | YAML parsed to dict 218 | 219 | Returns 220 | ------- 221 | str: 222 | string representation of number, e.g. "255.0" 223 | 224 | """ 225 | 226 | def _add_namespace(operation): 227 | return "torch::{}".format(operation) 228 | 229 | def _operation_with_arguments(operation, *values): 230 | return "{}({})".format( 231 | _add_namespace(operation), 232 | ",".join(map(str, [value for value in values if value])), 233 | ) 234 | 235 | def _no_arguments_multiple_operations(operations): 236 | output = _operation_with_arguments(operations[0], "output") 237 | for operation in operations[1:]: 238 | output = _operation_with_arguments(operation, output) 239 | return output 240 | 241 | def _wrap_in_list(value): 242 | if not isinstance(value, list): 243 | return [value] 244 | return value 245 | 246 | if settings["return"]["result"] is None: 247 | return "" 248 | 249 | if "code" in settings["return"]["result"]: 250 | return settings["return"]["result"]["code"] 251 | 252 | operations = settings["return"]["result"]["operations"] 253 | arguments = settings["return"]["result"]["arguments"] 254 | if arguments is None: 255 | if isinstance(operations, str): 256 | return "{}(output)".format(_add_namespace(operations)) 257 | return _no_arguments_multiple_operations(operations) 258 | 259 | operations, arguments = _wrap_in_list(operations), _wrap_in_list(arguments) 260 | output = _operation_with_arguments(operations[0], "output", arguments[0]) 261 | for operation, argument in itertools.zip_longest(operations[1:], arguments[1:]): 262 | output = _operation_with_arguments(operation, output, argument) 263 | return output 264 | 265 | 266 | def aws_function(settings, key: str, array: bool) -> str: 267 | """ 268 | Internal imputation specifying one of AWS SDK functions based on type. 269 | 270 | This function specifies which AWS SDK function will be used to create 271 | JSONValue (either as single item to return or as part of array to return). 272 | 273 | Looked for in return->output and return->result settings and returns one or both 274 | depending on which is specified. 275 | 276 | Parameters 277 | ---------- 278 | settings : typing.Dict 279 | YAML parsed to dict 280 | key : str 281 | Name of field to look for in type (either "output" or "result") 282 | array: bool 283 | Whether prefix should be tailored to array output (`As`) or item (`With`) 284 | 285 | Returns 286 | ------- 287 | str: 288 | "" or "As" or "With" AWS function name 289 | 290 | """ 291 | 292 | prefix = "As" if array else "With" 293 | type_mapping = { 294 | "int": "Integer", 295 | "long": "Int64", 296 | "double": "Double", 297 | } 298 | 299 | if settings["return"][key] is None: 300 | return "" 301 | return prefix + type_mapping[settings["return"][key]["type"].lower()] 302 | 303 | 304 | def field_if_exists(settings, key: str, name: str) -> str: 305 | """ 306 | Return value of nested fields if those are specified. 307 | 308 | Parameters 309 | ---------- 310 | settings : typing.Dict 311 | YAML parsed to dict 312 | key : str 313 | Name of field to look for in type (either "output" or "result") 314 | name: str 315 | Name of field to look for in one of "output" or "result" 316 | 317 | Returns 318 | ------- 319 | str: 320 | "" or value provided in field 321 | 322 | """ 323 | if settings["return"][key] is None: 324 | return "" 325 | return settings["return"][key][name] 326 | 327 | 328 | def model(settings) -> str: 329 | """ 330 | Return path to model specified by settings. 331 | 332 | Parameters 333 | ---------- 334 | settings : typing.Dict 335 | YAML parsed to dict 336 | 337 | Returns 338 | ------- 339 | str: 340 | "/path/to/model" 341 | 342 | """ 343 | return '"' + settings["model"] + '"' 344 | -------------------------------------------------------------------------------- /torchlambda/implementation/template.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import sys 3 | import typing 4 | 5 | import yaml 6 | 7 | from . import general, utils 8 | 9 | 10 | @general.message("reading YAML settings.") 11 | def read_settings(args) -> typing.Dict: 12 | """ 13 | Read user provided YAML settings. 14 | 15 | Parameters 16 | ---------- 17 | args : dict-like 18 | User provided arguments parsed by argparse.ArgumentParser instance. 19 | 20 | Returns 21 | ------- 22 | Dictionary containing settings to create template 23 | """ 24 | with open(args.yaml, "r") as file: 25 | try: 26 | return yaml.safe_load(file) 27 | except yaml.YAMLError as error: 28 | print("torchlambda:: Error during user settings parsing:") 29 | print(error) 30 | sys.exit(1) 31 | 32 | 33 | @general.message("validating YAML settings.") 34 | def validate(validator, settings: typing.Dict) -> None: 35 | """ 36 | Validate user provided YAML settings. 37 | 38 | Uses custom validator, see `utils.template/validator.py` for 39 | exact validation scheme. 40 | 41 | Parameters 42 | ---------- 43 | validator : cerberus.Validator 44 | Cerberus scheme validator 45 | settings : typing.Dict 46 | YAML parsed to dict 47 | 48 | """ 49 | if not validator.validate(settings, normalize=True): 50 | print("torchlambda:: Error during YAML validation:", file=sys.stderr) 51 | print(yaml.dump(validator.errors), file=sys.stderr) 52 | sys.exit(1) 53 | 54 | 55 | @general.message("creating .cpp source from YAML template.") 56 | def create_source(settings: typing.Dict) -> str: 57 | """ 58 | Create .cpp source code from template and settings 59 | 60 | Values are imputed using `format` to `main.cpp`. 61 | Each field is either a header (#define NAME , optional) 62 | or is directly imputed into source code. 63 | 64 | First case and all needed processing is located in `utils.template.header`, 65 | second case and all needed processing is located in `utils.template.imputation` 66 | 67 | `template/main.cpp` template file uses double curly brackets ({{}}) where normal 68 | C++ brackets are used in order to be compatible with `str.format`, see 69 | this: https://stackoverflow.com/questions/5466451/how-can-i-print-literal-curly-brace-characters-in-python-string-and-also-use-fo 70 | for some info. 71 | 72 | **DESCRIPTION OF HEADER FIELDS**: 73 | 74 | - STATIC - whether all shapes are static (e.g. no 75 | input dimension is dependent on value passed in request as field). 76 | True if `input_shape` only has integers (fixed input shape). 77 | 78 | - VALIDATE_INPUTS - whether fields provided in INPUT_SHAPE should be 79 | checked for correctness (they exist and are of integer type). 80 | Default: True 81 | 82 | - NO_GRAD - whether PyTorch's gradient should be disabled. 83 | Usually yes as AWS Lambda is mainly used for inference 84 | Default: True 85 | 86 | - CAST - to which type should the tensor be casted after creation from 87 | base64 encoded data (by default it's unsigned char which is rarely useful). 88 | Default: float 89 | 90 | - NORMALIZE - whether normalize input using NORMALIZE_MEANS and NORMALIZE_STDDEVS 91 | Used mainly for image inference. 92 | Default: False 93 | 94 | - DIVIDER - value by which input tensor will be divided after casting 95 | to CAST. 96 | Default: 255 (to bring `unsigned char` to `[0,1]` range, useful for image inference) 97 | 98 | - RETURN_OUTPUT - whether to return output as array. 99 | Exclusive with RETURN_OUTPUT_ITEM 100 | 101 | - RETURN_OUTPUT_ITEM - whether to return output as single item. 102 | Exclusive with RETURN_OUTPUT 103 | 104 | - RETURN_RESULT - whether to return result as array. 105 | Exclusive with RETURN_RESULT_ITEM 106 | 107 | - RETURN_RESULT_ITEM - whether to return output as single item. 108 | Exclusive with RETURN_RESULT 109 | 110 | **DESCRIPTION OF DIRECTLY IMPUTED FIELDS**: 111 | 112 | - DATA - Name of field providing data in request 113 | 114 | - FIELDS - Names (if any) of provided non-static fields 115 | 116 | - NORMALIZE_MEANS - Means to use for normalization (if any) 117 | 118 | - NORMALIZE_STDDEVS - Standard deviations to use for normalization (if any) 119 | 120 | - INPUTS - Input shapes used for reshape of tensor (including batch dimension) 121 | 122 | - IF RETURN_RESULT OR RETURN_RESULT_ITEM DEFINED: 123 | 124 | - OPERATIONS_AND_ARGUMENTS - Operations with arguments (if any) to apply over 125 | outputted tensor to create result. 126 | Mix of "operations" and "arguments" fields 127 | 128 | - IF RETURN_OUTPUT OR RETURN_OUTPUT_ITEM DEFINED: 129 | 130 | - AWS_OUTPUT_FUNCTION - Internal AWS SDK JsonValue function like WithInteger. 131 | Based on RETURN_OUTPUT_TYPE 132 | 133 | - IF RETURN_RESULT OR RETURN_RESULT_ITEM DEFINED: 134 | 135 | - AWS_RESULT_FUNCTION - Internal AWS SDK JsonValue function like WithInteger. 136 | Based on RETURN_RESULT_TYPE 137 | 138 | - IF RETURN_OUTPUT OR RETURN_OUTPUT_ITEM DEFINED: 139 | 140 | - OUTPUT_NAME - Key of returned JSON 141 | - OUTPUT_TYPE - Type of item of returned JSON 142 | 143 | - IF RETURN_RESULT OR RETURN_RESULT_ITEM DEFINED: 144 | 145 | - RESULT_NAME - Key of returned JSON 146 | - RESULT_TYPE - Type of item of returned JSON 147 | 148 | Parameters 149 | ---------- 150 | settings : typing.Dict 151 | YAML parsed to dict 152 | 153 | Returns 154 | ------- 155 | str: 156 | Source represented as string 157 | """ 158 | cwd = pathlib.Path(__file__).absolute().parent.parent 159 | with open(cwd / "templates/settings/main.cpp") as file: 160 | return file.read().format( 161 | # Top level defines 162 | STATIC=utils.template.header.static(settings), 163 | GRAD=utils.template.header.grad(settings), 164 | OPTIMIZE=utils.template.header.optimize(settings), 165 | VALIDATE_JSON=utils.template.header.validate_json(settings), 166 | BASE64=utils.template.header.base64(settings), 167 | VALIDATE_FIELD=utils.template.header.validate_field(settings), 168 | VALIDATE_SHAPE=utils.template.header.validate_shape(settings), 169 | NORMALIZE=utils.template.header.normalize(settings), 170 | CAST=utils.template.header.cast(settings), 171 | DIVIDE=utils.template.header.divide(settings), 172 | RETURN_OUTPUT=utils.template.header.return_output(settings), 173 | RETURN_OUTPUT_ITEM=utils.template.header.return_output_item(settings), 174 | RETURN_RESULT=utils.template.header.return_result(settings), 175 | RETURN_RESULT_ITEM=utils.template.header.return_result_item(settings), 176 | # Direct insertions 177 | DATA=utils.template.imputation.data(settings), 178 | FIELDS=utils.template.imputation.fields(settings), 179 | DATA_TYPE=utils.template.imputation.data_type(settings), 180 | DATA_FUNC=utils.template.imputation.data_func(settings), 181 | NORMALIZE_MEANS=utils.template.imputation.normalize(settings, key="means"), 182 | NORMALIZE_STDDEVS=utils.template.imputation.normalize( 183 | settings, key="stddevs" 184 | ), 185 | TORCH_DATA_TYPE=utils.template.imputation.torch_data_type(settings), 186 | INPUTS=utils.template.imputation.inputs(settings), 187 | OUTPUT_CAST=utils.template.imputation.aws_to_torch(settings, "output"), 188 | RESULT_CAST=utils.template.imputation.aws_to_torch(settings, "result"), 189 | OPERATIONS_AND_ARGUMENTS=utils.template.imputation.operations_and_arguments( 190 | settings 191 | ), 192 | AWS_OUTPUT_FUNCTION=utils.template.imputation.aws_function( 193 | settings, key="output", array=True, 194 | ), 195 | AWS_RESULT_FUNCTION=utils.template.imputation.aws_function( 196 | settings, key="result", array=True, 197 | ), 198 | AWS_OUTPUT_ITEM_FUNCTION=utils.template.imputation.aws_function( 199 | settings, key="output", array=False 200 | ), 201 | AWS_RESULT_ITEM_FUNCTION=utils.template.imputation.aws_function( 202 | settings, key="result", array=False 203 | ), 204 | TORCH_OUTPUT_TYPE=utils.template.imputation.torch_approximation( 205 | settings, key="output" 206 | ), 207 | OUTPUT_TYPE=utils.template.imputation.field_if_exists( 208 | settings, key="output", name="type" 209 | ), 210 | TORCH_RESULT_TYPE=utils.template.imputation.torch_approximation( 211 | settings, key="result" 212 | ), 213 | RESULT_TYPE=utils.template.imputation.field_if_exists( 214 | settings, key="result", name="type" 215 | ), 216 | OUTPUT_NAME=utils.template.imputation.field_if_exists( 217 | settings, key="output", name="name" 218 | ), 219 | RESULT_NAME=utils.template.imputation.field_if_exists( 220 | settings, key="result", name="name" 221 | ), 222 | MODEL_PATH=utils.template.imputation.model(settings), 223 | ) 224 | 225 | 226 | def save(source: str, args) -> None: 227 | """ 228 | Save created source file to specified destination. 229 | 230 | All needed folders will be created if necessary. 231 | 232 | Parameters 233 | ---------- 234 | source : str 235 | String representation of C++ deployment source code 236 | args : dict-like 237 | User provided arguments parsed by argparse.ArgumentParser instance. 238 | 239 | """ 240 | destination = pathlib.Path(args.destination).absolute() 241 | destination.mkdir(parents=True, exist_ok=True) 242 | with open(destination / "main.cpp", "w") as file: 243 | file.write(source) 244 | 245 | 246 | def create_template(args) -> None: 247 | """ 248 | Create and save template from user provided settings. 249 | 250 | Acts as an entrypoint if user specifies --yaml flag in `torchlambda scheme` 251 | command. 252 | 253 | Parameters 254 | ---------- 255 | args : dict-like 256 | User provided arguments parsed by argparse.ArgumentParser instance. 257 | 258 | """ 259 | settings = read_settings(args) 260 | validator = utils.template.validator.get() 261 | validate(validator, settings) 262 | settings = validator.normalized(settings) 263 | 264 | source = create_source(settings) 265 | save(source, args) 266 | -------------------------------------------------------------------------------- /BENCHMARKS.md: -------------------------------------------------------------------------------- 1 | # Performance matrix 2 | 3 | __You shouldn't base your settings on information provided in this section. Perform your 4 | own measurements, this section acts more as a rule of thumb__. 5 | 6 | Below matrices show latest performance measurements gathered from automated tests from last 7 | week. 8 | 9 | Around `3000` tests are performed daily which gives us around `30000` tests in total 10 | used for creation of this matrix 11 | 12 | __Each entry has the following format:__ 13 | 14 | ``` 15 | DurationType: Trait1 x Trait2 16 | ``` 17 | 18 | - __`DurationType`__: 19 | - `init` - how long it took to initialize Lambda function 20 | - `duration` - how long the request took 21 | - `billed` - same as `duration but rounded` to the next `100ms`. How much you would 22 | be charged for running this function. See how it translates to money on 23 | [AWS Lambda pricing](https://aws.amazon.com/lambda/pricing/) documentation 24 | - __`Trait1`__ - setting value by which analysis was taken. For `grad` it could 25 | be `True` and `False`. It could be `model_name` (one of five tested `torchvision` neural networks). 26 | - __`Trait2`__ - same as `Trait1` but different axis 27 | 28 | _See concrete comparisons below_ 29 | 30 | 31 | 32 | 33 | ## init: grad x type 34 | 35 | | | base64 | byte | char | short | int | long | float | double | 36 | |:------|---------:|--------:|--------:|--------:|--------:|--------:|--------:|---------:| 37 | | True | 262.794 | 268.382 | 272.844 | 272.637 | 270.627 | 272.486 | 279.914 | 283.939 | 38 | | False | 264.209 | 269.987 | 271.258 | 271.682 | 277.495 | 275.894 | 285.3 | 279.747 | 39 | 40 | ## init: grad x payload 41 | 42 | | | 128x128 | 256x256 | 512x512 | 1024x1024 | 43 | |:------|----------:|----------:|----------:|------------:| 44 | | True | 254.866 | 259.86 | 269.602 | 307.477 | 45 | | False | 258.506 | 261.383 | 272.267 | 306.083 | 46 | 47 | ## init: grad x model_name 48 | 49 | | | shufflenet_v2_x1_0 | resnet18 | mobilenet_v2 | mnasnet1_0 | mnasnet1_3 | 50 | |:------|---------------------:|-----------:|---------------:|-------------:|-------------:| 51 | | True | 225.229 | 297.9 | 271.317 | 278.306 | 293.555 | 52 | | False | 226.373 | 296.043 | 271.768 | 278.985 | 298.575 | 53 | 54 | ## init: type x payload 55 | 56 | | | 128x128 | 256x256 | 512x512 | 1024x1024 | 57 | |:-------|----------:|----------:|----------:|------------:| 58 | | base64 | 257.167 | 257.679 | 258.895 | 281.198 | 59 | | byte | 259.657 | 249.893 | 267.662 | 298.845 | 60 | | char | 255.026 | 259.244 | 270.006 | 304.967 | 61 | | short | 260.231 | 261.807 | 266.182 | 300.049 | 62 | | int | 257.553 | 257.348 | 274.804 | 307.239 | 63 | | long | 250.143 | 281.303 | 268.109 | 300.064 | 64 | | float | 257.498 | 261.89 | 280.604 | 329.155 | 65 | | double | 256.516 | 257.542 | 282.262 | 332.019 | 66 | 67 | ## init: type x model_name 68 | 69 | | | shufflenet_v2_x1_0 | resnet18 | mobilenet_v2 | mnasnet1_0 | mnasnet1_3 | 70 | |:-------|---------------------:|-----------:|---------------:|-------------:|-------------:| 71 | | base64 | 212.903 | 288.637 | 263.232 | 265.673 | 289.939 | 72 | | byte | 221.264 | 286.998 | 272.218 | 277.232 | 290.041 | 73 | | char | 222.558 | 290.604 | 262.26 | 282.246 | 303.873 | 74 | | short | 221.904 | 307.45 | 271.704 | 272.425 | 290.27 | 75 | | int | 225.391 | 305.489 | 275.713 | 273.626 | 291.12 | 76 | | long | 230.288 | 294.743 | 272.685 | 286.979 | 286.703 | 77 | | float | 235.465 | 300.092 | 274.945 | 289.429 | 311.165 | 78 | | double | 237.995 | 302.315 | 279.179 | 282.466 | 305.656 | 79 | 80 | ## init: payload x model_name 81 | 82 | | | shufflenet_v2_x1_0 | resnet18 | mobilenet_v2 | mnasnet1_0 | mnasnet1_3 | 83 | |:----------|---------------------:|-----------:|---------------:|-------------:|-------------:| 84 | | 128x128 | 205.434 | 281.052 | 253.675 | 264.836 | 279.517 | 85 | | 256x256 | 214.859 | 284.189 | 261.805 | 257.333 | 284.805 | 86 | | 512x512 | 228.727 | 294.48 | 260.142 | 277.278 | 295.323 | 87 | | 1024x1024 | 254.235 | 327.505 | 312.312 | 316.925 | 324.586 | 88 | 89 | ## duration: grad x type 90 | 91 | | | base64 | byte | char | short | int | long | float | double | 92 | |:------|---------:|--------:|--------:|--------:|--------:|--------:|--------:|---------:| 93 | | True | 432.529 | 696.713 | 710.694 | 680.162 | 715.483 | 691.227 | 709.192 | 743.032 | 94 | | False | 345.182 | 601.445 | 609.604 | 617.736 | 601.549 | 612.657 | 639.977 | 597.784 | 95 | 96 | ## duration: grad x payload 97 | 98 | | | 128x128 | 256x256 | 512x512 | 1024x1024 | 99 | |:------|----------:|----------:|----------:|------------:| 100 | | True | 142.531 | 221.485 | 522.29 | 1803.21 | 101 | | False | 120.025 | 193.554 | 453.129 | 1560.27 | 102 | 103 | ## duration: grad x model_name 104 | 105 | | | shufflenet_v2_x1_0 | resnet18 | mobilenet_v2 | mnasnet1_0 | mnasnet1_3 | 106 | |:------|---------------------:|-----------:|---------------:|-------------:|-------------:| 107 | | True | 468.485 | 556.825 | 723.169 | 719.833 | 888.704 | 108 | | False | 414.174 | 509.882 | 637.426 | 594.925 | 731.131 | 109 | 110 | ## duration: type x payload 111 | 112 | | | 128x128 | 256x256 | 512x512 | 1024x1024 | 113 | |:-------|----------:|----------:|----------:|------------:| 114 | | base64 | 120.622 | 165.184 | 311.129 | 995.249 | 115 | | byte | 133.315 | 203.628 | 498.391 | 1738.97 | 116 | | char | 128.331 | 209.306 | 517.482 | 1822.56 | 117 | | short | 135.859 | 207.389 | 497.818 | 1740.91 | 118 | | int | 133.42 | 216.163 | 519.502 | 1783.02 | 119 | | long | 126.979 | 228.497 | 516.98 | 1760.93 | 120 | | float | 135.825 | 223.045 | 515.245 | 1802.25 | 121 | | double | 137.281 | 209.267 | 536.959 | 1811.83 | 122 | 123 | ## duration: type x model_name 124 | 125 | | | shufflenet_v2_x1_0 | resnet18 | mobilenet_v2 | mnasnet1_0 | mnasnet1_3 | 126 | |:-------|---------------------:|-----------:|---------------:|-------------:|-------------:| 127 | | base64 | 219.756 | 278.199 | 442.88 | 437.775 | 571.823 | 128 | | byte | 449.97 | 571.625 | 679.552 | 695.264 | 865.326 | 129 | | char | 450.345 | 562.572 | 692.297 | 722.42 | 884.605 | 130 | | short | 462.636 | 600.807 | 706.776 | 660.66 | 815.31 | 131 | | int | 473.651 | 592.709 | 725.122 | 712.483 | 790.561 | 132 | | long | 482.671 | 546.883 | 741.208 | 648.576 | 827.792 | 133 | | float | 508.803 | 557.018 | 710.671 | 712.434 | 879.32 | 134 | | double | 500.988 | 556.12 | 754.763 | 682.386 | 850.336 | 135 | 136 | ## duration: payload x model_name 137 | 138 | | | shufflenet_v2_x1_0 | resnet18 | mobilenet_v2 | mnasnet1_0 | mnasnet1_3 | 139 | |:----------|---------------------:|-----------:|---------------:|-------------:|-------------:| 140 | | 128x128 | 111.577 | 88.7494 | 150.38 | 143.482 | 163.189 | 141 | | 256x256 | 168.288 | 153.559 | 231.17 | 218.247 | 263.759 | 142 | | 512x512 | 366.953 | 408.622 | 504.222 | 527.545 | 629.891 | 143 | | 1024x1024 | 1113.89 | 1459.65 | 1890.06 | 1798.75 | 2174.21 | 144 | 145 | ## billed: grad x type 146 | 147 | | | base64 | byte | char | short | int | long | float | double | 148 | |:------|---------:|--------:|--------:|--------:|--------:|--------:|--------:|---------:| 149 | | True | 466.967 | 731.787 | 745.211 | 714.972 | 750.411 | 726.061 | 743.148 | 777.318 | 150 | | False | 379.825 | 636.793 | 643.841 | 651.828 | 636.077 | 647.913 | 675.096 | 632.645 | 151 | 152 | ## billed: grad x payload 153 | 154 | | | 128x128 | 256x256 | 512x512 | 1024x1024 | 155 | |:------|----------:|----------:|----------:|------------:| 156 | | True | 179.779 | 254.078 | 556.66 | 1837.33 | 157 | | False | 152.366 | 229.676 | 489.205 | 1594.87 | 158 | 159 | ## billed: grad x model_name 160 | 161 | | | shufflenet_v2_x1_0 | resnet18 | mobilenet_v2 | mnasnet1_0 | mnasnet1_3 | 162 | |:------|---------------------:|-----------:|---------------:|-------------:|-------------:| 163 | | True | 496.569 | 595.265 | 759.582 | 757.506 | 921.477 | 164 | | False | 449.249 | 550.232 | 670.066 | 626.155 | 765.432 | 165 | 166 | ## billed: type x payload 167 | 168 | | | 128x128 | 256x256 | 512x512 | 1024x1024 | 169 | |:-------|----------:|----------:|----------:|------------:| 170 | | base64 | 157.573 | 197.474 | 344.856 | 1030.27 | 171 | | byte | 169.224 | 238.06 | 533.519 | 1774.34 | 172 | | char | 162.523 | 243.762 | 553.036 | 1855.85 | 173 | | short | 170.518 | 242.243 | 532.019 | 1775 | 174 | | int | 167.975 | 249.366 | 555.378 | 1818.43 | 175 | | long | 160.381 | 266.062 | 552.608 | 1794.81 | 176 | | float | 170.45 | 257.971 | 549.952 | 1836.1 | 177 | | double | 171.375 | 242.654 | 573.975 | 1845.74 | 178 | 179 | ## billed: type x model_name 180 | 181 | | | shufflenet_v2_x1_0 | resnet18 | mobilenet_v2 | mnasnet1_0 | mnasnet1_3 | 182 | |:-------|---------------------:|-----------:|---------------:|-------------:|-------------:| 183 | | base64 | 249.296 | 314.235 | 479.132 | 472.509 | 608.277 | 184 | | byte | 481.623 | 610.922 | 714.977 | 730.62 | 899.642 | 185 | | char | 480.69 | 601.661 | 725.461 | 757.319 | 919.109 | 186 | | short | 492.568 | 642.494 | 742.809 | 693.365 | 847.569 | 187 | | int | 506.9 | 634.132 | 758.885 | 745.333 | 823.212 | 188 | | long | 515.759 | 587.212 | 775.285 | 684.278 | 860.044 | 189 | | float | 541.204 | 594.824 | 744.811 | 748.252 | 911.824 | 190 | | double | 533.217 | 595.767 | 788.475 | 716.133 | 883.799 | 191 | 192 | ## billed: payload x model_name 193 | 194 | | | shufflenet_v2_x1_0 | resnet18 | mobilenet_v2 | mnasnet1_0 | mnasnet1_3 | 195 | |:----------|---------------------:|-----------:|---------------:|-------------:|-------------:| 196 | | 128x128 | 136.208 | 136.215 | 185.992 | 178.347 | 194.905 | 197 | | 256x256 | 201.749 | 193.313 | 264.962 | 249.489 | 297.343 | 198 | | 512x512 | 402.269 | 444.255 | 538.008 | 563.571 | 665.273 | 199 | | 1024x1024 | 1146.93 | 1494.11 | 1925.24 | 1834.52 | 2207.75 | 200 | 201 | _This file was auto-generated on 2020-05-30-11:22:06 based on 34415 tests_ -------------------------------------------------------------------------------- /torchlambda/arguments/subparsers.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | 4 | def settings(subparsers) -> None: 5 | """Create YAML settings to use with `torchlambda template --yaml`.""" 6 | 7 | description = "Create YAML settings to use with `torchlambda template --yaml`\n" 8 | "This is the easiest way to deploy model, just modify default settings provided by this comand.\n" 9 | "Not all provided fields are required, please see: https://github.com/szymonmaszke/torchlambda/wiki/Commands for more information." 10 | 11 | parser = subparsers.add_parser( 12 | "settings", 13 | description=description, 14 | help=description, 15 | formatter_class=argparse.RawTextHelpFormatter, 16 | ) 17 | 18 | parser.add_argument( 19 | "--destination", 20 | required=False, 21 | default="./torchlambda.yaml", 22 | help="""Path to file where YAML settings will be stored. Default: "./torchlambda.yaml" """, 23 | ) 24 | 25 | 26 | def template(subparsers) -> None: 27 | """Create C++ source code template used for model inference.""" 28 | 29 | description = ( 30 | "Create C++ deployment code scheme with AWS Lambda C++ SDK and PyTorch.\n" 31 | "In general users are advised to stick to YAML settings (--yaml) flag.\n" 32 | "Not all YAML fields are required, please see: https://github.com/szymonmaszke/torchlambda/wiki/Commands for more information.\n" 33 | "If --yaml unspecified, generate C++ code which one can use as a starting point for custom use cases." 34 | ) 35 | 36 | parser = subparsers.add_parser( 37 | "template", 38 | description=description, 39 | help=description, 40 | formatter_class=argparse.RawTextHelpFormatter, 41 | ) 42 | 43 | parser.add_argument( 44 | "--yaml", 45 | required=False, 46 | default=None, 47 | help="Path to YAML settings file from which code will be generated.\n" 48 | "See torchlambda settings and comments in generated YAML for more info.", 49 | ) 50 | 51 | parser.add_argument( 52 | "--destination", 53 | required=False, 54 | default="./torchlambda", 55 | help="""Path to folder where C++ deployment files will be created. Default: "./torchlambda" """, 56 | ) 57 | 58 | 59 | def build(subparsers) -> None: 60 | """Perform deployment of PyTorch C++ code to AWS Lambda.""" 61 | description = "Obtain AWS Lambda ready .zip package from C++ deployment source code and Torchscript compiled model.\n" 62 | "See following resources for more information:\n" 63 | "- Torchscript documentation: https://pytorch.org/docs/stable/jit.html\n" 64 | "- AWS Lambda Getting Started: https://aws.amazon.com/lambda/getting-started/\n" 65 | "- AWS SDK for C++: https://aws.amazon.com/sdk-for-cpp/" 66 | parser = subparsers.add_parser( 67 | "build", 68 | formatter_class=argparse.RawTextHelpFormatter, 69 | description=description, 70 | help=description, 71 | ) 72 | 73 | parser.add_argument( 74 | "source", 75 | help="Directory containing source files (possibly multiple) with C++ deployment code.\n" 76 | "cmake will build all files in this folder matching *.cpp, *.c, *.h or *.hpp extensions.\n" 77 | "If you are unsure how to create one, please use torchlambda scheme command.", 78 | ) 79 | 80 | parser.add_argument( 81 | "--destination", 82 | default="./torchlambda.zip", 83 | required=False, 84 | help="Path where created AWS Lambda deployment .zip package will be stored.\n" 85 | "Should be a filename ending with .zip extension", 86 | ) 87 | 88 | parser.add_argument( 89 | "--compilation", 90 | required=False, 91 | help="""Compilation flags used for inference source code.\n""" 92 | """Should be provided as string, e.g. "-Wall -Werror -O3".\n""" 93 | "By default no flags are passed to CMake targets.\n" 94 | 'If you want to pass a single flag you should add space after string, e.g. "-Wall "\n' 95 | "Command line defaults pass none additional flags as well.\n" 96 | "User might want to specify -Os for smaller size or -O2 for possibly increased performance.\n" 97 | "IMPORTANT: Due to linker flags smaller binary size may not be possible at the moment.", 98 | ) 99 | 100 | parser.add_argument( 101 | "--operations", 102 | required=False, 103 | help="Path containing exported model operations in .yaml format.\n" 104 | "See: https://pytorch.org/mobile/ios/#custom-build for more information.\n" 105 | "If specified, custom image will be build from scratch.\n" 106 | "Default: None (no operations)", 107 | ) 108 | 109 | parser.add_argument( 110 | "--pytorch", 111 | nargs="+", 112 | required=False, 113 | default=[], 114 | help="PyTorch's libtorch build flags.\n" 115 | "See PyTorch's CMakeLists.txt for all available flags: https://github.com/pytorch/pytorch/blob/master/CMakeLists.txt\n" 116 | "If specified, custom image will be build from scratch.\n" 117 | "Default build parameters used:\n" 118 | "-DBUILD_PYTHON=OFF\n" 119 | "-DUSE_MPI=OFF\n" 120 | "-DUSE_NUMPY=OFF\n" 121 | "-DUSE_ROCM=OFF\n" 122 | "-DUSE_NCCL=OFF\n" 123 | "-DUSE_NUMA=OFF\n" 124 | "-DUSE_MKLDNN=OFF\n" 125 | "-DUSE_GLOO=OFF\n" 126 | "-DUSE_OPENMP=OFF\n" 127 | "User can override defaults by providing multiple arguments WITHOUT -D, e.g. \n" 128 | "--pytorch USE_NUMPY=ON USE_OPENMP=ON\n" 129 | "Default additional command line options: None", 130 | ) 131 | 132 | parser.add_argument( 133 | "--pytorch-version", 134 | required=False, 135 | default=None, 136 | help="Commit or tag to which PyTorch will be set during build.\n" 137 | "See available releases at: https://github.com/pytorch/pytorch/releases (but any commit can be used)\n" 138 | 'Special value "None" allowed which leaves PyTorch at current head on master.\n' 139 | "If specified, custom image will be build from scratch.\n" 140 | "Default: latest ", 141 | ) 142 | 143 | parser.add_argument( 144 | "--aws", 145 | nargs="+", 146 | required=False, 147 | default=[], 148 | help="AWS C++ SDK build flags customizing dependency build.\n" 149 | "See: https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/cmake-params.html#cmake-build-only for more information.\n" 150 | "If specified, custom image will be built from scratch.\n" 151 | "Default build parameters used:\n" 152 | "-DBUILD_SHARED_LIBS=OFF (cannot be overriden)\n" 153 | "-DENABLE_UNITY_BUILD=ON (usually shouldn't be overriden)\n" 154 | "-DCUSTOM_MEMORY_MANAGEMENT=OFF\n" 155 | "-DCPP_STANDARD=17\n" 156 | "User can override defaults by providing multiple arguments WITHOUT -D, e.g. \n" 157 | "--aws CPP_STANDARD=11 CUSTOM_MEMORY_MANAGEMENT=ON\n" 158 | "`-DBUILD_ONLY` flag SHOULD NOT BE USED HERE, specify --aws-components instead\n" 159 | "Default additional command line options: None", 160 | ) 161 | 162 | parser.add_argument( 163 | "--aws-components", 164 | nargs="+", 165 | default=[], 166 | required=False, 167 | help="Components of AWS C++ SDK to build.\n" 168 | "If specified, custom image will be built from scratch.\n" 169 | "Acts as `-DBUILD_ONLY` during build, please see https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/cmake-params.html#cmake-build-only.\n" 170 | "Pass components as space separated arguments, e.g.\n" 171 | "--aws-components s3 dynamodb\n" 172 | "By default only core will be build." 173 | "Default additional command line options: None", 174 | ) 175 | 176 | parser.add_argument( 177 | "--image", 178 | required=False, 179 | default="torchlambda:custom", 180 | help="Name of Docker image to use for code building.\n" 181 | "If provided name image exists on localhost it will be used for `docker run` command.\n" 182 | "Otherwise AND IF custom build specified (either of --operations, --pytorch, --aws, --components or --build) image will be build from scratch.\n" 183 | "Otherwise prebuilt image szymonmaszke/torchlambda:latest will be downloaded and used.\n" 184 | "Default: torchlambda:custom", 185 | ) 186 | 187 | parser.add_argument( 188 | "--docker", 189 | required=False, 190 | help='Flags passed to "docker" command during build and run.\n' 191 | 'If you want to pass a single flag you should add space after string, e.g. "--debug "\n' 192 | """Flags should be passed as space separated string, e.g. "--debug --log-level debug".\nDefault: None""", 193 | ) 194 | 195 | parser.add_argument( 196 | "--docker-build", 197 | required=False, 198 | help="Flags passed to docker build command (custom image building).\n" 199 | """Flags should be passed as space separated string, e.g. "--compress --no-cache".\n""" 200 | 'If you want to pass a single flag you should add space after string, e.g. "--compress "\n' 201 | "`-t` flag SHOULD NOT BE USED, specify --image instead.\n" 202 | "Default: None", 203 | ) 204 | 205 | parser.add_argument( 206 | "--docker-run", 207 | required=False, 208 | help="Flags passed to docker run command (code deployment building).\n" 209 | "Flags should be passed as space separated string, e.g.\n" 210 | """--name deployment --mount source=myvol2,target=/home/app".\n""" 211 | 'If you want to pass a single flag you should add space after string, e.g. "--name my_name"\n' 212 | "Default: None", 213 | ) 214 | 215 | parser.add_argument( 216 | "--no-run", 217 | required=False, 218 | action="store_true", 219 | help="Do not run compilation of source code part.\n" 220 | "Can be used to only create Docker building image to be run later." 221 | "Default: False", 222 | ) 223 | 224 | 225 | def layer(subparsers) -> None: 226 | """Pack model as .zip file ready to deploy on AWS Lambda as layer.""" 227 | description = "Pack model as .zip file ready to deploy on AWS Lambda as layer." 228 | parser = subparsers.add_parser( 229 | "layer", 230 | description=description, 231 | help=description, 232 | formatter_class=argparse.RawTextHelpFormatter, 233 | ) 234 | 235 | parser.add_argument( 236 | "source", 237 | help="Path pointing to TorchScript compiled model.\n" 238 | "For more information check introduction to TorchScript:\n" 239 | "https://pytorch.org/tutorials/beginner/Intro_to_TorchScript_tutorial.html.", 240 | ) 241 | 242 | parser.add_argument( 243 | "--destination", 244 | required=False, 245 | default="./model.zip", 246 | help="Path where AWS Lambda layer containing model will be stored.\n" 247 | """Default: "./model.zip" """, 248 | ) 249 | 250 | parser.add_argument( 251 | "--directory", 252 | required=False, 253 | default=None, 254 | help="Directory where model will be stored. Usually you don't want to change that.\n" 255 | "Model will be unpacked to /opt/your/specified/directory and needs to be set analogously in C++/YAML settings.\n" 256 | "Default: None (model will be placed in /opt)", 257 | ) 258 | 259 | parser.add_argument( 260 | "--compression", 261 | required=False, 262 | default="STORED", 263 | choices=["STORED", "DEFLATED", "BZIP2", "LZMA"], 264 | type=str.upper, 265 | help="""Compression method used for model compression.\n""" 266 | "See: https://docs.python.org/3/library/zipfile.html#zipfile.ZIP_STORED for more information.\n" 267 | "IMPORTANT: It's best to use default (uncompressed archive stored in.zip).\n" 268 | "Model .ptc file will barely be compressed by any algorithm, while it may increase decompression speed on AWS Lambda.\n" 269 | "If you wish to compress your model please use quantization (https://pytorch.org/docs/stable/quantization.html) or related techniques instead.\n" 270 | """Default: "STORED" """, 271 | ) 272 | 273 | parser.add_argument( 274 | "--compression-level", 275 | required=False, 276 | default=None, 277 | choices=list(range(10)), 278 | type=int, 279 | help="""Level of compression used.\n""" 280 | "See: https://docs.python.org/3/library/zipfile.html#zipfile-objects for more information.\n" 281 | """Default: None (default for specified --compression) """, 282 | ) 283 | --------------------------------------------------------------------------------