├── scripts ├── linter │ ├── adapters │ │ ├── __init__.py │ │ ├── README.md │ │ ├── s3_init_config.json │ │ ├── exec_linter.py │ │ ├── pip_init.py │ │ ├── shellcheck_linter.py │ │ ├── cmake_linter.py │ │ ├── ufmt_linter.py │ │ ├── actionlint_linter.py │ │ ├── testowners_linter.py │ │ ├── newlines_linter.py │ │ └── mypy_linter.py │ └── clang_tidy │ │ ├── __init__.py │ │ └── generate_build_files.py ├── runtime_debugger.py ├── lint.sh └── auto_copyright.py ├── requirements.txt ├── compat-requirements.txt ├── dev-requirements.txt ├── multipy ├── runtime │ ├── example │ │ ├── simple.pt │ │ ├── fx │ │ │ ├── some_dependency.py │ │ │ └── examples.py │ │ ├── gpu_wrapper.py │ │ ├── tensorrt_example.py │ │ └── generate_examples.py │ ├── interpreter │ │ ├── third_party │ │ │ └── README.md │ │ ├── move_embedded_data_sections_implicit_linker_script.txt │ │ ├── import_find_sharedfuncptr.h │ │ ├── hide_symbols.script │ │ ├── register_pyyaml.cpp │ │ ├── plugin_torch.cpp │ │ ├── plugin_registry.h │ │ ├── plugin_registry.cpp │ │ ├── register_numpy.cpp │ │ ├── test_builtin_registry.cpp │ │ ├── import_find_sharedfuncptr.cpp │ │ ├── register_frozenpython.cpp │ │ ├── CMakeLists.txt │ │ └── builtin_registry.h │ ├── unity │ │ ├── tests │ │ │ ├── sum.py │ │ │ ├── test_unity.h │ │ │ ├── simple_model.py │ │ │ ├── test_unity_sum.cpp │ │ │ └── test_unity_simple_model.cpp │ │ ├── example.py │ │ ├── xar_environment.h │ │ └── main.cpp │ ├── path_environment.cpp │ ├── noop_environment.h │ ├── testdev │ │ └── test_deploy_from_python.py │ ├── test_deploy_missing_interpreter.cpp │ ├── test_deploy_python_ext.cpp │ ├── path_environment.h │ ├── embedded_file.h │ ├── __init__.py │ ├── test_deploy_imports.cpp │ ├── test_deploy_python.py │ ├── test_pybind.py │ ├── interactive_embedded_interpreter.cpp │ ├── test_compat.py │ ├── loader.h │ ├── README.md │ ├── pybind_init.cpp │ ├── Exception.h │ ├── mem_file.h │ ├── elf_file.h │ ├── embedded_file.cpp │ ├── remove_dt_needed.cpp │ ├── test_deploy_lib.cpp │ ├── elf_file.cpp │ ├── test_deploy_gpu.cpp │ ├── environment.h │ └── utils.cmake ├── utils │ ├── __init__.py │ └── _deploy.py ├── __init__.py └── version.py ├── .github ├── pyproject.toml ├── actions │ └── setup-ssh │ │ └── action.yml └── workflows │ ├── runtime_tests.yaml │ ├── runtime_nightly.yaml │ ├── lint.yaml │ ├── docs-build.yaml │ ├── test_docker_build.yml │ ├── build_test_release.yaml │ └── install_test.yaml ├── .gitignore ├── .gitmodules ├── docs ├── requirements.txt ├── source │ ├── tutorials │ │ ├── tutorial_root.rst │ │ ├── create_movable.rst │ │ ├── hello_world.rst │ │ └── quickstart.rst │ ├── index.rst │ └── Doxyfile ├── Makefile └── doc_push.sh ├── examples ├── quickstart │ ├── gen_package.py │ └── quickstart.cpp ├── hello_world │ └── hello_world_example.cpp ├── CMakeLists.txt └── movable_example │ └── movable_example.cpp ├── .flake8 ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE.md ├── .clang-format ├── CODE_OF_CONDUCT.md └── Dockerfile /scripts/linter/adapters/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scripts/linter/clang_tidy/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | dataclasses 2 | expecttest 3 | PyYAML==6.0 4 | numpy 5 | -------------------------------------------------------------------------------- /compat-requirements.txt: -------------------------------------------------------------------------------- 1 | tokenizers 2 | git+https://github.com/facebookresearch/pytorch3d.git 3 | -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | coverage 2 | codecov 3 | black==22.3.0 4 | usort==1.0.2 5 | flake8==3.9.0 6 | expecttest 7 | -------------------------------------------------------------------------------- /multipy/runtime/example/simple.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pytorch/multipy/HEAD/multipy/runtime/example/simple.pt -------------------------------------------------------------------------------- /.github/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["torch >= 1.11.0", "setuptools < 60.0"] 3 | 4 | [tool.usort] 5 | first_party_detection = false 6 | -------------------------------------------------------------------------------- /multipy/runtime/interpreter/third_party/README.md: -------------------------------------------------------------------------------- 1 | Python libraries that we want to package along with the Python implementation 2 | bundled in libinterpreter. 3 | -------------------------------------------------------------------------------- /multipy/runtime/interpreter/move_embedded_data_sections_implicit_linker_script.txt: -------------------------------------------------------------------------------- 1 | SECTIONS { 2 | .data_embedded_interpreter : { 3 | */embedded_interpreter_*.a:embedded_interpreter_*.o(.data) 4 | } 5 | } INSERT AFTER .bss 6 | -------------------------------------------------------------------------------- /multipy/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | -------------------------------------------------------------------------------- /multipy/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | from .version import __version__ # noqa F401 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .git 2 | .git/** 3 | **/__pycache__/** 4 | **.coverage 5 | **/build/** 6 | **/CMakeFiles/** 7 | multipy/runtime/example/generated/ 8 | examples/**/*.pt 9 | *.egg-info 10 | .DS_Store 11 | **/.DS_Store/** 12 | docs/source/api/ 13 | docs/src/ 14 | dist/ 15 | .lintbin 16 | .lintbin/** 17 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "multipy/runtime/third-party/fmt"] 2 | path = multipy/runtime/third-party/fmt 3 | url = https://github.com/fmtlib/fmt.git 4 | [submodule "multipy/runtime/third-party/pybind11"] 5 | path = multipy/runtime/third-party/pybind11 6 | url = https://github.com/pybind/pybind11.git 7 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx 2 | sphinx-autobuild 3 | -e git+http://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme 4 | sphinx-gallery<=0.7.0 5 | sphinxcontrib.katex 6 | matplotlib 7 | papermill 8 | jinja2<=3.0.3 9 | breathe 10 | exhale 11 | ipython_genutils 12 | ipykernel 13 | -------------------------------------------------------------------------------- /scripts/linter/adapters/README.md: -------------------------------------------------------------------------------- 1 | # lintrunner adapters 2 | 3 | These files adapt our various linters to work with `lintrunner`. 4 | 5 | ## Adding a new linter 6 | 1. init and linter 7 | 2. {{DRYRUN}} and {{PATHSFILE}} 8 | 3. never exit uncleanly 9 | 4. Communication protocol 10 | 5. Self-contained 11 | -------------------------------------------------------------------------------- /multipy/runtime/interpreter/import_find_sharedfuncptr.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | void loadSearchFile(const char* pathname); 8 | -------------------------------------------------------------------------------- /multipy/runtime/interpreter/hide_symbols.script: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | INTERPRETER_0.1 { 8 | global: newInterpreterImpl; 9 | local: *; 10 | }; 11 | -------------------------------------------------------------------------------- /multipy/runtime/example/fx/some_dependency.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # dependency for torch package 8 | 9 | 10 | def a_non_torch_leaf(a: int, b): 11 | return a * b 12 | -------------------------------------------------------------------------------- /docs/source/tutorials/tutorial_root.rst: -------------------------------------------------------------------------------- 1 | :github_url: https://github.com/pytorch/multipy 2 | 3 | ======================= 4 | torch::deploy tutorials 5 | ======================= 6 | 7 | 8 | 9 | Welcome to the tutorials for torch::deploy. 10 | 11 | 12 | .. toctree:: 13 | :maxdepth: 2 14 | :caption: Tutorials 15 | 16 | quickstart.rst 17 | hello_world.rst 18 | create_movable.rst 19 | -------------------------------------------------------------------------------- /multipy/runtime/unity/tests/sum.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | 10 | def func(*vlist): 11 | return sum(vlist) 12 | 13 | 14 | import sys 15 | 16 | print("byebye!", file=sys.stderr) 17 | -------------------------------------------------------------------------------- /multipy/runtime/unity/tests/test_unity.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #pragma once 8 | 9 | // NOLINTNEXTLINE 10 | static char TEST_PYTHON_APP_DIR_TEMP[] = 11 | "/tmp/torch_deploy_unity_unittest_XXXXXX"; 12 | -------------------------------------------------------------------------------- /.github/actions/setup-ssh/action.yml: -------------------------------------------------------------------------------- 1 | name: Setup SSH 2 | 3 | description: Adds ssh keys for current user to machine 4 | 5 | inputs: 6 | github-secret: 7 | description: GitHub token 8 | required: true 9 | 10 | runs: 11 | using: composite 12 | steps: 13 | - name: "Enable SSH (Click me for login details)" 14 | uses: seemethere/add-github-ssh-key@v1 15 | with: 16 | GITHUB_TOKEN: ${{ inputs.github-secret }} 17 | activate-with-label: false 18 | -------------------------------------------------------------------------------- /multipy/runtime/interpreter/register_pyyaml.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | #include 9 | 10 | extern "C" struct _frozen _PyImport_FrozenModules_pyyaml[]; 11 | 12 | REGISTER_TORCH_DEPLOY_BUILTIN(frozen_pyyaml, _PyImport_FrozenModules_pyyaml); 13 | -------------------------------------------------------------------------------- /examples/quickstart/gen_package.py: -------------------------------------------------------------------------------- 1 | # An example usage of `torch.package` which is 2 | # used for our quickstart example. 3 | 4 | import torchvision 5 | from torch.package import PackageExporter 6 | 7 | # Instantiate some model 8 | model = torchvision.models.resnet.resnet18() 9 | 10 | # Package and export it. 11 | with PackageExporter("my_package.pt") as e: 12 | e.intern("torchvision.**") 13 | e.extern("numpy.**") 14 | e.extern("sys") 15 | e.extern("PIL.*") 16 | e.extern("typing_extensions") 17 | e.save_pickle("model", "model.pkl", model) 18 | -------------------------------------------------------------------------------- /multipy/version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under the BSD-style license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | # Follows PEP-0440 version scheme guidelines 9 | # https://www.python.org/dev/peps/pep-0440/#version-scheme 10 | # 11 | # Examples: 12 | # 0.1.0.devN # Developmental release 13 | # 0.1.0aN # Alpha release 14 | # 0.1.0bN # Beta release 15 | # 0.1.0rcN # Release Candidate 16 | # 0.1.0 # Final release 17 | __version__ = "0.2.0.dev0" 18 | -------------------------------------------------------------------------------- /multipy/runtime/unity/example.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import numpy as np 10 | import scipy 11 | from scipy import linalg 12 | 13 | print("Hello, torch::deploy unity!") 14 | print(f"np.random.rand(5): {np.random.rand(5)}") 15 | print(f"scipy {scipy}") 16 | mat_a = np.array([[1, 0, 0, 0], [1, 1, 0, 0], [1, 2, 1, 0], [1, 3, 3, 1]]) 17 | mat_b = linalg.inv(mat_a) 18 | print(mat_b) 19 | -------------------------------------------------------------------------------- /.github/workflows/runtime_tests.yaml: -------------------------------------------------------------------------------- 1 | name: Multipy runtime tests 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - main 8 | workflow_dispatch: 9 | 10 | jobs: 11 | unittest: 12 | strategy: 13 | matrix: 14 | python3-minor-version: [7,8,9,10] 15 | platform: [linux.4xlarge.nvidia.gpu] 16 | fail-fast: false 17 | uses: ./.github/workflows/build_test_release.yaml 18 | with: 19 | python3-minor-version: ${{ matrix.python3-minor-version }} 20 | runner: ${{ matrix.platform }} 21 | compat-tests: true 22 | secrets: 23 | token: ${{ secrets.GITHUB_TOKEN }} 24 | -------------------------------------------------------------------------------- /multipy/runtime/path_environment.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | #include 9 | 10 | namespace torch { 11 | namespace deploy { 12 | 13 | void PathEnvironment::configureInterpreter(Interpreter* interp) { 14 | auto I = interp->acquireSession(); 15 | I.global("sys", "path").attr("append")({path_}); 16 | } 17 | 18 | } // namespace deploy 19 | } // namespace torch 20 | -------------------------------------------------------------------------------- /.github/workflows/runtime_nightly.yaml: -------------------------------------------------------------------------------- 1 | name: Multipy runtime nightly test + release 2 | 3 | on: 4 | schedule: 5 | - cron: '0 2 * * *' # run at 2 AM UTC 6 | workflow_dispatch: 7 | 8 | jobs: 9 | build-test-release: 10 | strategy: 11 | matrix: 12 | python3-minor-version: [7,8,9,10] 13 | platform: [linux.4xlarge.nvidia.gpu] 14 | fail-fast: false 15 | uses: ./.github/workflows/build_test_release.yaml 16 | with: 17 | python3-minor-version: ${{ matrix.python3-minor-version }} 18 | runner: ${{ matrix.platform }} 19 | release: true 20 | secrets: 21 | token: ${{ secrets.GITHUB_TOKEN }} 22 | -------------------------------------------------------------------------------- /multipy/runtime/noop_environment.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #pragma once 8 | 9 | #include 10 | 11 | namespace torch { 12 | namespace deploy { 13 | 14 | /// The local python Environment 15 | class NoopEnvironment : public Environment { 16 | public: 17 | /// no-op function as this is the no-op environment :) 18 | void configureInterpreter(Interpreter* /* interp */) override {} 19 | }; 20 | 21 | } // namespace deploy 22 | } // namespace torch 23 | -------------------------------------------------------------------------------- /multipy/runtime/testdev/test_deploy_from_python.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | 11 | # pyre-fixme[21]: Could not find module `test_deploy_python_ext`. 12 | # @manual=//multipy/runtime:test_deploy_python_ext 13 | import test_deploy_python_ext 14 | 15 | import torch # noqa: F401 16 | 17 | 18 | class TestDeployFromPython(unittest.TestCase): 19 | def test_deploy_from_python(self): 20 | self.assertTrue(test_deploy_python_ext.run()) 21 | -------------------------------------------------------------------------------- /multipy/runtime/test_deploy_missing_interpreter.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | #include 9 | 10 | int main(int argc, char* argv[]) { 11 | ::testing::InitGoogleTest(&argc, argv); 12 | int rc = RUN_ALL_TESTS(); 13 | return rc; 14 | } 15 | 16 | TEST(TorchDeployMissingInterpreter, Throws) { 17 | // NOLINTNEXTLINE(hicpp-avoid-goto,cppcoreguidelines-avoid-goto) 18 | EXPECT_THROW(torch::deploy::InterpreterManager(1), std::runtime_error); 19 | } 20 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | lint: 11 | runs-on: ubuntu-18.04 12 | steps: 13 | - name: Setup Python 14 | uses: actions/setup-python@v2 15 | with: 16 | python-version: 3.8 17 | architecture: x64 18 | - name: Checkout Multipy 19 | uses: actions/checkout@v2 20 | - name: Install Dependencies 21 | run: | 22 | set -eux 23 | pip install -r dev-requirements.txt 24 | - name: Run Lint 25 | run: | 26 | git config --global url."https://${{ secrets.GITHUB_TOKEN }}:x-oauth-basic@github.com/".insteadOf "https://github.com/" 27 | scripts/lint.sh 28 | -------------------------------------------------------------------------------- /examples/hello_world/hello_world_example.cpp: -------------------------------------------------------------------------------- 1 | /* Basic example of a single interpreter from `torch::deploy` 2 | to invoke python methods directly. Here we specifically 3 | invoke `print` to print out `Hello World`. 4 | */ 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | int main(int argc, const char* argv[]) { 14 | // create two interpreters 15 | multipy::runtime::InterpreterManager manager(2); 16 | 17 | // Acquire a session on one of the interpreters 18 | auto I = manager.acquireOne(); 19 | 20 | // from builtins import print 21 | // print("Hello world!") 22 | I.global("builtins", "print")({"Hello world!"}); 23 | } 24 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | select = B,C,E,F,P,T4,W,B9 3 | max-line-length = 120 4 | # C408 ignored because we like the dict keyword argument syntax 5 | # E501 is not flexible enough, we're using B950 instead 6 | ignore = 7 | E203,E305,E402,E501,E721,E741,F405,F821,F841,F999,W503,W504,C408,E302,W291,E303, 8 | # shebang has extra meaning in fbcode lints, so I think it's not worth trying 9 | # to line this up with executable bit 10 | EXE001, 11 | # these ignores are from flake8-bugbear; please fix! 12 | B007,B008, 13 | # these ignores are from flake8-comprehensions; please fix! 14 | C400,C401,C402,C403,C404,C405,C407,C411,C413,C414,C415 15 | optional-ascii-coding = True 16 | exclude = 17 | ./.git, 18 | ./docs 19 | ./build 20 | ./scripts, 21 | ./venv, 22 | *.pyi 23 | -------------------------------------------------------------------------------- /multipy/runtime/example/fx/examples.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import torch.fx 8 | 9 | try: 10 | from .some_dependency import a_non_torch_leaf 11 | except ImportError: 12 | # pyre-fixme[21]: Could not find module `some_dependency`. 13 | from some_dependency import a_non_torch_leaf 14 | 15 | 16 | torch.fx.wrap("a_non_torch_leaf") 17 | 18 | 19 | class SimpleWithLeaf(torch.nn.Module): 20 | def __init__(self, N, M): 21 | super().__init__() 22 | self.weight = torch.nn.Parameter(torch.rand(N, M)) 23 | 24 | def forward(self, input): 25 | output = self.weight + a_non_torch_leaf(1, input) 26 | return output 27 | -------------------------------------------------------------------------------- /multipy/runtime/unity/tests/simple_model.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import torch 10 | 11 | # pyre-fixme[21]: Could not find name `nn` in `torch`. 12 | from torch import nn 13 | 14 | 15 | # pyre-fixme[11]: Annotation `Module` is not defined as a type. 16 | class SimpleModel(nn.Module): 17 | def __init__(self): 18 | super(SimpleModel, self).__init__() 19 | self.fc = nn.Linear(256, 64) 20 | self.fc2 = nn.Linear(64, 10) 21 | 22 | def forward(self, X): 23 | X = self.fc(X) 24 | X = torch.relu(X) 25 | X = self.fc2(X) 26 | X = torch.softmax(X, dim=-1) 27 | return X 28 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | 3 | ## multipy-0.1.0 4 | 5 | This is the initial Beta release of `torch::deploy`. 6 | 7 | * PyTorch 1.13 support 8 | * Python 3.7-3.10 support 9 | * `torch::deploy` is now suitable for use in production environments. 10 | * `torch::deploy` uses the current Python environment and no longer 11 | requires building PyTorch, Python and C extensions from source. 12 | * C extensions can be installed via standard `pip`/`conda` and will be 13 | dynamically loaded at runtime. Popular PyTorch extensions have been tested but 14 | there may be some libraries that are incompatible. If you run into an 15 | incompatible library please file an issue. 16 | * Prototype aarch64 support 17 | * Improved performance and memory usage when keeping an InterpreterSession alive 18 | for a long time. 19 | * Supports all PyTorch core backends (CPU/CUDA/ROCm). 20 | -------------------------------------------------------------------------------- /multipy/runtime/test_deploy_python_ext.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | bool run() { 13 | torch::deploy::InterpreterManager m(2); 14 | m.registerModuleSource("check_none", "check = id(None)\n"); 15 | int64_t id0 = 0, id1 = 0; 16 | { 17 | auto I = m.allInstances()[0].acquireSession(); 18 | id0 = I.global("check_none", "check").toIValue().toInt(); 19 | } 20 | { 21 | auto I = m.allInstances()[1].acquireSession(); 22 | id1 = I.global("check_none", "check").toIValue().toInt(); 23 | } 24 | return id0 != id1; 25 | } 26 | 27 | PYBIND11_MODULE(test_deploy_python_ext, m) { 28 | m.def("run", run); 29 | } 30 | -------------------------------------------------------------------------------- /multipy/runtime/path_environment.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #pragma once 8 | 9 | #include 10 | #include 11 | 12 | namespace torch { 13 | namespace deploy { 14 | 15 | /// An Environment which is defined by a specific path to python code (ie. 16 | /// condas sitepackages) 17 | class PathEnvironment : public Environment { 18 | public: 19 | /// Environment constructor which takes a file name for the 20 | /// directory for the python modules. 21 | explicit PathEnvironment(std::string path) : path_(std::move(path)) {} 22 | /// Adds the path defined in the `PathEnvironment` to `interp` 23 | void configureInterpreter(Interpreter* interp) override; 24 | 25 | private: 26 | std::string path_; 27 | }; 28 | 29 | } // namespace deploy 30 | } // namespace torch 31 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = multipy 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | clean: 18 | @echo "Deleting build directory" 19 | rm -rf "$(BUILDDIR)" 20 | rm -rf _doxygen/ api/ 21 | 22 | # Catch-all target: route all unknown targets to Sphinx using the new 23 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 24 | %: Makefile 25 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 26 | 27 | # optional live version 28 | livehtml: 29 | sphinx-autobuild --watch ../multipy --watch ../examples --re-ignore ".*(build|examples_.*|api/.*|.new|source/.*(Dockerfile|.py))" "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 30 | -------------------------------------------------------------------------------- /multipy/runtime/unity/xar_environment.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #pragma once 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | namespace torch { 14 | namespace deploy { 15 | 16 | constexpr const char* DEFAULT_PYTHON_APP_DIR = "/tmp/torch_deploy_python_app"; 17 | 18 | class XarEnvironment : public Environment { 19 | public: 20 | explicit XarEnvironment( 21 | std::string exePath, 22 | std::string pythonAppDir = DEFAULT_PYTHON_APP_DIR); 23 | ~XarEnvironment() override; 24 | 25 | protected: 26 | void configureInterpreter(Interpreter* interp) override; 27 | 28 | private: 29 | void setupPythonApp(); 30 | void preloadSharedLibraries(); 31 | 32 | std::string exePath_; 33 | std::string pythonAppDir_; 34 | std::string pythonAppRoot_; 35 | bool alreadySetupPythonApp_ = false; 36 | }; 37 | 38 | } // namespace deploy 39 | } // namespace torch 40 | -------------------------------------------------------------------------------- /multipy/runtime/unity/tests/test_unity_sum.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | namespace torch { 13 | namespace deploy { 14 | 15 | const char* exePath = nullptr; 16 | 17 | TEST(UnityTest, TestUnitySum) { 18 | // use a different path for unit test. Normally don't specify the path will 19 | // use the default one. 20 | mkdtemp(TEST_PYTHON_APP_DIR_TEMP); 21 | std::shared_ptr env = 22 | std::make_shared(exePath, TEST_PYTHON_APP_DIR_TEMP); 23 | InterpreterManager m(2, env); 24 | 25 | auto I = m.acquireOne(); 26 | auto result = I.global("sum", "func")({1, 2, 3, 4}); 27 | EXPECT_EQ(10, result.toIValue().toInt()); 28 | } 29 | 30 | } // namespace deploy 31 | } // namespace torch 32 | 33 | int main(int argc, char** argv) { 34 | torch::deploy::exePath = argv[0]; 35 | ::testing::InitGoogleTest(&argc, argv); 36 | return RUN_ALL_TESTS(); 37 | } 38 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | :github_url: https://github.com/pytorch/multipy 2 | 3 | ``torch::deploy`` [Beta] 4 | ===================== 5 | 6 | ``torch::deploy`` is a system that allows you to load multiple python interpreters which execute PyTorch models, and run them in a single C++ process. Effectively, it allows people to multithread their pytorch models. 7 | For more information on how torch::deploy works please see the related `arXiv paper `_. We plan to further generalize ``torch::deploy`` into a more generic system, ``multipy::runtime``, 8 | which is more suitable for arbitrary python programs rather than just pytorch applications. 9 | 10 | 11 | Documentation 12 | --------------- 13 | 14 | .. toctree:: 15 | :maxdepth: 2 16 | :caption: Usage 17 | 18 | setup.md 19 | tutorials/tutorial_root 20 | api/library_root 21 | 22 | Acknowledgements 23 | ---------------- 24 | 25 | This documentation website for the MultiPy C++ API has been enabled by the 26 | `Exhale `_ project and generous investment 27 | of time and effort by its maintainer, `svenevs `_. 28 | We thank Stephen for his work and his efforts providing help with both the PyTorch and MultiPy C++ documentation. 29 | -------------------------------------------------------------------------------- /examples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.12 FATAL_ERROR) 2 | project(multipy_tutorial) 3 | 4 | set(MULTIPY_PATH ".." CACHE PATH "The repo where multipy is built or the PYTHONPATH") 5 | 6 | # include the multipy utils to help link against 7 | include(${MULTIPY_PATH}/multipy/runtime/utils.cmake) 8 | 9 | # add headers from multipy 10 | include_directories(${MULTIPY_PATH}) 11 | 12 | # link the multipy prebuilt binary 13 | add_library(multipy_internal STATIC IMPORTED) 14 | set_target_properties(multipy_internal 15 | PROPERTIES 16 | IMPORTED_LOCATION 17 | ${MULTIPY_PATH}/multipy/runtime/build/libtorch_deploy.a) 18 | caffe2_interface_library(multipy_internal multipy) 19 | 20 | # build our examples 21 | add_executable(hello_world_example hello_world/hello_world_example.cpp) 22 | target_link_libraries(hello_world_example PUBLIC "-Wl,--no-as-needed -rdynamic" dl pthread util multipy c10 torch_cpu) 23 | 24 | add_executable(quickstart quickstart/quickstart.cpp) 25 | target_link_libraries(quickstart PUBLIC "-Wl,--no-as-needed -rdynamic" dl pthread util multipy c10 torch_cpu) 26 | 27 | add_executable(movable_example movable_example/movable_example.cpp) 28 | target_link_libraries(movable_example PUBLIC "-Wl,--no-as-needed -rdynamic" dl pthread util multipy c10 torch_cpu) 29 | -------------------------------------------------------------------------------- /multipy/runtime/interpreter/plugin_torch.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "plugin_registry.h" 6 | 7 | namespace multipy { 8 | namespace torch { 9 | 10 | namespace { 11 | 12 | class TorchConverter : public Converter { 13 | public: 14 | TorchConverter() { 15 | registerConverter(this); 16 | } 17 | 18 | ~TorchConverter() override { 19 | deregisterConverter(this); 20 | } 21 | 22 | std::optional toTypeInferredIValue(py::handle input) override { 23 | return ::torch::jit::toTypeInferredIValue(input); 24 | } 25 | std::optional toPyObject(at::IValue ivalue) override { 26 | return ::torch::jit::toPyObject(ivalue); 27 | } 28 | std::optional createStorage(PyObject* obj) override { 29 | return ::torch::createStorage(obj); 30 | } 31 | std::optional createPyObject(const at::Storage& storage) override { 32 | return ::torch::createPyObject(storage); 33 | } 34 | std::optional getTHPDtype(at::ScalarType scalarType) override { 35 | return ::torch::getTHPDtype(scalarType); 36 | } 37 | }; 38 | 39 | TorchConverter converter; 40 | } // namespace 41 | } // namespace torch 42 | } // namespace multipy 43 | -------------------------------------------------------------------------------- /multipy/runtime/embedded_file.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #pragma once 8 | 9 | #include 10 | 11 | namespace torch { 12 | namespace deploy { 13 | 14 | /// Specifies which ELF section to load the interpreter from and the associated 15 | /// config. 16 | struct ExeSection { 17 | const char* sectionName; 18 | bool customLoader; 19 | }; 20 | 21 | /// Specifies which ELF symbols to load the interpreter from and the associated 22 | /// config. 23 | struct InterpreterSymbol { 24 | const char* startSym; 25 | const char* endSym; 26 | bool customLoader; 27 | }; 28 | 29 | /// EmbeddedFile makes it easier to load a custom interpreter embedded within 30 | /// the binary. 31 | struct EmbeddedFile { 32 | std::string libraryName; 33 | bool customLoader{false}; 34 | 35 | EmbeddedFile( 36 | std::string name, 37 | const std::initializer_list& sections, 38 | const std::initializer_list symbols); 39 | 40 | ~EmbeddedFile(); 41 | 42 | EmbeddedFile& operator=(const EmbeddedFile&) = delete; 43 | }; 44 | 45 | } // namespace deploy 46 | } // namespace torch 47 | -------------------------------------------------------------------------------- /multipy/runtime/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under the BSD-style license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | """ 9 | This contains a pybinded interface to MultiPy's subinterpreters. This enables 10 | running child interpreters from an existing python process without having any 11 | shared GIL. 12 | 13 | WARNING: This is currently a prototype and is subject to change at any point. 14 | 15 | Current limitations: 16 | * Obj must be destroyed/GCed before the InterpreterSession is destroyed. If left 17 | up to GC in most cases it will crash the program. 18 | * No pickle interface for smartly transferring models between interpreters 19 | 20 | See test_pybind.py for examples on how to use. 21 | """ 22 | 23 | import ctypes 24 | import os 25 | import os.path 26 | 27 | import torch 28 | 29 | # We need to load libtorch into the global symbol space so the subinterpreters 30 | # can use it. 31 | torch_dir = os.path.dirname(torch.__file__) 32 | libtorch_path = os.path.join(torch_dir, "lib", "libtorch.so") 33 | ctypes.CDLL(libtorch_path, mode=ctypes.RTLD_GLOBAL) 34 | 35 | from multipy.runtime.build.multipy_pybind import InterpreterManager # noqa: F401 36 | -------------------------------------------------------------------------------- /multipy/runtime/test_deploy_imports.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | void test_import(std::vector moduleNames) { 20 | // Test whether importing the python modules specified 21 | // in "moduleNames" work inside a deploy interpreter 22 | 23 | torch::deploy::InterpreterManager manager(1); 24 | auto I = manager.acquireOne(); 25 | for (const char* moduleName : moduleNames) { 26 | I.global("builtins", "__import__")({moduleName}); 27 | } 28 | } 29 | 30 | int main(int argc, char* argv[]) { 31 | ::testing::InitGoogleTest(&argc, argv); 32 | char tempeh[256]; 33 | getcwd(tempeh, 256); 34 | std::cout << "Current working directory: " << tempeh << std::endl; 35 | int rc = RUN_ALL_TESTS(); 36 | char tmp[256]; 37 | getcwd(tmp, 256); 38 | std::cout << "Current working directory: " << tmp << std::endl; 39 | return rc; 40 | } 41 | -------------------------------------------------------------------------------- /multipy/runtime/test_deploy_python.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # this is imported by test_deploy to do some checks in python 8 | import subprocess 9 | import sys 10 | from pathlib import Path 11 | 12 | 13 | # we've taken steps to clear out the embedded python environment, 14 | # so we have to go searching for real python to figure out where its libraries are installed. 15 | def python_path(cpath): 16 | for maybe in cpath.split(":"): 17 | candidate = Path(maybe) / "python" 18 | if candidate.exists(): 19 | cmd = [str(candidate), "-c", 'import sys; print(":".join(sys.path))'] 20 | return subprocess.check_output(cmd).decode("utf-8").strip("\n").split(":") 21 | raise RuntimeError("could not find real python") 22 | 23 | 24 | def setup(path): 25 | sys.path.extend(python_path(path)) 26 | sys.path.append("build/lib") # for our test python extension 27 | 28 | 29 | # smoke test the numpy extension loading works 30 | def numpy_test(x): 31 | import numpy as np 32 | 33 | xs = [np.array([x, x]), np.array([x, x])] 34 | for i in range(10): 35 | xs.append(xs[-1] + xs[-2]) 36 | return int(xs[-1][0]) 37 | -------------------------------------------------------------------------------- /multipy/runtime/unity/main.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | namespace torch { 12 | namespace deploy { 13 | 14 | // the way we lookup main module follows how an xar file is setup 15 | std::string lookupMainModule(InterpreterManager& m) { 16 | auto I = m.acquireOne(); 17 | auto mainModule = 18 | I.global("__manifest__", "fbmake").attr("get")({"main_module"}); 19 | std::ostringstream ss; 20 | ss << mainModule.toIValue(); 21 | LOG(INFO) << "main module is " << ss.str(); 22 | return ss.str(); 23 | } 24 | 25 | int doMain(int /* argc */, char** argv) { 26 | std::shared_ptr env = std::make_shared(argv[0]); 27 | InterpreterManager m(2, env); 28 | 29 | auto mainModule = lookupMainModule(m); 30 | auto I = m.acquireOne(); 31 | I.global("runpy", "run_module")({mainModule}); 32 | return 0; 33 | } 34 | 35 | } // namespace deploy 36 | } // namespace torch 37 | 38 | // NOLINTNEXTLINE(bugprone-exception-escape) 39 | int main(int argc, char** argv) { 40 | return torch::deploy::doMain(argc, argv); 41 | } 42 | -------------------------------------------------------------------------------- /examples/movable_example/movable_example.cpp: -------------------------------------------------------------------------------- 1 | // Basic example of using `ReplicatedObject` in `torch::deploy`. 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | int main(int argc, const char* argv[]) { 11 | torch::deploy::InterpreterManager m(4); 12 | 13 | try { 14 | // Load the model from the torch.package. 15 | auto I = m.acquireOne(); 16 | std::vector constructor_inputs; 17 | auto model_obj = I.global("torch.nn", "Conv2d")({6, 2, 2, 1}); 18 | auto rObj = m.createMovable(model_obj, &I); 19 | auto I2 = m.acquireOne(); 20 | auto model_obj2 = I2.fromMovable(rObj); 21 | rObj.unload(); // free the replicated object 22 | 23 | // Create a vector of inputs. 24 | std::vector inputs; 25 | inputs.push_back(torch::ones({1, 6, 6, 6})); 26 | 27 | // Execute the model and turn its output into a tensor. 28 | at::Tensor output = model_obj2(inputs).toIValue().toTensor(); 29 | std::cout << output.slice(/*dim=*/1, /*start=*/0, /*end=*/5) << '\n'; 30 | 31 | } catch (const c10::Error& e) { 32 | std::cerr << "error creating movable\n"; 33 | std::cerr << e.msg(); 34 | return -1; 35 | } 36 | 37 | std::cout << "ok\n"; 38 | } 39 | -------------------------------------------------------------------------------- /docs/source/tutorials/create_movable.rst: -------------------------------------------------------------------------------- 1 | Passing Python Objects from one Interpreter to Another 2 | ====================================================== 3 | 4 | Here we use ``torch::deploy`` to create a `ReplicatedObj` in order to pass 5 | an `Obj` from one interpreter to another. 6 | 7 | Moving Python Objects Between Interpreters 8 | ------------------------------------------ 9 | 10 | .. literalinclude:: ../../../examples/movable_example/movable_example.cpp 11 | :language: c++ 12 | 13 | Here we highlight ``torch::deploy::InterpreterManager::create_movable(Obj, InterpreterSession*)`` 14 | and ``InterpreterSession::fromMovable(const ReplicatedObj&)``. These functions allow conversions 15 | between ``Obj`s` which are speciifc to an interpreter and ``ReplicatedObj``s 16 | which are replicated across multiple interpreters. 17 | 18 | Build and execute 19 | ----------------- 20 | 21 | Assuming the above C++ program was stored in a file called 22 | ``movable_example.cpp``, a minimal ``CMakeLists.txt`` file would 23 | look like: 24 | 25 | .. literalinclude:: ../../../examples/CMakeLists.txt 26 | :language: cmake 27 | :lines: 1-20,27,28 28 | 29 | From here we execute the hello world program 30 | 31 | .. code:: bash 32 | 33 | mkdir build 34 | cd build 35 | cmake -S . -B build/ -DMULTIPY_PATH="" -DPython3_EXECUTABLE="$(which python3)" && \ 36 | cmake --build build/ --config Release -j 37 | ./hello_world 38 | -------------------------------------------------------------------------------- /multipy/runtime/test_pybind.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under the BSD-style license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | import unittest 9 | 10 | import torch 11 | 12 | from multipy.runtime import InterpreterManager 13 | 14 | 15 | class TestPybind(unittest.TestCase): 16 | @classmethod 17 | def setUpClass(cls): 18 | cls.manager = InterpreterManager(1) 19 | 20 | def test_print(self): 21 | I = self.manager.acquire_one() 22 | iprint = I.global_("builtins", "print") 23 | iprint("hello!") 24 | 25 | def test_tensor_passing(self): 26 | I = self.manager.acquire_one() 27 | model = I.global_("torch.nn", "Conv2d")(6, 2, 2, 1) 28 | out = model(torch.ones((1, 6, 6, 6))) 29 | tensor = out.deref() 30 | self.assertIsInstance(tensor, torch.Tensor) 31 | del out 32 | del model 33 | del I 34 | print("exit2") 35 | 36 | def test_multiple(self): 37 | m = InterpreterManager(2) 38 | self.assertEqual(len(m), 2) 39 | for i in range(len(m)): 40 | I = m[i] 41 | iprint = I.global_("builtins", "print") 42 | iprint(f"hello {i}") 43 | del iprint 44 | del I 45 | 46 | 47 | if __name__ == "__main__": 48 | unittest.main() 49 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to MultiPy 2 | We want to make contributing to this project as easy and transparent as 3 | possible. 4 | 5 | ## Our Development Process 6 | ... (in particular how this is synced with internal changes to the project) 7 | 8 | ## Pull Requests 9 | We actively welcome your pull requests. 10 | 11 | 1. Fork the repo and create your branch from `main`. 12 | 2. If you've added code that should be tested, add tests. 13 | 3. If you've changed APIs, update the documentation. 14 | 4. Ensure the test suite passes. 15 | 5. Make sure your code lints. 16 | 6. If you haven't already, complete the Contributor License Agreement ("CLA"). 17 | 18 | ## Contributor License Agreement ("CLA") 19 | In order to accept your pull request, we need you to submit a CLA. You only need 20 | to do this once to work on any of Meta's open source projects. 21 | 22 | Complete your CLA here: 23 | 24 | ## Issues 25 | We use GitHub issues to track public bugs. Please ensure your description is 26 | clear and has sufficient instructions to be able to reproduce the issue. 27 | 28 | Meta has a [bounty program](https://www.facebook.com/whitehat/) for the safe 29 | disclosure of security bugs. In those cases, please go through the process 30 | outlined on that page and do not file a public issue. 31 | 32 | ## Coding Style 33 | * 2 spaces for indentation rather than tabs 34 | * 80 character line length 35 | * ... 36 | 37 | ## License 38 | By contributing to __________, you agree that your contributions will be licensed 39 | under the LICENSE file in the root directory of this source tree. 40 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | BSD License 2 | 3 | For MultiPy software 4 | 5 | Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without modification, 8 | are permitted provided that the following conditions are met: 9 | 10 | * Redistributions of source code must retain the above copyright notice, this 11 | list of conditions and the following disclaimer. 12 | 13 | * Redistributions in binary form must reproduce the above copyright notice, 14 | this list of conditions and the following disclaimer in the documentation 15 | and/or other materials provided with the distribution. 16 | 17 | * Neither the name Meta nor the names of its contributors may be used to 18 | endorse or promote products derived from this software without specific 19 | prior written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 22 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 23 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 25 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 26 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 28 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 30 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /multipy/runtime/unity/tests/test_unity_simple_model.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | namespace torch { 13 | namespace deploy { 14 | 15 | const char* exePath = nullptr; 16 | 17 | TEST(UnityTest, TestUnitySimpleModel) { 18 | // use a different path for unit test. Normally don't specify the path will 19 | // use the default one. 20 | mkdtemp(TEST_PYTHON_APP_DIR_TEMP); 21 | std::shared_ptr env = 22 | std::make_shared(exePath, TEST_PYTHON_APP_DIR_TEMP); 23 | InterpreterManager m(2, env); 24 | 25 | auto I = m.acquireOne(); 26 | 27 | auto noArgs = at::ArrayRef(); 28 | auto input = I.global("torch", "randn")({32, 256}); 29 | auto model = I.global("simple_model", "SimpleModel")(noArgs); 30 | 31 | auto output = model({input}); // implicitly calls model's forward method 32 | EXPECT_EQ(2, output.attr("shape").attr("__len__")(noArgs).toIValue().toInt()); 33 | EXPECT_EQ( 34 | 32, output.attr("shape").attr("__getitem__")({0}).toIValue().toInt()); 35 | EXPECT_EQ( 36 | 10, output.attr("shape").attr("__getitem__")({1}).toIValue().toInt()); 37 | } 38 | 39 | } // namespace deploy 40 | } // namespace torch 41 | 42 | int main(int argc, char** argv) { 43 | torch::deploy::exePath = argv[0]; 44 | ::testing::InitGoogleTest(&argc, argv); 45 | return RUN_ALL_TESTS(); 46 | } 47 | -------------------------------------------------------------------------------- /examples/quickstart/quickstart.cpp: -------------------------------------------------------------------------------- 1 | /* quickstart.cpp highlights some of the most basic concepts of `torch::deploy`. 2 | Specifically loading a pytorch model which was serialized by `torch.package` 3 | and running it. 4 | 5 | In order to run this file, one needs to provide an archive produced by 6 | `torch.package`. The one used in our example is created by `gen_package.py` 7 | which produces my_package.pt. This program takes the path to the archive as 8 | an argument. 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include 17 | #include 18 | 19 | int main(int argc, const char* argv[]) { 20 | if (argc != 2) { 21 | std::cerr << "usage: example-app \n"; 22 | return -1; 23 | } 24 | 25 | // Start an interpreter manager governing 4 embedded interpreters. 26 | torch::deploy::InterpreterManager manager(4); 27 | torch::deploy::ReplicatedObj model; 28 | try { 29 | // Load the model from the multipy.package. 30 | torch::deploy::Package package = manager.loadPackage(argv[1]); 31 | model = package.loadPickle("model", "model.pkl"); 32 | } catch (const c10::Error& e) { 33 | std::cerr << "error loading the model\n"; 34 | std::cerr << e.msg(); 35 | return -1; 36 | } 37 | 38 | // Create a vector of inputs. 39 | std::vector inputs; 40 | inputs.push_back(torch::ones({1, 3, 224, 224})); 41 | 42 | // Execute the model and turn its output into a tensor. 43 | at::Tensor output = model(inputs).toTensor(); 44 | std::cout << output.slice(/*dim=*/1, /*start=*/0, /*end=*/5) << '\n'; 45 | 46 | std::cout << "ok\n"; 47 | } 48 | -------------------------------------------------------------------------------- /scripts/runtime_debugger.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under the BSD-style license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | import lldb # type: ignore[import] 9 | 10 | # load into lldb instance with: 11 | # command script import tools/lldb/deploy_debugger.py 12 | 13 | target = lldb.debugger.GetSelectedTarget() 14 | bp = target.BreakpointCreateByRegex("__deploy_register_code") 15 | bp.SetScriptCallbackBody( 16 | """\ 17 | process = frame.thread.GetProcess() 18 | target = process.target 19 | symbol_addr = frame.module.FindSymbol("__deploy_module_info").GetStartAddress() 20 | info_addr = symbol_addr.GetLoadAddress(target) 21 | e = lldb.SBError() 22 | ptr_size = 8 23 | str_addr = process.ReadPointerFromMemory(info_addr, e) 24 | file_addr = process.ReadPointerFromMemory(info_addr + ptr_size, e) 25 | file_size = process.ReadPointerFromMemory(info_addr + 2*ptr_size, e) 26 | load_bias = process.ReadPointerFromMemory(info_addr + 3*ptr_size, e) 27 | name = process.ReadCStringFromMemory(str_addr, 512, e) 28 | r = process.ReadMemory(file_addr, file_size, e) 29 | from tempfile import NamedTemporaryFile 30 | from pathlib import Path 31 | stem = Path(name).stem 32 | with NamedTemporaryFile(prefix=stem, suffix='.so', delete=False) as tf: 33 | tf.write(r) 34 | print("torch_deploy registering debug inforation for ", tf.name) 35 | cmd1 = f"target modules add {tf.name}" 36 | # print(cmd1) 37 | lldb.debugger.HandleCommand(cmd1) 38 | cmd2 = f"target modules load -f {tf.name} -s {hex(load_bias)}" 39 | # print(cmd2) 40 | lldb.debugger.HandleCommand(cmd2) 41 | 42 | return False 43 | """ 44 | ) 45 | -------------------------------------------------------------------------------- /multipy/runtime/interactive_embedded_interpreter.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | /* 8 | * The tool provides a shell to the embedded interpreter. Useful to inspect the 9 | * state of the embedding interpreter interactively. 10 | */ 11 | #include 12 | #include 13 | #include 14 | 15 | C10_DEFINE_string( 16 | python_path, 17 | "", 18 | "The root of the installed python libraries in the system"); 19 | C10_DEFINE_string(pyscript, "", "The path of the python script to execute"); 20 | 21 | // NOLINTNEXTLINE(bugprone-exception-escape) 22 | int main(int argc, char** argv) { 23 | c10::ParseCommandLineFlags(&argc, &argv); 24 | 25 | if (FLAGS_python_path.size() > 0) { 26 | LOG(INFO) << "Will add " << FLAGS_python_path << " to python sys.path"; 27 | } 28 | std::shared_ptr env = 29 | std::make_shared(FLAGS_python_path); 30 | // create multiple interpreter instances so the tool does not just cover the 31 | // simplest case with a single interpreter instance. 32 | torch::deploy::InterpreterManager m(2, env); 33 | auto I = m.acquireOne(); 34 | 35 | if (FLAGS_pyscript.size() > 0) { 36 | auto realpath = 37 | I.global("os", "path").attr("expanduser")({FLAGS_pyscript}).toIValue(); 38 | I.global("runpy", "run_path") 39 | .callKwargs({realpath}, {{"run_name", "__main__"}}); 40 | } else { 41 | c10::ArrayRef noArgs; 42 | I.global("pdb", "set_trace")(noArgs); 43 | } 44 | return 0; 45 | } 46 | -------------------------------------------------------------------------------- /docs/source/tutorials/hello_world.rst: -------------------------------------------------------------------------------- 1 | Hello World (Using InterpreterSession directly) 2 | =============================================== 3 | 4 | Here we use ``torch::deploy`` to print ``Hello World`` to the console 5 | without using ``torch.package``. Instead we simply acquire an individual 6 | ``InterpreterSession``, and use it to print ``Hello World`` directly. 7 | 8 | Printing Hello World with ``torch::deploy`` 9 | ------------------------------------------- 10 | 11 | .. literalinclude:: ../../../examples/hello_world/hello_world_example.cpp 12 | :language: c++ 13 | :lines: 5- 14 | 15 | Here we introduce the pythonic nature of ``torch::deploy``\ ’s 16 | ``InterpreterSession``s by using them in a similar fasion to 17 | python objects. This allows us to add further flexibility to the code 18 | exported by ``torch.package`` by interacting with it in C++. 19 | 20 | ``manager.acquireOne`` allows us to create an individual subinterpreter 21 | we can interact with. 22 | 23 | ``InterpreterSession::global(const char* module, const char* name)`` 24 | allows us to access python modules such as ``builtins`` and their 25 | attributes such as ``print``. This function outputs an ``Obj`` 26 | which is a wrapper around ``print``. From here we call ``print`` by 27 | using ``{"Hello world!"}`` as its argument(s). 28 | 29 | Build and execute 30 | ----------------- 31 | 32 | Assuming the above C++ program was stored in a file called 33 | ``hello_world_example.cpp``, a minimal ``CMakeLists.txt`` file would 34 | look like: 35 | 36 | .. literalinclude:: ../../../examples/CMakeLists.txt 37 | :language: cmake 38 | :lines: 1-20,21,22 39 | 40 | From here we execute the hello world program 41 | 42 | .. code:: bash 43 | 44 | mkdir build 45 | cd build 46 | cmake -S . -B build/ -DMULTIPY_PATH="" -DPython3_EXECUTABLE="$(which python3)" && \ 47 | cmake --build build/ --config Release -j 48 | ./hello_world 49 | -------------------------------------------------------------------------------- /multipy/runtime/interpreter/plugin_registry.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | namespace py = pybind11; 12 | 13 | namespace multipy { 14 | 15 | /// A `Converter` is used in order to convert `PyObject`s/`py::object` into 16 | /// an `IValue` or some other representation such as storage. 17 | class Converter { 18 | public: 19 | virtual ~Converter() = default; 20 | 21 | /// Converts a `py::handle` to an `IValue` 22 | virtual std::optional toTypeInferredIValue(py::handle input) = 0; 23 | 24 | /// Converts an `IValue` into a `py::object` 25 | virtual std::optional toPyObject(at::IValue ivalue) = 0; 26 | 27 | /// Converts an `PyObject` into a `Storage` 28 | virtual std::optional createStorage(PyObject* obj) = 0; 29 | 30 | /// Creates a `PyObject` from `storage` 31 | virtual std::optional createPyObject( 32 | const at::Storage& storage) = 0; 33 | 34 | // Returns the `THPDtype` of `scalarType` 35 | virtual std::optional getTHPDtype(at::ScalarType scalarType) = 0; 36 | }; 37 | 38 | /// Registers a converter to be used by torch::deploy / multipy. 39 | /// The order of the registration of the converters is dictated by the order of 40 | /// compilation. 41 | void registerConverter(Converter*); 42 | /// Deregisters a converter from torch::deploy / multipy 43 | /// The order of the deregistration of the converters is dictated by the order 44 | /// of compilation. 45 | void deregisterConverter(Converter*); 46 | 47 | at::IValue toTypeInferredIValue(py::handle input); 48 | py::object toPyObject(at::IValue ivalue); 49 | at::Storage createStorage(PyObject* obj); 50 | PyObject* createPyObject(const at::Storage& storage); 51 | THPDtype* getTHPDtype(at::ScalarType scalarType); 52 | } // namespace multipy 53 | -------------------------------------------------------------------------------- /multipy/runtime/interpreter/plugin_registry.cpp: -------------------------------------------------------------------------------- 1 | #include "multipy/runtime/interpreter/plugin_registry.h" 2 | 3 | #include 4 | 5 | namespace multipy { 6 | 7 | std::vector& getConverters() { 8 | static std::vector converters; 9 | return converters; 10 | } 11 | 12 | void registerConverter(Converter* c) { 13 | getConverters().emplace_back(c); 14 | } 15 | 16 | void deregisterConverter(Converter* c) { 17 | auto& converters = getConverters(); 18 | auto it = std::find(converters.begin(), converters.end(), c); 19 | if (it != converters.end()) { 20 | converters.erase(it); 21 | } 22 | } 23 | 24 | at::IValue toTypeInferredIValue(py::handle input) { 25 | for (auto c : getConverters()) { 26 | auto out = c->toTypeInferredIValue(input); 27 | if (out) { 28 | return *out; 29 | } 30 | } 31 | throw std::runtime_error("failed to convert to IValue"); 32 | } 33 | py::object toPyObject(at::IValue ivalue) { 34 | for (auto c : getConverters()) { 35 | auto out = c->toPyObject(ivalue); 36 | if (out) { 37 | return *out; 38 | } 39 | } 40 | throw std::runtime_error("failed to convert to py::object"); 41 | } 42 | at::Storage createStorage(PyObject* obj) { 43 | for (auto c : getConverters()) { 44 | auto out = c->createStorage(obj); 45 | if (out) { 46 | return *out; 47 | } 48 | } 49 | throw std::runtime_error("failed to createStorage"); 50 | } 51 | PyObject* createPyObject(const at::Storage& storage) { 52 | for (auto c : getConverters()) { 53 | auto out = c->createPyObject(storage); 54 | if (out) { 55 | return *out; 56 | } 57 | } 58 | throw std::runtime_error("failed to createPyObject"); 59 | } 60 | THPDtype* getTHPDtype(at::ScalarType scalarType) { 61 | for (auto c : getConverters()) { 62 | auto out = c->getTHPDtype(scalarType); 63 | if (out) { 64 | return *out; 65 | } 66 | } 67 | throw std::runtime_error("failed to createPyObject"); 68 | } 69 | } // namespace multipy 70 | -------------------------------------------------------------------------------- /scripts/linter/clang_tidy/generate_build_files.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import sys 4 | from typing import List 5 | 6 | 7 | def run_cmd(cmd: List[str]) -> None: 8 | print(f"Running: {cmd}") 9 | result = subprocess.run( 10 | cmd, 11 | stdout=subprocess.PIPE, 12 | stderr=subprocess.PIPE, 13 | ) 14 | stdout, stderr = ( 15 | result.stdout.decode("utf-8").strip(), 16 | result.stderr.decode("utf-8").strip(), 17 | ) 18 | print(stdout) 19 | print(stderr) 20 | if result.returncode != 0: 21 | print(f"Failed to run {cmd}") 22 | exit(1) 23 | 24 | 25 | def run_timed_cmd(cmd: List[str]) -> None: 26 | run_cmd(["time"] + cmd) 27 | 28 | 29 | def update_submodules() -> None: 30 | run_cmd(["git", "submodule", "update", "--init", "--recursive"]) 31 | 32 | 33 | def gen_compile_commands() -> None: 34 | os.environ["USE_NCCL"] = "0" 35 | os.environ["CC"] = "clang" 36 | os.environ["CXX"] = "clang++" 37 | run_timed_cmd([sys.executable, "setup.py", "--cmake-only", "build"]) 38 | 39 | 40 | def run_autogen() -> None: 41 | run_timed_cmd( 42 | [ 43 | sys.executable, 44 | "-m", 45 | "torchgen.gen", 46 | "-s", 47 | "aten/src/ATen", 48 | "-d", 49 | "build/aten/src/ATen", 50 | "--per-operator-headers", 51 | ] 52 | ) 53 | 54 | run_timed_cmd( 55 | [ 56 | sys.executable, 57 | "tools/setup_helpers/generate_code.py", 58 | "--native-functions-path", 59 | "aten/src/ATen/native/native_functions.yaml", 60 | "--tags-path", 61 | "aten/src/ATen/native/tags.yaml", 62 | "--gen_lazy_ts_backend", 63 | ] 64 | ) 65 | 66 | 67 | def generate_build_files() -> None: 68 | update_submodules() 69 | gen_compile_commands() 70 | run_autogen() 71 | 72 | 73 | if __name__ == "__main__": 74 | generate_build_files() 75 | -------------------------------------------------------------------------------- /multipy/runtime/test_compat.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import unittest 8 | 9 | import torch 10 | import torch._dynamo 11 | 12 | 13 | class TestCompat(unittest.TestCase): 14 | def test_torchvision(self): 15 | import torchvision # noqa: F401 16 | 17 | def test_torchaudio(self): 18 | import torchaudio # noqa: F401 19 | 20 | def test_pytorch3d(self): 21 | import pytorch3d # noqa: F401 22 | 23 | def test_hf_tokenizers(self): 24 | import tokenizers # noqa: F401 25 | 26 | def test_torchdynamo_eager(self): 27 | torch._dynamo.reset() 28 | 29 | def fn(x, y): 30 | a = torch.cos(x) 31 | b = torch.sin(y) 32 | return a + b 33 | 34 | c_fn = torch.compile(fn, backend="eager") 35 | c_fn(torch.randn(10), torch.randn(10)) 36 | 37 | def test_torchdynamo_ofi(self): 38 | torch._dynamo.reset() 39 | 40 | def fn(x, y): 41 | a = torch.cos(x) 42 | b = torch.sin(y) 43 | return a + b 44 | 45 | c_fn = torch.compile(fn, backend="ofi") 46 | c_fn(torch.randn(10), torch.randn(10)) 47 | 48 | def test_torchdynamo_inductor(self): 49 | torch._dynamo.reset() 50 | 51 | def fn(x, y): 52 | a = torch.cos(x) 53 | b = torch.sin(y) 54 | return a + b 55 | 56 | c_fn = torch.compile(fn) 57 | c_fn(torch.randn(10), torch.randn(10)) 58 | 59 | def test_functional_collectives(self): 60 | with self.assertWarns(Warning): 61 | # Functional collectives don't work with torchdeploy 62 | # but we want to make sure the import doesn't totally crash 63 | import torch.distributed._functional_collectives 64 | 65 | 66 | if __name__ == "__main__": 67 | unittest.main() 68 | -------------------------------------------------------------------------------- /multipy/runtime/loader.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #pragma once 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | namespace torch { 15 | namespace deploy { 16 | 17 | struct DeployLoaderError : public std::runtime_error { 18 | using std::runtime_error::runtime_error; 19 | }; 20 | 21 | struct TLSIndex { 22 | size_t module_id; // if module_id & TLS_LOCAL_FLAG, then module_id & 23 | // ~TLS_LOCAL_FLAG is a TLSMemory*; 24 | size_t offset; 25 | }; 26 | 27 | struct SymbolProvider { 28 | SymbolProvider() = default; 29 | virtual std::optional sym( 30 | const char* name, 31 | const char* version = nullptr) const = 0; 32 | virtual std::optional tls_sym(const char* name) const = 0; 33 | SymbolProvider(const SymbolProvider&) = delete; 34 | SymbolProvider& operator=(const SymbolProvider&) = delete; 35 | virtual ~SymbolProvider() = default; 36 | }; 37 | 38 | // RAII wrapper around dlopen 39 | struct SystemLibrary : public SymbolProvider { 40 | // create a wrapper around an existing handle returned from dlopen 41 | // if steal == true, then this will dlclose the handle when it is destroyed. 42 | static std::shared_ptr create( 43 | void* handle = RTLD_DEFAULT, 44 | bool steal = false); 45 | static std::shared_ptr create(const char* path, int flags); 46 | }; 47 | 48 | struct CustomLibrary : public SymbolProvider { 49 | static std::shared_ptr 50 | create(const char* filename, int argc = 0, const char** argv = nullptr); 51 | virtual void add_search_library(std::shared_ptr lib) = 0; 52 | virtual void load() = 0; 53 | }; 54 | 55 | using SystemLibraryPtr = std::shared_ptr; 56 | using CustomLibraryPtr = std::shared_ptr; 57 | 58 | } // namespace deploy 59 | } // namespace torch 60 | -------------------------------------------------------------------------------- /scripts/linter/adapters/s3_init_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "HOW TO UPDATE THE BINARIES": [ 3 | "Upload the new file to S3 under a new folder with the version number embedded in (see actionlint for an example).", 4 | "(Don't override the old files, otherwise you'll break `lintrunner install` for anyone using an older commit of pytorch.)", 5 | "'Hash' is the sha256 of the uploaded file.", 6 | "Validate the new download url and hash by running 'lintrunner init' to pull the new binaries and then run 'lintrunner' to try linting the files." 7 | ], 8 | "clang-format": { 9 | "Darwin": { 10 | "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/mac/clang-format-mojave", 11 | "hash": "1485a242a96c737ba7cdd9f259114f2201accdb46d87ac7a8650b1a814cd4d4d" 12 | }, 13 | "Linux": { 14 | "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64", 15 | "hash": "e1c8b97b919541a99e0a355df5c3f9e8abebc64259dbee6f8c68e1ef90582856" 16 | } 17 | }, 18 | "clang-tidy": { 19 | "Darwin": { 20 | "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/macos/clang-tidy", 21 | "hash": "541797a7b8fa795e2f3c1adcd8236cc336a40aa927028dc5bc79172e1d9eca36" 22 | }, 23 | "Linux": { 24 | "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-tidy", 25 | "hash": "e4a1537ee997aa486a67bcc06d050b1aa6cfb14aa3073c08f19123ac990ab2f7" 26 | } 27 | }, 28 | "actionlint": { 29 | "Darwin": { 30 | "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/actionlint/1.6.21/Darwin_amd64/actionlint", 31 | "hash": "b354db83815384d3c3a07f68f44b30cb0a70899757a0d185d7322de9952e8813" 32 | }, 33 | "Linux": { 34 | "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/actionlint/1.6.21/Linux_arm64/actionlint", 35 | "hash": "025ac157db121b33971ef24af72d73d71cda3cb1e3a94795bb2708ef4032ca76" 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /multipy/runtime/README.md: -------------------------------------------------------------------------------- 1 | # Torch Deploy 2 | This is an experimental feature to embed multiple python interpreters inside the torch library, 3 | providing a solution to the 'GIL problem' for multithreading with the convenience of python 4 | and eager or torchscripted pytorch programs. 5 | 6 | # libinterpreter 7 | This is an internal library used behind the scenes to enable multiple python interpreters in 8 | a single deploy runtime. libinterpreter.so is DLOPENed multiple times by the deploy library. 9 | Each copy of libinterpreter exposes a simple interpreter interface but hides its python and other 10 | internal symbols, preventing the different python instances from seeing each other. 11 | 12 | # CPython build 13 | Torch Deploy builds CPython from source as part of the embedded python interpreter. CPython has a flexible build system that builds successfully with or without a variety of dependencies installed - if missing, the resulting CPython build simply omits optional functionality, meaning some stdlib modules/libs are not present. 14 | 15 | Currently, the torch deploy build setup assumes the full CPython build is present. This matters because there is a [hardcoded list of python stdlib modules](https://github.com/pytorch/pytorch/blob/2662e34e9287a72e96dabb590e7732f9d4a6b37b/torch/csrc/deploy/interpreter/interpreter_impl.cpp#L35) that are explicitly loaded from the embedded binary at runtime. 16 | 17 | ### rebuilding CPython after installing missing dependencies 18 | Because CPython builds successfully when optional dependencies are missing, the cmake wrapper currently doesn't know if you need to rebuild CPython after adding missing dependencies (or whether dependencies were missing in the first place). 19 | 20 | To be safe, install the [complete list of dependencies for CPython](https://devguide.python.org/setup/#install-dependencies) for your platform, before trying to build torch with USE_DEPLOY=1. 21 | 22 | If you already built CPython without all the dependencies and want to fix it, just blow away the CPython folder under torch/csrc/deploy/third_party, install the missing system dependencies, and re-attempt the pytorch build command. 23 | -------------------------------------------------------------------------------- /multipy/runtime/interpreter/register_numpy.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | #include 9 | 10 | extern "C" struct _frozen _PyImport_FrozenModules_numpy[]; 11 | 12 | extern "C" PyObject* PyInit__multiarray_umath(void); 13 | extern "C" PyObject* PyInit__multiarray_tests(void); 14 | extern "C" PyObject* PyInit_lapack_lite(void); 15 | extern "C" PyObject* PyInit__umath_linalg(void); 16 | extern "C" PyObject* PyInit__pocketfft_internal(void); 17 | extern "C" PyObject* PyInit_mtrand(void); 18 | extern "C" PyObject* PyInit_bit_generator(void); 19 | extern "C" PyObject* PyInit__common(void); 20 | extern "C" PyObject* PyInit__bounded_integers(void); 21 | extern "C" PyObject* PyInit__mt19937(void); 22 | extern "C" PyObject* PyInit__philox(void); 23 | extern "C" PyObject* PyInit__pcg64(void); 24 | extern "C" PyObject* PyInit__sfc64(void); 25 | extern "C" PyObject* PyInit__generator(void); 26 | 27 | REGISTER_TORCH_DEPLOY_BUILTIN( 28 | frozen_numpy, 29 | _PyImport_FrozenModules_numpy, 30 | "numpy.core._multiarray_umath", 31 | PyInit__multiarray_umath, 32 | "numpy.core._multiarray_tests", 33 | PyInit__multiarray_tests, 34 | "numpy.linalg.lapack_lite", 35 | PyInit_lapack_lite, 36 | "numpy.linalg._umath_linalg", 37 | PyInit__umath_linalg, 38 | "numpy.fft._pocketfft_internal", 39 | PyInit__pocketfft_internal, 40 | "numpy.random.mtrand", 41 | PyInit_mtrand, 42 | "numpy.random.bit_generator", 43 | PyInit_bit_generator, 44 | "numpy.random._common", 45 | PyInit__common, 46 | "numpy.random._bounded_integers", 47 | PyInit__bounded_integers, 48 | "numpy.random._mt19937", 49 | PyInit__mt19937, 50 | "numpy.random._philox", 51 | PyInit__philox, 52 | "numpy.random._pcg64", 53 | PyInit__pcg64, 54 | "numpy.random._sfc64", 55 | PyInit__sfc64, 56 | "numpy.random._generator", 57 | PyInit__generator); 58 | -------------------------------------------------------------------------------- /.github/workflows/docs-build.yaml: -------------------------------------------------------------------------------- 1 | name: Docs Build 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | docbuild: 11 | runs-on: ubuntu-18.04 12 | steps: 13 | - name: Setup Python 14 | uses: actions/setup-python@v2 15 | with: 16 | python-version: 3.8 17 | architecture: x64 18 | - name: Checkout MultiPy 19 | uses: actions/checkout@v2 20 | with: 21 | submodules: true 22 | - name: Install Dependencies 23 | run: | 24 | set -eux 25 | sudo apt-get install -y pandoc doxygen 26 | pip install -r docs/requirements.txt 27 | - name: Doc Test 28 | run: | 29 | cd docs 30 | make doctest 31 | - name: Linkcheck 32 | run: | 33 | cd docs 34 | make linkcheck 35 | - name: Doc Build 36 | run: | 37 | cd docs 38 | make html 39 | 40 | docpush: 41 | runs-on: ubuntu-18.04 42 | needs: docbuild 43 | if: ${{ github.ref == 'refs/heads/main' }} 44 | steps: 45 | - name: Setup Python 46 | uses: actions/setup-python@v2 47 | with: 48 | python-version: 3.8 49 | architecture: x64 50 | - name: Checkout MultiPy 51 | uses: actions/checkout@v2 52 | - name: Set Identity 53 | run: | 54 | set -ex 55 | git config --global user.email "runner@github.com" 56 | git config --global user.name "MultiPy CI Runner" 57 | - name: Install Dependencies 58 | run: | 59 | set -eux 60 | sudo apt-get install -y pandoc doxygen 61 | pip install -r docs/requirements.txt 62 | - name: Build 63 | run: | 64 | set -ex 65 | sudo bash docs/doc_push.sh --dry-run 66 | - name: Push 67 | run: | 68 | set -ex 69 | cd /tmp/multipy_docs_tmp/multipy_gh_pages 70 | sudo git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} 71 | if git diff --exit-code gh-pages origin/gh-pages; then 72 | echo "Docs are unchanged, skipping push" 73 | exit 74 | fi 75 | git push 76 | -------------------------------------------------------------------------------- /multipy/runtime/interpreter/test_builtin_registry.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | namespace torch { 12 | namespace deploy { 13 | 14 | bool allowLibrary(const std::string& libname) { 15 | return libname == "lib1" || libname == "lib2"; 16 | } 17 | 18 | // NOLINTNEXTLINE(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) 19 | struct _frozen lib1FrozenModules[] = { 20 | {"mod1", nullptr, 0}, 21 | {nullptr, nullptr, 0}}; 22 | 23 | // NOLINTNEXTLINE(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) 24 | struct _frozen lib2FrozenModules[] = { 25 | {"mod2", nullptr, 0}, 26 | {"mod3", nullptr, 0}, 27 | {nullptr, nullptr, 0}}; 28 | 29 | void builtin1() {} 30 | void builtin2() {} 31 | REGISTER_TORCH_DEPLOY_BUILTIN( 32 | lib1, 33 | lib1FrozenModules, 34 | "lib1.builtin1", 35 | builtin1, 36 | "lib1.builtin2", 37 | builtin2); 38 | REGISTER_TORCH_DEPLOY_BUILTIN(lib2, lib2FrozenModules); 39 | 40 | TEST(BuiltinRegistryTest, SimpleTest) { 41 | const auto& items = BuiltinRegistry::items(); 42 | EXPECT_EQ(2, items.size()); 43 | EXPECT_EQ(lib1FrozenModules, items[0]->frozenModules); 44 | EXPECT_EQ(lib2FrozenModules, items[1]->frozenModules); 45 | 46 | struct _frozen* allFrozenModules = BuiltinRegistry::getAllFrozenModules(); 47 | EXPECT_EQ("mod1", allFrozenModules[0].name); 48 | EXPECT_EQ("mod2", allFrozenModules[1].name); 49 | EXPECT_EQ("mod3", allFrozenModules[2].name); 50 | EXPECT_EQ(nullptr, allFrozenModules[3].name); 51 | 52 | auto allBuiltinModules = BuiltinRegistry::getAllBuiltinModules(); 53 | EXPECT_EQ(2, allBuiltinModules.size()); 54 | EXPECT_EQ("lib1.builtin1", allBuiltinModules[0].first); 55 | EXPECT_EQ(builtin1, allBuiltinModules[0].second); 56 | EXPECT_EQ("lib1.builtin2", allBuiltinModules[1].first); 57 | EXPECT_EQ(builtin2, allBuiltinModules[1].second); 58 | 59 | std::string expectedBuiltinModulesCSV = "'lib1.builtin1', 'lib1.builtin2'"; 60 | EXPECT_EQ(expectedBuiltinModulesCSV, BuiltinRegistry::getBuiltinModulesCSV()); 61 | } 62 | 63 | } // namespace deploy 64 | } // namespace torch 65 | -------------------------------------------------------------------------------- /multipy/runtime/pybind_init.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | namespace py = pybind11; 7 | 8 | using namespace torch::deploy; 9 | 10 | namespace { 11 | at::IValue detachIValue(at::IValue&& iv) { 12 | if (iv.isTensor()) { 13 | // detach tensors to avoid cross interpreter autograd state 14 | return std::move(iv).toTensor().detach(); 15 | } 16 | return iv; 17 | } 18 | at::IValue toIValue(const py::handle& obj) { 19 | return detachIValue(torch::jit::toTypeInferredIValue(obj)); 20 | } 21 | } // namespace 22 | 23 | PYBIND11_MODULE(multipy_pybind, m) { 24 | m.doc() = "multipy python bindings"; 25 | 26 | py::class_(m, "InterpreterManager") 27 | .def(py::init()) 28 | .def("acquire_one", &InterpreterManager::acquireOne) 29 | .def( 30 | "__len__", 31 | [](InterpreterManager& self) -> int { 32 | return self.allInstances().size(); 33 | }) 34 | .def( 35 | "__getitem__", 36 | [](InterpreterManager& self, int i) -> InterpreterSession { 37 | return self.allInstances().at(i).acquireSession(); 38 | }); 39 | 40 | py::class_(m, "Interpreter") 41 | .def("acquire_session", &Interpreter::acquireSession); 42 | 43 | py::class_(m, "InterpreterSession") 44 | .def("global_", &InterpreterSession::global); 45 | 46 | py::class_(m, "Obj") 47 | .def( 48 | "__call__", 49 | [](Obj& self, py::args args, const py::kwargs& kwargs) -> Obj { 50 | std::vector iargs; 51 | std::unordered_map ikwargs; 52 | 53 | for (auto& arg : args) { 54 | iargs.emplace_back(toIValue(arg)); 55 | } 56 | for (auto& arg : kwargs) { 57 | ikwargs.emplace( 58 | arg.first.cast(), toIValue(arg.second)); 59 | } 60 | 61 | return self.callKwargs(iargs, ikwargs); 62 | }) 63 | .def( 64 | "__getattr__", 65 | [](Obj& self, std::string attr) -> Obj { 66 | return self.attr(attr.c_str()); 67 | }) 68 | .def("deref", [](Obj& self) -> py::object { 69 | return ::torch::jit::toPyObject(detachIValue(self.toIValue())); 70 | }); 71 | } 72 | -------------------------------------------------------------------------------- /scripts/linter/adapters/exec_linter.py: -------------------------------------------------------------------------------- 1 | """ 2 | EXEC: Ensure that source files are not executable. 3 | """ 4 | 5 | import argparse 6 | import json 7 | import logging 8 | import os 9 | import sys 10 | 11 | from enum import Enum 12 | from typing import NamedTuple, Optional 13 | 14 | LINTER_CODE = "EXEC" 15 | 16 | 17 | class LintSeverity(str, Enum): 18 | ERROR = "error" 19 | WARNING = "warning" 20 | ADVICE = "advice" 21 | DISABLED = "disabled" 22 | 23 | 24 | class LintMessage(NamedTuple): 25 | path: Optional[str] 26 | line: Optional[int] 27 | char: Optional[int] 28 | code: str 29 | severity: LintSeverity 30 | name: str 31 | original: Optional[str] 32 | replacement: Optional[str] 33 | description: Optional[str] 34 | 35 | 36 | def check_file(filename: str) -> Optional[LintMessage]: 37 | is_executable = os.access(filename, os.X_OK) 38 | if is_executable: 39 | return LintMessage( 40 | path=filename, 41 | line=None, 42 | char=None, 43 | code=LINTER_CODE, 44 | severity=LintSeverity.ERROR, 45 | name="executable-permissions", 46 | original=None, 47 | replacement=None, 48 | description="This file has executable permission; please remove it by using `chmod -x`.", 49 | ) 50 | return None 51 | 52 | 53 | if __name__ == "__main__": 54 | parser = argparse.ArgumentParser( 55 | description="exec linter", 56 | fromfile_prefix_chars="@", 57 | ) 58 | parser.add_argument( 59 | "--verbose", 60 | action="store_true", 61 | ) 62 | parser.add_argument( 63 | "filenames", 64 | nargs="+", 65 | help="paths to lint", 66 | ) 67 | 68 | args = parser.parse_args() 69 | 70 | logging.basicConfig( 71 | format="<%(threadName)s:%(levelname)s> %(message)s", 72 | level=( 73 | logging.NOTSET 74 | if args.verbose 75 | else logging.DEBUG 76 | if len(args.filenames) < 1000 77 | else logging.INFO 78 | ), 79 | stream=sys.stderr, 80 | ) 81 | 82 | lint_messages = [] 83 | for filename in args.filenames: 84 | lint_message = check_file(filename) 85 | if lint_message is not None: 86 | lint_messages.append(lint_message) 87 | 88 | for lint_message in lint_messages: 89 | print(json.dumps(lint_message._asdict()), flush=True) 90 | -------------------------------------------------------------------------------- /multipy/runtime/Exception.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #ifndef MULTIPY_EXCEPTION_H 8 | #define MULTIPY_EXCEPTION_H 9 | 10 | #include 11 | 12 | #define MULTIPY_INTERNAL_ASSERT_WITH_MESSAGE(condition, message) \ 13 | if (!(condition)) { \ 14 | throw std::runtime_error( \ 15 | "Internal Assertion failed: (" + std::string(#condition) + "), " + \ 16 | "function " + __FUNCTION__ + ", file " + __FILE__ + ", line " + \ 17 | std::to_string(__LINE__) + ".\n" + "Please report bug to Pytorch.\n" + \ 18 | message + "\n"); \ 19 | } 20 | 21 | #define MULTIPY_INTERNAL_ASSERT_NO_MESSAGE(condition) \ 22 | MULTIPY_INTERNAL_ASSERT_WITH_MESSAGE(condition, #condition) 23 | 24 | #define MULTIPY_INTERNAL_ASSERT_(x, condition, message, FUNC, ...) FUNC 25 | 26 | #define MULTIPY_INTERNAL_ASSERT(...) \ 27 | MULTIPY_INTERNAL_ASSERT_( \ 28 | , \ 29 | ##__VA_ARGS__, \ 30 | MULTIPY_INTERNAL_ASSERT_WITH_MESSAGE(__VA_ARGS__), \ 31 | MULTIPY_INTERNAL_ASSERT_NO_MESSAGE(__VA_ARGS__)); 32 | 33 | #define MULTIPY_CHECK_WITH_MESSAGE(condition, message) \ 34 | if (!(condition)) { \ 35 | throw std::runtime_error( \ 36 | "Check failed: (" + std::string(#condition) + "), " + "function " + \ 37 | __FUNCTION__ + ", file " + __FILE__ + ", line " + \ 38 | std::to_string(__LINE__) + ".\n" + message + "\n"); \ 39 | } 40 | 41 | #define MULTIPY_CHECK_NO_MESSAGE(condition) \ 42 | MULTIPY_CHECK_WITH_MESSAGE(condition, #condition) 43 | 44 | #define MULTIPY_CHECK_(x, condition, message, FUNC, ...) FUNC 45 | 46 | #define MULTIPY_CHECK(...) \ 47 | MULTIPY_CHECK_( \ 48 | , \ 49 | ##__VA_ARGS__, \ 50 | MULTIPY_CHECK_WITH_MESSAGE(__VA_ARGS__), \ 51 | MULTIPY_CHECK_NO_MESSAGE(__VA_ARGS__)); 52 | 53 | #endif // MULTIPY_EXCEPTION_H 54 | -------------------------------------------------------------------------------- /multipy/runtime/example/gpu_wrapper.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # used by the benchmarking program to wrap cpu models for GPU use 8 | from copy import deepcopy 9 | 10 | import torch 11 | 12 | 13 | def to_device(i, d): 14 | if isinstance(i, torch.Tensor): 15 | return i.to(device=d) 16 | elif isinstance(i, (tuple, list)): 17 | return tuple(to_device(e, d) for e in i) 18 | else: 19 | raise RuntimeError("inputs are weird") 20 | 21 | 22 | class GPUWrapper(torch.nn.Module): 23 | def __init__(self, root): 24 | super().__init__() 25 | self.models = [] 26 | self.streams = {} 27 | for i in range(torch.cuda.device_count()): 28 | m = deepcopy(root) if i != 0 else root 29 | d = f"cuda:{i}" 30 | m.to(device=d) 31 | self.models.append((m, d)) 32 | 33 | def __getstate__(self): 34 | return self.models 35 | 36 | def __setstate__(self, models): 37 | super().__init__() 38 | self.models = models 39 | self.streams = {} 40 | for m, d in models: 41 | torch.cuda.synchronize(d) 42 | 43 | # roi_align, 2210 count, ROIAlign_cuda.cu: add threadsync: problem goes away, return rand problem goes away, 44 | # use different streams here, problem goes away. 45 | def forward(self, tid, *args): 46 | m, d = self.models[tid % len(self.models)] 47 | if tid not in self.streams: 48 | self.streams[tid] = torch.cuda.Stream(d) 49 | s = self.streams[tid] 50 | with torch.cuda.stream(s): 51 | iput = to_device(args, d) 52 | r = to_device(m(*iput), "cpu") 53 | return r 54 | 55 | 56 | if __name__ == "__main__": 57 | 58 | def check_close(a, b): 59 | if isinstance(a, (list, tuple)): 60 | for ae, be in zip(a, b): 61 | check_close(ae, be) 62 | else: 63 | print(torch.max(torch.abs(a - b))) 64 | assert torch.allclose(a, b) 65 | 66 | import sys 67 | 68 | from torch.package import PackageImporter 69 | 70 | i = PackageImporter(sys.argv[1]) 71 | torch.version.interp = 0 72 | model = i.loadPickle("model", "model.pkl") 73 | eg = i.loadPickle("model", "example.pkl") 74 | r = model(*eg) 75 | 76 | gpu_model = GPUWrapper(model) 77 | r2 = gpu_model(*eg) 78 | check_close(r, r2) 79 | -------------------------------------------------------------------------------- /scripts/lint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under the BSD-style license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | set -e 9 | 10 | if [ ! "$(black --version)" ] 11 | then 12 | echo "Please install black." 13 | exit 1 14 | fi 15 | if [ ! "$(usort --version)" ] 16 | then 17 | echo "Please install usort." 18 | exit 1 19 | fi 20 | if [ ! "$(flake8 --version)" ] 21 | then 22 | echo "Please install flake8." 23 | exit 1 24 | fi 25 | 26 | # cd to the project directory 27 | cd "$(dirname "$0")/.." || exit 1 28 | 29 | GIT_URL_1="https://github.com/pytorch/MultiPy.git" 30 | GIT_URL_2="git@github.com:pytorch/MultiPy.git" 31 | 32 | UPSTREAM_URL="$(git config remote.upstream.url)" || true 33 | 34 | if [ -z "$UPSTREAM_URL" ] 35 | then 36 | echo "Setting upstream remote to $GIT_URL_1" 37 | git remote add upstream "$GIT_URL_1" 38 | elif [ "$UPSTREAM_URL" != "$GIT_URL_1" ] && \ 39 | [ "$UPSTREAM_URL" != "$GIT_URL_2" ] 40 | then 41 | echo "upstream remote set to $UPSTREAM_URL." 42 | echo "Please delete the upstream remote or set it to $GIT_URL_1 to use this script." 43 | exit 1 44 | fi 45 | 46 | git fetch upstream 47 | 48 | CHANGED_FILES="$(git diff --diff-filter=ACMRT --name-only upstream/main | grep '\.py$' | tr '\n' ' ')" 49 | 50 | if [ "$CHANGED_FILES" != "" ] 51 | then 52 | # Processing files one by one since passing directly $CHANGED_FILES will 53 | # treat the whole variable as a single file. 54 | echo "Running linters ..." 55 | for file in $CHANGED_FILES 56 | do 57 | echo "Checking $file" 58 | usort format "$file" 59 | black "$file" -q 60 | flake8 "$file" || LINT_ERRORS=1 61 | scripts/auto_copyright.py "$file" || LINT_ERRORS=1 62 | done 63 | else 64 | echo "No changes made to any Python files. Nothing to do." 65 | exit 0 66 | fi 67 | 68 | if [ "$LINT_ERRORS" != "" ] 69 | then 70 | echo "One of the linters returned an error. See above output." 71 | # need this so that CI fails 72 | exit 1 73 | fi 74 | 75 | # Check if any files were modified by running usort + black 76 | # If so, then the files were formatted incorrectly (e.g. did not pass lint) 77 | CHANGED_FILES="$(git diff --name-only | grep '\.py$' | tr '\n' ' ')" 78 | if [ "$CHANGED_FILES" != "" ] 79 | then 80 | git diff --name-only 81 | echo "There are uncommitted changes on disk likely caused by the linters." 82 | # need this so that CI fails 83 | exit 1 84 | fi 85 | -------------------------------------------------------------------------------- /multipy/runtime/mem_file.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #pragma once 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | namespace torch { 20 | namespace deploy { 21 | 22 | /// Memory maps a file into the address space read-only, and manages the 23 | /// lifetime of the mapping. Here are a few use cases: 24 | /// 1. Used in the loader to read in initial image, and to inspect 25 | // ELF files for dependencies before callling dlopen. 26 | /// 27 | /// 2. Used in unity to load the elf file. 28 | struct MemFile { 29 | explicit MemFile(const char* filename_) 30 | : fd_(0), mem_(nullptr), n_bytes_(0), name_(filename_) { 31 | fd_ = open(filename_, O_RDONLY); 32 | MULTIPY_CHECK( 33 | fd_ != -1, "failed to open {}: {}" + filename_ + strerror(errno)); 34 | // NOLINTNEXTLINE 35 | struct stat s; 36 | if (-1 == fstat(fd_, &s)) { 37 | close(fd_); // destructors don't run during exceptions 38 | MULTIPY_CHECK( 39 | false, "failed to stat {}: {}" + filename_ + strerror(errno)); 40 | } 41 | n_bytes_ = s.st_size; 42 | mem_ = mmap(nullptr, n_bytes_, PROT_READ, MAP_SHARED, fd_, 0); 43 | if (MAP_FAILED == mem_) { 44 | close(fd_); 45 | MULTIPY_CHECK( 46 | false, "failed to mmap {}: {}" + filename_ + strerror(errno)); 47 | } 48 | } 49 | MemFile(const MemFile&) = delete; 50 | MemFile& operator=(const MemFile&) = delete; 51 | [[nodiscard]] const char* data() const { 52 | return (const char*)mem_; 53 | } 54 | 55 | /// Returns whether or not the file descriptor 56 | /// of the underlying file is valid. 57 | int valid() { 58 | return fcntl(fd_, F_GETFD) != -1 || errno != EBADF; 59 | } 60 | ~MemFile() { 61 | if (mem_) { 62 | munmap((void*)mem_, n_bytes_); 63 | } 64 | if (fd_) { 65 | close(fd_); 66 | } 67 | } 68 | 69 | /// Returns the size of the underlying file defined by the `MemFile` 70 | size_t size() { 71 | return n_bytes_; 72 | } 73 | [[nodiscard]] int fd() const { 74 | return fd_; 75 | } 76 | 77 | private: 78 | int fd_; 79 | void* mem_; 80 | size_t n_bytes_; 81 | std::string name_; 82 | }; 83 | 84 | } // namespace deploy 85 | } // namespace torch 86 | -------------------------------------------------------------------------------- /multipy/runtime/interpreter/import_find_sharedfuncptr.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | using torch::deploy::CustomLibrary; 15 | using torch::deploy::CustomLibraryPtr; 16 | using torch::deploy::SystemLibrary; 17 | 18 | // NOLINTNEXTLINE 19 | std::vector loaded_files_; 20 | std::vector search_files_; 21 | // NOLINTNEXTLINE 22 | static void* deploy_self = nullptr; 23 | 24 | void loadSearchFile(const char* pathname) { 25 | const char* args[] = {"deploy"}; 26 | search_files_.emplace_back(CustomLibrary::create(pathname, 1, args)); 27 | CustomLibrary& lib = *search_files_.back(); 28 | lib.add_search_library(SystemLibrary::create(deploy_self)); 29 | lib.add_search_library(SystemLibrary::create()); 30 | for (const auto& f : search_files_) { 31 | if (f.get() == &lib) { 32 | continue; 33 | } 34 | lib.add_search_library(f); 35 | } 36 | lib.load(); 37 | } 38 | 39 | extern "C" { 40 | 41 | __attribute__((visibility("default"))) void deploy_set_self(void* v) { 42 | deploy_self = v; 43 | } 44 | 45 | typedef void (*dl_funcptr)(); 46 | extern "C" dl_funcptr _PyImport_FindSharedFuncptr( 47 | const char* prefix, 48 | const char* shortname, 49 | const char* pathname, 50 | FILE* fp) { 51 | const char* args[] = {"deploy"}; 52 | // XXX: we have to manually flush loaded_files_ (see deploy_flush_python_libs) 53 | // when the manager unloads. Otherwise some libraries can live longer than 54 | // they are needed, and the process of unloading them might use functionality 55 | // that itself gets unloaded. 56 | loaded_files_.emplace_back(CustomLibrary::create(pathname, 1, args)); 57 | CustomLibrary& lib = *loaded_files_.back(); 58 | assert(deploy_self); 59 | lib.add_search_library(SystemLibrary::create(deploy_self)); 60 | lib.add_search_library(SystemLibrary::create()); 61 | for (const auto& f : search_files_) { 62 | lib.add_search_library(f); 63 | } 64 | lib.load(); 65 | std::stringstream ss; 66 | ss << prefix << "_" << shortname; 67 | auto r = (dl_funcptr)lib.sym(ss.str().c_str()).value(); 68 | assert(r); 69 | return r; 70 | } 71 | __attribute__((visibility("default"))) void deploy_flush_python_libs() { 72 | loaded_files_.clear(); 73 | search_files_.clear(); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /multipy/runtime/elf_file.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #pragma once 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | namespace torch { 18 | namespace deploy { 19 | 20 | /// A section of an ElfFile. 21 | struct Section { 22 | Section() = default; 23 | explicit Section( 24 | std::shared_ptr _memfile, 25 | const char* _name, 26 | const char* _start, 27 | size_t _len = 0) 28 | : memfile(_memfile), name(_name), start(_start), len(_len) {} 29 | 30 | std::shared_ptr memfile; 31 | const char* name{nullptr}; 32 | const char* start{nullptr}; 33 | size_t len{0}; 34 | 35 | operator bool() const { 36 | return start != nullptr; 37 | } 38 | }; 39 | 40 | // TODO: consolidate other ELF file related functions in loader.cpp to this file 41 | 42 | /* 43 | * This class provie utilities to handle ELF file. Only support 64bit ELF file. 44 | */ 45 | class ElfFile { 46 | public: 47 | /// Constructs an Elffile with the corresponding `filename` 48 | explicit ElfFile(const char* filename); 49 | 50 | /// Finds and return a `Section` with the corresponding `name`. If nothing is 51 | /// found, then a `std::nullopt` is returned. 52 | std::optional
findSection(const char* name) const; 53 | 54 | private: 55 | Section toSection(Elf64_Shdr* shdr) { 56 | auto nameOff = shdr->sh_name; 57 | auto shOff = shdr->sh_offset; 58 | auto len = shdr->sh_size; 59 | const char* name = ""; 60 | 61 | if (strtabSection_) { 62 | MULTIPY_CHECK(nameOff >= 0 && nameOff < strtabSection_.len); 63 | name = strtabSection_.start + nameOff; 64 | } 65 | const char* start = memFile_->data() + shOff; 66 | return Section{memFile_, name, start, len}; 67 | } 68 | 69 | [[nodiscard]] const char* str(size_t off) const { 70 | MULTIPY_CHECK(off < strtabSection_.len, "String table index out of range"); 71 | return strtabSection_.start + off; 72 | } 73 | void checkFormat() const; 74 | std::shared_ptr memFile_; 75 | Elf64_Ehdr* ehdr_; 76 | Elf64_Shdr* shdrList_; 77 | size_t numSections_; 78 | 79 | Section strtabSection_; 80 | std::vector
sections_; 81 | }; 82 | 83 | std::optional
searchForSection(const char* name); 84 | 85 | } // namespace deploy 86 | } // namespace torch 87 | -------------------------------------------------------------------------------- /multipy/runtime/embedded_file.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | namespace torch { 16 | namespace deploy { 17 | 18 | EmbeddedFile::EmbeddedFile( 19 | std::string name, 20 | const std::initializer_list& sections, 21 | const std::initializer_list symbols) 22 | : libraryName("/tmp/multipy_" + name + "XXXXXX") { 23 | int fd = mkstemp(libraryName.data()); 24 | MULTIPY_INTERNAL_ASSERT(fd != -1, "failed to create temporary file"); 25 | FILE* dst = fdopen(fd, "wb"); 26 | MULTIPY_INTERNAL_ASSERT(dst); 27 | 28 | const char* payloadStart = nullptr; 29 | size_t size = 0; 30 | // payloadSection needs to be kept to ensure the source file is still mapped. 31 | std::optional
payloadSection; 32 | for (const auto& s : sections) { 33 | payloadSection = searchForSection(s.sectionName); 34 | if (payloadSection != std::nullopt) { 35 | payloadStart = payloadSection->start; 36 | customLoader = s.customLoader; 37 | size = payloadSection->len; 38 | MULTIPY_CHECK(payloadSection.has_value(), "Missing the payload section"); 39 | break; 40 | } 41 | } 42 | if (payloadStart == nullptr) { 43 | const char* libStart = nullptr; 44 | const char* libEnd = nullptr; 45 | for (const auto& s : symbols) { 46 | libStart = (const char*)dlsym(nullptr, s.startSym); 47 | if (libStart) { 48 | libEnd = (const char*)dlsym(nullptr, s.endSym); 49 | customLoader = s.customLoader; 50 | break; 51 | } 52 | } 53 | MULTIPY_CHECK( 54 | libStart != nullptr && libEnd != nullptr, 55 | "torch::deploy requires a build-time dependency on " 56 | "embedded_interpreter or embedded_interpreter_cuda, neither of which " 57 | "were found. name=" + 58 | name + " torch::cuda::is_available()=" + 59 | std::to_string(torch::cuda::is_available())); 60 | 61 | size = libEnd - libStart; 62 | payloadStart = libStart; 63 | } 64 | size_t written = fwrite(payloadStart, 1, size, dst); 65 | MULTIPY_INTERNAL_ASSERT(size == written, "expected written == size"); 66 | 67 | fclose(dst); 68 | } 69 | 70 | EmbeddedFile::~EmbeddedFile() { 71 | unlink(libraryName.c_str()); 72 | } 73 | 74 | } // namespace deploy 75 | } // namespace torch 76 | -------------------------------------------------------------------------------- /scripts/auto_copyright.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under the BSD-style license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | import os 9 | import sys 10 | from typing import List, Tuple 11 | 12 | COPYRIGHT_NOTICE_PYTHON: str = """ 13 | # Copyright (c) Meta Platforms, Inc. and affiliates. 14 | # All rights reserved. 15 | # 16 | # This source code is licensed under the BSD-style license found in the 17 | # LICENSE file in the root directory of this source tree. 18 | """.strip() 19 | 20 | COPYRIGHT_NOTICE_C: str = """ 21 | // Copyright (c) Meta Platforms, Inc. and affiliates. 22 | // All rights reserved. 23 | // 24 | // This source code is licensed under the BSD-style license found in the 25 | // LICENSE file in the root directory of this source tree. 26 | """.strip() 27 | 28 | 29 | def get_all_files_with_extension(directories: List[str], extensions: Tuple[str]): 30 | filelist = [] 31 | for directory in directories: 32 | for path, _, files in os.walk(directory): 33 | for file in files: 34 | if file.endswith(extensions): 35 | filelist.append(path + os.sep + file) 36 | return filelist 37 | 38 | 39 | def apply_copywrite( 40 | directories: List[str], COPYRIGHT_NOTICE: str, extensions: Tuple[str] 41 | ) -> None: 42 | missing = [] 43 | filelist = get_all_files_with_extension(directories, extensions) 44 | 45 | for path in filelist: 46 | with open(path, "rt") as f: 47 | data = f.read() 48 | 49 | data = data.strip() 50 | # don't lint empty files 51 | if len(data) == 0: 52 | continue 53 | 54 | # skip the interpreter command and formatting 55 | while data.startswith("#!") or data.startswith("# -*-"): 56 | data = data[data.index("\n") + 1 :] 57 | 58 | if not data.startswith(COPYRIGHT_NOTICE): 59 | missing.append(path) 60 | 61 | for filename in missing: 62 | try: 63 | inbuffer = open(filename, "U").readlines() 64 | outbuffer = [COPYRIGHT_NOTICE] + ["\n\n"] + inbuffer 65 | 66 | open(filename, "w").write("".join([str(line) for line in outbuffer])) 67 | except IOError: 68 | print( 69 | f"Please check the files, there was an error when trying to open {filename}..." 70 | ) 71 | raise 72 | 73 | 74 | if __name__ == "__main__": 75 | files = sys.argv[1:] 76 | apply_copywrite(files, COPYRIGHT_NOTICE_PYTHON, (".py")) 77 | apply_copywrite(files, COPYRIGHT_NOTICE_C, (".c", ".cpp", ".hpp", ".h")) 78 | -------------------------------------------------------------------------------- /.github/workflows/test_docker_build.yml: -------------------------------------------------------------------------------- 1 | name: Docker tests 2 | on: 3 | pull_request: 4 | push: 5 | branches: 6 | - main 7 | workflow_dispatch: 8 | jobs: 9 | unittest: 10 | strategy: 11 | matrix: 12 | python3-minor-version: [7,8,9,10] 13 | platform: [linux.4xlarge.nvidia.gpu] 14 | fail-fast: false 15 | runs-on: ${{ matrix.platform }} 16 | steps: 17 | - name: Checkout MultiPy 18 | uses: actions/checkout@v2 19 | with: 20 | submodules: true 21 | - name: Install nvidia driver, nvidia-docker runtime, set GPU_FLAG 22 | uses: pytorch/test-infra/.github/actions/setup-nvidia@main 23 | - name: Setup SSH (Click me for login details) 24 | uses: ./.github/actions/setup-ssh 25 | with: 26 | github-secret: ${{ secrets.GITHUB_TOKEN }} 27 | 28 | - name: Build Image 29 | env: 30 | DOCKER_BUILDKIT: 1 31 | run: nvidia-docker build -t multipy --progress=plain --build-arg PYTHON_3_MINOR_VERSION=${{ matrix.python3-minor-version }} --build-arg BUILD_CUDA_TESTS=1 . 32 | 33 | - name: Run Tests and Examples 34 | run: | 35 | docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && multipy/runtime/build/test_deploy" 36 | nvidia-docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && multipy/runtime/build/test_deploy_gpu" 37 | docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && python multipy/runtime/test_pybind.py" 38 | docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && examples/build/hello_world_example" 39 | docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && python3 examples/quickstart/gen_package.py && ./examples/build/quickstart my_package.pt" 40 | docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && ./examples/build/movable_example" 41 | docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && ./multipy/runtime/build/deploy_benchmark 2 none jit multipy/runtime/example/generated/resnet" 42 | docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -gt 7 ]]; then pip install -r compat-requirements.txt && multipy/runtime/build/interactive_embedded_interpreter --pyscript multipy/runtime/test_compat.py; fi" 43 | -------------------------------------------------------------------------------- /multipy/runtime/remove_dt_needed.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | 21 | #define ERROR(msg_fmt, ...) \ 22 | throw std::runtime_error(fmt::format(msg_fmt, ##__VA_ARGS__)) 23 | 24 | #define CHECK(cond, fmt, ...) \ 25 | if (!(cond)) { \ 26 | ERROR(fmt, ##__VA_ARGS__); \ 27 | } 28 | 29 | // NOLINTNEXTLINE 30 | int main(int argc, const char** argv) { 31 | if (argc != 3) { 32 | std::cout << "usage: " << argv[0] << " \n"; 33 | return 1; 34 | } 35 | const char* filename = argv[1]; 36 | int fd_ = open(filename, O_RDONLY); 37 | CHECK(fd_ != -1, "failed to open {}: {}", filename, strerror(errno)); 38 | struct stat s = {0}; 39 | if (-1 == fstat(fd_, &s)) { 40 | close(fd_); // destructors don't run during exceptions 41 | ERROR("failed to stat {}: {}", filename, strerror(errno)); 42 | } 43 | size_t n_bytes = s.st_size; 44 | void* mem = 45 | mmap(nullptr, n_bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd_, 0); 46 | if (MAP_FAILED == mem) { 47 | close(fd_); 48 | ERROR("failed to mmap {}: {}", filename, strerror(errno)); 49 | } 50 | 51 | char* data = (char*)mem; 52 | auto header = (Elf64_Ehdr*)data; 53 | auto program_headers = (Elf64_Phdr*)(data + header->e_phoff); 54 | auto n_program_headers = header->e_phnum; 55 | Elf64_Dyn* dynamic = nullptr; 56 | for (decltype(n_program_headers) i = 0; i < n_program_headers; ++i) { 57 | const Elf64_Phdr* phdr = &program_headers[i]; 58 | if (phdr->p_type == PT_DYNAMIC) { 59 | dynamic = reinterpret_cast(data + phdr->p_offset); 60 | break; 61 | } 62 | } 63 | CHECK( 64 | dynamic, 65 | "{}: could not load dynamic section for looking up DT_NEEDED", 66 | filename); 67 | std::vector entries; 68 | for (const Elf64_Dyn* d = dynamic; d->d_tag != DT_NULL; ++d) { 69 | entries.push_back(*d); 70 | } 71 | Elf64_Dyn* w = dynamic; 72 | for (const Elf64_Dyn& e : entries) { 73 | if (e.d_tag != DT_NEEDED) { 74 | *w++ = e; 75 | } 76 | } 77 | auto nwritten = w - dynamic; 78 | memset(w, 0, sizeof(Elf64_Dyn) * (entries.size() - nwritten)); 79 | 80 | FILE* dst = fopen(argv[2], "w"); 81 | CHECK(dst != nullptr, "{}: {}", argv[2], strerror(errno)); 82 | fwrite(mem, n_bytes, 1, dst); 83 | fclose(dst); 84 | munmap(mem, n_bytes); 85 | close(fd_); 86 | return 0; 87 | } 88 | -------------------------------------------------------------------------------- /scripts/linter/adapters/pip_init.py: -------------------------------------------------------------------------------- 1 | """ 2 | Initializer script that installs stuff to pip. 3 | """ 4 | 5 | import argparse 6 | import logging 7 | import os 8 | import subprocess 9 | import sys 10 | import time 11 | 12 | from typing import List 13 | 14 | 15 | def run_command(args: List[str]) -> "subprocess.CompletedProcess[bytes]": 16 | logging.debug("$ %s", " ".join(args)) 17 | start_time = time.monotonic() 18 | try: 19 | return subprocess.run(args, check=True) 20 | finally: 21 | end_time = time.monotonic() 22 | logging.debug("took %dms", (end_time - start_time) * 1000) 23 | 24 | 25 | if __name__ == "__main__": 26 | parser = argparse.ArgumentParser(description="pip initializer") 27 | parser.add_argument( 28 | "packages", 29 | nargs="+", 30 | help="pip packages to install", 31 | ) 32 | parser.add_argument( 33 | "--verbose", 34 | action="store_true", 35 | help="verbose logging", 36 | ) 37 | parser.add_argument( 38 | "--dry-run", help="do not install anything, just print what would be done." 39 | ) 40 | parser.add_argument( 41 | "--no-black-binary", 42 | help="do not use pre-compiled binaries from pip for black.", 43 | action="store_true", 44 | ) 45 | 46 | args = parser.parse_args() 47 | 48 | logging.basicConfig( 49 | format="<%(threadName)s:%(levelname)s> %(message)s", 50 | level=logging.NOTSET if args.verbose else logging.DEBUG, 51 | stream=sys.stderr, 52 | ) 53 | 54 | pip_args = ["pip3", "install"] 55 | 56 | # If we are in a global install, use `--user` to install so that you do not 57 | # need root access in order to initialize linters. 58 | # 59 | # However, `pip install --user` interacts poorly with virtualenvs (see: 60 | # https://bit.ly/3vD4kvl) and conda (see: https://bit.ly/3KG7ZfU). So in 61 | # these cases perform a regular installation. 62 | in_conda = os.environ.get("CONDA_PREFIX") is not None 63 | in_virtualenv = os.environ.get("VIRTUAL_ENV") is not None 64 | if not in_conda and not in_virtualenv: 65 | pip_args.append("--user") 66 | 67 | pip_args.extend(args.packages) 68 | 69 | for package in args.packages: 70 | package_name, _, version = package.partition("=") 71 | if version == "": 72 | raise RuntimeError( 73 | "Package {package_name} did not have a version specified. " 74 | "Please specify a version to produce a consistent linting experience." 75 | ) 76 | if args.no_black_binary and "black" in package_name: 77 | pip_args.append(f"--no-binary={package_name}") 78 | 79 | dry_run = args.dry_run == "1" 80 | if dry_run: 81 | print(f"Would have run: {pip_args}") 82 | sys.exit(0) 83 | 84 | run_command(pip_args) 85 | -------------------------------------------------------------------------------- /multipy/runtime/example/tensorrt_example.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import pickle 8 | from typing import Any, List 9 | 10 | import torch 11 | 12 | 13 | class TestTRTModule(torch.nn.Module): 14 | def __init__(self, engine, input_names=None, output_names=None, fp16_output=False): 15 | super(TestTRTModule, self).__init__() 16 | self.engine = engine 17 | self.input_names = input_names 18 | self.output_names = output_names 19 | 20 | # Indicate output is in fp16 21 | self.fp16_output = fp16_output 22 | 23 | def forward(self, *inputs): 24 | batch_size = inputs[0].shape[0] 25 | contiguous_inputs: List[torch.Tensor] = [i.contiguous() for i in inputs] 26 | bindings: List[Any] = [None] * (len(self.input_names) + len(self.output_names)) 27 | 28 | # create output tensors 29 | outputs: List[torch.Tensor] = [] 30 | for _, output_name in enumerate(self.output_names): 31 | idx: int = self.engine.get_binding_index(output_name) 32 | shape = (batch_size,) + tuple(self.engine.get_binding_shape(idx)) 33 | output = torch.empty(size=shape, dtype=torch.float32, device="cuda") 34 | outputs.append(output) 35 | bindings[idx] = output.data_ptr() 36 | 37 | for i, input_name in enumerate(self.input_names): 38 | idx = self.engine.get_binding_index(input_name) 39 | bindings[idx] = contiguous_inputs[i].data_ptr() 40 | 41 | context = self.engine.create_execution_context() 42 | context.execute_async( 43 | batch_size, bindings, torch.cuda.current_stream().cuda_stream 44 | ) 45 | 46 | if len(outputs) == 1: 47 | return outputs[0] 48 | 49 | return tuple(outputs) 50 | 51 | 52 | def make_trt_module(): 53 | import tensorrt as trt 54 | 55 | logger = trt.Logger(trt.Logger.WARNING) 56 | builder = trt.Builder(logger) 57 | network = builder.create_network() 58 | 59 | x = network.add_input("x", shape=(1, 2, 3), dtype=trt.float32) 60 | layer = network.add_elementwise(x, x, trt.ElementWiseOperation.SUM) 61 | layer.name = "add" 62 | output = layer.get_output(0) 63 | output.name = "output" 64 | network.mark_output(output) 65 | output.dtype = trt.float32 66 | 67 | builder.max_batch_size = 1024 68 | builder_config = builder.create_builder_config() 69 | builder_config.max_workspace_size = 1 << 25 70 | # Test engine can be serialized and loaded correctly. 71 | serialized_engine = pickle.dumps(builder.build_engine(network, builder_config)) 72 | return TestTRTModule(pickle.loads(serialized_engine), ["x"], ["output"]) 73 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | AccessModifierOffset: -1 3 | AlignAfterOpenBracket: AlwaysBreak 4 | AlignConsecutiveAssignments: false 5 | AlignConsecutiveDeclarations: false 6 | AlignEscapedNewlinesLeft: true 7 | AlignOperands: false 8 | AlignTrailingComments: false 9 | AllowAllParametersOfDeclarationOnNextLine: false 10 | AllowShortBlocksOnASingleLine: false 11 | AllowShortCaseLabelsOnASingleLine: false 12 | AllowShortFunctionsOnASingleLine: Empty 13 | AllowShortIfStatementsOnASingleLine: false 14 | AllowShortLoopsOnASingleLine: false 15 | AlwaysBreakAfterReturnType: None 16 | AlwaysBreakBeforeMultilineStrings: true 17 | AlwaysBreakTemplateDeclarations: true 18 | BinPackArguments: false 19 | BinPackParameters: false 20 | BraceWrapping: 21 | AfterClass: false 22 | AfterControlStatement: false 23 | AfterEnum: false 24 | AfterFunction: false 25 | AfterNamespace: false 26 | AfterObjCDeclaration: false 27 | AfterStruct: false 28 | AfterUnion: false 29 | BeforeCatch: false 30 | BeforeElse: false 31 | IndentBraces: false 32 | BreakBeforeBinaryOperators: None 33 | BreakBeforeBraces: Attach 34 | BreakBeforeTernaryOperators: true 35 | BreakConstructorInitializersBeforeComma: false 36 | BreakAfterJavaFieldAnnotations: false 37 | BreakStringLiterals: false 38 | ColumnLimit: 80 39 | CommentPragmas: '^ IWYU pragma:' 40 | CompactNamespaces: false 41 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 42 | ConstructorInitializerIndentWidth: 4 43 | ContinuationIndentWidth: 4 44 | Cpp11BracedListStyle: true 45 | DerivePointerAlignment: false 46 | DisableFormat: false 47 | ForEachMacros: [ FOR_EACH_RANGE, FOR_EACH, ] 48 | IncludeCategories: 49 | - Regex: '^<.*\.h(pp)?>' 50 | Priority: 1 51 | - Regex: '^<.*' 52 | Priority: 2 53 | - Regex: '.*' 54 | Priority: 3 55 | IndentCaseLabels: true 56 | IndentWidth: 2 57 | IndentWrappedFunctionNames: false 58 | KeepEmptyLinesAtTheStartOfBlocks: false 59 | MacroBlockBegin: '' 60 | MacroBlockEnd: '' 61 | MaxEmptyLinesToKeep: 1 62 | NamespaceIndentation: None 63 | ObjCBlockIndentWidth: 2 64 | ObjCSpaceAfterProperty: false 65 | ObjCSpaceBeforeProtocolList: false 66 | PenaltyBreakBeforeFirstCallParameter: 1 67 | PenaltyBreakComment: 300 68 | PenaltyBreakFirstLessLess: 120 69 | PenaltyBreakString: 1000 70 | PenaltyExcessCharacter: 1000000 71 | PenaltyReturnTypeOnItsOwnLine: 2000000 72 | PointerAlignment: Left 73 | ReflowComments: true 74 | SortIncludes: true 75 | SpaceAfterCStyleCast: false 76 | SpaceBeforeAssignmentOperators: true 77 | SpaceBeforeParens: ControlStatements 78 | SpaceInEmptyParentheses: false 79 | SpacesBeforeTrailingComments: 1 80 | SpacesInAngles: false 81 | SpacesInContainerLiterals: true 82 | SpacesInCStyleCastParentheses: false 83 | SpacesInParentheses: false 84 | SpacesInSquareBrackets: false 85 | Standard: Cpp11 86 | TabWidth: 8 87 | UseTab: Never 88 | ... 89 | -------------------------------------------------------------------------------- /multipy/runtime/test_deploy_lib.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | namespace py = pybind11; 12 | 13 | int foo_constructed = 0; 14 | int bar_constructed = 0; 15 | int bar_destructed = 0; 16 | 17 | struct Foo { 18 | Foo() { 19 | ++foo_constructed; 20 | } 21 | int v = -1; 22 | }; 23 | 24 | Foo f; 25 | 26 | struct Bar { 27 | Bar() { 28 | ++bar_constructed; 29 | } 30 | ~Bar() { 31 | ++bar_destructed; 32 | } 33 | int v = 14; 34 | }; 35 | 36 | static thread_local int first = 1; // local TLS, probably offset 0 37 | static thread_local int second = 2; // local TLS, probably offset 4 38 | thread_local int bss_local; // local TLS, bss initialized so it probably comes 39 | // after the initialized stuff 40 | thread_local int third = 3; // local TLS, but extern declared so it will look 41 | // for the symbol third globally, but not find it 42 | static thread_local Bar bar; // local TLS, with a constructor that should run 43 | thread_local int 44 | in_another_module; // non local TLS, this is defined in test_deploy.cpp 45 | 46 | struct MyError : public std::runtime_error { 47 | using std::runtime_error::runtime_error; 48 | }; 49 | 50 | bool raise_and_catch_exception(bool except) { 51 | try { 52 | if (except) { 53 | throw MyError("yep"); 54 | } 55 | return false; 56 | } catch (MyError&) { 57 | return true; 58 | } 59 | } 60 | bool raise_exception() { 61 | throw MyError("yet"); // caught in test_deploy 62 | } 63 | 64 | bool check_initial_state() { 65 | bool bv = bar.v == 14; // unless we reference bar it is unspecified whether it 66 | // should have been constructed 67 | return bv && first == 1 && second == 2 && bss_local == 0 && third == 3 && 68 | bar_constructed == 1 && foo_constructed == 1 && bar_destructed == 0; 69 | } 70 | 71 | int get_in_another_module() { 72 | return in_another_module; 73 | } 74 | 75 | void set_in_another_module(int x) { 76 | in_another_module = x; 77 | } 78 | int get_bar() { 79 | return bar.v; 80 | } 81 | void set_bar(int v) { 82 | bar.v = v; 83 | } 84 | int get_bar_destructed() { 85 | return bar_destructed; 86 | } 87 | 88 | int simple_add(int a, int b) { 89 | return a + b; 90 | } 91 | 92 | PYBIND11_MODULE(libtest_deploy_lib, m) { 93 | m.def("raise_and_catch_exception", raise_and_catch_exception); 94 | m.def("raise_exception", raise_exception); 95 | m.def("check_initial_state", check_initial_state); 96 | m.def("get_in_another_module", get_in_another_module); 97 | m.def("set_in_another_module", set_in_another_module); 98 | m.def("get_bar", get_bar); 99 | m.def("set_bar", set_bar); 100 | m.def("get_bar_destructed", get_bar_destructed); 101 | m.def("simple_add", simple_add); 102 | } 103 | -------------------------------------------------------------------------------- /docs/source/tutorials/quickstart.rst: -------------------------------------------------------------------------------- 1 | Quickstart 2 | ========== 3 | 4 | Packaging a model ``for torch::deploy`` 5 | --------------------------------------- 6 | 7 | ``torch::deploy`` can load and run Python models that are packaged with 8 | ``torch.package``. You can find ``torch.package``'s documentation 9 | `here `__. 10 | 11 | For now, let’s create a simple model that we can load and run in 12 | ``torch::deploy``. 13 | 14 | .. literalinclude:: ../../../examples/quickstart/gen_package.py 15 | :language: python 16 | :lines: 4- 17 | 18 | Note that since "numpy", "sys", "PIL" were marked as "extern", 19 | ``torch.package`` will look for these dependencies on the system that 20 | loads this package. They will not be packaged with the model. 21 | 22 | Now, there should be a file named ``my_package.pt`` in your working 23 | directory. 24 | 25 | Load the model in C++ 26 | --------------------- 27 | 28 | .. literalinclude:: ../../../examples/quickstart/quickstart.cpp 29 | :language: c++ 30 | :lines: 11- 31 | 32 | This small program introduces many of the core concepts of 33 | ``torch::deploy``. 34 | 35 | An ``InterpreterManager`` abstracts over a collection of independent 36 | Python interpreters, allowing you to load balance across them when 37 | running your code. 38 | 39 | Using the ``InterpreterManager::loadPackage`` method, you can load a 40 | ``torch.package`` from disk and make it available to all interpreters. 41 | 42 | ``Package::loadPickle`` allows you to retrieve specific Python objects 43 | from the package, like the ResNet model we saved earlier. 44 | 45 | Finally, the model itself is a ``ReplicatedObj``. This is an abstract 46 | handle to an object that is replicated across multiple interpreters. 47 | When you interact with a ``ReplicatedObj`` (for example, by calling 48 | ``forward``), it will select an free interpreter to execute that 49 | interaction. 50 | 51 | Build and execute the C++ example 52 | --------------------------------- 53 | 54 | Assuming the above C++ program was stored in a file called 55 | ``example-app.cpp``, a minimal ``CMakeLists.txt`` file would look like: 56 | 57 | .. literalinclude:: ../../../examples/CMakeLists.txt 58 | :language: cmake 59 | :lines: 1-20,24,25 60 | 61 | The ``-rdynamic`` and ``--no-as-needed`` flags are needed when linking to the 62 | executable to ensure that symbols are exported to the dynamic table, 63 | making them accessible to the ``torch::deploy`` interpreters (which are dynamically 64 | loaded). 65 | 66 | The last step is configuring and building the project. Assuming that our 67 | code directory is laid out like this: 68 | 69 | .. code:: shell 70 | 71 | example-app/ 72 | CMakeLists.txt 73 | quickstart.cpp 74 | 75 | We can now run the following commands to build the application from 76 | within the ``example-app/`` folder: 77 | 78 | .. code:: bash 79 | 80 | cmake -S . -B build -DMULTIPY_PATH="/home/user/repos/multipy" # the parent directory of multipy (i.e. the git repo) 81 | cmake --build build --config Release -j 82 | 83 | Now we can run our app: 84 | 85 | .. code:: bash 86 | 87 | ./example-app /path/to/my_package.pt 88 | -------------------------------------------------------------------------------- /multipy/runtime/interpreter/register_frozenpython.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | #include 9 | 10 | #define FOREACH_LIBRARY(_) \ 11 | _(array) \ 12 | _(_asyncio) \ 13 | _(audioop) \ 14 | _(binascii) \ 15 | _(_bisect) \ 16 | _(_blake2) \ 17 | _(_bz2) \ 18 | _(cmath) \ 19 | _(_codecs_cn) \ 20 | _(_codecs_hk) \ 21 | _(_codecs_iso2022) \ 22 | _(_codecs_jp) \ 23 | _(_codecs_kr) \ 24 | _(_codecs_tw) \ 25 | _(_contextvars) \ 26 | _(_crypt) \ 27 | _(_csv) \ 28 | _(_ctypes) \ 29 | _(_ctypes_test) \ 30 | _(_curses) \ 31 | _(_curses_panel) \ 32 | _(_datetime) \ 33 | _(_decimal) \ 34 | _(_elementtree) \ 35 | _(fcntl) \ 36 | _(grp) \ 37 | _(_hashlib) \ 38 | _(_heapq) \ 39 | _(_json) \ 40 | _(_lsprof) \ 41 | _(_lzma) \ 42 | _(math) \ 43 | _(_md5) \ 44 | _(mmap) \ 45 | _(_multibytecodec) \ 46 | _(_multiprocessing) \ 47 | _(nis) \ 48 | _(_opcode) \ 49 | _(ossaudiodev) \ 50 | _(_pickle) \ 51 | _(_posixsubprocess) \ 52 | _(pyexpat) \ 53 | _(_queue) \ 54 | _(_random) \ 55 | _(readline) \ 56 | _(resource) \ 57 | _(select) \ 58 | _(_sha1) \ 59 | _(_sha256) \ 60 | _(_sha3) \ 61 | _(_sha512) \ 62 | _(_socket) \ 63 | _(spwd) \ 64 | _(_ssl) \ 65 | _(_struct) \ 66 | _(syslog) \ 67 | _(termios) \ 68 | _(_testbuffer) \ 69 | _(_testcapi) \ 70 | _(_testimportmultiple) \ 71 | _(_testmultiphase) \ 72 | _(unicodedata) \ 73 | _(xxlimited) \ 74 | _(_xxtestfuzz) \ 75 | _(zlib) 76 | 77 | #define DECLARE_LIBRARY_INIT(name) extern "C" PyObject* PyInit_##name(void); 78 | FOREACH_LIBRARY(DECLARE_LIBRARY_INIT) 79 | #undef DECLARE_LIBRARY_INIT 80 | 81 | extern "C" struct _frozen _PyImport_FrozenModules[]; 82 | 83 | #define STD_LIBARY_PARMS(name) , #name, PyInit_##name 84 | #ifdef INCLUDE_LEGACY_PARSER_MODULE 85 | extern "C" PyObject* PyInit_parser(void); 86 | REGISTER_TORCH_DEPLOY_BUILTIN( 87 | frozenpython, 88 | _PyImport_FrozenModules FOREACH_LIBRARY(STD_LIBARY_PARMS), 89 | "parser", 90 | PyInit_parser); 91 | #else 92 | REGISTER_TORCH_DEPLOY_BUILTIN( 93 | frozenpython, 94 | _PyImport_FrozenModules FOREACH_LIBRARY(STD_LIBARY_PARMS)); 95 | #endif 96 | #undef STD_LIBARY_PARMS 97 | -------------------------------------------------------------------------------- /multipy/runtime/elf_file.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | #include 15 | 16 | namespace torch { 17 | namespace deploy { 18 | 19 | ElfFile::ElfFile(const char* filename) 20 | : memFile_(std::make_shared(filename)) { 21 | const char* fileData = memFile_->data(); 22 | ehdr_ = (Elf64_Ehdr*)fileData; 23 | checkFormat(); 24 | 25 | numSections_ = ehdr_->e_shnum; 26 | shdrList_ = (Elf64_Shdr*)(fileData + ehdr_->e_shoff); 27 | 28 | auto strtabSecNo = ehdr_->e_shstrndx; 29 | MULTIPY_CHECK( 30 | strtabSecNo >= 0 && strtabSecNo < numSections_, 31 | "e_shstrndx out of range"); 32 | 33 | strtabSection_ = toSection(&shdrList_[strtabSecNo]); 34 | 35 | sections_.reserve(numSections_); 36 | for (const auto i : c10::irange(numSections_)) { 37 | sections_.emplace_back(toSection(&shdrList_[i])); 38 | } 39 | } 40 | 41 | std::optional
ElfFile::findSection(const char* name) const { 42 | MULTIPY_CHECK(name != nullptr, "Null name"); 43 | std::optional
found = std::nullopt; 44 | for (const auto& section : sections_) { 45 | if (strcmp(name, section.name) == 0) { 46 | found = section; 47 | break; 48 | } 49 | } 50 | 51 | return found; 52 | } 53 | 54 | void ElfFile::checkFormat() const { 55 | // check the magic numbers 56 | MULTIPY_CHECK( 57 | (ehdr_->e_ident[EI_MAG0] == ELFMAG0) && 58 | (ehdr_->e_ident[EI_MAG1] == ELFMAG1) && 59 | (ehdr_->e_ident[EI_MAG2] == ELFMAG2) && 60 | (ehdr_->e_ident[EI_MAG3] == ELFMAG3), 61 | "Unexpected magic numbers"); 62 | MULTIPY_CHECK( 63 | ehdr_->e_ident[EI_CLASS] == ELFCLASS64, "Only support 64bit ELF file"); 64 | } 65 | 66 | namespace { 67 | // from https://stackoverflow.com/a/12774387/4722305 68 | // MIT license 69 | inline bool exists(const char* name) { 70 | std::ifstream f(name); 71 | return f.good(); 72 | } 73 | } // namespace 74 | 75 | std::optional
searchForSection(const char* name) { 76 | { 77 | ElfFile elfFile("/proc/self/exe"); 78 | auto section = elfFile.findSection(name); 79 | if (section) { 80 | return section; 81 | } 82 | } 83 | 84 | struct context { 85 | const char* name; 86 | std::optional
section{}; 87 | }; 88 | context ctx; 89 | ctx.name = name; 90 | 91 | dl_iterate_phdr( 92 | [](struct dl_phdr_info* info, size_t, void* data) { 93 | if (!exists(info->dlpi_name)) { 94 | return 0; 95 | } 96 | ElfFile elfFile(info->dlpi_name); 97 | auto localCtx = static_cast(data); 98 | localCtx->section = elfFile.findSection(localCtx->name); 99 | if (localCtx->section) { 100 | return 1; 101 | } 102 | return 0; 103 | }, 104 | &ctx); 105 | return ctx.section; 106 | } 107 | 108 | } // namespace deploy 109 | } // namespace torch 110 | -------------------------------------------------------------------------------- /multipy/runtime/test_deploy_gpu.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | int main(int argc, char* argv[]) { 17 | ::testing::InitGoogleTest(&argc, argv); 18 | int rc = RUN_ALL_TESTS(); 19 | return rc; 20 | } 21 | 22 | const char* simple = "multipy/runtime/example/generated/simple"; 23 | const char* simple_jit = "multipy/runtime/example/generated/simple_jit"; 24 | 25 | const char* path(const char* envname, const char* path) { 26 | const char* e = getenv(envname); 27 | return e ? e : path; 28 | } 29 | 30 | TEST(TorchDeployGPUTest, SimpleModel) { 31 | if (!torch::cuda::is_available()) { 32 | GTEST_SKIP(); 33 | } 34 | const char* model_filename = path("SIMPLE", simple); 35 | const char* jit_filename = path("SIMPLE_JIT", simple_jit); 36 | 37 | // Test 38 | torch::deploy::InterpreterManager m(2); 39 | torch::deploy::Package p = m.loadPackage(model_filename); 40 | auto model = p.loadPickle("model", "model.pkl"); 41 | { 42 | auto M = model.acquireSession(); 43 | M.self.attr("to")({"cuda"}); 44 | } 45 | // NOLINTNEXTLINE(cppcoreguidelines-init-variables) 46 | std::vector inputs; 47 | { 48 | auto I = p.acquireSession(); 49 | auto eg = I.self.attr("load_pickle")({"model", "example.pkl"}).toIValue(); 50 | inputs = eg.toTupleRef().elements(); 51 | inputs[0] = inputs[0].toTensor().to("cuda"); 52 | } 53 | at::Tensor output = model(inputs).toTensor(); 54 | ASSERT_TRUE(output.device().is_cuda()); 55 | 56 | // Reference 57 | auto ref_model = torch::jit::load(jit_filename); 58 | ref_model.to(torch::kCUDA); 59 | at::Tensor ref_output = ref_model.forward(inputs).toTensor(); 60 | 61 | ASSERT_TRUE(ref_output.allclose(output, 1e-03, 1e-05)); 62 | } 63 | 64 | TEST(TorchDeployGPUTest, UsesCuda) { 65 | #ifdef FBCODE_CAFFE2 66 | if (!torch::cuda::is_available()) { 67 | GTEST_SKIP(); 68 | } 69 | #endif 70 | 71 | const auto model_filename = 72 | path("USES_CUDA", "multipy/runtime/example/generated/uses_cuda"); 73 | torch::deploy::InterpreterManager m(2); 74 | torch::deploy::Package p = m.loadPackage(model_filename); 75 | { 76 | auto I = p.acquireSession(); 77 | I.self.attr("import_module")({"uses_cuda"}); 78 | } 79 | } 80 | 81 | #ifdef FBCODE_CAFFE2 82 | TEST(TorchDeployGPUTest, TensorRT) { 83 | if (!torch::cuda::is_available()) { 84 | GTEST_SKIP(); 85 | } 86 | auto packagePath = path( 87 | "MAKE_TRT_MODULE", "multipy/runtime/example/generated/make_trt_module"); 88 | torch::deploy::InterpreterManager m(2); 89 | torch::deploy::Package p = m.loadPackage(packagePath); 90 | auto makeModel = p.loadPickle("make_trt_module", "model.pkl"); 91 | { 92 | auto I = makeModel.acquireSession(); 93 | auto model = I.self(at::ArrayRef{}); 94 | auto input = at::ones({1, 2, 3}).cuda(); 95 | auto output = input * 2; 96 | ASSERT_TRUE( 97 | output.allclose(model(at::IValue{input}).toIValue().toTensor())); 98 | } 99 | } 100 | #endif 101 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to make participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies within all project spaces, and it also applies when 49 | an individual is representing the project or its community in public spaces. 50 | Examples of representing a project or community include using an official 51 | project e-mail address, posting via an official social media account, or acting 52 | as an appointed representative at an online or offline event. Representation of 53 | a project may be further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at . All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /scripts/linter/adapters/shellcheck_linter.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import logging 4 | import shutil 5 | import subprocess 6 | import time 7 | from enum import Enum 8 | from typing import List, NamedTuple, Optional 9 | 10 | 11 | LINTER_CODE = "SHELLCHECK" 12 | 13 | 14 | class LintSeverity(str, Enum): 15 | ERROR = "error" 16 | WARNING = "warning" 17 | ADVICE = "advice" 18 | DISABLED = "disabled" 19 | 20 | 21 | class LintMessage(NamedTuple): 22 | path: Optional[str] 23 | line: Optional[int] 24 | char: Optional[int] 25 | code: str 26 | severity: LintSeverity 27 | name: str 28 | original: Optional[str] 29 | replacement: Optional[str] 30 | description: Optional[str] 31 | 32 | 33 | def run_command( 34 | args: List[str], 35 | ) -> "subprocess.CompletedProcess[bytes]": 36 | logging.debug("$ %s", " ".join(args)) 37 | start_time = time.monotonic() 38 | try: 39 | return subprocess.run( 40 | args, 41 | stdout=subprocess.PIPE, 42 | stderr=subprocess.PIPE, 43 | ) 44 | finally: 45 | end_time = time.monotonic() 46 | logging.debug("took %dms", (end_time - start_time) * 1000) 47 | 48 | 49 | def check_files( 50 | files: List[str], 51 | ) -> List[LintMessage]: 52 | try: 53 | proc = run_command( 54 | ["shellcheck", "--external-sources", "--format=json1"] + files 55 | ) 56 | except OSError as err: 57 | return [ 58 | LintMessage( 59 | path=None, 60 | line=None, 61 | char=None, 62 | code=LINTER_CODE, 63 | severity=LintSeverity.ERROR, 64 | name="command-failed", 65 | original=None, 66 | replacement=None, 67 | description=(f"Failed due to {err.__class__.__name__}:\n{err}"), 68 | ) 69 | ] 70 | stdout = str(proc.stdout, "utf-8").strip() 71 | results = json.loads(stdout)["comments"] 72 | return [ 73 | LintMessage( 74 | path=result["file"], 75 | name=f"SC{result['code']}", 76 | description=result["message"], 77 | line=result["line"], 78 | char=result["column"], 79 | code=LINTER_CODE, 80 | severity=LintSeverity.ERROR, 81 | original=None, 82 | replacement=None, 83 | ) 84 | for result in results 85 | ] 86 | 87 | 88 | if __name__ == "__main__": 89 | parser = argparse.ArgumentParser( 90 | description="shellcheck runner", 91 | fromfile_prefix_chars="@", 92 | ) 93 | parser.add_argument( 94 | "filenames", 95 | nargs="+", 96 | help="paths to lint", 97 | ) 98 | 99 | if shutil.which("shellcheck") is None: 100 | err_msg = LintMessage( 101 | path="", 102 | line=None, 103 | char=None, 104 | code=LINTER_CODE, 105 | severity=LintSeverity.ERROR, 106 | name="command-failed", 107 | original=None, 108 | replacement=None, 109 | description="shellcheck is not installed, did you forget to run `lintrunner init`?", 110 | ) 111 | print(json.dumps(err_msg._asdict()), flush=True) 112 | exit(0) 113 | 114 | args = parser.parse_args() 115 | 116 | lint_messages = check_files(args.filenames) 117 | for lint_message in lint_messages: 118 | print(json.dumps(lint_message._asdict()), flush=True) 119 | -------------------------------------------------------------------------------- /multipy/runtime/environment.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | #pragma once 8 | #include 9 | #include 10 | #include 11 | 12 | namespace torch { 13 | namespace deploy { 14 | 15 | class Interpreter; 16 | 17 | /** 18 | * An environment is the concept to decribe the circumstances in which a 19 | * torch::deploy interpreter runs. In can be an xar file embedded in the binary, 20 | * a filesystem path for the installed libraries etc. 21 | */ 22 | class Environment { 23 | std::vector extraPythonPaths_; 24 | // all zipped python libraries will be written 25 | // under this directory 26 | std::string extraPythonLibrariesDir_; 27 | std::string getZippedArchive( 28 | const char* zipped_torch_name, 29 | const std::string& pythonAppDir) { 30 | // load the zipped torch modules 31 | auto zippedTorchSection = searchForSection(zipped_torch_name); 32 | MULTIPY_CHECK( 33 | zippedTorchSection.has_value(), "Missing the zipped torch section"); 34 | const char* zippedTorchStart = zippedTorchSection->start; 35 | auto zippedTorchSize = zippedTorchSection->len; 36 | 37 | std::string zipArchive = pythonAppDir; 38 | auto zippedFile = fopen(zipArchive.c_str(), "wb"); 39 | MULTIPY_CHECK( 40 | zippedFile != nullptr, "Fail to create file: ", strerror(errno)); 41 | fwrite(zippedTorchStart, 1, zippedTorchSize, zippedFile); 42 | fclose(zippedFile); 43 | return zipArchive; 44 | } 45 | 46 | void setupZippedPythonModules(const std::string& pythonAppDir) { 47 | #ifdef FBCODE_CAFFE2 48 | extraPythonPaths_.push_back(getZippedArchive( 49 | ".mpmath_python_modules", 50 | std::string(pythonAppDir) + "/mpmath_python_modules.zip")); 51 | extraPythonPaths_.push_back(getZippedArchive( 52 | ".sympy_python_modules", 53 | std::string(pythonAppDir) + "/sympy_python_modules.zip")); 54 | extraPythonPaths_.push_back(getZippedArchive( 55 | ".torch_python_modules", 56 | std::string(pythonAppDir) + "/torch_python_modules.zip")); 57 | extraPythonPaths_.push_back(getZippedArchive( 58 | ".multipy_python_modules", 59 | std::string(pythonAppDir) + "/multipy_python_modules.zip")); 60 | extraPythonPaths_.push_back(getZippedArchive( 61 | ".torchgen_python_modules", 62 | std::string(pythonAppDir) + "/torchgen_python_modules.zip")); 63 | 64 | #endif 65 | extraPythonLibrariesDir_ = pythonAppDir; 66 | } 67 | 68 | public: 69 | /// Environment constructor which creates a random temporary directory as 70 | /// a directory for the zipped python modules. 71 | explicit Environment() { 72 | char tempDirName[] = "/tmp/torch_deploy_zipXXXXXX"; 73 | char* tempDirectory = mkdtemp(tempDirName); 74 | setupZippedPythonModules(tempDirectory); 75 | } 76 | /// Environment constructor which takes a file name for the 77 | /// directory for the python modules. 78 | explicit Environment(const std::string& pythonAppDir) { 79 | setupZippedPythonModules(pythonAppDir); 80 | } 81 | 82 | virtual ~Environment() { 83 | auto rmCmd = "rm -rf " + extraPythonLibrariesDir_; 84 | (void)system(rmCmd.c_str()); 85 | } 86 | virtual const std::vector& getExtraPythonPaths() { 87 | return extraPythonPaths_; 88 | } 89 | /// Gives information to the interpreter about the 90 | /// Environment if necessary 91 | virtual void configureInterpreter(Interpreter* interp) = 0; 92 | }; 93 | 94 | } // namespace deploy 95 | } // namespace torch 96 | -------------------------------------------------------------------------------- /docs/doc_push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under the BSD-style license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | # 9 | # Builds docs from the checkedout HEAD 10 | # and pushes the artifacts to gh-pages branch in github.com/pytorch/multipy 11 | # 12 | # 1. sphinx generated docs are copied to / 13 | # 2. if a release tag is found on HEAD then redirects are copied to /latest 14 | # 3. if no release tag is found on HEAD then redirects are copied to /main 15 | # 16 | # gh-pages branch should look as follows: 17 | # 18 | # |- 0.1.0rc2 19 | # |- 0.1.0rc3 20 | # |- 21 | # |- main (redirects to the most recent ver in trunk, including release) 22 | # |- latest (redirects to the most recent release) 23 | # If the most recent release is 0.1.0 and main is at 0.1.1rc1 then, 24 | # https://pytorch.org/multipy/main -> https://pytorch.org/multipy/0.1.1rc1 25 | # https://pytorch.org/multipy/latest -> https://pytorch.org/multipy/0.1.0 26 | # 27 | # Redirects are done via Jekyll redirect-from plugin. See: 28 | # sources/scripts/create_redirect_md.py 29 | # Makefile (redirect target) 30 | # (on gh-pages branch) _layouts/docs_redirect.html 31 | 32 | set -ex 33 | 34 | dry_run=0 35 | for arg in "$@"; do 36 | shift 37 | case "$arg" in 38 | "--dry-run") dry_run=1 ;; 39 | "--help") echo "Usage $0 [--dry-run]"; exit 0 ;; 40 | esac 41 | done 42 | 43 | repo_origin="$(git remote get-url origin)" 44 | repo_root=$(git rev-parse --show-toplevel) 45 | branch=$(git rev-parse --abbrev-ref HEAD) 46 | commit_id=$(git rev-parse --short HEAD) 47 | 48 | if ! release_tag=$(git describe --tags --exact-match HEAD 2>/dev/null); then 49 | echo "No release tag found, building docs for main..." 50 | redirects=(main) 51 | release_tag="main" 52 | else 53 | echo "Release tag $release_tag found, building docs for release..." 54 | redirects=(latest main) 55 | fi 56 | 57 | echo "Installing multipy from $repo_root..." 58 | cd "$repo_root" || exit 59 | 60 | # For some reason the CI uses python 3.6 for the script, so we can't build multipy. 61 | # Fortunately, we just need the version. 62 | multipy_ver=$(python3 -c "from multipy.version import __version__; print(__version__)") 63 | 64 | echo "Building multipy-$multipy_ver docs..." 65 | docs_dir=$repo_root/docs 66 | build_dir=$docs_dir/build 67 | cd "$docs_dir" || exit 68 | python3 -m pip install setuptools 69 | python3 -m pip install -r requirements.txt 70 | make clean html 71 | echo "Doc build complete" 72 | 73 | tmp_dir=/tmp/multipy_docs_tmp 74 | rm -rf "${tmp_dir:?}" 75 | 76 | echo "Checking out gh-pages branch..." 77 | gh_pages_dir="$tmp_dir/multipy_gh_pages" 78 | git clone -b gh-pages --single-branch "$repo_origin" $gh_pages_dir 79 | 80 | echo "Copying doc pages for $multipy_ver into $gh_pages_dir..." 81 | rm -rf "${gh_pages_dir:?}/${multipy_ver:?}" 82 | cp -R "$build_dir/html" "$gh_pages_dir/$multipy_ver" 83 | 84 | cd $gh_pages_dir || exit 85 | 86 | for redirect in "${redirects[@]}"; do 87 | echo "Creating redirect symlinks for: $redirect -> $multipy_ver..." 88 | rm -rf "${gh_pages_dir:?}/${redirect:?}" 89 | ln -s "$multipy_ver" "$redirect" 90 | done 91 | 92 | git add . 93 | git commit --allow-empty --quiet -m "[doc_push][$release_tag] built from $commit_id ($branch). Redirects: ${redirects[*]} -> $multipy_ver." 94 | 95 | if [ $dry_run -eq 1 ]; then 96 | echo "*** --dry-run mode, skipping push to gh-pages branch. To publish run: cd ${gh_pages_dir} && git push" 97 | exit 98 | fi 99 | 100 | git push 101 | -------------------------------------------------------------------------------- /scripts/linter/adapters/cmake_linter.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import concurrent.futures 3 | import json 4 | import logging 5 | import os 6 | import re 7 | import subprocess 8 | import time 9 | from enum import Enum 10 | from typing import List, NamedTuple, Optional, Pattern 11 | 12 | 13 | LINTER_CODE = "CMAKE" 14 | 15 | 16 | class LintSeverity(str, Enum): 17 | ERROR = "error" 18 | WARNING = "warning" 19 | ADVICE = "advice" 20 | DISABLED = "disabled" 21 | 22 | 23 | class LintMessage(NamedTuple): 24 | path: Optional[str] 25 | line: Optional[int] 26 | char: Optional[int] 27 | code: str 28 | severity: LintSeverity 29 | name: str 30 | original: Optional[str] 31 | replacement: Optional[str] 32 | description: Optional[str] 33 | 34 | 35 | # CMakeLists.txt:901: Lines should be <= 80 characters long [linelength] 36 | RESULTS_RE: Pattern[str] = re.compile( 37 | r"""(?mx) 38 | ^ 39 | (?P.*?): 40 | (?P\d+): 41 | \s(?P.*) 42 | \s(?P\[.*\]) 43 | $ 44 | """ 45 | ) 46 | 47 | 48 | def run_command( 49 | args: List[str], 50 | ) -> "subprocess.CompletedProcess[bytes]": 51 | logging.debug("$ %s", " ".join(args)) 52 | start_time = time.monotonic() 53 | try: 54 | return subprocess.run( 55 | args, 56 | stdout=subprocess.PIPE, 57 | stderr=subprocess.PIPE, 58 | ) 59 | finally: 60 | end_time = time.monotonic() 61 | logging.debug("took %dms", (end_time - start_time) * 1000) 62 | 63 | 64 | def check_file( 65 | filename: str, 66 | config: str, 67 | ) -> List[LintMessage]: 68 | try: 69 | proc = run_command( 70 | ["cmakelint", f"--config={config}", filename], 71 | ) 72 | except OSError as err: 73 | return [ 74 | LintMessage( 75 | path=None, 76 | line=None, 77 | char=None, 78 | code=LINTER_CODE, 79 | severity=LintSeverity.ERROR, 80 | name="command-failed", 81 | original=None, 82 | replacement=None, 83 | description=(f"Failed due to {err.__class__.__name__}:\n{err}"), 84 | ) 85 | ] 86 | stdout = str(proc.stdout, "utf-8").strip() 87 | return [ 88 | LintMessage( 89 | path=match["file"], 90 | name=match["code"], 91 | description=match["message"], 92 | line=int(match["line"]), 93 | char=None, 94 | code=LINTER_CODE, 95 | severity=LintSeverity.ERROR, 96 | original=None, 97 | replacement=None, 98 | ) 99 | for match in RESULTS_RE.finditer(stdout) 100 | ] 101 | 102 | 103 | if __name__ == "__main__": 104 | parser = argparse.ArgumentParser( 105 | description="cmakelint runner", 106 | fromfile_prefix_chars="@", 107 | ) 108 | parser.add_argument( 109 | "--config", 110 | required=True, 111 | help="location of cmakelint config", 112 | ) 113 | parser.add_argument( 114 | "filenames", 115 | nargs="+", 116 | help="paths to lint", 117 | ) 118 | 119 | args = parser.parse_args() 120 | 121 | with concurrent.futures.ThreadPoolExecutor( 122 | max_workers=os.cpu_count(), 123 | thread_name_prefix="Thread", 124 | ) as executor: 125 | futures = { 126 | executor.submit( 127 | check_file, 128 | filename, 129 | args.config, 130 | ): filename 131 | for filename in args.filenames 132 | } 133 | for future in concurrent.futures.as_completed(futures): 134 | try: 135 | for lint_message in future.result(): 136 | print(json.dumps(lint_message._asdict()), flush=True) 137 | except Exception: 138 | logging.critical('Failed at "%s".', futures[future]) 139 | raise 140 | -------------------------------------------------------------------------------- /scripts/linter/adapters/ufmt_linter.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import concurrent.futures 3 | import json 4 | import logging 5 | import os 6 | import sys 7 | from enum import Enum 8 | from pathlib import Path 9 | from typing import Any, List, NamedTuple, Optional 10 | 11 | from ufmt.core import make_black_config, ufmt_string 12 | from usort import Config as UsortConfig 13 | 14 | 15 | IS_WINDOWS: bool = os.name == "nt" 16 | 17 | 18 | def eprint(*args: Any, **kwargs: Any) -> None: 19 | print(*args, file=sys.stderr, flush=True, **kwargs) 20 | 21 | 22 | class LintSeverity(str, Enum): 23 | ERROR = "error" 24 | WARNING = "warning" 25 | ADVICE = "advice" 26 | DISABLED = "disabled" 27 | 28 | 29 | class LintMessage(NamedTuple): 30 | path: Optional[str] 31 | line: Optional[int] 32 | char: Optional[int] 33 | code: str 34 | severity: LintSeverity 35 | name: str 36 | original: Optional[str] 37 | replacement: Optional[str] 38 | description: Optional[str] 39 | 40 | 41 | def as_posix(name: str) -> str: 42 | return name.replace("\\", "/") if IS_WINDOWS else name 43 | 44 | 45 | def format_error_message(filename: str, err: Exception) -> LintMessage: 46 | return LintMessage( 47 | path=filename, 48 | line=None, 49 | char=None, 50 | code="UFMT", 51 | severity=LintSeverity.ADVICE, 52 | name="command-failed", 53 | original=None, 54 | replacement=None, 55 | description=(f"Failed due to {err.__class__.__name__}:\n{err}"), 56 | ) 57 | 58 | 59 | def check_file( 60 | filename: str, 61 | ) -> List[LintMessage]: 62 | with open(filename, "rb") as f: 63 | original = f.read().decode("utf-8") 64 | 65 | try: 66 | path = Path(filename) 67 | 68 | usort_config = UsortConfig.find(path) 69 | black_config = make_black_config(path) 70 | 71 | # Use UFMT API to call both usort and black 72 | replacement = ufmt_string( 73 | path=path, 74 | content=original, 75 | usort_config=usort_config, 76 | black_config=black_config, 77 | ) 78 | 79 | if original == replacement: 80 | return [] 81 | 82 | return [ 83 | LintMessage( 84 | path=filename, 85 | line=None, 86 | char=None, 87 | code="UFMT", 88 | severity=LintSeverity.WARNING, 89 | name="format", 90 | original=original, 91 | replacement=replacement, 92 | description="Run `lintrunner -a` to apply this patch.", 93 | ) 94 | ] 95 | except Exception as err: 96 | return [format_error_message(filename, err)] 97 | 98 | 99 | def main() -> None: 100 | parser = argparse.ArgumentParser( 101 | description="Format files with ufmt (black + usort).", 102 | fromfile_prefix_chars="@", 103 | ) 104 | parser.add_argument( 105 | "--verbose", 106 | action="store_true", 107 | help="verbose logging", 108 | ) 109 | parser.add_argument( 110 | "filenames", 111 | nargs="+", 112 | help="paths to lint", 113 | ) 114 | args = parser.parse_args() 115 | 116 | logging.basicConfig( 117 | format="<%(threadName)s:%(levelname)s> %(message)s", 118 | level=( 119 | logging.NOTSET 120 | if args.verbose 121 | else logging.DEBUG 122 | if len(args.filenames) < 1000 123 | else logging.INFO 124 | ), 125 | stream=sys.stderr, 126 | ) 127 | 128 | with concurrent.futures.ThreadPoolExecutor( 129 | max_workers=os.cpu_count(), 130 | thread_name_prefix="Thread", 131 | ) as executor: 132 | futures = {executor.submit(check_file, x): x for x in args.filenames} 133 | for future in concurrent.futures.as_completed(futures): 134 | try: 135 | for lint_message in future.result(): 136 | print(json.dumps(lint_message._asdict()), flush=True) 137 | except Exception: 138 | logging.critical('Failed at "%s".', futures[future]) 139 | raise 140 | 141 | 142 | if __name__ == "__main__": 143 | main() 144 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE=nvidia/cuda:11.6.1-devel-ubuntu18.04 2 | 3 | FROM ${BASE_IMAGE} as dev-base 4 | 5 | SHELL ["/bin/bash", "-c"] 6 | 7 | # Install system dependencies 8 | RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \ 9 | apt update && DEBIAN_FRONTEND=noninteractive apt install -yq --no-install-recommends \ 10 | build-essential \ 11 | ca-certificates \ 12 | ccache \ 13 | curl \ 14 | cmake-mozilla \ 15 | wget \ 16 | git \ 17 | libjpeg-dev \ 18 | xz-utils \ 19 | bzip2 \ 20 | libbz2-dev \ 21 | liblzma-dev \ 22 | libreadline6-dev \ 23 | libexpat1-dev \ 24 | libgdbm-dev \ 25 | glibc-source \ 26 | libgmp-dev \ 27 | libffi-dev \ 28 | libgl-dev \ 29 | ncurses-dev \ 30 | libncursesw5-dev \ 31 | libncurses5-dev \ 32 | gnome-panel \ 33 | libssl-dev \ 34 | tcl-dev \ 35 | tix-dev \ 36 | libgtest-dev \ 37 | tk-dev \ 38 | libsqlite3-dev \ 39 | zlib1g-dev \ 40 | llvm \ 41 | python-openssl \ 42 | apt-transport-https \ 43 | ca-certificates \ 44 | gnupg \ 45 | software-properties-common \ 46 | python-pip \ 47 | python3-pip && \ 48 | rm -rf /var/lib/apt/lists/* 49 | RUN /usr/sbin/update-ccache-symlinks 50 | RUN mkdir /opt/ccache && ccache --set-config=cache_dir=/opt/ccache 51 | ENV PATH /opt/conda/bin:$PATH 52 | 53 | # get the repo 54 | FROM dev-base as submodule-update 55 | WORKDIR /opt/multipy 56 | 57 | # add files incrementally 58 | COPY .git .git 59 | COPY .gitmodules .gitmodules 60 | COPY multipy multipy 61 | COPY compat-requirements.txt compat-requirements.txt 62 | COPY setup.py setup.py 63 | COPY README.md README.md 64 | COPY dev-requirements.txt dev-requirements.txt 65 | 66 | RUN git -c fetch.parallel=0 -c submodule.fetchJobs=0 submodule update --init --recursive 67 | 68 | # Install conda/pyenv + necessary python dependencies 69 | FROM dev-base as conda-pyenv 70 | ARG PYTHON_3_MINOR_VERSION=8 71 | ARG BUILD_CUDA_TESTS=0 72 | ENV PYTHON_3_MINOR_VERSION=${PYTHON_3_MINOR_VERSION} 73 | ENV PYTHON_VERSION=3.${PYTHON_3_MINOR_VERSION} 74 | RUN if [[ ${PYTHON_3_MINOR_VERSION} -gt 7 ]]; then \ 75 | curl -fsSL -v -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ 76 | chmod +x ~/miniconda.sh && \ 77 | /bin/bash ~/miniconda.sh -b -p /opt/conda && \ 78 | rm ~/miniconda.sh && \ 79 | /opt/conda/bin/conda install -y python=${PYTHON_VERSION} mkl mkl-include conda-build pyyaml numpy ipython && \ 80 | /opt/conda/bin/conda install -y -c conda-forge libpython-static=${PYTHON_VERSION} && \ 81 | /opt/conda/bin/conda install -y pytorch torchvision torchaudio pytorch-cuda=11.6 -c pytorch-nightly -c nvidia && \ 82 | /opt/conda/bin/conda clean -ya; \ 83 | else \ 84 | pip3 install virtualenv && \ 85 | git clone https://github.com/pyenv/pyenv.git ~/.pyenv && \ 86 | export CFLAGS="-fPIC -g" && \ 87 | ~/.pyenv/bin/pyenv install --force 3.7.10 && \ 88 | virtualenv -p ~/.pyenv/versions/3.7.10/bin/python3 ~/venvs/multipy && \ 89 | source ~/venvs/multipy/bin/activate && \ 90 | pip3 install --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cu116; \ 91 | fi 92 | 93 | FROM conda-pyenv as build 94 | COPY --from=conda-pyenv /opt/conda* /opt/conda 95 | COPY --from=submodule-update /opt/multipy /opt/multipy 96 | 97 | WORKDIR /opt/multipy 98 | 99 | # Build Multipy 100 | RUN ls && pwd && rm -rf multipy/runtime/build && \ 101 | if [[ ${PYTHON_3_MINOR_VERSION} -lt 8 ]]; then \ 102 | source ~/venvs/multipy/bin/activate; \ 103 | fi && \ 104 | if [[ ${BUILD_CUDA_TESTS} -eq 1 ]]; then \ 105 | BUILD_CUDA_TESTS=1 python -m pip install -e .; \ 106 | else \ 107 | python -m pip install -e .; \ 108 | fi && \ 109 | python multipy/runtime/example/generate_examples.py 110 | 111 | # Build examples 112 | COPY examples examples 113 | RUN cd examples && \ 114 | rm -r build; \ 115 | if [[ ${PYTHON_3_MINOR_VERSION} -lt 8 ]]; then \ 116 | source ~/venvs/multipy/bin/activate; \ 117 | else \ 118 | source /opt/conda/bin/activate; \ 119 | fi && \ 120 | cmake -S . -B build/ -DMULTIPY_PATH=".." && \ 121 | cmake --build build/ --config Release -j 122 | 123 | ENV PYTHONPATH=. LIBTEST_DEPLOY_LIB=multipy/runtime/build/libtest_deploy_lib.so 124 | 125 | 126 | RUN mkdir /opt/dist && cp -r multipy/runtime/build/dist/* /opt/dist/ 127 | -------------------------------------------------------------------------------- /scripts/linter/adapters/actionlint_linter.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import concurrent.futures 3 | import json 4 | import logging 5 | import os 6 | import re 7 | import subprocess 8 | import time 9 | from enum import Enum 10 | from typing import List, NamedTuple, Optional, Pattern 11 | 12 | 13 | LINTER_CODE = "ACTIONLINT" 14 | 15 | 16 | class LintSeverity(str, Enum): 17 | ERROR = "error" 18 | WARNING = "warning" 19 | ADVICE = "advice" 20 | DISABLED = "disabled" 21 | 22 | 23 | class LintMessage(NamedTuple): 24 | path: Optional[str] 25 | line: Optional[int] 26 | char: Optional[int] 27 | code: str 28 | severity: LintSeverity 29 | name: str 30 | original: Optional[str] 31 | replacement: Optional[str] 32 | description: Optional[str] 33 | 34 | 35 | RESULTS_RE: Pattern[str] = re.compile( 36 | r"""(?mx) 37 | ^ 38 | (?P.*?): 39 | (?P\d+): 40 | (?P\d+): 41 | \s(?P.*) 42 | \s(?P\[.*\]) 43 | $ 44 | """ 45 | ) 46 | 47 | 48 | def run_command( 49 | args: List[str], 50 | ) -> "subprocess.CompletedProcess[bytes]": 51 | logging.debug("$ %s", " ".join(args)) 52 | start_time = time.monotonic() 53 | try: 54 | return subprocess.run( 55 | args, 56 | stdout=subprocess.PIPE, 57 | stderr=subprocess.PIPE, 58 | ) 59 | finally: 60 | end_time = time.monotonic() 61 | logging.debug("took %dms", (end_time - start_time) * 1000) 62 | 63 | 64 | def check_file( 65 | binary: str, 66 | file: str, 67 | ) -> List[LintMessage]: 68 | try: 69 | proc = run_command([binary, file]) 70 | except OSError as err: 71 | return [ 72 | LintMessage( 73 | path=None, 74 | line=None, 75 | char=None, 76 | code=LINTER_CODE, 77 | severity=LintSeverity.ERROR, 78 | name="command-failed", 79 | original=None, 80 | replacement=None, 81 | description=(f"Failed due to {err.__class__.__name__}:\n{err}"), 82 | ) 83 | ] 84 | stdout = str(proc.stdout, "utf-8").strip() 85 | return [ 86 | LintMessage( 87 | path=match["file"], 88 | name=match["code"], 89 | description=match["message"], 90 | line=int(match["line"]), 91 | char=int(match["char"]), 92 | code=LINTER_CODE, 93 | severity=LintSeverity.ERROR, 94 | original=None, 95 | replacement=None, 96 | ) 97 | for match in RESULTS_RE.finditer(stdout) 98 | ] 99 | 100 | 101 | if __name__ == "__main__": 102 | parser = argparse.ArgumentParser( 103 | description="actionlint runner", 104 | fromfile_prefix_chars="@", 105 | ) 106 | parser.add_argument( 107 | "--binary", 108 | required=True, 109 | help="actionlint binary path", 110 | ) 111 | parser.add_argument( 112 | "filenames", 113 | nargs="+", 114 | help="paths to lint", 115 | ) 116 | 117 | args = parser.parse_args() 118 | 119 | if not os.path.exists(args.binary): 120 | err_msg = LintMessage( 121 | path="", 122 | line=None, 123 | char=None, 124 | code=LINTER_CODE, 125 | severity=LintSeverity.ERROR, 126 | name="command-failed", 127 | original=None, 128 | replacement=None, 129 | description=( 130 | f"Could not find actionlint binary at {args.binary}," 131 | " you may need to run `lintrunner init`." 132 | ), 133 | ) 134 | print(json.dumps(err_msg._asdict()), flush=True) 135 | exit(0) 136 | 137 | with concurrent.futures.ThreadPoolExecutor( 138 | max_workers=os.cpu_count(), 139 | thread_name_prefix="Thread", 140 | ) as executor: 141 | futures = { 142 | executor.submit( 143 | check_file, 144 | args.binary, 145 | filename, 146 | ): filename 147 | for filename in args.filenames 148 | } 149 | for future in concurrent.futures.as_completed(futures): 150 | try: 151 | for lint_message in future.result(): 152 | print(json.dumps(lint_message._asdict()), flush=True) 153 | except Exception: 154 | logging.critical('Failed at "%s".', futures[future]) 155 | raise 156 | -------------------------------------------------------------------------------- /multipy/utils/_deploy.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import io 8 | 9 | import torch 10 | 11 | import torch.package 12 | from torch.package import Importer, OrderedImporter, PackageImporter, sys_importer 13 | from torch.package._package_pickler import create_pickler 14 | from torch.package._package_unpickler import PackageUnpickler 15 | from torch.serialization import _maybe_decode_ascii 16 | 17 | # For < pytorch 1.13 compatibility. We can likely delete this after it's release. 18 | try: 19 | STORAGE_TYPE = torch.storage._TypedStorage 20 | except: 21 | STORAGE_TYPE = torch.storage.TypedStorage 22 | 23 | 24 | def _save_storages(importer, obj): 25 | serialized_storages = [] 26 | serialized_dtypes = [] 27 | 28 | importer = importer if isinstance(importer, torch.package.PackageImporter) else None 29 | importers: Importer 30 | if importer is not None: 31 | importers = OrderedImporter(importer, sys_importer) 32 | else: 33 | importers = sys_importer 34 | 35 | def persistent_id(obj): 36 | if torch.is_storage(obj) or isinstance(obj, STORAGE_TYPE): 37 | if isinstance(obj, STORAGE_TYPE): 38 | # TODO: Once we decide to break serialization FC, we can 39 | # remove this case 40 | storage = obj.untyped() 41 | dtype = obj.dtype 42 | else: 43 | storage = obj 44 | dtype = torch.uint8 45 | 46 | serialized_storages.append(obj) 47 | serialized_dtypes.append(dtype) 48 | return ("storage", len(serialized_storages) - 1) 49 | 50 | if hasattr(obj, "__reduce_deploy__"): 51 | if _serialized_reduces.get(id(obj)) is None: 52 | _serialized_reduces[id(obj)] = ( 53 | "reduce_deploy", 54 | id(obj), 55 | *obj.__reduce_deploy__(importers), 56 | ) 57 | return _serialized_reduces[id(obj)] 58 | 59 | return None 60 | 61 | # Write the pickle data for `obj` 62 | data_buf = io.BytesIO() 63 | pickler = create_pickler(data_buf, importers) 64 | pickler.persistent_id = persistent_id 65 | pickler.dump(obj) 66 | data_value = data_buf.getvalue() 67 | return ( 68 | data_value, 69 | serialized_storages, 70 | serialized_dtypes, 71 | importer.zip_reader if importer else None, 72 | ) 73 | 74 | 75 | def _load_storages(id, zip_reader, obj_bytes, serialized_storages, serialized_dtypes): 76 | def persistent_load(saved_id): 77 | assert isinstance(saved_id, tuple) 78 | typename = _maybe_decode_ascii(saved_id[0]) 79 | data = saved_id[1:] 80 | 81 | if typename == "storage": 82 | # TODO: Once we decide to break serialization FC, we can 83 | # stop wrapping with _TypedStorage 84 | storage = serialized_storages[data[0]] 85 | dtype = serialized_dtypes[data[0]] 86 | 87 | # For < pytorch 1.13 compatibility. We can likely remove after it's release. 88 | try: 89 | storage_untyped = storage.untyped 90 | except: 91 | storage_untyped = storage._untyped 92 | 93 | return STORAGE_TYPE(wrap_storage=storage_untyped(), dtype=dtype) 94 | 95 | if typename == "reduce_deploy": 96 | reduce_id, func, args = data 97 | if reduce_id not in _loaded_reduces: 98 | _loaded_reduces[reduce_id] = func(_raw_packages[zip_reader], *args) 99 | return _loaded_reduces[reduce_id] 100 | 101 | return None 102 | 103 | importer: Importer 104 | if zip_reader is not None: 105 | importer = OrderedImporter(_get_package(zip_reader), sys_importer) 106 | else: 107 | importer = sys_importer 108 | 109 | unpickler = PackageUnpickler(importer, io.BytesIO(obj_bytes)) 110 | unpickler.persistent_load = persistent_load # type: ignore[assignment] 111 | result = _deploy_objects[id] = unpickler.load() 112 | return result 113 | 114 | 115 | def _get_package(zip_reader): 116 | if zip_reader not in _raw_packages: 117 | _raw_packages[zip_reader] = PackageImporter(zip_reader) 118 | return _raw_packages[zip_reader] 119 | 120 | 121 | _raw_packages: dict = {} 122 | _deploy_objects: dict = {} 123 | _serialized_reduces: dict = {} 124 | _loaded_reduces: dict = {} 125 | -------------------------------------------------------------------------------- /multipy/runtime/interpreter/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | set(CMAKE_POSITION_INDEPENDENT_CODE ON) 8 | 9 | SET(INTERPRETER_DIR "${DEPLOY_DIR}/interpreter" ) 10 | SET(INTERPRETER_DIR "${DEPLOY_DIR}/interpreter" PARENT_SCOPE) 11 | 12 | SET(MULTIPY_UTILS "${CMAKE_CURRENT_SOURCE_DIR}/../../utils") 13 | 14 | message(STATUS "Python3_EXECUTABLE - ${Python3_EXECUTABLE}" ) 15 | message(STATUS "Python3_SITELIB - ${Python3_SITELIB}" ) 16 | message(STATUS "Python3_LIBRARIES - ${Python3_LIBRARIES}" ) 17 | 18 | # Find static libpython with a preference for nolto since cross compiler version 19 | # LTO is problematic. We also recursively search in the directory for static 20 | # libpython, as it's not guarenteed to be in the same directory as libpython such 21 | # as with pyenv. 22 | 23 | FUNCTION(find_file_with_different_suffix VAR FILE ORIGINAL_SUFFIX NEW_SUFFIX) 24 | string(REPLACE ${ORIGINAL_SUFFIX} ${NEW_SUFFIX} MODIFIED_FILE ${FILE}) 25 | 26 | if (NOT EXISTS ${MODIFIED_FILE}) 27 | get_filename_component(MODIFIED_FILENAME ${MODIFIED_FILE} NAME) 28 | string(REPLACE "${MODIFIED_FILENAME}" "**/${MODIFIED_FILENAME}" MODIFIED_FILE_GLOB ${MODIFIED_FILE} ) 29 | file(GLOB_RECURSE MODIFIED_FILE ${MODIFIED_FILE_GLOB}) 30 | endif() 31 | set(${VAR} ${MODIFIED_FILE} PARENT_SCOPE) 32 | ENDFUNCTION() 33 | 34 | find_file_with_different_suffix(Python3_STATIC_LIBRARIES ${Python3_LIBRARIES} ".so" ".nolto.a") 35 | 36 | if (NOT EXISTS ${Python3_STATIC_LIBRARIES}) 37 | find_file_with_different_suffix(Python3_STATIC_LIBRARIES ${Python3_LIBRARIES} ".so" ".a") 38 | endif() 39 | 40 | # If we still can't find the library, we fail early 41 | if (NOT EXISTS ${Python3_STATIC_LIBRARIES}) 42 | get_filename_component(Python_FILENAME ${Python3_LIBRARIES} NAME) 43 | string(REPLACE ".so" ".nolto.a" Python3_STATIC_NOLTO_FILENAME ${Python_FILENAME}) 44 | string(REPLACE ".so" ".a" Python3_STATIC_FILENAME ${Python_FILENAME}) 45 | message(FATAL_ERROR "Cannot find ${Python3_STATIC_NOLTO_FILENAME} or ${Python3_STATIC_FILENAME} in ${libpython_DIR}. Are you using a static version of python? Please refer to https://github.com/pytorch/multipy for install instructions using conda and pyenv.") 46 | endif() 47 | 48 | message(STATUS "Python3_STATIC_LIBRARIES - ${Python3_STATIC_LIBRARIES}" ) 49 | 50 | include_directories(BEFORE ${CMAKE_SOURCE_DIR}/../..) 51 | 52 | # add gtest dependency 53 | include(FetchContent) 54 | FetchContent_Declare( 55 | googletest 56 | URL https://github.com/google/googletest/archive/609281088cfefc76f9d0ce82e1ff6c30cc3591e5.zip 57 | ) 58 | FetchContent_MakeAvailable(googletest) 59 | include(GoogleTest) 60 | 61 | IF(CMAKE_BUILD_TYPE MATCHES Debug) 62 | set(OBJCOPY_FLAGS "") 63 | else() 64 | # binutils prior to 2.32 have issues linking against newer libraries due to 65 | # debug info changes. By default we can strip these out to ensure it 66 | # compiles. 67 | set(OBJCOPY_FLAGS "--strip-debug") 68 | endif() 69 | 70 | # instantiate a library based on the objects that make up torch_python 71 | # make sure system python isn't used here 72 | add_custom_command( 73 | OUTPUT libpython_multipy.a 74 | COMMAND cp ${Python3_STATIC_LIBRARIES} libpython_multipy.a 75 | COMMAND chmod +w libpython_multipy.a 76 | COMMAND "${CMAKE_OBJCOPY}" ${OBJCOPY_FLAGS} --weaken-symbol=_PyImport_FindSharedFuncptr libpython_multipy.a 77 | ) 78 | add_custom_target(libpython_multipy DEPENDS libpython_multipy.a) 79 | 80 | # Build the interpreter lib, designed to be standalone and dlopened 81 | # We bake the python and torch_python binding objs into libinterpreter 82 | set(INTERPRETER_LIB_SOURCES 83 | ${INTERPRETER_DIR}/interpreter_impl.cpp 84 | ${INTERPRETER_DIR}/builtin_registry.cpp 85 | ${INTERPRETER_DIR}/import_find_sharedfuncptr.cpp 86 | ${INTERPRETER_DIR}/plugin_registry.cpp 87 | ${INTERPRETER_DIR}/../loader.cpp 88 | ${LINKER_SCRIPT} 89 | ) 90 | add_library(torch_deployinterpreter SHARED ${INTERPRETER_LIB_SOURCES} ${LINKER_SCRIPT}) 91 | add_library(multipy_torch SHARED plugin_torch.cpp) 92 | 93 | add_dependencies(torch_deployinterpreter libpython_multipy) 94 | target_link_libraries(torch_deployinterpreter PRIVATE "-Wl,--no-as-needed -rdynamic" ${CMAKE_CURRENT_BINARY_DIR}/libpython_multipy.a) 95 | 96 | # need to ensure headers are present before any .cpp in interpreter are compiled, 97 | # but cpp themselves don't clearly depend on cpython so there is a race otherwise 98 | if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") 99 | target_compile_options(torch_deployinterpreter PRIVATE -fno-gnu-unique) 100 | endif() 101 | target_include_directories(torch_deployinterpreter PRIVATE ${INTERPRETER_DIR}) 102 | target_include_directories(torch_deployinterpreter BEFORE PUBLIC ${Python3_INCLUDE_DIRS}) 103 | 104 | target_link_libraries(torch_deployinterpreter PRIVATE fmt::fmt-header-only) 105 | target_link_libraries(torch_deployinterpreter PRIVATE gtest) 106 | target_link_libraries(torch_deployinterpreter PRIVATE torch_python) 107 | target_link_libraries(torch_deployinterpreter PRIVATE multipy_torch) 108 | -------------------------------------------------------------------------------- /multipy/runtime/example/generate_examples.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | """ 8 | Generate the example files that torchpy_test uses. 9 | """ 10 | 11 | import argparse 12 | from pathlib import Path 13 | 14 | import torch 15 | from torch.fx import symbolic_trace 16 | from torch.package import PackageExporter 17 | 18 | try: 19 | from .examples import ( 20 | BatchedModel, 21 | load_library, 22 | multi_return_metadata, 23 | MultiReturn, 24 | resnet18, 25 | Simple, 26 | ) 27 | except ImportError: 28 | # pyre-fixme[21]: Could not find module `examples`. 29 | from examples import ( 30 | BatchedModel, 31 | load_library, 32 | multi_return_metadata, 33 | MultiReturn, 34 | resnet18, 35 | Simple, 36 | ) 37 | 38 | try: 39 | from .fx.examples import SimpleWithLeaf 40 | except ImportError: 41 | # pyre-fixme[21]: Could not find module `fx.examples`. 42 | from fx.examples import SimpleWithLeaf 43 | 44 | try: 45 | from .tensorrt_example import make_trt_module 46 | except ImportError: 47 | # pyre-fixme[21]: Could not find module `tensorrt_example`. 48 | from tensorrt_example import make_trt_module 49 | 50 | 51 | def generate_fx_example(): 52 | name = "simple_leaf" 53 | model = SimpleWithLeaf(5, 10) 54 | graph_module: torch.fx.GraphModule = symbolic_trace(model) 55 | with PackageExporter(str(p / (name + "_fx"))) as e: 56 | e.intern("**") 57 | e.save_pickle("model", "model.pkl", graph_module) 58 | 59 | model_jit = torch.jit.script(model) 60 | model_jit.save(str(p / (name + "_jit"))) 61 | 62 | 63 | def save( 64 | name, 65 | model, 66 | model_jit=None, 67 | eg=None, 68 | featurestore_meta=None, 69 | text_in_extra_file=None, 70 | binary_in_extra_file=None, 71 | ): 72 | with PackageExporter(str(p / name)) as e: 73 | e.mock("iopath.**") 74 | e.intern("**") 75 | e.save_pickle("model", "model.pkl", model) 76 | if eg: 77 | e.save_pickle("model", "example.pkl", eg) 78 | if featurestore_meta: 79 | # TODO(whc) can this name come from buck somehow, 80 | # so it's consistent with predictor_config_constants::METADATA_FILE_NAME()? 81 | e.save_text("extra_files", "metadata.json", featurestore_meta) 82 | if text_in_extra_file: 83 | e.save_text("extra_files", "text", text_in_extra_file) 84 | if binary_in_extra_file: 85 | e.save_binary("extra_files", "binary", binary_in_extra_file) 86 | 87 | if model_jit: 88 | model_jit.save(str(p / (name + "_jit"))) 89 | 90 | 91 | parser = argparse.ArgumentParser(description="Generate Examples") 92 | parser.add_argument("--install_dir", help="Root directory for all output files") 93 | 94 | 95 | def main() -> None: 96 | global p 97 | args = parser.parse_args() 98 | if args.install_dir is None: 99 | # pyre-fixme[10]: Name `p` is used but not defined. 100 | p = Path(__file__).parent / "generated" 101 | p.mkdir(exist_ok=True) 102 | else: 103 | p = Path(args.install_dir) 104 | 105 | resnet = resnet18() 106 | resnet.eval() 107 | resnet_eg = torch.rand(1, 3, 224, 224) 108 | resnet_traced = torch.jit.trace(resnet, resnet_eg) 109 | save("resnet", resnet, resnet_traced, (resnet_eg,)) 110 | 111 | simple = Simple(10, 20) 112 | save( 113 | name="simple", 114 | model=simple, 115 | model_jit=torch.jit.script(simple), 116 | eg=(torch.rand(10, 20),), 117 | text_in_extra_file="hello", 118 | binary_in_extra_file=b"hello", 119 | ) 120 | 121 | multi_return = MultiReturn() 122 | save( 123 | "multi_return", 124 | multi_return, 125 | torch.jit.script(multi_return), 126 | (torch.rand(10, 20),), 127 | multi_return_metadata, 128 | ) 129 | 130 | # used for torch deploy/package tests in predictor 131 | batched_model = BatchedModel() 132 | save("batched_model", batched_model) 133 | 134 | with PackageExporter(str(p / "load_library")) as e: 135 | e.mock("iopath.**") 136 | e.intern("**") 137 | e.save_pickle("fn", "fn.pkl", load_library) 138 | 139 | generate_fx_example() 140 | 141 | with PackageExporter(p / "uses_distributed") as e: 142 | e.save_source_string( 143 | "uses_distributed", 144 | "import torch.distributed; assert torch.distributed.is_available()", 145 | ) 146 | 147 | with PackageExporter(p / "uses_cuda") as e: 148 | e.save_source_string( 149 | "uses_cuda", 150 | "import torch; assert torch.cuda.is_available()", 151 | ) 152 | 153 | with PackageExporter(str(p / "make_trt_module")) as e: 154 | e.extern("tensorrt") 155 | e.add_dependency("tensorrt") 156 | e.mock("iopath.**") 157 | e.intern("**") 158 | e.save_pickle("make_trt_module", "model.pkl", make_trt_module) 159 | 160 | 161 | if __name__ == "__main__": 162 | main() # pragma: no cover 163 | -------------------------------------------------------------------------------- /multipy/runtime/interpreter/builtin_registry.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // 4 | // This source code is licensed under the BSD-style license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | /* 8 | * The torch::deploy builtin registry library is used to make adding new bultins 9 | * to torch::deploy easy and clean. 10 | * 11 | * Under the hood, to add a torch::deploy builtin, the following things need to 12 | * be done 13 | * 1. merge the frozen modules for the builtin into PyImport_FrozenModules 14 | * 2. appending PyInit methods for modules implemented in C++ to the CPython 15 | * builtin module list via methods like PyImport_AppendInittab 16 | * 3. tweak the sys.meta_path a bit to force loading non-toplevel moduels for 17 | * the torch::deploy builtin via the CPython builtin module importer. 18 | * 19 | * Doing all these things again and again manually is cumbersome and 20 | * error-prone. This builtin registry library supports open registration for 21 | * torch::deploy builtins. It does the work above by a single line of code 22 | * invoking REGISTER_TORCH_DEPLOY_BUILTIN macro. Here is an example for numpy: 23 | * 24 | * REGISTER_TORCH_DEPLOY_BUILTIN(numpy, numpy_frozen_modules, ) 26 | * 27 | * Calling REGISTER_TORCH_DEPLOY_BUILTIN macro will instantiate a 28 | * BuiltinRegisterer object. The constructor of BuiltinRegisterer does the real 29 | * registration work. 30 | */ 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | struct _frozen; 38 | 39 | namespace torch { 40 | namespace deploy { 41 | 42 | /* 43 | * This data structure describes a torch::deploy builtin being registered to 44 | * the registry. 45 | * 46 | * Each torch::deploy builtin contains the following basically information: 47 | * - a name for the builtin. It's usually the name of the library like numpy 48 | * - the lsit of frozen modules 49 | * - the list of builtin modules 50 | */ 51 | struct BuiltinRegistryItem { 52 | explicit BuiltinRegistryItem( 53 | const char* _name, 54 | const struct _frozen* _frozenModules, 55 | std::vector>&& _builtinModules); 56 | const char* name; 57 | const struct _frozen* frozenModules; 58 | unsigned numModules; 59 | std::vector> builtinModules; 60 | }; 61 | 62 | /* 63 | * BuiltinRegistry maintains all the registered torch::deploy builtins. This 64 | * class is a singleton. Calling BuiltinRegistry::get() returns the single 65 | * object instance. 66 | * 67 | * The state of this class is basically a list of BuiltinRegistryItem registered 68 | * so far. 69 | */ 70 | class BuiltinRegistry { 71 | public: 72 | static void runPreInitialization(); 73 | static void runPostInitialization(); 74 | 75 | private: 76 | static struct _frozen* getAllFrozenModules(); 77 | // call this after all the registration is done. 78 | static void sanityCheck(); 79 | static void appendCPythonInittab(); 80 | static std::string getBuiltinModulesCSV(); 81 | 82 | static void registerBuiltin(std::unique_ptr item); 83 | static const std::vector>& items() { 84 | return get()->items_; 85 | } 86 | static unsigned totalNumModules(); 87 | static BuiltinRegistry* get(); 88 | static BuiltinRegistryItem* getItem(const std::string& name); 89 | static std::vector> getAllBuiltinModules(); 90 | 91 | explicit BuiltinRegistry() = default; 92 | std::unordered_map name2idx_; 93 | std::vector> items_; 94 | 95 | friend class BuiltinRegisterer; 96 | FRIEND_TEST(BuiltinRegistryTest, SimpleTest); 97 | }; 98 | 99 | /* 100 | * If nobody defines allowLibrary method, allowLibrary will be evaluated to 101 | * 0 and we allow registering any libraries. If someone defines allowLibrary, 102 | * we respect that and only registering libraries that get true from calling 103 | * allowLibrary(libname). 104 | * 105 | * Currently used in unit test so we can fully control the registered libraries. 106 | */ 107 | __attribute__((weak)) bool allowLibrary(const std::string& libname); 108 | 109 | /* 110 | * This class implements RAII (resource acquisition is initialization) to 111 | * register a bulitin to the registry. 112 | */ 113 | class BuiltinRegisterer { 114 | public: 115 | explicit BuiltinRegisterer( 116 | const char* name, 117 | const struct _frozen* frozenModules...); 118 | }; 119 | 120 | } // namespace deploy 121 | } // namespace torch 122 | 123 | #define CONCAT_IMPL(s1, s2) s1##s2 124 | #define CONCAT(s1, s2) CONCAT_IMPL(s1, s2) 125 | #define ANONYMOUS_VARIABLE(str) CONCAT(str, __LINE__) 126 | 127 | /* there can be a variable list of builtin modules following frozen_modules 128 | * A typical usage of this macro is: 129 | * 130 | * REGISTER_TORCH_DEPLOY_BUILTIN(library_name_without_quote, 131 | * frozen_modules_list, builtin_module_name_1, builtin_module_init_function_1, 132 | * ..., builtin_module_name_N, builtin_module_init_function_N) 133 | */ 134 | #define REGISTER_TORCH_DEPLOY_BUILTIN(libname, frozenModules...) \ 135 | static torch::deploy::BuiltinRegisterer ANONYMOUS_VARIABLE( \ 136 | BuiltinRegisterer)(#libname, frozenModules, nullptr) 137 | -------------------------------------------------------------------------------- /.github/workflows/build_test_release.yaml: -------------------------------------------------------------------------------- 1 | name: Reusable build, test and release 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | python3-minor-version: 7 | required: true 8 | type: string 9 | runner: 10 | required: true 11 | type: string 12 | compat-tests: 13 | default: false 14 | type: boolean 15 | release: 16 | default: false 17 | type: boolean 18 | secrets: 19 | token: 20 | required: true 21 | 22 | jobs: 23 | build-test: 24 | uses: pytorch/test-infra/.github/workflows/linux_job.yml@main 25 | with: 26 | runner: ${{ inputs.runner }} 27 | upload-artifact: multipy_runtime_python3.${{ inputs.python3-minor-version }} 28 | repository: pytorch/multipy 29 | gpu-arch-type: cuda 30 | gpu-arch-version: 116 31 | script: | 32 | python_version=3.${{ inputs.python3-minor-version }} 33 | 34 | echo "::group::Install runtime build dependencies" 35 | xargs yum install -y < build-requirements-centos7.txt 36 | echo "::endgroup::" 37 | 38 | echo "::group::Sync git submodules" 39 | git -c fetch.parallel=0 -c submodule.fetchJobs=0 submodule update --init --recursive 40 | echo "::endgroup::" 41 | 42 | echo "::group::Setup virtual environment" 43 | if [[ ${{ inputs.python3-minor-version }} -gt 7 ]]; 44 | then 45 | conda install -y python=${python_version} mkl mkl-include conda-build pyyaml numpy ipython 46 | conda install -y -c conda-forge libpython-static=${python_version} 47 | conda install -y pytorch torchvision torchaudio cpuonly -c pytorch 48 | conda clean -ya; 49 | else 50 | conda deactivate 51 | pip install virtualenv 52 | git clone https://github.com/pyenv/pyenv.git ~/.pyenv 53 | export CFLAGS="-fPIC -g" 54 | ~/.pyenv/bin/pyenv install --force ${python_version} 55 | virtualenv -p ~/.pyenv/versions/$(~/.pyenv/bin/pyenv latest ${python_version})/bin/python3 ~/venvs/multipy 56 | source ~/venvs/multipy/bin/activate 57 | pip install \ 58 | torch torchvision torchaudio \ 59 | --extra-index-url https://download.pytorch.org/whl/cpu; 60 | fi 61 | echo "::endgroup::" 62 | 63 | echo "::group::Install" 64 | BUILD_CUDA_TESTS=1 python -m pip install -e . 65 | echo "::endgroup::" 66 | 67 | echo "::group::Generate examples" 68 | python multipy/runtime/example/generate_examples.py 69 | cd examples 70 | cmake -S . -B build/ -DMULTIPY_PATH=".." && cmake --build build/ --config Release -j 71 | cd - 72 | echo "::endgroup::" 73 | 74 | export PYTHONPATH=$(pwd) 75 | export LIBTEST_DEPLOY_LIB=$(pwd)/multipy/runtime/build/libtest_deploy_lib.so 76 | export LD_LIBRARY_PATH=/opt/conda/lib/:$LD_LIBRARY_PATH 77 | 78 | echo "::group::Test C++" 79 | multipy/runtime/build/test_deploy 80 | echo "::endgroup::" 81 | 82 | echo "::group::Test Pybind" 83 | python multipy/runtime/test_pybind.py 84 | echo "::endgroup::" 85 | 86 | echo "::group::Run examples" 87 | examples/build/hello_world_example 88 | python3 examples/quickstart/gen_package.py 89 | ./examples/build/quickstart my_package.pt 90 | ./examples/build/movable_example 91 | echo "::endgroup::" 92 | 93 | echo "::group::Benchmark" 94 | ./multipy/runtime/build/deploy_benchmark 2 none jit multipy/runtime/example/generated/resnet 95 | echo "::endgroup::" 96 | 97 | if [[ ${{ inputs.python3-minor-version }} -gt 7 ]]; 98 | then 99 | echo "::group::Compat test" 100 | pip install -r compat-requirements.txt 101 | multipy/runtime/build/interactive_embedded_interpreter --pyscript multipy/runtime/test_compat.py 102 | echo "::endgroup::"; 103 | fi 104 | 105 | echo "::group::Test GPU" 106 | # Separating GPU tests due to issues with py37 and py38: https://github.com/pytorch/multipy/issues/239 107 | pip install --upgrade --force-reinstall torch \ 108 | --extra-index-url https://download.pytorch.org/whl/cu116 109 | multipy/runtime/build/test_deploy_gpu 110 | echo "::endgroup::" 111 | 112 | echo "::group::Create tarball" 113 | cp -r multipy/runtime/build/dist/* . 114 | rm -rf multipy/runtime/build 115 | tar -czf multipy_runtime_python${python_version}.tar.gz multipy/ 116 | echo "::endgroup::" 117 | 118 | echo "::group::Move artifact for upload" 119 | mv multipy_runtime_python${python_version}.tar.gz ${RUNNER_ARTIFACT_DIR}/ 120 | echo "::endgroup::" 121 | 122 | release: 123 | needs: [build-test] 124 | runs-on: ubuntu-latest 125 | if: ${{ inputs.release }} 126 | steps: 127 | - name: Download artifact 128 | uses: actions/download-artifact@v3 129 | with: 130 | name: multipy_runtime_python3.${{ inputs.python3-minor-version }} 131 | path: ./ 132 | 133 | - name: Update nightly release 134 | uses: pyTooling/Actions/releaser@main 135 | with: 136 | tag: nightly-runtime-python3.${{ inputs.python3-minor-version }} 137 | rm: true 138 | token: ${{ secrets.token }} 139 | files: multipy_runtime_python3.${{ inputs.python3-minor-version }}.tar.gz 140 | -------------------------------------------------------------------------------- /scripts/linter/adapters/testowners_linter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Test ownership was introduced in https://github.com/pytorch/pytorch/issues/66232. 4 | 5 | This lint verifies that every Python test file (file that matches test_*.py or *_test.py in the test folder) 6 | has valid ownership information in a comment header. Valid means: 7 | - The format of the header follows the pattern "# Owner(s): ["list", "of owner", "labels"] 8 | - Each owner label actually exists in PyTorch 9 | - Each owner label starts with "module: " or "oncall: " or is in ACCEPTABLE_OWNER_LABELS 10 | """ 11 | 12 | import argparse 13 | import json 14 | from enum import Enum 15 | from typing import Any, List, NamedTuple, Optional 16 | from urllib.request import urlopen 17 | 18 | 19 | LINTER_CODE = "TESTOWNERS" 20 | 21 | 22 | class LintSeverity(str, Enum): 23 | ERROR = "error" 24 | WARNING = "warning" 25 | ADVICE = "advice" 26 | DISABLED = "disabled" 27 | 28 | 29 | class LintMessage(NamedTuple): 30 | path: Optional[str] 31 | line: Optional[int] 32 | char: Optional[int] 33 | code: str 34 | severity: LintSeverity 35 | name: str 36 | original: Optional[str] 37 | replacement: Optional[str] 38 | description: Optional[str] 39 | 40 | 41 | # Team/owner labels usually start with "module: " or "oncall: ", but the following are acceptable exceptions 42 | ACCEPTABLE_OWNER_LABELS = ["NNC", "high priority"] 43 | OWNERS_PREFIX = "# Owner(s): " 44 | 45 | 46 | def get_pytorch_labels() -> Any: 47 | labels = ( 48 | urlopen("https://ossci-metrics.s3.amazonaws.com/pytorch_labels.json") 49 | .read() 50 | .decode("utf-8") 51 | ) 52 | return json.loads(labels) 53 | 54 | 55 | PYTORCH_LABELS = get_pytorch_labels() 56 | # Team/owner labels usually start with "module: " or "oncall: ", but the following are acceptable exceptions 57 | ACCEPTABLE_OWNER_LABELS = ["NNC", "high priority"] 58 | GLOB_EXCEPTIONS = ["**/test/run_test.py"] 59 | 60 | 61 | def check_labels( 62 | labels: List[str], filename: str, line_number: int 63 | ) -> List[LintMessage]: 64 | lint_messages = [] 65 | for label in labels: 66 | if label not in PYTORCH_LABELS: 67 | lint_messages.append( 68 | LintMessage( 69 | path=filename, 70 | line=line_number, 71 | char=None, 72 | code=LINTER_CODE, 73 | severity=LintSeverity.ERROR, 74 | name="[invalid-label]", 75 | original=None, 76 | replacement=None, 77 | description=( 78 | f"{label} is not a PyTorch label " 79 | "(please choose from https://github.com/pytorch/pytorch/labels)" 80 | ), 81 | ) 82 | ) 83 | 84 | if ( 85 | label.startswith("module:") 86 | or label.startswith("oncall:") 87 | or label in ACCEPTABLE_OWNER_LABELS 88 | ): 89 | continue 90 | 91 | lint_messages.append( 92 | LintMessage( 93 | path=filename, 94 | line=line_number, 95 | char=None, 96 | code=LINTER_CODE, 97 | severity=LintSeverity.ERROR, 98 | name="[invalid-owner]", 99 | original=None, 100 | replacement=None, 101 | description=( 102 | f"{label} is not an acceptable owner " 103 | "(please update to another label or edit ACCEPTABLE_OWNERS_LABELS " 104 | "in tools/linters/adapters/testowners_linter.py" 105 | ), 106 | ) 107 | ) 108 | 109 | return lint_messages 110 | 111 | 112 | def check_file(filename: str) -> List[LintMessage]: 113 | lint_messages = [] 114 | has_ownership_info = False 115 | 116 | with open(filename) as f: 117 | for idx, line in enumerate(f): 118 | if not line.startswith(OWNERS_PREFIX): 119 | continue 120 | 121 | has_ownership_info = True 122 | labels = json.loads(line[len(OWNERS_PREFIX) :]) 123 | lint_messages.extend(check_labels(labels, filename, idx + 1)) 124 | 125 | if has_ownership_info is False: 126 | lint_messages.append( 127 | LintMessage( 128 | path=filename, 129 | line=None, 130 | char=None, 131 | code=LINTER_CODE, 132 | severity=LintSeverity.ERROR, 133 | name="[no-owner-info]", 134 | original=None, 135 | replacement=None, 136 | description="Missing a comment header with ownership information.", 137 | ) 138 | ) 139 | 140 | return lint_messages 141 | 142 | 143 | def main() -> None: 144 | parser = argparse.ArgumentParser( 145 | description="test ownership linter", 146 | fromfile_prefix_chars="@", 147 | ) 148 | parser.add_argument( 149 | "filenames", 150 | nargs="+", 151 | help="paths to lint", 152 | ) 153 | 154 | args = parser.parse_args() 155 | lint_messages = [] 156 | 157 | for filename in args.filenames: 158 | lint_messages.extend(check_file(filename)) 159 | 160 | for lint_message in lint_messages: 161 | print(json.dumps(lint_message._asdict()), flush=True) 162 | 163 | 164 | if __name__ == "__main__": 165 | main() 166 | -------------------------------------------------------------------------------- /scripts/linter/adapters/newlines_linter.py: -------------------------------------------------------------------------------- 1 | """ 2 | NEWLINE: Checks files to make sure there are no trailing newlines. 3 | """ 4 | 5 | import argparse 6 | import json 7 | import logging 8 | import sys 9 | 10 | from enum import Enum 11 | from typing import List, NamedTuple, Optional 12 | 13 | NEWLINE = 10 # ASCII "\n" 14 | CARRIAGE_RETURN = 13 # ASCII "\r" 15 | LINTER_CODE = "NEWLINE" 16 | 17 | 18 | class LintSeverity(str, Enum): 19 | ERROR = "error" 20 | WARNING = "warning" 21 | ADVICE = "advice" 22 | DISABLED = "disabled" 23 | 24 | 25 | class LintMessage(NamedTuple): 26 | path: Optional[str] 27 | line: Optional[int] 28 | char: Optional[int] 29 | code: str 30 | severity: LintSeverity 31 | name: str 32 | original: Optional[str] 33 | replacement: Optional[str] 34 | description: Optional[str] 35 | 36 | 37 | def check_file(filename: str) -> Optional[LintMessage]: 38 | logging.debug("Checking file %s", filename) 39 | 40 | with open(filename, "rb") as f: 41 | lines = f.readlines() 42 | 43 | if len(lines) == 0: 44 | # File is empty, just leave it alone. 45 | return None 46 | 47 | if len(lines) == 1 and len(lines[0]) == 1: 48 | # file is wrong whether or not the only byte is a newline 49 | return LintMessage( 50 | path=filename, 51 | line=None, 52 | char=None, 53 | code=LINTER_CODE, 54 | severity=LintSeverity.ERROR, 55 | name="testestTrailing newline", 56 | original=None, 57 | replacement=None, 58 | description="Trailing newline found. Run `lintrunner --take NEWLINE -a` to apply changes.", 59 | ) 60 | 61 | if len(lines[-1]) == 1 and lines[-1][0] == NEWLINE: 62 | try: 63 | original = b"".join(lines).decode("utf-8") 64 | except Exception as err: 65 | return LintMessage( 66 | path=filename, 67 | line=None, 68 | char=None, 69 | code=LINTER_CODE, 70 | severity=LintSeverity.ERROR, 71 | name="Decoding failure", 72 | original=None, 73 | replacement=None, 74 | description=f"utf-8 decoding failed due to {err.__class__.__name__}:\n{err}", 75 | ) 76 | 77 | return LintMessage( 78 | path=filename, 79 | line=None, 80 | char=None, 81 | code=LINTER_CODE, 82 | severity=LintSeverity.ERROR, 83 | name="Trailing newline", 84 | original=original, 85 | replacement=original.rstrip("\n") + "\n", 86 | description="Trailing newline found. Run `lintrunner --take NEWLINE -a` to apply changes.", 87 | ) 88 | has_changes = False 89 | original_lines: Optional[List[bytes]] = None 90 | for idx, line in enumerate(lines): 91 | if len(line) >= 2 and line[-1] == NEWLINE and line[-2] == CARRIAGE_RETURN: 92 | if not has_changes: 93 | original_lines = list(lines) 94 | has_changes = True 95 | lines[idx] = line[:-2] + b"\n" 96 | 97 | if has_changes: 98 | try: 99 | assert original_lines is not None 100 | original = b"".join(original_lines).decode("utf-8") 101 | replacement = b"".join(lines).decode("utf-8") 102 | except Exception as err: 103 | return LintMessage( 104 | path=filename, 105 | line=None, 106 | char=None, 107 | code=LINTER_CODE, 108 | severity=LintSeverity.ERROR, 109 | name="Decoding failure", 110 | original=None, 111 | replacement=None, 112 | description=f"utf-8 decoding failed due to {err.__class__.__name__}:\n{err}", 113 | ) 114 | return LintMessage( 115 | path=filename, 116 | line=None, 117 | char=None, 118 | code=LINTER_CODE, 119 | severity=LintSeverity.ERROR, 120 | name="DOS newline", 121 | original=original, 122 | replacement=replacement, 123 | description="DOS newline found. Run `lintrunner --take NEWLINE -a` to apply changes.", 124 | ) 125 | 126 | return None 127 | 128 | 129 | if __name__ == "__main__": 130 | parser = argparse.ArgumentParser( 131 | description="native functions linter", 132 | fromfile_prefix_chars="@", 133 | ) 134 | parser.add_argument( 135 | "--verbose", 136 | action="store_true", 137 | help="location of native_functions.yaml", 138 | ) 139 | parser.add_argument( 140 | "filenames", 141 | nargs="+", 142 | help="paths to lint", 143 | ) 144 | 145 | args = parser.parse_args() 146 | 147 | logging.basicConfig( 148 | format="<%(threadName)s:%(levelname)s> %(message)s", 149 | level=( 150 | logging.NOTSET 151 | if args.verbose 152 | else logging.DEBUG 153 | if len(args.filenames) < 1000 154 | else logging.INFO 155 | ), 156 | stream=sys.stderr, 157 | ) 158 | 159 | lint_messages = [] 160 | for filename in args.filenames: 161 | lint_message = check_file(filename) 162 | if lint_message is not None: 163 | lint_messages.append(lint_message) 164 | 165 | for lint_message in lint_messages: 166 | print(json.dumps(lint_message._asdict()), flush=True) 167 | -------------------------------------------------------------------------------- /docs/source/Doxyfile: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Primary project setup. # 3 | ################################################################################ 4 | PROJECT_NAME = "MultiPy" 5 | # Parent directory of where Doxygen should output files. Cannot be for example 6 | # `../build/doxygen` because if `../build` does not exist Doxygen will error. 7 | OUTPUT_DIRECTORY = ../build 8 | # XML output stored in ${OUTPUT_DIRECTORY}/${XML_OUTPUT}. Must agree 9 | # with what is specified to `breathe_projects` in `conf.py`. 10 | XML_OUTPUT = xml 11 | # HTML output stored in ${OUTPUT_DIRECTORY}/${HTML_OUTPUT}. 12 | # NOTE: it can be useful to have Doxygen generate HTML to verify what sphinx is 13 | # creating, or to generate HTML documentation to check a docstring you are 14 | # writing without needing to wait for Sphinx. 15 | # 16 | # # Must run from this directory (see relative paths below...) 17 | # $ cd /path/to/pytorch/docs/cpp/source 18 | # 19 | # # Send configs in this file plus `GENERATE_HTML=YES` to doxygen on stdin 20 | # $ (cat Doxyfile; echo 'GENERATE_HTML = YES') | doxygen - 21 | # 22 | # # View the doxygen results. 23 | # $ open ../build/doxygen_html/index.html 24 | # 25 | # This sends everything in this file plus `GENERATE_HTML = YES` to doxygen 26 | # on stdin. Without needing to edit this `Doxyfile` directly. 27 | HTML_OUTPUT = doxygen_html 28 | # Strip the path prefix. *MUST* agree with `exhale_args` in conf.py. 29 | # {repo_root}/docs/cpp/source/../../.. -> {repo_root} 30 | STRIP_FROM_PATH = ../../.. 31 | # What folders / files Doxygen should process. 32 | INPUT = ../../multipy/runtime/deploy.h \ 33 | ../../multipy/runtime/interpreter/plugin_registry.h \ 34 | ../../multipy/runtime/interpreter/interpreter_impl.h \ 35 | ../../multipy/runtime/environment.h \ 36 | ../../multipy/runtime/path_environment.h \ 37 | ../../multipy/runtime/noop_environment.h \ 38 | # Don't include .cpp files! 39 | FILE_PATTERNS = *.h 40 | # If you need this to be YES, exhale will probably break. 41 | CREATE_SUBDIRS = NO 42 | # So that Doxygen does not trim paths, which affects the file hierarchy 43 | FULL_PATH_NAMES = YES 44 | # Nested folders will be ignored without this. 45 | RECURSIVE = YES 46 | # Increase the max node size for our large files 47 | DOT_GRAPH_MAX_NODES = 100 48 | ################################################################################ 49 | # Output formats for Doxygen to create. # 50 | ################################################################################ 51 | # Set to YES if you are debugging or want to compare. 52 | GENERATE_HTML = NO 53 | # Unless you want it... 54 | GENERATE_LATEX = NO 55 | # Both breathe and exhale need the xml. 56 | GENERATE_XML = YES 57 | # Set to NO if you do not want the Doxygen program listing included. 58 | # NOTE: setting to NO may result in unrecovered file relationships 59 | # (which file defined which compound) 60 | XML_PROGRAMLISTING = YES 61 | ################################################################################ 62 | # Doxygen preprocessor / parser control. # 63 | ################################################################################ 64 | ENABLE_PREPROCESSING = YES 65 | MACRO_EXPANSION = YES 66 | EXPAND_ONLY_PREDEF = NO 67 | SKIP_FUNCTION_MACROS = NO 68 | # Extra defs for to help with building the _right_ version of the docs 69 | PREDEFINED = DOXYGEN_DOCUMENTATION_BUILD 70 | PREDEFINED += DOXYGEN_SHOULD_SKIP_THIS 71 | # Symbol expansion to follow for #include statements (not input) 72 | SEARCH_INCLUDES = YES 73 | # INCLUDE_PATH = ../.. \ 74 | # ../../torch/csrc/api/include \ 75 | ################################################################################ 76 | # Compound extraction control. # 77 | ################################################################################ 78 | EXTRACT_ALL = YES 79 | EXTRACT_PACKAGE = YES 80 | EXTRACT_STATIC = YES 81 | CASE_SENSE_NAMES = NO 82 | EXCLUDE_SYMBOLS = caffe2::* cereal* DL* TH* cudnn* std::* *Impl* *InterpreterObj* *PickledObject 83 | ################################################################################ 84 | # Docstring control / customization. # 85 | ################################################################################ 86 | # First line of /** comment */ treated as \brief 87 | JAVADOC_AUTOBRIEF = YES 88 | # Allow for rst directives and advanced functions e.g. grid tables 89 | # Example: 90 | # /** 91 | # * \rst 92 | # * .. code-block:: cpp 93 | # * 94 | # * int main() { 95 | # * return 0; 96 | # * } 97 | # * 98 | # * \endrst 99 | # */ 100 | # NOTE: 101 | # 1. \rst and \endrst must be on their own line. 102 | # 2. leading-asterisk required. 103 | ALIASES = "rst=\verbatim embed:rst:leading-asterisk" 104 | ALIASES += "endrst=\endverbatim" 105 | ################################################################################ 106 | # Warning suppression. # 107 | ################################################################################ 108 | QUIET = YES 109 | WARN_IF_UNDOCUMENTED = NO 110 | -------------------------------------------------------------------------------- /scripts/linter/adapters/mypy_linter.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import logging 4 | import os 5 | import re 6 | import subprocess 7 | import sys 8 | import time 9 | from enum import Enum 10 | from pathlib import Path 11 | from typing import Any, Dict, List, NamedTuple, Optional, Pattern 12 | 13 | 14 | IS_WINDOWS: bool = os.name == "nt" 15 | 16 | 17 | def eprint(*args: Any, **kwargs: Any) -> None: 18 | print(*args, file=sys.stderr, flush=True, **kwargs) 19 | 20 | 21 | class LintSeverity(str, Enum): 22 | ERROR = "error" 23 | WARNING = "warning" 24 | ADVICE = "advice" 25 | DISABLED = "disabled" 26 | 27 | 28 | class LintMessage(NamedTuple): 29 | path: Optional[str] 30 | line: Optional[int] 31 | char: Optional[int] 32 | code: str 33 | severity: LintSeverity 34 | name: str 35 | original: Optional[str] 36 | replacement: Optional[str] 37 | description: Optional[str] 38 | 39 | 40 | def as_posix(name: str) -> str: 41 | return name.replace("\\", "/") if IS_WINDOWS else name 42 | 43 | 44 | # tools/linter/flake8_linter.py:15:13: error: Incompatibl...int") [assignment] 45 | RESULTS_RE: Pattern[str] = re.compile( 46 | r"""(?mx) 47 | ^ 48 | (?P.*?): 49 | (?P\d+): 50 | (?:(?P-?\d+):)? 51 | \s(?P\S+?):? 52 | \s(?P.*) 53 | \s(?P\[.*\]) 54 | $ 55 | """ 56 | ) 57 | 58 | 59 | def run_command( 60 | args: List[str], 61 | *, 62 | extra_env: Optional[Dict[str, str]], 63 | retries: int, 64 | ) -> "subprocess.CompletedProcess[bytes]": 65 | logging.debug("$ %s", " ".join(args)) 66 | start_time = time.monotonic() 67 | try: 68 | return subprocess.run( 69 | args, 70 | stdout=subprocess.PIPE, 71 | stderr=subprocess.PIPE, 72 | ) 73 | finally: 74 | end_time = time.monotonic() 75 | logging.debug("took %dms", (end_time - start_time) * 1000) 76 | 77 | 78 | # Severity is either "error" or "note": 79 | # https://github.com/python/mypy/blob/8b47a032e1317fb8e3f9a818005a6b63e9bf0311/mypy/errors.py#L46-L47 80 | severities = { 81 | "error": LintSeverity.ERROR, 82 | "note": LintSeverity.ADVICE, 83 | } 84 | 85 | 86 | def check_files( 87 | filenames: List[str], 88 | config: str, 89 | retries: int, 90 | ) -> List[LintMessage]: 91 | try: 92 | proc = run_command( 93 | [sys.executable, "-mmypy", f"--config={config}"] + filenames, 94 | extra_env={}, 95 | retries=retries, 96 | ) 97 | except OSError as err: 98 | return [ 99 | LintMessage( 100 | path=None, 101 | line=None, 102 | char=None, 103 | code="MYPY", 104 | severity=LintSeverity.ERROR, 105 | name="command-failed", 106 | original=None, 107 | replacement=None, 108 | description=(f"Failed due to {err.__class__.__name__}:\n{err}"), 109 | ) 110 | ] 111 | stdout = str(proc.stdout, "utf-8").strip() 112 | return [ 113 | LintMessage( 114 | path=match["file"], 115 | name=match["code"], 116 | description=match["message"], 117 | line=int(match["line"]), 118 | char=( 119 | int(match["column"]) 120 | if match["column"] is not None and not match["column"].startswith("-") 121 | else None 122 | ), 123 | code="MYPY", 124 | severity=severities.get(match["severity"], LintSeverity.ERROR), 125 | original=None, 126 | replacement=None, 127 | ) 128 | for match in RESULTS_RE.finditer(stdout) 129 | ] 130 | 131 | 132 | def main() -> None: 133 | parser = argparse.ArgumentParser( 134 | description="mypy wrapper linter.", 135 | fromfile_prefix_chars="@", 136 | ) 137 | parser.add_argument( 138 | "--retries", 139 | default=3, 140 | type=int, 141 | help="times to retry timed out mypy", 142 | ) 143 | parser.add_argument( 144 | "--config", 145 | required=True, 146 | help="path to an mypy .ini config file", 147 | ) 148 | parser.add_argument( 149 | "--verbose", 150 | action="store_true", 151 | help="verbose logging", 152 | ) 153 | parser.add_argument( 154 | "filenames", 155 | nargs="+", 156 | help="paths to lint", 157 | ) 158 | args = parser.parse_args() 159 | 160 | logging.basicConfig( 161 | format="<%(threadName)s:%(levelname)s> %(message)s", 162 | level=( 163 | logging.NOTSET 164 | if args.verbose 165 | else logging.DEBUG 166 | if len(args.filenames) < 1000 167 | else logging.INFO 168 | ), 169 | stream=sys.stderr, 170 | ) 171 | 172 | # Use a dictionary here to preserve order. mypy cares about order, 173 | # tragically, e.g. https://github.com/python/mypy/issues/2015 174 | filenames: Dict[str, bool] = {} 175 | 176 | # If a stub file exists, have mypy check it instead of the original file, in 177 | # accordance with PEP-484 (see https://www.python.org/dev/peps/pep-0484/#stub-files) 178 | for filename in args.filenames: 179 | if filename.endswith(".pyi"): 180 | filenames[filename] = True 181 | continue 182 | 183 | stub_filename = filename.replace(".py", ".pyi") 184 | if Path(stub_filename).exists(): 185 | filenames[stub_filename] = True 186 | else: 187 | filenames[filename] = True 188 | 189 | lint_messages = check_files(list(filenames), args.config, args.retries) 190 | for lint_message in lint_messages: 191 | print(json.dumps(lint_message._asdict()), flush=True) 192 | 193 | 194 | if __name__ == "__main__": 195 | main() 196 | -------------------------------------------------------------------------------- /.github/workflows/install_test.yaml: -------------------------------------------------------------------------------- 1 | name: Install tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | schedule: 10 | - cron: '0 2 * * *' # run at 2 AM UTC 11 | workflow_dispatch: 12 | 13 | jobs: 14 | installtest: 15 | strategy: 16 | matrix: 17 | python3-minor-version: [7,8,9,10] 18 | platform: [ubuntu-18.04] 19 | fail-fast: false 20 | runs-on: ${{ matrix.platform }} 21 | steps: 22 | - name: Set Python Version 23 | run: | 24 | echo "python-version=3.${{ matrix.python3-minor-version }}" >> "$GITHUB_ENV" 25 | - name: Setup Python ${{ env.python-version }} 26 | uses: actions/setup-python@v2 27 | with: 28 | python-version: ${{ env.python-version }} 29 | architecture: x64 30 | 31 | - name: Checkout MultiPy 32 | uses: actions/checkout@v2 33 | with: 34 | submodules: true 35 | 36 | - name: Install runtime build dependencies 37 | run: | 38 | set -eux 39 | sudo apt update 40 | xargs sudo apt install -y -qq --no-install-recommends < build-requirements-debian.txt 41 | git -c fetch.parallel=0 -c submodule.fetchJobs=0 submodule update --init --recursive 42 | 43 | - name: Update cmake 44 | run: | 45 | wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo gpg --dearmor -o /usr/share/keyrings/magic-key.gpg 46 | echo "deb [arch=amd64,arm64 signed-by=/usr/share/keyrings/magic-key.gpg] https://apt.kitware.com/ubuntu/ bionic main" | sudo tee -a /etc/apt/sources.list 47 | echo "deb http://security.ubuntu.com/ubuntu focal-security main" | sudo tee -a /etc/apt/sources.list 48 | sudo apt update 49 | sudo apt install -y binutils cmake 50 | 51 | - name: Set up virtual environment 52 | run: | 53 | set -eux 54 | pip3 install virtualenv 55 | git clone https://github.com/pyenv/pyenv.git ~/.pyenv 56 | export CFLAGS="-fPIC -g" 57 | declare -A py_version_dict 58 | py_version_dict[7]='3.7.10' 59 | py_version_dict[8]='3.8.13' 60 | py_version_dict[9]='3.9.13' 61 | py_version_dict[10]='3.10.6' 62 | py_install_version=${py_version_dict[${{ matrix.python3-minor-version }}]} 63 | ~/.pyenv/bin/pyenv install --force ${py_install_version} 64 | virtualenv -p ~/.pyenv/versions/${py_install_version}/bin/python3 ~/venvs/multipy 65 | 66 | - name: Install dependencies 67 | run: | 68 | set -eux 69 | source "$HOME/venvs/multipy/bin/activate" 70 | python -m pip install -r dev-requirements.txt 71 | pip3 install --pre torch 72 | deactivate 73 | 74 | - name: Run lightweight pip install within virtualenv 75 | run: | 76 | set -eux 77 | source "$HOME/venvs/multipy/bin/activate" 78 | INSTALL_PYTHON_ONLY=1 python -m pip install -e . 79 | deactivate 80 | 81 | - name: Run pip editable install within virtualenv 82 | run: | 83 | set -eux 84 | source "$HOME/venvs/multipy/bin/activate" 85 | rm -rf multipy/runtime/build* 86 | pip install -e . 87 | deactivate 88 | 89 | - name: Run setup.py install within virtualenv 90 | run: | 91 | set -eux 92 | source "$HOME/venvs/multipy/bin/activate" 93 | python setup.py install 94 | test -e ~/venvs/multipy/lib/python3.${{ matrix.python3-minor-version }}/site-packages/multipy/runtime/build/libtorch_deploy.a 95 | deactivate 96 | 97 | - name: Install sdist 98 | run: | 99 | set -eux 100 | source "$HOME/venvs/multipy/bin/activate" 101 | python setup.py sdist 102 | pip install dist/*.tar.gz --force-reinstall 103 | test -e ~/venvs/multipy/lib/python3.${{ matrix.python3-minor-version }}/site-packages/multipy/runtime/build/libtorch_deploy.a 104 | deactivate 105 | 106 | - name: Install bdist_wheel 107 | run: | 108 | set -eux 109 | source "$HOME/venvs/multipy/bin/activate" 110 | python setup.py bdist_wheel 111 | pip install dist/*.whl --force-reinstall 112 | test -e ~/venvs/multipy/lib/python3.${{ matrix.python3-minor-version }}/site-packages/multipy/runtime/build/libtorch_deploy.a 113 | deactivate 114 | 115 | - name: Install bdist_wheel with cuda tests 116 | run: | 117 | set -eux 118 | source "$HOME/venvs/multipy/bin/activate" 119 | python setup.py bdist_wheel 120 | BUILD_CUDA_TESTS=1 pip install dist/*.whl "--force-reinstall" 121 | test -e ~/venvs/multipy/lib/python3.${{ matrix.python3-minor-version }}/site-packages/multipy/runtime/build/libtorch_deploy.a 122 | deactivate 123 | 124 | - name: Install dependencies with conda for 3.8+ 125 | run: | 126 | set -eux 127 | if [[ ${{ matrix.python3-minor-version }} -gt 7 ]]; then \ 128 | curl -fsSL -v -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ 129 | chmod +x ~/miniconda.sh && \ 130 | /bin/bash ~/miniconda.sh -b -p /opt/conda && \ 131 | rm ~/miniconda.sh && \ 132 | /opt/conda/bin/conda install -y python=${{ env.python-version }} mkl mkl-include conda-build pyyaml numpy ipython && \ 133 | /opt/conda/bin/conda install -y -c conda-forge libpython-static=${{ env.python-version }} && \ 134 | /opt/conda/bin/conda install -y pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch-nightly && \ 135 | /opt/conda/bin/conda clean -ya; \ 136 | fi 137 | 138 | - name: Run pip install with conda for 3.8+ 139 | run: | 140 | set -eux 141 | if [[ ${{ matrix.python3-minor-version }} -gt 7 ]]; then 142 | source /opt/conda/bin/activate 143 | pip install -e . 144 | fi 145 | -------------------------------------------------------------------------------- /multipy/runtime/utils.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # prefer the active Python version instead of the latest -- only works on cmake 8 | # 3.15+ 9 | # https://cmake.org/cmake/help/latest/module/FindPython3.html#hints 10 | if (NOT DEFINED _GLIBCXX_USE_CXX11_ABI) 11 | # infer the ABI setting from the installed version of PyTorch 12 | execute_process( 13 | COMMAND python -c "import torch; print(1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0)" 14 | OUTPUT_VARIABLE _GLIBCXX_USE_CXX11_ABI 15 | OUTPUT_STRIP_TRAILING_WHITESPACE 16 | RESULT_VARIABLE ret 17 | ) 18 | if(ret EQUAL "1") 19 | message(FATAL_ERROR "Failed to detect ABI version") 20 | endif() 21 | endif() 22 | string(APPEND CMAKE_CXX_FLAGS " -D_GLIBCXX_USE_CXX11_ABI=${_GLIBCXX_USE_CXX11_ABI}") 23 | add_definitions(-D_GLIBCXX_USE_CXX11_ABI=${_GLIBCXX_USE_CXX11_ABI}) 24 | message(STATUS "_GLIBCXX_USE_CXX11_ABI - ${_GLIBCXX_USE_CXX11_ABI}") 25 | 26 | if (NOT DEFINED _CXX_ABI_VERSION) 27 | # infer the ABI setting from the installed version of PyTorch 28 | execute_process( 29 | COMMAND python -c "import torch; print(torch._C._PYBIND11_BUILD_ABI[-2:])" 30 | OUTPUT_VARIABLE _CXX_ABI_VERSION 31 | OUTPUT_STRIP_TRAILING_WHITESPACE 32 | RESULT_VARIABLE ret 33 | ) 34 | if(ret EQUAL "1") 35 | message(WARNING "Failed to detect ABI version, reseting to 11") 36 | set(_CXX_ABI_VERSION 11) 37 | endif() 38 | endif() 39 | string(APPEND CMAKE_CXX_FLAGS " -fabi-version=${_CXX_ABI_VERSION}") 40 | message(STATUS "_CXX_ABI_VERSION - ${_CXX_ABI_VERSION}") 41 | 42 | set(Python3_FIND_STRATEGY LOCATION) 43 | 44 | find_package (Python3 COMPONENTS Interpreter Development) 45 | set(PYTORCH_ROOT "${Python3_SITELIB}") 46 | 47 | # if pytorch was installed in develop mode we need to resolve the egg-link 48 | set(PYTORCH_EGG_LINK "${PYTORCH_ROOT}/torch.egg-link") 49 | if (EXISTS "${PYTORCH_EGG_LINK}") 50 | file (STRINGS "${PYTORCH_EGG_LINK}" PYTORCH_ROOT LIMIT_COUNT 1) 51 | endif() 52 | 53 | message(STATUS "PYTORCH_ROOT - ${PYTORCH_ROOT}" ) 54 | 55 | include_directories(BEFORE "${PYTORCH_ROOT}/torch/include") 56 | include_directories(BEFORE "${PYTORCH_ROOT}/torch/include/torch/csrc/api/include/") 57 | include_directories(BEFORE "${Python3_INCLUDE_DIRS}") 58 | LINK_DIRECTORIES("${PYTORCH_ROOT}/torch/lib") 59 | 60 | macro(caffe2_interface_library SRC DST) 61 | add_library(${DST} INTERFACE) 62 | add_dependencies(${DST} ${SRC}) 63 | # Depending on the nature of the source library as well as the compiler, 64 | # determine the needed compilation flags. 65 | get_target_property(__src_target_type ${SRC} TYPE) 66 | # Depending on the type of the source library, we will set up the 67 | # link command for the specific SRC library. 68 | if(${__src_target_type} STREQUAL "STATIC_LIBRARY") 69 | # In the case of static library, we will need to add whole-static flags. 70 | if(APPLE) 71 | target_link_libraries( 72 | ${DST} INTERFACE -Wl,-force_load,\"$\") 73 | elseif(MSVC) 74 | # In MSVC, we will add whole archive in default. 75 | target_link_libraries( 76 | ${DST} INTERFACE "$") 77 | target_link_options( 78 | ${DST} INTERFACE "-WHOLEARCHIVE:$") 79 | else() 80 | # Assume everything else is like gcc 81 | target_link_libraries(${DST} INTERFACE 82 | "-Wl,--whole-archive,\"$\" -Wl,--no-whole-archive") 83 | endif() 84 | # Link all interface link libraries of the src target as well. 85 | # For static library, we need to explicitly depend on all the libraries 86 | # that are the dependent library of the source library. Note that we cannot 87 | # use the populated INTERFACE_LINK_LIBRARIES property, because if one of the 88 | # dependent library is not a target, cmake creates a $ wrapper 89 | # and then one is not able to find target "src". For more discussions, check 90 | # https://gitlab.kitware.com/cmake/cmake/issues/15415 91 | # https://cmake.org/pipermail/cmake-developers/2013-May/019019.html 92 | # Specifically the following quote 93 | # 94 | # """ 95 | # For STATIC libraries we can define that the PUBLIC/PRIVATE/INTERFACE keys 96 | # are ignored for linking and that it always populates both LINK_LIBRARIES 97 | # LINK_INTERFACE_LIBRARIES. Note that for STATIC libraries the 98 | # LINK_LIBRARIES property will not be used for anything except build-order 99 | # dependencies. 100 | # """ 101 | target_link_libraries(${DST} INTERFACE 102 | $) 103 | elseif(${__src_target_type} STREQUAL "SHARED_LIBRARY") 104 | if("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") 105 | target_link_libraries(${DST} INTERFACE 106 | "-Wl,--no-as-needed,\"$\" -Wl,--as-needed") 107 | else() 108 | target_link_libraries(${DST} INTERFACE ${SRC}) 109 | endif() 110 | # Link all interface link libraries of the src target as well. 111 | # For shared libraries, we can simply depend on the INTERFACE_LINK_LIBRARIES 112 | # property of the target. 113 | target_link_libraries(${DST} INTERFACE 114 | $) 115 | else() 116 | message(FATAL_ERROR 117 | "You made a CMake build file error: target " ${SRC} 118 | " must be of type either STATIC_LIBRARY or SHARED_LIBRARY. However, " 119 | "I got " ${__src_target_type} ".") 120 | endif() 121 | # For all other interface properties, manually inherit from the source target. 122 | set_target_properties(${DST} PROPERTIES 123 | INTERFACE_COMPILE_DEFINITIONS 124 | $ 125 | INTERFACE_COMPILE_OPTIONS 126 | $ 127 | INTERFACE_INCLUDE_DIRECTORIES 128 | $ 129 | INTERFACE_SYSTEM_INCLUDE_DIRECTORIES 130 | $) 131 | endmacro() 132 | --------------------------------------------------------------------------------