├── .circleci ├── config.yml └── unittest │ ├── linux │ └── scripts │ │ ├── environment.yml │ │ ├── install.sh │ │ ├── post_process.sh │ │ ├── run_test.sh │ │ └── setup_env.sh │ └── windows │ └── scripts │ ├── environment.yml │ ├── install.sh │ ├── install_conda.bat │ ├── post_process.sh │ ├── run_test.sh │ ├── setup_env.sh │ └── vc_env_helper.bat ├── .github └── csprng_architecture.png ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── examples ├── csprng.ipynb └── encrypt_decrypt.ipynb ├── packaging ├── README.md ├── build_conda.sh ├── build_wheel.sh ├── conda │ ├── build_csprng.sh │ ├── install_conda.bat │ └── switch_cuda_version.sh ├── pkg_helpers.bash ├── torchcsprng │ ├── bld.bat │ ├── conda_build_config.yaml │ └── meta.yaml ├── vs2017 │ ├── activate.bat │ ├── conda_build_config.yaml │ ├── install_activate.bat │ ├── install_runtime.bat │ └── meta.yaml ├── vs2019 │ ├── activate.bat │ ├── conda_build_config.yaml │ ├── install_activate.bat │ ├── install_runtime.bat │ └── meta.yaml ├── wheel │ ├── linux_manywheel.sh │ ├── osx_wheel.sh │ └── relocate.py └── windows │ ├── azure-pipelines-ci.yml │ ├── azure-pipelines.yml │ ├── build_csprng.bat │ ├── cpu.bat │ ├── cuda101.bat │ ├── cuda102.bat │ ├── cuda92.bat │ ├── internal │ ├── auth.bat │ ├── build_conda.bat │ ├── build_wheels.bat │ ├── check_deps.bat │ ├── check_opts.bat │ ├── clean.bat │ ├── clone.bat │ ├── copy.bat │ ├── copy_cpu.bat │ ├── cuda_install.bat │ ├── dep_install.bat │ ├── env_fix.bat │ ├── nightly_defaults.bat │ ├── publish.bat │ ├── setup.bat │ ├── test.bat │ ├── upload.bat │ ├── vc_env_helper.bat │ ├── vc_install_helper.sh │ ├── vs2017_install.ps1 │ ├── vs2019_install.ps1 │ └── vs_install.bat │ ├── old │ ├── cuda100.bat │ └── cuda90.bat │ └── templates │ ├── auth_task.yml │ ├── build_conda.yml │ ├── build_task.yml │ ├── build_wheels.yml │ ├── linux_build_task.yml │ ├── override_pytorch_version.yml │ ├── publish_packages.yml │ ├── publish_test_results.yml │ ├── setup_env_for_msagent.yml │ ├── setup_nightly_variables.yml │ ├── upload_to_conda.yml │ ├── upload_to_s3.yml │ └── vsts_auth.yml ├── setup.py ├── test ├── __init__.py └── test_csprng.py ├── torchcsprng ├── __init__.py ├── __init__.pyi └── csrc │ ├── OffsetCalculator.cuh │ ├── THCIntegerDivider.cuh │ ├── aes.inc │ ├── block_cipher.h │ ├── cpu │ ├── kernels.cpp │ └── kernels.h │ ├── csprng.cpp │ ├── cuda │ ├── kernels.cu │ └── kernels.cuh │ ├── kernels_body.inc │ ├── kernels_commons.h │ ├── kernels_decls.inc │ └── macros.h └── version.txt /.circleci/unittest/linux/scripts/environment.yml: -------------------------------------------------------------------------------- 1 | channels: 2 | - pytorch 3 | - conda-forge 4 | - defaults 5 | dependencies: 6 | - numpy 7 | - pytest 8 | - pytest-cov 9 | - codecov 10 | - pip 11 | - ca-certificates 12 | - pycrypto 13 | - pip: 14 | - future 15 | - scipy 16 | -------------------------------------------------------------------------------- /.circleci/unittest/linux/scripts/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | unset PYTORCH_VERSION 4 | # For unittest, nightly PyTorch is used as the following section, 5 | # so no need to set PYTORCH_VERSION. 6 | # In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. 7 | 8 | set -e 9 | 10 | eval "$(./conda/bin/conda shell.bash hook)" 11 | conda activate ./env 12 | 13 | if [ "${CU_VERSION:-}" == cpu ] ; then 14 | cudatoolkit="cpuonly" 15 | else 16 | if [[ ${#CU_VERSION} -eq 4 ]]; then 17 | CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" 18 | elif [[ ${#CU_VERSION} -eq 5 ]]; then 19 | CUDA_VERSION="${CU_VERSION:2:2}.${CU_VERSION:4:1}" 20 | fi 21 | echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION" 22 | version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" 23 | cudatoolkit="cudatoolkit=${version}" 24 | fi 25 | printf "Installing PyTorch with %s\n" "${cudatoolkit}" 26 | conda install -y -c pytorch-nightly pytorch "${cudatoolkit}" 27 | 28 | printf "* Installing torchcsprng\n" 29 | python setup.py develop -------------------------------------------------------------------------------- /.circleci/unittest/linux/scripts/post_process.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | eval "$(./conda/bin/conda shell.bash hook)" 6 | conda activate ./env 7 | 8 | codecov -------------------------------------------------------------------------------- /.circleci/unittest/linux/scripts/run_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | eval "$(./conda/bin/conda shell.bash hook)" 6 | conda activate ./env 7 | 8 | python -m torch.utils.collect_env 9 | pytest --cov=torchcsprng --junitxml=test-results/junit.xml -v --durations 20 test -------------------------------------------------------------------------------- /.circleci/unittest/linux/scripts/setup_env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script is for setting up environment in which unit test is ran. 4 | # To speed up the CI time, the resulting environment is cached. 5 | # 6 | # Do not install PyTorch and torchcsprng here, otherwise they also get cached. 7 | 8 | set -e 9 | 10 | this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 11 | root_dir="$(git rev-parse --show-toplevel)" 12 | conda_dir="${root_dir}/conda" 13 | env_dir="${root_dir}/env" 14 | 15 | cd "${root_dir}" 16 | 17 | case "$(uname -s)" in 18 | Darwin*) os=MacOSX;; 19 | *) os=Linux 20 | esac 21 | 22 | # 1. Install conda at ./conda 23 | if [ ! -d "${conda_dir}" ]; then 24 | printf "* Installing conda\n" 25 | wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh" 26 | bash ./miniconda.sh -b -f -p "${conda_dir}" 27 | fi 28 | eval "$(${conda_dir}/bin/conda shell.bash hook)" 29 | 30 | # 2. Create test environment at ./env 31 | if [ ! -d "${env_dir}" ]; then 32 | printf "* Creating a test environment\n" 33 | conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" 34 | fi 35 | conda activate "${env_dir}" 36 | 37 | # 3. Install Conda dependencies 38 | printf "* Installing dependencies (except PyTorch)\n" 39 | conda env update --file "${this_dir}/environment.yml" --prune 40 | -------------------------------------------------------------------------------- /.circleci/unittest/windows/scripts/environment.yml: -------------------------------------------------------------------------------- 1 | channels: 2 | - pytorch 3 | - conda-forge 4 | - defaults 5 | dependencies: 6 | - numpy 7 | - pytest 8 | - pytest-cov 9 | - codecov 10 | - pip 11 | - ca-certificates 12 | - pycrypto 13 | - pip: 14 | - future 15 | - scipy 16 | -------------------------------------------------------------------------------- /.circleci/unittest/windows/scripts/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | unset PYTORCH_VERSION 4 | # For unittest, nightly PyTorch is used as the following section, 5 | # so no need to set PYTORCH_VERSION. 6 | # In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. 7 | 8 | set -e 9 | 10 | this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 11 | 12 | eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" 13 | conda activate ./env 14 | 15 | if [ "${CU_VERSION:-}" == cpu ] ; then 16 | cudatoolkit="cpuonly" 17 | else 18 | if [[ ${#CU_VERSION} -eq 4 ]]; then 19 | CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" 20 | elif [[ ${#CU_VERSION} -eq 5 ]]; then 21 | CUDA_VERSION="${CU_VERSION:2:2}.${CU_VERSION:4:1}" 22 | fi 23 | echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION" 24 | version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" 25 | cudatoolkit="cudatoolkit=${version}" 26 | fi 27 | printf "Installing PyTorch with %s\n" "${cudatoolkit}" 28 | conda install -y -c pytorch-nightly pytorch "${cudatoolkit}" 29 | 30 | printf "* Installing torchcsprng\n" 31 | "$this_dir/vc_env_helper.bat" python setup.py develop -------------------------------------------------------------------------------- /.circleci/unittest/windows/scripts/install_conda.bat: -------------------------------------------------------------------------------- 1 | start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda% -------------------------------------------------------------------------------- /.circleci/unittest/windows/scripts/post_process.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" 6 | conda activate ./env 7 | 8 | #codecov 9 | -------------------------------------------------------------------------------- /.circleci/unittest/windows/scripts/run_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" 6 | conda activate ./env 7 | 8 | python -m torch.utils.collect_env 9 | pytest --cov=torchcsprng --junitxml=test-results/junit.xml -v --durations 20 test -------------------------------------------------------------------------------- /.circleci/unittest/windows/scripts/setup_env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script is for setting up environment in which unit test is ran. 4 | # To speed up the CI time, the resulting environment is cached. 5 | # 6 | # Do not install PyTorch and torchcsprng here, otherwise they also get cached. 7 | 8 | set -e 9 | 10 | this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 11 | root_dir="$(git rev-parse --show-toplevel)" 12 | conda_dir="${root_dir}/conda" 13 | env_dir="${root_dir}/env" 14 | 15 | cd "${root_dir}" 16 | 17 | # 1. Install conda at ./conda 18 | if [ ! -d "${conda_dir}" ]; then 19 | printf "* Installing conda\n" 20 | export tmp_conda="$(echo $conda_dir | tr '/' '\\')" 21 | export miniconda_exe="$(echo $root_dir | tr '/' '\\')\\miniconda.exe" 22 | curl --output miniconda.exe https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -O 23 | "$this_dir/install_conda.bat" 24 | unset tmp_conda 25 | unset miniconda_exe 26 | fi 27 | 28 | eval "$(${conda_dir}/Scripts/conda.exe 'shell.bash' 'hook')" 29 | 30 | # 2. Create test environment at ./env 31 | if [ ! -d "${env_dir}" ]; then 32 | printf "* Creating a test environment\n" 33 | conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" 34 | fi 35 | conda activate "${env_dir}" 36 | 37 | # 3. Install Conda dependencies 38 | printf "* Installing dependencies (except PyTorch)\n" 39 | conda env update --file "${this_dir}/environment.yml" --prune -------------------------------------------------------------------------------- /.circleci/unittest/windows/scripts/vc_env_helper.bat: -------------------------------------------------------------------------------- 1 | @echo on 2 | 3 | set VC_VERSION_LOWER=16 4 | set VC_VERSION_UPPER=17 5 | 6 | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do ( 7 | if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( 8 | set "VS15INSTALLDIR=%%i" 9 | set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" 10 | goto vswhere 11 | ) 12 | ) 13 | 14 | :vswhere 15 | if "%VSDEVCMD_ARGS%" == "" ( 16 | call "%VS15VCVARSALL%" x64 || exit /b 1 17 | ) else ( 18 | call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1 19 | ) 20 | 21 | @echo on 22 | 23 | set DISTUTILS_USE_SDK=1 24 | 25 | set args=%1 26 | shift 27 | :start 28 | if [%1] == [] goto done 29 | set args=%args% %1 30 | shift 31 | goto start 32 | 33 | :done 34 | if "%args%" == "" ( 35 | echo Usage: vc_env_helper.bat [command] [args] 36 | echo e.g. vc_env_helper.bat cl /c test.cpp 37 | ) 38 | 39 | %args% || exit /b 1 40 | -------------------------------------------------------------------------------- /.github/csprng_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pytorch/csprng/3f17f1f96f6d3bd8db535e2f59aa1338c7337f85/.github/csprng_architecture.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .vscode 3 | # do not change or delete this comment - `python setup.py clean` deletes everything after this line 4 | dist/ 5 | build/ 6 | *.egg-info/ 7 | torchcsprng/version.py 8 | */__pycache__ 9 | .pytest_cache 10 | *.so 11 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to make participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies within all project spaces, and it also applies when 49 | an individual is representing the project or its community in public spaces. 50 | Examples of representing a project or community include using an official 51 | project e-mail address, posting via an official social media account, or acting 52 | as an appointed representative at an online or offline event. Representation of 53 | a project may be further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at . All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to csprng 2 | We want to make contributing to this project as easy and transparent as 3 | possible. 4 | 5 | ## Pull Requests 6 | We actively welcome your pull requests. 7 | 8 | 1. Fork the repo and create your branch from `master`. 9 | 2. If you've added code that should be tested, add tests. 10 | 3. If you've changed APIs, update the documentation. 11 | 4. Ensure the test suite passes. 12 | 5. Make sure your code lints. 13 | 6. If you haven't already, complete the Contributor License Agreement ("CLA"). 14 | 15 | ## Contributor License Agreement ("CLA") 16 | In order to accept your pull request, we need you to submit a CLA. You only need 17 | to do this once to work on any of Facebook's open source projects. 18 | 19 | Complete your CLA here: 20 | 21 | ## Issues 22 | We use GitHub issues to track public bugs. Please ensure your description is 23 | clear and has sufficient instructions to be able to reproduce the issue. 24 | 25 | Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe 26 | disclosure of security bugs. In those cases, please go through the process 27 | outlined on that page and do not file a public issue. 28 | 29 | ## License 30 | By contributing to csprng, you agree that your contributions will be licensed 31 | under the LICENSE file in the root directory of this source tree. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2020, pytorch 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /examples/csprng.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "csprng.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [] 9 | }, 10 | "kernelspec": { 11 | "name": "python3", 12 | "display_name": "Python 3" 13 | } 14 | }, 15 | "cells": [ 16 | { 17 | "cell_type": "markdown", 18 | "metadata": { 19 | "id": "Lpno_zUJT8ms" 20 | }, 21 | "source": [ 22 | "# Cryptographically secure pseudorandom number generators for PyTorch\n", 23 | "\n", 24 | "The torchcsprng API is available in `torchcsprng` module:\n" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "metadata": { 30 | "id": "db4YYky-PDI_" 31 | }, 32 | "source": [ 33 | "!pip install torchcsprng==0.2.0 torch==1.8.0 -f https://download.pytorch.org/whl/cu101/torch_stable.html" 34 | ], 35 | "execution_count": null, 36 | "outputs": [] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "metadata": { 41 | "id": "O1s_j8CPPHSn" 42 | }, 43 | "source": [ 44 | "import torch\n", 45 | "import torchcsprng as csprng" 46 | ], 47 | "execution_count": null, 48 | "outputs": [] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "metadata": { 53 | "id": "o1Kz25IoS9m-" 54 | }, 55 | "source": [ 56 | "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" 57 | ], 58 | "execution_count": null, 59 | "outputs": [] 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "metadata": { 64 | "id": "HLlLxkDIUWCG" 65 | }, 66 | "source": [ 67 | "Create crypto-secure PRNG from /dev/urandom:" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "metadata": { 73 | "id": "yyyYlq5kUQss" 74 | }, 75 | "source": [ 76 | "urandom_gen = csprng.create_random_device_generator('/dev/urandom')" 77 | ], 78 | "execution_count": null, 79 | "outputs": [] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": { 84 | "id": "xbUCnJfkUdUI" 85 | }, 86 | "source": [ 87 | "Create empty boolean tensor on the `device` and initialize it with random values from `urandom_gen`:\n" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "metadata": { 93 | "id": "zmj_VlIzUYIO" 94 | }, 95 | "source": [ 96 | "torch.empty(10, dtype=torch.bool, device=device).random_(generator=urandom_gen)" 97 | ], 98 | "execution_count": null, 99 | "outputs": [] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": { 104 | "id": "ycODsYhtUud9" 105 | }, 106 | "source": [ 107 | "Create empty int16 tensor on the `device` and initialize it with random values in range [0, 100) from `urandom_gen`:" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "metadata": { 113 | "id": "uel-jbW9UlZH" 114 | }, 115 | "source": [ 116 | "torch.empty(10, dtype=torch.int16, device=device).random_(100, generator=urandom_gen)" 117 | ], 118 | "execution_count": null, 119 | "outputs": [] 120 | }, 121 | { 122 | "cell_type": "markdown", 123 | "metadata": { 124 | "id": "1jXW1FEmVMW_" 125 | }, 126 | "source": [ 127 | "Create non-crypto-secure MT19937 PRNG:" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "metadata": { 133 | "id": "sL-cwFGfVOrp" 134 | }, 135 | "source": [ 136 | "mt19937_gen = csprng.create_mt19937_generator()\n", 137 | "torch.empty(10, dtype=torch.int64, device=device).random_(torch.iinfo(torch.int64).min, to=None, generator=mt19937_gen)" 138 | ], 139 | "execution_count": null, 140 | "outputs": [] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": { 145 | "id": "KW96wT4UVXBm" 146 | }, 147 | "source": [ 148 | "Create crypto-secure PRNG from default random device:" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "metadata": { 154 | "id": "tjwbuE6FVRgm" 155 | }, 156 | "source": [ 157 | "default_device_gen = csprng.create_random_device_generator()\n", 158 | "torch.randn(10, device=device, generator=default_device_gen)" 159 | ], 160 | "execution_count": null, 161 | "outputs": [] 162 | }, 163 | { 164 | "cell_type": "markdown", 165 | "metadata": { 166 | "id": "qYgdkZAYVfZT" 167 | }, 168 | "source": [ 169 | "Create non-crypto-secure MT19937 PRNG with seed:" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "metadata": { 175 | "id": "xjOsYOxxVbzg" 176 | }, 177 | "source": [ 178 | "mt19937_gen = csprng.create_mt19937_generator(42)\n", 179 | "first = torch.empty(10, device=device).geometric_(p=0.2, generator=mt19937_gen)" 180 | ], 181 | "execution_count": null, 182 | "outputs": [] 183 | }, 184 | { 185 | "cell_type": "markdown", 186 | "metadata": { 187 | "id": "cV77v7tHVlRd" 188 | }, 189 | "source": [ 190 | "Recreate MT19937 PRNG with the same seed:" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "metadata": { 196 | "id": "i0O2lC0hVjAg" 197 | }, 198 | "source": [ 199 | "mt19937_gen = csprng.create_mt19937_generator(42)\n", 200 | "second = torch.empty(10, device=device).geometric_(p=0.2, generator=mt19937_gen)" 201 | ], 202 | "execution_count": null, 203 | "outputs": [] 204 | }, 205 | { 206 | "cell_type": "markdown", 207 | "metadata": { 208 | "id": "OcgSK0mejcef" 209 | }, 210 | "source": [ 211 | "Check that `first` equals to `second`:" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "metadata": { 217 | "id": "vMx1BRO3jh7L" 218 | }, 219 | "source": [ 220 | "assert (first == second).all()" 221 | ], 222 | "execution_count": null, 223 | "outputs": [] 224 | } 225 | ] 226 | } 227 | -------------------------------------------------------------------------------- /examples/encrypt_decrypt.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "encrypt_decrypt.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [] 9 | }, 10 | "kernelspec": { 11 | "name": "python3", 12 | "display_name": "Python 3" 13 | } 14 | }, 15 | "cells": [ 16 | { 17 | "cell_type": "markdown", 18 | "metadata": { 19 | "id": "4JG-7IJgz_dK" 20 | }, 21 | "source": [ 22 | "# PyTorch/CSPRNG encrypt/decrypt examples" 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "metadata": { 28 | "id": "H8TZemj30JvQ" 29 | }, 30 | "source": [ 31 | "torchcsprng 0.2.0 exposes new API for tensor encryption/decryption. Tensor encryption/decryption API is dtype agnostic, so a tensor of any dtype can be encrypted and the result can be stored to a tensor of any dtype. An encryption key also can be a tensor of any dtype. Currently torchcsprng supports AES cipher with 128-bit key in two modes: ECB and CTR." 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "metadata": { 37 | "id": "jC1O-C25vI0W" 38 | }, 39 | "source": [ 40 | "!pip install torchcsprng==0.2.0 torch==1.8.0 -f https://download.pytorch.org/whl/cu101/torch_stable.html" 41 | ], 42 | "execution_count": null, 43 | "outputs": [] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "metadata": { 48 | "id": "su2RWWdOrWFU" 49 | }, 50 | "source": [ 51 | "import torch\n", 52 | "import torchcsprng as csprng" 53 | ], 54 | "execution_count": null, 55 | "outputs": [] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "metadata": { 60 | "id": "NHTOLPZ_3254" 61 | }, 62 | "source": [ 63 | "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" 64 | ], 65 | "execution_count": null, 66 | "outputs": [] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "metadata": { 71 | "id": "17L0sgmy0R6o" 72 | }, 73 | "source": [ 74 | "torchcsprng implementation of AES with 128 bit key requires to have a key tensor of 16 bytes but of any dtype" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "metadata": { 80 | "id": "rw7WYZ-50To9" 81 | }, 82 | "source": [ 83 | "key = torch.empty(16, dtype=torch.uint8, device=device).random_(0, 256)\n", 84 | "key" 85 | ], 86 | "execution_count": null, 87 | "outputs": [] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "metadata": { 92 | "id": "RRfvyfHM4MY1" 93 | }, 94 | "source": [ 95 | "Alternatively it can be a tensor of 8 elements of `torch.int16` or even 4 elements of `torch.float32`" 96 | ] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "metadata": { 101 | "id": "rCy01t1-0dtO" 102 | }, 103 | "source": [ 104 | "The size of input tensor is 42 * (32/8) = 168 bytes. AES 128 operates with 16-bytes blocks, so zero-padding of 8 bytes will be used to form 176 bytes(eleven 16-bytes blocks)" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "metadata": { 110 | "id": "LcuVmhyU0WTn" 111 | }, 112 | "source": [ 113 | "initial = torch.empty(42, dtype=torch.float32, device=device).normal_(-24.0, 42.0)\n", 114 | "initial" 115 | ], 116 | "execution_count": null, 117 | "outputs": [] 118 | }, 119 | { 120 | "cell_type": "markdown", 121 | "metadata": { 122 | "id": "rPNq2u4e3tlJ" 123 | }, 124 | "source": [ 125 | "torchcsprng requires output tensor to be of the same size in bytes as input tensor rounded up to 16 bytes(AES 128 block size), so if `torch.int64` is dtype of the destination tensor size must be 176 / (64/8) = 22" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "metadata": { 131 | "id": "RAJya9GT0gb4" 132 | }, 133 | "source": [ 134 | "encrypted = torch.empty(22, dtype=torch.int64, device=device)" 135 | ], 136 | "execution_count": null, 137 | "outputs": [] 138 | }, 139 | { 140 | "cell_type": "markdown", 141 | "metadata": { 142 | "id": "-DCI4QOh4oGX" 143 | }, 144 | "source": [ 145 | "Call `torchcsprng.encrypt` to encrypt `initial` tensor in [ECB](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Electronic_codebook_(ECB)) mode with 128-bit `key` tensor and store the result to `encrypted` tensor." 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "metadata": { 151 | "id": "TK4OjPRq4lsJ" 152 | }, 153 | "source": [ 154 | "csprng.encrypt(initial, encrypted, key, \"aes128\", \"ecb\")" 155 | ], 156 | "execution_count": null, 157 | "outputs": [] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "metadata": { 162 | "id": "yXUAwFHh5PSy" 163 | }, 164 | "source": [ 165 | "Create an output tensor" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "metadata": { 171 | "id": "4LtJ-kD446DJ" 172 | }, 173 | "source": [ 174 | "decrypted = torch.empty_like(initial)" 175 | ], 176 | "execution_count": null, 177 | "outputs": [] 178 | }, 179 | { 180 | "cell_type": "markdown", 181 | "metadata": { 182 | "id": "8VcF04mf6Rn5" 183 | }, 184 | "source": [ 185 | "Call `torchcsprng.decrypt` to decrypt `encrypted` tensor in ECB mode with 128-bit `key` tensor and store the result to `decrypted` tensor." 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "metadata": { 191 | "id": "kojXCFGK5v6l" 192 | }, 193 | "source": [ 194 | "csprng.decrypt(encrypted, decrypted, key, \"aes128\", \"ecb\")" 195 | ], 196 | "execution_count": null, 197 | "outputs": [] 198 | }, 199 | { 200 | "cell_type": "markdown", 201 | "metadata": { 202 | "id": "9dEBSPD6EFSu" 203 | }, 204 | "source": [ 205 | "Let's check that `decrypted` equals to `initial`:" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "metadata": { 211 | "id": "yOc1ftnM5yyj" 212 | }, 213 | "source": [ 214 | "assert (decrypted == initial).all()" 215 | ], 216 | "execution_count": null, 217 | "outputs": [] 218 | }, 219 | { 220 | "cell_type": "markdown", 221 | "metadata": { 222 | "id": "cQWyteLlE4mQ" 223 | }, 224 | "source": [ 225 | "Another example is to use [CTR](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_(CTR)) mode with 128-bit `key` tensor of 4 elements of dtype `dtype=torch.float32`:" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "metadata": { 231 | "id": "ZFInqYawD7ks" 232 | }, 233 | "source": [ 234 | "key = torch.empty(4, dtype=torch.float32, device=device).random_()\n", 235 | "key" 236 | ], 237 | "execution_count": null, 238 | "outputs": [] 239 | }, 240 | { 241 | "cell_type": "markdown", 242 | "metadata": { 243 | "id": "FRz94NaZGyRS" 244 | }, 245 | "source": [ 246 | "Let's encrypt 100 elements `torch.bool` tensor and store the result in 56 elements `torch.int16` tensor:" 247 | ] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "metadata": { 252 | "id": "8uiqxiehF_is" 253 | }, 254 | "source": [ 255 | "initial = torch.empty(100, dtype=torch.bool, device=device).random_()\n", 256 | "initial" 257 | ], 258 | "execution_count": null, 259 | "outputs": [] 260 | }, 261 | { 262 | "cell_type": "code", 263 | "metadata": { 264 | "id": "G0URlmQYGfcW" 265 | }, 266 | "source": [ 267 | "encrypted = torch.empty(56, dtype=torch.int16, device=device)\n", 268 | "csprng.encrypt(initial, encrypted, key, \"aes128\", \"ctr\")" 269 | ], 270 | "execution_count": null, 271 | "outputs": [] 272 | }, 273 | { 274 | "cell_type": "markdown", 275 | "metadata": { 276 | "id": "U9Zz2oXoHw9Q" 277 | }, 278 | "source": [ 279 | "Decrypt it back and check that `decrypted` equals to `initial`:" 280 | ] 281 | }, 282 | { 283 | "cell_type": "code", 284 | "metadata": { 285 | "id": "YXNcdUbXHoPC" 286 | }, 287 | "source": [ 288 | "decrypted = torch.empty_like(initial)\n", 289 | "csprng.decrypt(encrypted, decrypted, key, \"aes128\", \"ctr\")\n", 290 | "decrypted" 291 | ], 292 | "execution_count": null, 293 | "outputs": [] 294 | }, 295 | { 296 | "cell_type": "code", 297 | "metadata": { 298 | "id": "ie7epw1SKrdQ" 299 | }, 300 | "source": [ 301 | "assert (decrypted == initial).all()" 302 | ], 303 | "execution_count": null, 304 | "outputs": [] 305 | } 306 | ] 307 | } 308 | -------------------------------------------------------------------------------- /packaging/README.md: -------------------------------------------------------------------------------- 1 | # Building torchcsprng packages for release 2 | 3 | ## Anaconda packages 4 | 5 | ### Linux 6 | 7 | ```bash 8 | nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/conda-cuda bash 9 | pushd remote/conda 10 | 11 | ./build_csprng.sh 9.0 12 | ./build_csprng.sh 10.0 13 | ./build_csprng.sh cpu 14 | 15 | # copy packages over to /remote 16 | # exit docker 17 | # anaconda upload -u pytorch torchcsprng*.bz2 18 | ``` 19 | 20 | ### OSX 21 | 22 | ```bash 23 | # create a fresh anaconda environment / install and activate it 24 | conda install -y conda-build anaconda-client 25 | ./build_csprng.sh cpu 26 | 27 | # copy packages over to /remote 28 | # exit docker 29 | # anaconda upload -u pytorch torchcsprng*.bz2 30 | ``` 31 | 32 | ### Windows 33 | 34 | ```bash 35 | # Open `Git Bash` and change dir to `conda` 36 | ./build_csprng.sh 9.0 37 | ./build_csprng.sh 10.0 38 | ./build_csprng.sh cpu 39 | 40 | # copy packages to a output directory 41 | # anaconda upload -u pytorch torchcsprng*.bz2 42 | ``` 43 | 44 | ## Wheels 45 | 46 | ### Linux 47 | 48 | pushd wheel 49 | 50 | ```bash 51 | nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/manylinux-cuda90:latest bash 52 | cd remote 53 | ./linux_manywheel.sh cu90 54 | 55 | rm -rf /usr/local/cuda* 56 | ./linux_manywheel.sh cpu 57 | ``` 58 | 59 | ```bash 60 | nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/manylinux-cuda100:latest bash 61 | cd remote 62 | ./linux_manywheel.sh cu100 63 | ``` 64 | 65 | wheels are in the folders `cpu`, `cu90`, `cu100`. 66 | 67 | You can upload the `cu90` wheels to twine with `twine upload *.whl`. 68 | Which wheels we upload depends on which wheels PyTorch uploads as default, and right now, it's `cu90`. 69 | 70 | ### OSX 71 | 72 | ```bash 73 | pushd wheel 74 | ./osx_wheel.sh 75 | ``` 76 | 77 | ### Windows 78 | 79 | ```cmd 80 | set PYTORCH_REPO=pytorch 81 | 82 | pushd windows 83 | call build_csprng.bat 90 0.3.0 1 84 | call build_csprng.bat 100 0.3.0 1 85 | call build_csprng.bat cpu 0.3.0 1 86 | ``` 87 | 88 | wheels are in the current folder. 89 | 90 | You can upload them to twine with `twine upload *.whl` 91 | -------------------------------------------------------------------------------- /packaging/build_conda.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 5 | . "$script_dir/pkg_helpers.bash" 6 | 7 | export BUILD_TYPE=conda 8 | setup_env $(cat "version.txt" | sed "s/\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/g") 9 | export SOURCE_ROOT_DIR="$PWD" 10 | setup_conda_pytorch_constraint 11 | setup_conda_cudatoolkit_constraint 12 | setup_visual_studio_constraint 13 | setup_junit_results_folder 14 | conda build $CONDA_CHANNEL_FLAGS -c defaults -c conda-forge --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchcsprng 15 | -------------------------------------------------------------------------------- /packaging/build_wheel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 5 | . "$script_dir/pkg_helpers.bash" 6 | 7 | export BUILD_TYPE=wheel 8 | setup_env $(cat "version.txt" | sed "s/\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/g") 9 | setup_wheel_python 10 | pip_install numpy pyyaml future ninja 11 | setup_pip_pytorch_version 12 | python setup.py clean 13 | 14 | # Copy binaries to be included in the wheel distribution 15 | if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then 16 | python_exec="$(which python)" 17 | bin_path=$(dirname $python_exec) 18 | env_path=$(dirname $bin_path) 19 | if [[ "$(uname)" == Darwin ]]; then 20 | # Install delocate to relocate the required binaries 21 | pip_install delocate 22 | fi 23 | else 24 | # Install auditwheel to get some inspection utilities 25 | pip_install auditwheel 26 | 27 | # Point to custom libraries 28 | export LD_LIBRARY_PATH=$(pwd)/ext_libraries/lib:$LD_LIBRARY_PATH 29 | export TORCHCSPRNG_INCLUDE=$(pwd)/ext_libraries/include 30 | export TORCHCSPRNG_LIBRARY=$(pwd)/ext_libraries/lib 31 | fi 32 | 33 | if [[ "$OSTYPE" == "msys" ]]; then 34 | IS_WHEEL=1 "$script_dir/windows/internal/vc_env_helper.bat" python setup.py bdist_wheel 35 | else 36 | IS_WHEEL=1 python setup.py bdist_wheel 37 | fi 38 | 39 | 40 | if [[ "$(uname)" == Darwin ]]; then 41 | pushd dist/ 42 | python_exec="$(which python)" 43 | bin_path=$(dirname $python_exec) 44 | env_path=$(dirname $bin_path) 45 | for whl in *.whl; do 46 | DYLD_LIBRARY_PATH="$env_path/lib/:$DYLD_LIBRARY_PATH" delocate-wheel -v $whl 47 | done 48 | else 49 | if [[ "$OSTYPE" == "msys" ]]; then 50 | "$script_dir/windows/internal/vc_env_helper.bat" python $script_dir/wheel/relocate.py 51 | else 52 | LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python $script_dir/wheel/relocate.py 53 | fi 54 | fi 55 | -------------------------------------------------------------------------------- /packaging/conda/build_csprng.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | if [[ -x "/remote/anaconda_token" ]]; then 3 | . /remote/anaconda_token || true 4 | fi 5 | 6 | set -ex 7 | 8 | if [[ "$CIRCLECI" == 'true' ]]; then 9 | export PATH="/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:.:$PATH" 10 | fi 11 | 12 | # Function to retry functions that sometimes timeout or have flaky failures 13 | retry () { 14 | $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) 15 | } 16 | 17 | # Parse arguments and determmine version 18 | ########################################################### 19 | if [[ -n "$DESIRED_CUDA" && -n "$TORCHCSPRNG_BUILD_VERSION" && -n "$TORCHCSPRNG_BUILD_NUMBER" ]]; then 20 | desired_cuda="$DESIRED_CUDA" 21 | build_version="$PYTORCH_BUILD_VERSION" 22 | build_number="$PYTORCH_BUILD_NUMBER" 23 | else 24 | if [ "$#" -ne 3 ]; then 25 | echo "Illegal number of parameters. Pass cuda version, pytorch version, build number" 26 | echo "CUDA version should be Mm with no dot, e.g. '80'" 27 | echo "DESIRED_PYTHON should be M.m, e.g. '2.7'" 28 | exit 1 29 | fi 30 | 31 | desired_cuda="$1" 32 | build_version="$2" 33 | build_number="$3" 34 | fi 35 | if [[ "$desired_cuda" != cpu ]]; then 36 | desired_cuda="$(echo $desired_cuda | tr -d cuda. )" 37 | fi 38 | echo "Building cuda version $desired_cuda and torchcsprng version: $build_version build_number: $build_number" 39 | 40 | if [[ "$desired_cuda" == 'cpu' ]]; then 41 | cpu_only=1 42 | cuver="cpu" 43 | else 44 | # Switch desired_cuda to be M.m to be consistent with other scripts in 45 | # pytorch/builder 46 | export FORCE_CUDA=1 47 | cuda_nodot="$desired_cuda" 48 | 49 | if [[ ${#cuda_nodot} -eq 2 ]]; then 50 | desired_cuda="${desired_cuda:0:1}.${desired_cuda:1:1}" 51 | elif [[ ${#cuda_nodot} -eq 3 ]]; then 52 | desired_cuda="${desired_cuda:0:2}.${desired_cuda:2:1}" 53 | else 54 | echo "unknown cuda version $cuda_nodot" 55 | exit 1 56 | fi 57 | 58 | cuver="cu$cuda_nodot" 59 | fi 60 | 61 | export TORCHCSPRNG_BUILD_VERSION=$build_version 62 | export TORCHCSPRNG_BUILD_NUMBER=$build_number 63 | 64 | if [[ -z "$DESIRED_PYTHON" ]]; then 65 | DESIRED_PYTHON=('3.5' '3.6' '3.7') 66 | fi 67 | 68 | SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 69 | 70 | if [[ -z "$WIN_PACKAGE_WORK_DIR" ]]; then 71 | WIN_PACKAGE_WORK_DIR="$(echo $(pwd -W) | tr '/' '\\')\\tmp_conda_$(date +%H%M%S)" 72 | fi 73 | 74 | mkdir -p "$WIN_PACKAGE_WORK_DIR" || true 75 | csprng_rootdir="$(realpath ${WIN_PACKAGE_WORK_DIR})/torchcsprng-src" 76 | git config --system core.longpaths true 77 | 78 | if [[ ! -d "$csprng_rootdir" ]]; then 79 | rm -rf "$csprng_rootdir" 80 | git clone "https://github.com/pytorch/csprng" "$csprng_rootdir" 81 | pushd "$csprng_rootdir" 82 | git checkout $PYTORCH_BRANCH 83 | popd 84 | fi 85 | 86 | cd "$SOURCE_DIR" 87 | 88 | export tmp_conda="${WIN_PACKAGE_WORK_DIR}\\conda" 89 | export miniconda_exe="${WIN_PACKAGE_WORK_DIR}\\miniconda.exe" 90 | rm -rf "$tmp_conda" 91 | rm -f "$miniconda_exe" 92 | curl -sSk https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "$miniconda_exe" 93 | "$SOURCE_DIR/install_conda.bat" && rm "$miniconda_exe" 94 | pushd $tmp_conda 95 | export PATH="$(pwd):$(pwd)/Library/usr/bin:$(pwd)/Library/bin:$(pwd)/Scripts:$(pwd)/bin:$PATH" 96 | popd 97 | retry conda install -yq conda-build 98 | 99 | ANACONDA_USER=pytorch-nightly 100 | conda config --set anaconda_upload no 101 | 102 | 103 | export TORCHCSPRNG_PACKAGE_SUFFIX="" 104 | if [[ "$desired_cuda" == 'cpu' ]]; then 105 | export CONDA_CUDATOOLKIT_CONSTRAINT="" 106 | export CONDA_CPUONLY_FEATURE="- cpuonly # [not osx]" 107 | export CUDA_VERSION="None" 108 | else 109 | export CONDA_CPUONLY_FEATURE="" 110 | . ./switch_cuda_version.sh $desired_cuda 111 | if [[ "$desired_cuda" == "10.2" ]]; then 112 | export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.2,<10.3 # [not osx]" 113 | elif [[ "$desired_cuda" == "10.1" ]]; then 114 | export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.1,<10.2 # [not osx]" 115 | elif [[ "$desired_cuda" == "10.0" ]]; then 116 | export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.0,<10.1 # [not osx]" 117 | elif [[ "$desired_cuda" == "9.2" ]]; then 118 | export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=9.2,<9.3 # [not osx]" 119 | elif [[ "$desired_cuda" == "9.0" ]]; then 120 | export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=9.0,<9.1 # [not osx]" 121 | elif [[ "$desired_cuda" == "8.0" ]]; then 122 | export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=8.0,<8.1 # [not osx]" 123 | else 124 | echo "unhandled desired_cuda: $desired_cuda" 125 | exit 1 126 | fi 127 | fi 128 | 129 | if [[ -z "$PYTORCH_VERSION" ]]; then 130 | export CONDA_CHANNEL_FLAGS="-c pytorch-nightly -c pytorch" 131 | export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | \ 132 | python -c "import os, sys, json, re; cuver = '$cuver'; \ 133 | cuver = cuver.replace('cu', 'cuda') if cuver != 'cpu' else cuver; \ 134 | print(re.sub(r'\\+.*$', '', \ 135 | [x['version'] for x in json.load(sys.stdin)['pytorch'] \ 136 | if (x['platform'] == 'darwin' or cuver in x['fn']) \ 137 | and 'py' + os.environ['DESIRED_PYTHON'] in x['fn']][-1]))")" 138 | if [[ -z "$PYTORCH_VERSION" ]]; then 139 | echo "PyTorch version auto detection failed" 140 | echo "No package found for desired_cuda=$desired_cuda and DESIRED_PYTHON=$DESIRED_PYTHON" 141 | exit 1 142 | fi 143 | else 144 | export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-nightly" 145 | fi 146 | if [[ "$desired_cuda" == 'cpu' ]]; then 147 | export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION" 148 | export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION" 149 | else 150 | export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}" 151 | export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}" 152 | fi 153 | 154 | # Loop through all Python versions to build a package for each 155 | for py_ver in "${DESIRED_PYTHON[@]}"; do 156 | build_string="py${py_ver}_${build_string_suffix}" 157 | folder_tag="${build_string}_$(date +'%Y%m%d')" 158 | 159 | # Create the conda package into this temporary folder. This is so we can find 160 | # the package afterwards, as there's no easy way to extract the final filename 161 | # from conda-build 162 | output_folder="out_$folder_tag" 163 | rm -rf "$output_folder" 164 | mkdir "$output_folder" 165 | 166 | if [[ "$py_ver" == 3.5 ]]; then 167 | export CONDA_TYPING_CONSTRAINT="- typing" 168 | else 169 | export CONDA_TYPING_CONSTRAINT="" 170 | fi 171 | 172 | export VSTOOLCHAIN_PACKAGE=vs2017 173 | 174 | # We need to build the compiler activation scripts first on Windows 175 | time VSDEVCMD_ARGS=${VSDEVCMD_ARGS[@]} \ 176 | conda build -c "$ANACONDA_USER" \ 177 | --no-anaconda-upload \ 178 | --output-folder "$output_folder" \ 179 | ../$VSTOOLCHAIN_PACKAGE 180 | 181 | cp ../$VSTOOLCHAIN_PACKAGE/conda_build_config.yaml ../torchcsprng/conda_build_config.yaml 182 | 183 | conda config --set anaconda_upload no 184 | echo "Calling conda-build at $(date)" 185 | if [[ "$desired_cuda" == "9.2" ]]; then 186 | time CMAKE_ARGS=${CMAKE_ARGS[@]} \ 187 | BUILD_VERSION="$TORCHCSPRNG_BUILD_VERSION" \ 188 | CU_VERSION="$cuver" \ 189 | SOURCE_ROOT_DIR="$csprng_rootdir" \ 190 | conda build -c "$ANACONDA_USER" \ 191 | -c defaults \ 192 | -c conda-forge \ 193 | -c "numba/label/dev" \ 194 | --no-anaconda-upload \ 195 | --python "$py_ver" \ 196 | --output-folder "$output_folder" \ 197 | --no-verify \ 198 | --no-test \ 199 | ../torchcsprng 200 | else 201 | time CMAKE_ARGS=${CMAKE_ARGS[@]} \ 202 | BUILD_VERSION="$TORCHCSPRNG_BUILD_VERSION" \ 203 | CU_VERSION="$cuver" \ 204 | SOURCE_ROOT_DIR="$csprng_rootdir" \ 205 | conda build -c "$ANACONDA_USER" \ 206 | -c defaults \ 207 | -c conda-forge \ 208 | --no-anaconda-upload \ 209 | --python "$py_ver" \ 210 | --output-folder "$output_folder" \ 211 | --no-verify \ 212 | --no-test \ 213 | ../torchcsprng 214 | fi 215 | echo "Finished conda-build at $(date)" 216 | 217 | # Extract the package for testing 218 | ls -lah "$output_folder" 219 | built_package="$(find $output_folder/ -name '*torchcsprng*.tar.bz2')" 220 | 221 | # Copy the built package to the host machine for persistence before testing 222 | if [[ -n "$PYTORCH_FINAL_PACKAGE_DIR" ]]; then 223 | mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR" || true 224 | cp "$built_package" "$PYTORCH_FINAL_PACKAGE_DIR/" 225 | fi 226 | done 227 | 228 | 229 | set +e 230 | -------------------------------------------------------------------------------- /packaging/conda/install_conda.bat: -------------------------------------------------------------------------------- 1 | start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda% 2 | -------------------------------------------------------------------------------- /packaging/conda/switch_cuda_version.sh: -------------------------------------------------------------------------------- 1 | if [[ "$OSTYPE" == "msys" ]]; then 2 | CUDA_DIR="/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v$1" 3 | else 4 | CUDA_DIR="/usr/local/cuda-$1" 5 | fi 6 | 7 | if ! ls "$CUDA_DIR" 8 | then 9 | echo "folder $CUDA_DIR not found to switch" 10 | fi 11 | 12 | echo "Switching symlink to $CUDA_DIR" 13 | mkdir -p /usr/local 14 | rm -fr /usr/local/cuda 15 | ln -s "$CUDA_DIR" /usr/local/cuda 16 | 17 | if [[ "$OSTYPE" == "msys" ]]; then 18 | export CUDA_VERSION=`ls /usr/local/cuda/bin/cudart64*.dll | head -1 | tr '._' ' ' | cut -d ' ' -f2` 19 | export CUDNN_VERSION=`ls /usr/local/cuda/bin/cudnn64*.dll | head -1 | tr '._' ' ' | cut -d ' ' -f2` 20 | else 21 | export CUDA_VERSION=$(ls /usr/local/cuda/lib64/libcudart.so.*|sort|tac | head -1 | rev | cut -d"." -f -3 | rev) 22 | export CUDNN_VERSION=$(ls /usr/local/cuda/lib64/libcudnn.so.*|sort|tac | head -1 | rev | cut -d"." -f -3 | rev) 23 | fi 24 | 25 | ls -alh /usr/local/cuda 26 | 27 | echo "CUDA_VERSION=$CUDA_VERSION" 28 | echo "CUDNN_VERSION=$CUDNN_VERSION" 29 | -------------------------------------------------------------------------------- /packaging/torchcsprng/bld.bat: -------------------------------------------------------------------------------- 1 | @echo on 2 | 3 | set TORCHCSPRNG_BUILD_VERSION=%PKG_VERSION% 4 | set TORCHCSPRNG_BUILD_NUMBER=%PKG_BUILDNUM% 5 | 6 | set build_with_cuda= 7 | 8 | if "%CUDA_VERSION%" == "None" goto cuda_flags_end 9 | if "%CUDA_VERSION%" == "cpu" goto cuda_flags_end 10 | if "%CUDA_VERSION%" == "" goto cuda_flags_end 11 | 12 | set build_with_cuda=1 13 | set desired_cuda=%CUDA_VERSION:~0,-1%.%CUDA_VERSION:~-1,1% 14 | 15 | set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v%desired_cuda% 16 | set CUDA_BIN_PATH=%CUDA_PATH%\bin 17 | set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr 18 | if "%desired_cuda%" == "9.0" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50 19 | if "%desired_cuda%" == "9.2" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50 20 | if "%desired_cuda%" == "10.0" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50 21 | if "%desired_cuda%" == "10.1" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50 22 | if "%desired_cuda%" == "10.2" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50 23 | 24 | :cuda_flags_end 25 | 26 | python setup.py install --single-version-externally-managed --record=record.txt 27 | if errorlevel 1 exit /b 1 28 | -------------------------------------------------------------------------------- /packaging/torchcsprng/conda_build_config.yaml: -------------------------------------------------------------------------------- 1 | channel_sources: 2 | - pytorch-nightly,pytorch,defaults 3 | blas_impl: 4 | - mkl # [x86_64] 5 | c_compiler: 6 | - vs2017 # [win] 7 | cxx_compiler: 8 | - vs2017 # [win] 9 | python: 10 | - 3.5 11 | - 3.6 12 | # This differs from target_platform in that it determines what subdir the compiler 13 | # will target, not what subdir the compiler package will be itself. 14 | # For example, we need a win-64 vs2008_win-32 package, so that we compile win-32 15 | # code on win-64 miniconda. 16 | cross_compiler_target_platform: 17 | - win-64 # [win] 18 | target_platform: 19 | - win-64 # [win] 20 | vc: 21 | - 14 22 | zip_keys: 23 | - # [win] 24 | - vc # [win] 25 | - c_compiler # [win] 26 | - cxx_compiler # [win] 27 | -------------------------------------------------------------------------------- /packaging/torchcsprng/meta.yaml: -------------------------------------------------------------------------------- 1 | package: 2 | name: torchcsprng 3 | version: "{{ environ.get('BUILD_VERSION') }}" 4 | 5 | source: 6 | path: "{{ environ.get('SOURCE_ROOT_DIR') }}" 7 | 8 | requirements: 9 | build: 10 | - {{ compiler('c') }} # [win] 11 | - {{ compiler('cxx') }} 12 | 13 | host: 14 | - python 15 | - setuptools 16 | {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }} 17 | {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }} 18 | {{ environ.get('CONDA_CPUONLY_FEATURE') }} 19 | 20 | run: 21 | - python 22 | - pillow >=4.1.1 23 | - numpy >=1.11 24 | {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }} 25 | {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }} 26 | 27 | build: 28 | string: py{{py}}_{{ environ['CU_VERSION'] }} 29 | script: python setup.py install --single-version-externally-managed --record=record.txt # [not win] 30 | script_env: 31 | - CU_VERSION 32 | - CUDA_HOME 33 | - FORCE_CUDA 34 | - NVCC_FLAGS 35 | - BUILD_VERSION 36 | features: 37 | {{ environ.get('CONDA_CPUONLY_FEATURE') }} 38 | 39 | #test: 40 | # imports: 41 | # - torch 42 | # - torchcsprng 43 | # source_files: 44 | # - test 45 | # requires: 46 | # - pytest 47 | # - scipy 48 | # - pycrypto 49 | # commands: 50 | # pytest . --verbose 51 | 52 | about: 53 | home: https://github.com/pytorch/csprng 54 | license: BSD 55 | license_file: LICENSE 56 | summary: 'Cryptographically secure pseudorandom number generators for PyTorch' 57 | -------------------------------------------------------------------------------- /packaging/vs2017/activate.bat: -------------------------------------------------------------------------------- 1 | :: Set env vars that tell distutils to use the compiler that we put on path 2 | SET DISTUTILS_USE_SDK=1 3 | SET MSSdk=1 4 | 5 | SET "VS_VERSION=15.0" 6 | SET "VS_MAJOR=15" 7 | SET "VS_YEAR=2017" 8 | 9 | set "MSYS2_ARG_CONV_EXCL=/AI;/AL;/OUT;/out" 10 | set "MSYS2_ENV_CONV_EXCL=CL" 11 | 12 | :: For Python 3.5+, ensure that we link with the dynamic runtime. See 13 | :: http://stevedower.id.au/blog/building-for-python-3-5-part-two/ for more info 14 | set "PY_VCRUNTIME_REDIST=%PREFIX%\\bin\\vcruntime140.dll" 15 | 16 | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do ( 17 | if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( 18 | set "VSINSTALLDIR=%%i\" 19 | goto :vswhere 20 | ) 21 | ) 22 | 23 | :vswhere 24 | 25 | :: Shorten PATH to avoid the `input line too long` error. 26 | SET MyPath=%PATH% 27 | 28 | setlocal EnableDelayedExpansion 29 | 30 | SET TempPath="%MyPath:;=";"%" 31 | SET var= 32 | FOR %%a IN (%TempPath%) DO ( 33 | IF EXIST %%~sa ( 34 | SET "var=!var!;%%~sa" 35 | ) 36 | ) 37 | 38 | set "TempPath=!var:~1!" 39 | endlocal & set "PATH=%TempPath%" 40 | 41 | :: Shorten current directory too 42 | FOR %%A IN (.) DO CD "%%~sA" 43 | 44 | :: other things added by install_activate.bat at package build time 45 | -------------------------------------------------------------------------------- /packaging/vs2017/conda_build_config.yaml: -------------------------------------------------------------------------------- 1 | blas_impl: 2 | - mkl # [x86_64] 3 | c_compiler: 4 | - vs2017 # [win] 5 | cxx_compiler: 6 | - vs2017 # [win] 7 | python: 8 | - 3.5 9 | - 3.6 10 | # This differs from target_platform in that it determines what subdir the compiler 11 | # will target, not what subdir the compiler package will be itself. 12 | # For example, we need a win-64 vs2008_win-32 package, so that we compile win-32 13 | # code on win-64 miniconda. 14 | cross_compiler_target_platform: 15 | - win-64 # [win] 16 | target_platform: 17 | - win-64 # [win] 18 | vc: 19 | - 14 20 | zip_keys: 21 | - # [win] 22 | - vc # [win] 23 | - c_compiler # [win] 24 | - cxx_compiler # [win] 25 | -------------------------------------------------------------------------------- /packaging/vs2017/install_activate.bat: -------------------------------------------------------------------------------- 1 | set YEAR=2017 2 | set VER=15 3 | 4 | mkdir "%PREFIX%\etc\conda\activate.d" 5 | COPY "%RECIPE_DIR%\activate.bat" "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 6 | 7 | IF "%cross_compiler_target_platform%" == "win-64" ( 8 | set "target_platform=amd64" 9 | echo SET "CMAKE_GENERATOR=Visual Studio %VER% %YEAR% Win64" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 10 | echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 11 | IF "%VSDEVCMD_ARGS%" == "" ( 12 | echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x64 >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 13 | echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 14 | echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 15 | echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 16 | ) ELSE ( 17 | echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x64 %VSDEVCMD_ARGS% >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 18 | echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 19 | echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 20 | echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 %VSDEVCMD_ARGS% >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 21 | ) 22 | echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 23 | ) else ( 24 | set "target_platform=x86" 25 | echo SET "CMAKE_GENERATOR=Visual Studio %VER% %YEAR%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 26 | echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 27 | echo CALL "VC\Auxiliary\Build\vcvars32.bat" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 28 | echo popd 29 | ) 30 | 31 | -------------------------------------------------------------------------------- /packaging/vs2017/install_runtime.bat: -------------------------------------------------------------------------------- 1 | set VC_PATH=x86 2 | if "%ARCH%"=="64" ( 3 | set VC_PATH=x64 4 | ) 5 | 6 | set MSC_VER=2017 7 | 8 | rem :: This should always be present for VC installed with VS. Not sure about VC installed with Visual C++ Build Tools 2015 9 | rem FOR /F "usebackq tokens=3*" %%A IN (`REG QUERY "HKEY_LOCAL_MACHINE\Software\Microsoft\DevDiv\VC\Servicing\14.0\IDE.x64" /v UpdateVersion`) DO ( 10 | rem set SP=%%A 11 | rem ) 12 | 13 | rem if not "%SP%" == "%PKG_VERSION%" ( 14 | rem echo "Version detected from registry: %SP%" 15 | rem echo "does not match version of package being built (%PKG_VERSION%)" 16 | rem echo "Do you have current updates for VS 2015 installed?" 17 | rem exit 1 18 | rem ) 19 | 20 | 21 | REM ========== REQUIRES Win 10 SDK be installed, or files otherwise copied to location below! 22 | robocopy "C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\%VC_PATH%" "%LIBRARY_BIN%" *.dll /E 23 | robocopy "C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\%VC_PATH%" "%PREFIX%" *.dll /E 24 | if %ERRORLEVEL% GEQ 8 exit 1 25 | 26 | REM ========== This one comes from visual studio 2017 27 | set "VC_VER=141" 28 | 29 | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do ( 30 | if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( 31 | set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" 32 | goto :eof 33 | ) 34 | ) 35 | 36 | @setlocal 37 | call "%VS15VARSALL%" x64 38 | 39 | set "REDIST_ROOT=%VCToolsRedistDir%%VC_PATH%" 40 | 41 | robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.CRT" "%LIBRARY_BIN%" *.dll /E 42 | if %ERRORLEVEL% LSS 8 exit 0 43 | robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.CRT" "%PREFIX%" *.dll /E 44 | if %ERRORLEVEL% LSS 8 exit 0 45 | robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.OpenMP" "%LIBRARY_BIN%" *.dll /E 46 | if %ERRORLEVEL% LSS 8 exit 0 47 | robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.OpenMP" "%PREFIX%" *.dll /E 48 | if %ERRORLEVEL% LSS 8 exit 0 49 | @endlocal 50 | -------------------------------------------------------------------------------- /packaging/vs2017/meta.yaml: -------------------------------------------------------------------------------- 1 | {% set vcver="14.1" %} 2 | {% set vcfeature="14" %} 3 | {% set vsyear="2017" %} 4 | {% set fullver="15.4.27004.2010" %} 5 | 6 | package: 7 | name: vs{{ vsyear }} 8 | version: {{ fullver }} 9 | 10 | build: 11 | skip: True [not win] 12 | script_env: 13 | - VSDEVCMD_ARGS # [win] 14 | 15 | outputs: 16 | - name: vs{{ vsyear }}_{{ cross_compiler_target_platform }} 17 | script: install_activate.bat 18 | track_features: 19 | # VS 2017 is binary-compatible with VS 2015/vc14. Tools are "v141". 20 | strong: 21 | - vc{{ vcfeature }} 22 | about: 23 | summary: Activation and version verification of MSVC {{ vcver }} (VS {{ vsyear }}) compiler 24 | license: BSD 3-clause 25 | -------------------------------------------------------------------------------- /packaging/vs2019/activate.bat: -------------------------------------------------------------------------------- 1 | :: Set env vars that tell distutils to use the compiler that we put on path 2 | SET DISTUTILS_USE_SDK=1 3 | SET MSSdk=1 4 | 5 | SET "VS_VERSION=16.0" 6 | SET "VS_MAJOR=16" 7 | SET "VS_YEAR=2019" 8 | 9 | set "MSYS2_ARG_CONV_EXCL=/AI;/AL;/OUT;/out" 10 | set "MSYS2_ENV_CONV_EXCL=CL" 11 | 12 | :: For Python 3.5+, ensure that we link with the dynamic runtime. See 13 | :: http://stevedower.id.au/blog/building-for-python-3-5-part-two/ for more info 14 | set "PY_VCRUNTIME_REDIST=%PREFIX%\\bin\\vcruntime140.dll" 15 | 16 | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [16^,17^) -property installationPath`) do ( 17 | if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( 18 | set "VSINSTALLDIR=%%i\" 19 | goto :vswhere 20 | ) 21 | ) 22 | 23 | :vswhere 24 | 25 | :: Shorten PATH to avoid the `input line too long` error. 26 | SET MyPath=%PATH% 27 | 28 | setlocal EnableDelayedExpansion 29 | 30 | SET TempPath="%MyPath:;=";"%" 31 | SET var= 32 | FOR %%a IN (%TempPath%) DO ( 33 | IF EXIST %%~sa ( 34 | SET "var=!var!;%%~sa" 35 | ) 36 | ) 37 | 38 | set "TempPath=!var:~1!" 39 | endlocal & set "PATH=%TempPath%" 40 | 41 | :: Shorten current directory too 42 | FOR %%A IN (.) DO CD "%%~sA" 43 | 44 | :: other things added by install_activate.bat at package build time 45 | -------------------------------------------------------------------------------- /packaging/vs2019/conda_build_config.yaml: -------------------------------------------------------------------------------- 1 | blas_impl: 2 | - mkl # [x86_64] 3 | c_compiler: 4 | - vs2019 # [win] 5 | cxx_compiler: 6 | - vs2019 # [win] 7 | python: 8 | - 3.5 9 | - 3.6 10 | # This differs from target_platform in that it determines what subdir the compiler 11 | # will target, not what subdir the compiler package will be itself. 12 | # For example, we need a win-64 vs2008_win-32 package, so that we compile win-32 13 | # code on win-64 miniconda. 14 | cross_compiler_target_platform: 15 | - win-64 # [win] 16 | target_platform: 17 | - win-64 # [win] 18 | vc: 19 | - 14 20 | zip_keys: 21 | - # [win] 22 | - vc # [win] 23 | - c_compiler # [win] 24 | - cxx_compiler # [win] 25 | -------------------------------------------------------------------------------- /packaging/vs2019/install_activate.bat: -------------------------------------------------------------------------------- 1 | set YEAR=2019 2 | set VER=16 3 | 4 | mkdir "%PREFIX%\etc\conda\activate.d" 5 | COPY "%RECIPE_DIR%\activate.bat" "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 6 | 7 | IF "%cross_compiler_target_platform%" == "win-64" ( 8 | set "target_platform=amd64" 9 | echo SET "CMAKE_GENERATOR=Visual Studio %VER% %YEAR% Win64" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 10 | echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 11 | IF "%VSDEVCMD_ARGS%" == "" ( 12 | echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x64 >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 13 | echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 14 | echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 15 | echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 16 | ) ELSE ( 17 | echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x64 %VSDEVCMD_ARGS% >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 18 | echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 19 | echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 20 | echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 %VSDEVCMD_ARGS% >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 21 | ) 22 | echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 23 | ) else ( 24 | set "target_platform=x86" 25 | echo SET "CMAKE_GENERATOR=Visual Studio %VER% %YEAR%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 26 | echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 27 | echo CALL "VC\Auxiliary\Build\vcvars32.bat" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" 28 | echo popd 29 | ) 30 | 31 | -------------------------------------------------------------------------------- /packaging/vs2019/install_runtime.bat: -------------------------------------------------------------------------------- 1 | set VC_PATH=x86 2 | if "%ARCH%"=="64" ( 3 | set VC_PATH=x64 4 | ) 5 | 6 | set MSC_VER=2019 7 | 8 | rem :: This should always be present for VC installed with VS. Not sure about VC installed with Visual C++ Build Tools 2015 9 | rem FOR /F "usebackq tokens=3*" %%A IN (`REG QUERY "HKEY_LOCAL_MACHINE\Software\Microsoft\DevDiv\VC\Servicing\14.0\IDE.x64" /v UpdateVersion`) DO ( 10 | rem set SP=%%A 11 | rem ) 12 | 13 | rem if not "%SP%" == "%PKG_VERSION%" ( 14 | rem echo "Version detected from registry: %SP%" 15 | rem echo "does not match version of package being built (%PKG_VERSION%)" 16 | rem echo "Do you have current updates for VS 2015 installed?" 17 | rem exit 1 18 | rem ) 19 | 20 | 21 | REM ========== REQUIRES Win 10 SDK be installed, or files otherwise copied to location below! 22 | robocopy "C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\%VC_PATH%" "%LIBRARY_BIN%" *.dll /E 23 | robocopy "C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\%VC_PATH%" "%PREFIX%" *.dll /E 24 | if %ERRORLEVEL% GEQ 8 exit 1 25 | 26 | REM ========== This one comes from visual studio 2019 27 | set "VC_VER=142" 28 | 29 | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [16^,17^) -property installationPath`) do ( 30 | if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( 31 | set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" 32 | goto :eof 33 | ) 34 | ) 35 | 36 | @setlocal 37 | call "%VS15VARSALL%" x64 38 | 39 | set "REDIST_ROOT=%VCToolsRedistDir%%VC_PATH%" 40 | 41 | robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.CRT" "%LIBRARY_BIN%" *.dll /E 42 | if %ERRORLEVEL% LSS 8 exit 0 43 | robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.CRT" "%PREFIX%" *.dll /E 44 | if %ERRORLEVEL% LSS 8 exit 0 45 | robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.OpenMP" "%LIBRARY_BIN%" *.dll /E 46 | if %ERRORLEVEL% LSS 8 exit 0 47 | robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.OpenMP" "%PREFIX%" *.dll /E 48 | if %ERRORLEVEL% LSS 8 exit 0 49 | @endlocal 50 | -------------------------------------------------------------------------------- /packaging/vs2019/meta.yaml: -------------------------------------------------------------------------------- 1 | {% set vcver="14.2" %} 2 | {% set vcfeature="14" %} 3 | {% set vsyear="2019" %} 4 | {% set fullver="15.4.27004.2010" %} 5 | 6 | package: 7 | name: vs{{ vsyear }} 8 | version: {{ fullver }} 9 | 10 | build: 11 | skip: True [not win] 12 | script_env: 13 | - VSDEVCMD_ARGS # [win] 14 | 15 | outputs: 16 | - name: vs{{ vsyear }}_{{ cross_compiler_target_platform }} 17 | script: install_activate.bat 18 | track_features: 19 | # VS 2019 is binary-compatible with VS 2017/vc 14.1 and 2015/vc14. Tools are "v142". 20 | strong: 21 | - vc{{ vcfeature }} 22 | about: 23 | summary: Activation and version verification of MSVC {{ vcver }} (VS {{ vsyear }}) compiler 24 | license: BSD 3-clause 25 | -------------------------------------------------------------------------------- /packaging/wheel/linux_manywheel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | if [ "$#" -ne 1 ]; then 5 | echo "Illegal number of parameters. Pass cuda version" 6 | echo "CUDA version should be cu92, cu100 or cpu" 7 | exit 1 8 | fi 9 | export CUVER="$1" # cu[0-9]* cpu 10 | 11 | if [[ "$CUVER" == "cu102" ]]; then 12 | cu_suffix="" 13 | else 14 | cu_suffix="+$CUVER" 15 | fi 16 | 17 | export TORCHCSPRNG_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")${cu_suffix}" 18 | export TORCHCSPRNG_BUILD_NUMBER="1" 19 | export TORCHCSPRNG_LOCAL_VERSION_LABEL="$CUVER" 20 | export OUT_DIR="/remote/$CUVER" 21 | 22 | pushd /opt/python 23 | DESIRED_PYTHON=(*/) 24 | popd 25 | for desired_py in "${DESIRED_PYTHON[@]}"; do 26 | python_installations+=("/opt/python/$desired_py") 27 | done 28 | 29 | OLD_PATH=$PATH 30 | cd /tmp 31 | rm -rf csprng 32 | git clone https://github.com/pytorch/csprng 33 | 34 | cd /tmp/csprng 35 | 36 | for PYDIR in "${python_installations[@]}"; do 37 | export PATH=$PYDIR/bin:$OLD_PATH 38 | pip install --upgrade pip 39 | pip install numpy pyyaml future 40 | 41 | pip uninstall -y torch || true 42 | pip uninstall -y torch_nightly || true 43 | 44 | export TORCHCSPRNG_PYTORCH_DEPENDENCY_NAME=torch_nightly 45 | pip install torch_nightly -f https://download.pytorch.org/whl/nightly/$CUVER/torch_nightly.html 46 | # CPU/CUDA variants of PyTorch have ABI compatible PyTorch for 47 | # the CPU only bits. Therefore, we 48 | # strip off the local package qualifier, but ONLY if we're 49 | # doing a CPU build. 50 | if [[ "$CUVER" == "cpu" ]]; then 51 | export TORCHCSPRNG_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version: \+//' | sed 's/+.\+//')" 52 | else 53 | export TORCHCSPRNG_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version: \+//')" 54 | fi 55 | echo "Building against ${TORCHCSPRNG_PYTORCH_DEPENDENCY_VERSION}" 56 | 57 | pip install ninja 58 | python setup.py clean 59 | python setup.py bdist_wheel 60 | mkdir -p $OUT_DIR 61 | cp dist/*.whl $OUT_DIR/ 62 | done 63 | -------------------------------------------------------------------------------- /packaging/wheel/osx_wheel.sh: -------------------------------------------------------------------------------- 1 | if [[ ":$PATH:" == *"conda"* ]]; then 2 | echo "existing anaconda install in PATH, remove it and run script" 3 | exit 1 4 | fi 5 | # download and activate anaconda 6 | rm -rf ~/minconda_wheel_env_tmp 7 | wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh && \ 8 | chmod +x Miniconda3-latest-MacOSX-x86_64.sh && \ 9 | ./Miniconda3-latest-MacOSX-x86_64.sh -b -p ~/minconda_wheel_env_tmp && \ 10 | rm Miniconda3-latest-MacOSX-x86_64.sh 11 | 12 | . ~/minconda_wheel_env_tmp/bin/activate 13 | 14 | 15 | export TORCHCSPRNG_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")" 16 | export TORCHCSPRNG_BUILD_NUMBER="1" 17 | export OUT_DIR=~/torchcsprng_wheels 18 | 19 | export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ 20 | 21 | pushd /tmp 22 | rm -rf csprng 23 | git clone https://github.com/pytorch/csprng 24 | pushd csprng 25 | 26 | desired_pythons=( "2.7" "3.5" "3.6" "3.7" ) 27 | # for each python 28 | for desired_python in "${desired_pythons[@]}" 29 | do 30 | # create and activate python env 31 | env_name="env$desired_python" 32 | conda create -yn $env_name python="$desired_python" 33 | conda activate $env_name 34 | 35 | pip uninstall -y torch || true 36 | pip uninstall -y torch_nightly || true 37 | 38 | export TORCHCSPRNG_PYTORCH_DEPENDENCY_NAME=torch_nightly 39 | pip install torch_nightly -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html 40 | export TORCHCSPRNG_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version: *//')" 41 | echo "Building against ${TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION}" 42 | 43 | # install torchcsprng dependencies 44 | pip install ninja scipy pytest pycrypto 45 | 46 | python setup.py clean 47 | python setup.py bdist_wheel 48 | mkdir -p $OUT_DIR 49 | cp dist/*.whl $OUT_DIR/ 50 | done 51 | popd 52 | popd 53 | -------------------------------------------------------------------------------- /packaging/wheel/relocate.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Helper script to package wheels and relocate binaries.""" 4 | 5 | import glob 6 | import hashlib 7 | import io 8 | 9 | # Standard library imports 10 | import os 11 | import os.path as osp 12 | import platform 13 | import shutil 14 | import subprocess 15 | import sys 16 | import zipfile 17 | from base64 import urlsafe_b64encode 18 | 19 | # Third party imports 20 | if sys.platform == "linux": 21 | from auditwheel.lddtree import lddtree 22 | 23 | 24 | ALLOWLIST = { 25 | "libgcc_s.so.1", 26 | "libstdc++.so.6", 27 | "libm.so.6", 28 | "libdl.so.2", 29 | "librt.so.1", 30 | "libc.so.6", 31 | "libnsl.so.1", 32 | "libutil.so.1", 33 | "libpthread.so.0", 34 | "libresolv.so.2", 35 | "libX11.so.6", 36 | "libXext.so.6", 37 | "libXrender.so.1", 38 | "libICE.so.6", 39 | "libSM.so.6", 40 | "libGL.so.1", 41 | "libgobject-2.0.so.0", 42 | "libgthread-2.0.so.0", 43 | "libglib-2.0.so.0", 44 | "ld-linux-x86-64.so.2", 45 | "ld-2.17.so", 46 | } 47 | 48 | WINDOWS_ALLOWLIST = { 49 | "MSVCP140.dll", 50 | "KERNEL32.dll", 51 | "VCRUNTIME140_1.dll", 52 | "VCRUNTIME140.dll", 53 | "api-ms-win-crt-heap-l1-1-0.dll", 54 | "api-ms-win-crt-runtime-l1-1-0.dll", 55 | "api-ms-win-crt-stdio-l1-1-0.dll", 56 | "api-ms-win-crt-filesystem-l1-1-0.dll", 57 | "api-ms-win-crt-string-l1-1-0.dll", 58 | "api-ms-win-crt-environment-l1-1-0.dll", 59 | "api-ms-win-crt-math-l1-1-0.dll", 60 | "api-ms-win-crt-convert-l1-1-0.dll", 61 | } 62 | 63 | 64 | HERE = osp.dirname(osp.abspath(__file__)) 65 | PACKAGE_ROOT = osp.dirname(osp.dirname(HERE)) 66 | PLATFORM_ARCH = platform.machine() 67 | PYTHON_VERSION = sys.version_info 68 | 69 | 70 | def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): 71 | """Yield pieces of data from a file-like object until EOF.""" 72 | while True: 73 | chunk = file.read(size) 74 | if not chunk: 75 | break 76 | yield chunk 77 | 78 | 79 | def rehash(path, blocksize=1 << 20): 80 | """Return (hash, length) for path using hashlib.sha256()""" 81 | h = hashlib.sha256() 82 | length = 0 83 | with open(path, "rb") as f: 84 | for block in read_chunks(f, size=blocksize): 85 | length += len(block) 86 | h.update(block) 87 | digest = "sha256=" + urlsafe_b64encode(h.digest()).decode("latin1").rstrip("=") 88 | # unicode/str python2 issues 89 | return (digest, str(length)) # type: ignore 90 | 91 | 92 | def unzip_file(file, dest): 93 | """Decompress zip `file` into directory `dest`.""" 94 | with zipfile.ZipFile(file, "r") as zip_ref: 95 | zip_ref.extractall(dest) 96 | 97 | 98 | def is_program_installed(basename): 99 | """ 100 | Return program absolute path if installed in PATH. 101 | Otherwise, return None 102 | On macOS systems, a .app is considered installed if 103 | it exists. 104 | """ 105 | if sys.platform == "darwin" and basename.endswith(".app") and osp.exists(basename): 106 | return basename 107 | 108 | for path in os.environ["PATH"].split(os.pathsep): 109 | abspath = osp.join(path, basename) 110 | if osp.isfile(abspath): 111 | return abspath 112 | 113 | 114 | def find_program(basename): 115 | """ 116 | Find program in PATH and return absolute path 117 | Try adding .exe or .bat to basename on Windows platforms 118 | (return None if not found) 119 | """ 120 | names = [basename] 121 | if os.name == "nt": 122 | # Windows platforms 123 | extensions = (".exe", ".bat", ".cmd", ".dll") 124 | if not basename.endswith(extensions): 125 | names = [basename + ext for ext in extensions] + [basename] 126 | for name in names: 127 | path = is_program_installed(name) 128 | if path: 129 | return path 130 | 131 | 132 | def patch_new_path(library_path, new_dir): 133 | library = osp.basename(library_path) 134 | name, *rest = library.split(".") 135 | rest = ".".join(rest) 136 | hash_id = hashlib.sha256(library_path.encode("utf-8")).hexdigest()[:8] 137 | new_name = ".".join([name, hash_id, rest]) 138 | return osp.join(new_dir, new_name) 139 | 140 | 141 | def find_dll_dependencies(dumpbin, binary): 142 | out = subprocess.run([dumpbin, "/dependents", binary], stdout=subprocess.PIPE) 143 | out = out.stdout.strip().decode("utf-8") 144 | start_index = out.find("dependencies:") + len("dependencies:") 145 | end_index = out.find("Summary") 146 | dlls = out[start_index:end_index].strip() 147 | dlls = dlls.split(os.linesep) 148 | dlls = [dll.strip() for dll in dlls] 149 | return dlls 150 | 151 | 152 | def relocate_elf_library(patchelf, output_dir, output_library, binary): 153 | """ 154 | Relocate an ELF shared library to be packaged on a wheel. 155 | 156 | Given a shared library, find the transitive closure of its dependencies, 157 | rename and copy them into the wheel while updating their respective rpaths. 158 | """ 159 | 160 | print("Relocating {0}".format(binary)) 161 | binary_path = osp.join(output_library, binary) 162 | 163 | ld_tree = lddtree(binary_path) 164 | tree_libs = ld_tree["libs"] 165 | 166 | binary_queue = [(n, binary) for n in ld_tree["needed"]] 167 | binary_paths = {binary: binary_path} 168 | binary_dependencies = {} 169 | 170 | while binary_queue != []: 171 | library, parent = binary_queue.pop(0) 172 | library_info = tree_libs[library] 173 | print(library) 174 | 175 | if library_info["path"] is None: 176 | print("Omitting {0}".format(library)) 177 | continue 178 | 179 | if library in ALLOWLIST: 180 | # Omit glibc/gcc/system libraries 181 | print("Omitting {0}".format(library)) 182 | continue 183 | 184 | parent_dependencies = binary_dependencies.get(parent, []) 185 | parent_dependencies.append(library) 186 | binary_dependencies[parent] = parent_dependencies 187 | 188 | if library in binary_paths: 189 | continue 190 | 191 | binary_paths[library] = library_info["path"] 192 | binary_queue += [(n, library) for n in library_info["needed"]] 193 | 194 | print("Copying dependencies to wheel directory") 195 | new_libraries_path = osp.join(output_dir, "torchcsprng.libs") 196 | os.makedirs(new_libraries_path) 197 | 198 | new_names = {binary: binary_path} 199 | 200 | for library in binary_paths: 201 | if library != binary: 202 | library_path = binary_paths[library] 203 | new_library_path = patch_new_path(library_path, new_libraries_path) 204 | print("{0} -> {1}".format(library, new_library_path)) 205 | shutil.copyfile(library_path, new_library_path) 206 | new_names[library] = new_library_path 207 | 208 | print("Updating dependency names by new files") 209 | for library in binary_paths: 210 | if library != binary: 211 | if library not in binary_dependencies: 212 | continue 213 | library_dependencies = binary_dependencies[library] 214 | new_library_name = new_names[library] 215 | for dep in library_dependencies: 216 | new_dep = osp.basename(new_names[dep]) 217 | print("{0}: {1} -> {2}".format(library, dep, new_dep)) 218 | subprocess.check_output( 219 | [patchelf, "--replace-needed", dep, new_dep, new_library_name], 220 | cwd=new_libraries_path, 221 | ) 222 | 223 | print("Updating library rpath") 224 | subprocess.check_output( 225 | [patchelf, "--set-rpath", "$ORIGIN", new_library_name], 226 | cwd=new_libraries_path, 227 | ) 228 | 229 | subprocess.check_output( 230 | [patchelf, "--print-rpath", new_library_name], cwd=new_libraries_path 231 | ) 232 | 233 | print("Update library dependencies") 234 | library_dependencies = binary_dependencies[binary] 235 | for dep in library_dependencies: 236 | new_dep = osp.basename(new_names[dep]) 237 | print("{0}: {1} -> {2}".format(binary, dep, new_dep)) 238 | subprocess.check_output( 239 | [patchelf, "--replace-needed", dep, new_dep, binary], cwd=output_library 240 | ) 241 | 242 | print("Update library rpath") 243 | subprocess.check_output( 244 | [patchelf, "--set-rpath", "$ORIGIN:$ORIGIN/../torchcsprng.libs", binary_path], 245 | cwd=output_library, 246 | ) 247 | 248 | 249 | def relocate_dll_library(dumpbin, output_dir, output_library, binary): 250 | """ 251 | Relocate a DLL/PE shared library to be packaged on a wheel. 252 | 253 | Given a shared library, find the transitive closure of its dependencies, 254 | rename and copy them into the wheel. 255 | """ 256 | print("Relocating {0}".format(binary)) 257 | binary_path = osp.join(output_library, binary) 258 | 259 | library_dlls = find_dll_dependencies(dumpbin, binary_path) 260 | binary_queue = [(dll, binary) for dll in library_dlls] 261 | binary_paths = {binary: binary_path} 262 | binary_dependencies = {} 263 | 264 | while binary_queue != []: 265 | library, parent = binary_queue.pop(0) 266 | if library in WINDOWS_ALLOWLIST or library.startswith("api-ms-win"): 267 | print("Omitting {0}".format(library)) 268 | continue 269 | 270 | library_path = find_program(library) 271 | if library_path is None: 272 | print("{0} not found".format(library)) 273 | continue 274 | 275 | if osp.basename(osp.dirname(library_path)) == "system32": 276 | continue 277 | 278 | print("{0}: {1}".format(library, library_path)) 279 | parent_dependencies = binary_dependencies.get(parent, []) 280 | parent_dependencies.append(library) 281 | binary_dependencies[parent] = parent_dependencies 282 | 283 | if library in binary_paths: 284 | continue 285 | 286 | binary_paths[library] = library_path 287 | downstream_dlls = find_dll_dependencies(dumpbin, library_path) 288 | binary_queue += [(n, library) for n in downstream_dlls] 289 | 290 | print("Copying dependencies to wheel directory") 291 | package_dir = osp.join(output_dir, "torchcsprng") 292 | for library in binary_paths: 293 | if library != binary: 294 | library_path = binary_paths[library] 295 | new_library_path = osp.join(package_dir, library) 296 | print("{0} -> {1}".format(library, new_library_path)) 297 | shutil.copyfile(library_path, new_library_path) 298 | 299 | 300 | def compress_wheel(output_dir, wheel, wheel_dir, wheel_name): 301 | """Create RECORD file and compress wheel distribution.""" 302 | print("Update RECORD file in wheel") 303 | dist_info = glob.glob(osp.join(output_dir, "*.dist-info"))[0] 304 | record_file = osp.join(dist_info, "RECORD") 305 | 306 | with open(record_file, "w") as f: 307 | for root, _, files in os.walk(output_dir): 308 | for this_file in files: 309 | full_file = osp.join(root, this_file) 310 | rel_file = osp.relpath(full_file, output_dir) 311 | if full_file == record_file: 312 | f.write("{0},,\n".format(rel_file)) 313 | else: 314 | digest, size = rehash(full_file) 315 | f.write("{0},{1},{2}\n".format(rel_file, digest, size)) 316 | 317 | print("Compressing wheel") 318 | base_wheel_name = osp.join(wheel_dir, wheel_name) 319 | shutil.make_archive(base_wheel_name, "zip", output_dir) 320 | os.remove(wheel) 321 | shutil.move("{0}.zip".format(base_wheel_name), wheel) 322 | shutil.rmtree(output_dir) 323 | 324 | 325 | def patch_linux(): 326 | # Get patchelf location 327 | patchelf = find_program("patchelf") 328 | if patchelf is None: 329 | raise FileNotFoundError( 330 | "Patchelf was not found in the system, please" 331 | " make sure that is available on the PATH." 332 | ) 333 | 334 | # Find wheel 335 | print("Finding wheels...") 336 | wheels = glob.glob(osp.join(PACKAGE_ROOT, "dist", "*.whl")) 337 | output_dir = osp.join(PACKAGE_ROOT, "dist", ".wheel-process") 338 | 339 | image_binary = "image.so" 340 | video_binary = "video_reader.so" 341 | torchcsprng_binaries = [image_binary, video_binary] 342 | for wheel in wheels: 343 | if osp.exists(output_dir): 344 | shutil.rmtree(output_dir) 345 | 346 | os.makedirs(output_dir) 347 | 348 | print("Unzipping wheel...") 349 | wheel_file = osp.basename(wheel) 350 | wheel_dir = osp.dirname(wheel) 351 | print("{0}".format(wheel_file)) 352 | wheel_name, _ = osp.splitext(wheel_file) 353 | unzip_file(wheel, output_dir) 354 | 355 | print("Finding ELF dependencies...") 356 | output_library = osp.join(output_dir, "torchcsprng") 357 | for binary in torchcsprng_binaries: 358 | if osp.exists(osp.join(output_library, binary)): 359 | relocate_elf_library(patchelf, output_dir, output_library, binary) 360 | 361 | compress_wheel(output_dir, wheel, wheel_dir, wheel_name) 362 | 363 | 364 | def patch_win(): 365 | # Get dumpbin location 366 | dumpbin = find_program("dumpbin") 367 | if dumpbin is None: 368 | raise FileNotFoundError( 369 | "Dumpbin was not found in the system, please" 370 | " make sure that is available on the PATH." 371 | ) 372 | 373 | # Find wheel 374 | print("Finding wheels...") 375 | wheels = glob.glob(osp.join(PACKAGE_ROOT, "dist", "*.whl")) 376 | output_dir = osp.join(PACKAGE_ROOT, "dist", ".wheel-process") 377 | 378 | image_binary = "image.pyd" 379 | video_binary = "video_reader.pyd" 380 | torchcsprng_binaries = [image_binary, video_binary] 381 | for wheel in wheels: 382 | if osp.exists(output_dir): 383 | shutil.rmtree(output_dir) 384 | 385 | os.makedirs(output_dir) 386 | 387 | print("Unzipping wheel...") 388 | wheel_file = osp.basename(wheel) 389 | wheel_dir = osp.dirname(wheel) 390 | print("{0}".format(wheel_file)) 391 | wheel_name, _ = osp.splitext(wheel_file) 392 | unzip_file(wheel, output_dir) 393 | 394 | print("Finding DLL/PE dependencies...") 395 | output_library = osp.join(output_dir, "torchcsprng") 396 | for binary in torchcsprng_binaries: 397 | if osp.exists(osp.join(output_library, binary)): 398 | relocate_dll_library(dumpbin, output_dir, output_library, binary) 399 | 400 | compress_wheel(output_dir, wheel, wheel_dir, wheel_name) 401 | 402 | 403 | if __name__ == "__main__": 404 | if sys.platform == "linux": 405 | patch_linux() 406 | elif sys.platform == "win32": 407 | patch_win() 408 | -------------------------------------------------------------------------------- /packaging/windows/azure-pipelines-ci.yml: -------------------------------------------------------------------------------- 1 | 2 | # Turn off auto builds for commits 3 | trigger: none 4 | pr: none 5 | 6 | jobs: 7 | - template: templates/build_task.yml 8 | parameters: 9 | package: 'Wheels' 10 | spec: 'CPU' 11 | msagent: true 12 | -------------------------------------------------------------------------------- /packaging/windows/azure-pipelines.yml: -------------------------------------------------------------------------------- 1 | 2 | # Turn off auto builds for commits 3 | trigger: none 4 | pr: none 5 | 6 | jobs: 7 | - template: templates/auth_task.yml 8 | 9 | - template: templates/build_task.yml 10 | parameters: 11 | package: 'Wheels' 12 | spec: 'CPU' 13 | msagent: true 14 | 15 | - template: templates/build_task.yml 16 | parameters: 17 | package: 'Conda' 18 | spec: 'CPU' 19 | msagent: true 20 | 21 | - template: templates/build_task.yml 22 | parameters: 23 | package: 'Wheels' 24 | spec: 'CUDA' 25 | msagent: true 26 | 27 | - template: templates/build_task.yml 28 | parameters: 29 | package: 'Conda' 30 | spec: 'CUDA' 31 | msagent: true 32 | 33 | - template: templates/linux_build_task.yml 34 | parameters: 35 | msagent: $(ms.hosted.agent.cpu) 36 | -------------------------------------------------------------------------------- /packaging/windows/build_csprng.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | :: This script parses args, installs required libraries (miniconda, MKL, 4 | :: Magma), and then delegates to cpu.bat, cuda80.bat, etc. 5 | 6 | IF NOT "%CUDA_VERSION%" == "" IF NOT "%TORCHCSPRNG_BUILD_VERSION%" == "" if NOT "%TORCHCSPRNG_BUILD_NUMBER%" == "" goto env_end 7 | if "%~1"=="" goto arg_error 8 | if "%~2"=="" goto arg_error 9 | if "%~3"=="" goto arg_error 10 | if NOT "%~4"=="" goto arg_error 11 | goto arg_end 12 | 13 | :arg_error 14 | 15 | echo Illegal number of parameters. Pass cuda version, pytorch version, build number 16 | echo CUDA version should be Mm with no dot, e.g. '80' 17 | echo DESIRED_PYTHON should be M.m, e.g. '2.7' 18 | exit /b 1 19 | 20 | :arg_end 21 | 22 | set CUDA_VERSION=%~1 23 | set TORCHCSPRNG_BUILD_VERSION=%~2 24 | set TORCHCSPRNG_BUILD_NUMBER=%~3 25 | 26 | set BUILD_VERSION=%TORCHCSPRNG_BUILD_VERSION% 27 | 28 | :env_end 29 | 30 | if NOT "%CUDA_VERSION%" == "cpu" ( 31 | set CUDA_PREFIX=cuda%CUDA_VERSION% 32 | set CUVER=cu%CUDA_VERSION% 33 | set FORCE_CUDA=1 34 | ) else ( 35 | set CUDA_PREFIX=cpu 36 | set CUVER=cpu 37 | ) 38 | 39 | set BUILD_CSPRNG=1 40 | REM set TORCH_WHEEL=torch -f https://download.pytorch.org/whl/%CUVER%/stable.html --no-index 41 | 42 | IF "%DESIRED_PYTHON%" == "" set DESIRED_PYTHON=3.5;3.6;3.7 43 | set DESIRED_PYTHON_PREFIX=%DESIRED_PYTHON:.=% 44 | set DESIRED_PYTHON_PREFIX=py%DESIRED_PYTHON_PREFIX:;=;py% 45 | 46 | set SRC_DIR=%~dp0 47 | pushd %SRC_DIR% 48 | 49 | :: Install Miniconda3 50 | set "CONDA_HOME=%CD%\conda" 51 | set "tmp_conda=%CONDA_HOME%" 52 | set "miniconda_exe=%CD%\miniconda.exe" 53 | rmdir /s /q conda 54 | del miniconda.exe 55 | curl -k https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "%miniconda_exe%" 56 | call ..\conda\install_conda.bat 57 | IF ERRORLEVEL 1 exit /b 1 58 | set "ORIG_PATH=%PATH%" 59 | set "PATH=%CONDA_HOME%;%CONDA_HOME%\scripts;%CONDA_HOME%\Library\bin;%PATH%" 60 | 61 | :: Create a new conda environment 62 | setlocal EnableDelayedExpansion 63 | FOR %%v IN (%DESIRED_PYTHON%) DO ( 64 | set PYTHON_VERSION_STR=%%v 65 | set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=! 66 | conda remove -n py!PYTHON_VERSION_STR! --all -y || rmdir %CONDA_HOME%\envs\py!PYTHON_VERSION_STR! /s 67 | conda create -n py!PYTHON_VERSION_STR! -y -q -c defaults -c conda-forge numpy>=1.11 mkl>=2018 python=%%v ca-certificates scipy pycrypto 68 | ) 69 | 70 | :: Uncomment for stable releases 71 | :: FOR %%v IN (%DESIRED_PYTHON%) DO ( 72 | :: set PYTHON_VERSION_STR=%%v 73 | :: set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=! 74 | :: set "PATH=%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\scripts;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\Library\bin;%ORIG_PATH%" 75 | 76 | :: if "%CUDA_VERSION%" == "100" ( 77 | :: set TORCH_WHEEL=https://download.pytorch.org/whl/%CUVER%/torch-1.2.0-cp!PYTHON_VERSION_STR!-cp!PYTHON_VERSION_STR!m-win_amd64.whl 78 | :: ) else ( 79 | :: set TORCH_WHEEL=https://download.pytorch.org/whl/%CUVER%/torch-1.2.0%%2B%CUVER%-cp!PYTHON_VERSION_STR!-cp!PYTHON_VERSION_STR!m-win_amd64.whl 80 | :: ) 81 | :: echo Installing !TORCH_WHEEL!... 82 | :: pip install "!TORCH_WHEEL!" 83 | :: ) 84 | 85 | :: Uncomment for nightly releases 86 | FOR %%v IN (%DESIRED_PYTHON%) DO ( 87 | set PYTHON_VERSION_STR=%%v 88 | set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=! 89 | set "PATH=%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\scripts;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\Library\bin;%ORIG_PATH%" 90 | 91 | set TORCH_WHEEL=torch --pre -f https://download.pytorch.org/whl/nightly/%CUVER%/torch_nightly.html 92 | echo Installing !TORCH_WHEEL!... 93 | pip install !TORCH_WHEEL! 94 | ) 95 | 96 | endlocal 97 | 98 | if "%DEBUG%" == "1" ( 99 | set BUILD_TYPE=debug 100 | ) ELSE ( 101 | set BUILD_TYPE=release 102 | ) 103 | 104 | :: Install sccache 105 | if "%USE_SCCACHE%" == "1" ( 106 | mkdir %CD%\tmp_bin 107 | curl -k https://s3.amazonaws.com/ossci-windows/sccache.exe --output %CD%\tmp_bin\sccache.exe 108 | if not "%CUDA_VERSION%" == "" ( 109 | copy %CD%\tmp_bin\sccache.exe %CD%\tmp_bin\nvcc.exe 110 | 111 | set CUDA_NVCC_EXECUTABLE=%CD%\tmp_bin\nvcc 112 | set "PATH=%CD%\tmp_bin;%PATH%" 113 | ) 114 | ) 115 | 116 | for %%v in (%DESIRED_PYTHON_PREFIX%) do ( 117 | :: Activate Python Environment 118 | set PYTHON_PREFIX=%%v 119 | set "PATH=%CONDA_HOME%\envs\%%v;%CONDA_HOME%\envs\%%v\scripts;%CONDA_HOME%\envs\%%v\Library\bin;%ORIG_PATH%" 120 | if defined INCLUDE ( 121 | set "INCLUDE=%INCLUDE%;%CONDA_HOME%\envs\%%v\Library\include" 122 | ) else ( 123 | set "INCLUDE=%CONDA_HOME%\envs\%%v\Library\include" 124 | ) 125 | if defined LIB ( 126 | set "LIB=%LIB%;%CONDA_HOME%\envs\%%v\Library\lib" 127 | ) else ( 128 | set "LIB=%CONDA_HOME%\envs\%%v\Library\lib" 129 | ) 130 | @setlocal 131 | :: Set Flags 132 | if NOT "%CUDA_VERSION%"=="cpu" ( 133 | set CUDNN_VERSION=7 134 | ) 135 | call %CUDA_PREFIX%.bat 136 | IF ERRORLEVEL 1 exit /b 1 137 | call internal\test.bat 138 | IF ERRORLEVEL 1 exit /b 1 139 | @endlocal 140 | ) 141 | 142 | set "PATH=%ORIG_PATH%" 143 | popd 144 | 145 | IF ERRORLEVEL 1 exit /b 1 146 | -------------------------------------------------------------------------------- /packaging/windows/cpu.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | IF NOT "%BUILD_CSPRNG%" == "" ( 4 | set MODULE_NAME=csprng 5 | ) ELSE ( 6 | set MODULE_NAME=pytorch 7 | ) 8 | 9 | IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" ( 10 | call internal\clone.bat 11 | cd .. 12 | IF ERRORLEVEL 1 goto eof 13 | ) ELSE ( 14 | call internal\clean.bat 15 | ) 16 | 17 | call internal\check_deps.bat 18 | IF ERRORLEVEL 1 goto eof 19 | 20 | REM Check for optional components 21 | 22 | echo Disabling CUDA 23 | set NO_CUDA=1 24 | set USE_CUDA=0 25 | 26 | IF "%BUILD_CSPRNG%" == "" ( 27 | call internal\check_opts.bat 28 | IF ERRORLEVEL 1 goto eof 29 | 30 | call internal\copy_cpu.bat 31 | IF ERRORLEVEL 1 goto eof 32 | ) 33 | 34 | call internal\setup.bat 35 | IF ERRORLEVEL 1 goto eof 36 | 37 | :eof 38 | -------------------------------------------------------------------------------- /packaging/windows/cuda101.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | IF NOT "%BUILD_CSPRNG%" == "" ( 4 | set MODULE_NAME=csprng 5 | ) ELSE ( 6 | set MODULE_NAME=pytorch 7 | ) 8 | 9 | IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" ( 10 | call internal\clone.bat 11 | cd .. 12 | IF ERRORLEVEL 1 goto eof 13 | ) ELSE ( 14 | call internal\clean.bat 15 | ) 16 | 17 | call internal\check_deps.bat 18 | IF ERRORLEVEL 1 goto eof 19 | 20 | REM Check for optional components 21 | 22 | set NO_CUDA= 23 | set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 24 | 25 | IF "%NVTOOLSEXT_PATH%"=="" ( 26 | echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing 27 | exit /b 1 28 | goto optcheck 29 | ) 30 | 31 | IF "%CUDA_PATH_V10_1%"=="" ( 32 | echo CUDA 10.1 not found, failing 33 | exit /b 1 34 | ) ELSE ( 35 | IF "%BUILD_CSPRNG%" == "" ( 36 | set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0;7.5 37 | set TORCH_NVCC_FLAGS=-Xfatbin -compress-all 38 | ) ELSE ( 39 | set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50 40 | ) 41 | 42 | set "CUDA_PATH=%CUDA_PATH_V10_1%" 43 | set "PATH=%CUDA_PATH_V10_1%\bin;%PATH%" 44 | ) 45 | 46 | :optcheck 47 | 48 | IF "%BUILD_CSPRNG%" == "" ( 49 | call internal\check_opts.bat 50 | IF ERRORLEVEL 1 goto eof 51 | 52 | call internal\copy.bat 53 | IF ERRORLEVEL 1 goto eof 54 | ) 55 | 56 | call internal\setup.bat 57 | IF ERRORLEVEL 1 goto eof 58 | 59 | :eof 60 | -------------------------------------------------------------------------------- /packaging/windows/cuda102.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | IF NOT "%BUILD_CSPRNG%" == "" ( 4 | set MODULE_NAME=csprng 5 | ) ELSE ( 6 | set MODULE_NAME=pytorch 7 | ) 8 | 9 | IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" ( 10 | call internal\clone.bat 11 | cd .. 12 | IF ERRORLEVEL 1 goto eof 13 | ) ELSE ( 14 | call internal\clean.bat 15 | ) 16 | 17 | call internal\check_deps.bat 18 | IF ERRORLEVEL 1 goto eof 19 | 20 | REM Check for optional components 21 | 22 | set NO_CUDA= 23 | set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 24 | 25 | IF "%NVTOOLSEXT_PATH%"=="" ( 26 | echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing 27 | exit /b 1 28 | goto optcheck 29 | ) 30 | 31 | IF "%CUDA_PATH_V10_2%"=="" ( 32 | echo CUDA 10.2 not found, failing 33 | exit /b 1 34 | ) ELSE ( 35 | IF "%BUILD_CSPRNG%" == "" ( 36 | set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0;7.5 37 | set TORCH_NVCC_FLAGS=-Xfatbin -compress-all 38 | ) ELSE ( 39 | set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50 40 | ) 41 | 42 | set "CUDA_PATH=%CUDA_PATH_V10_2%" 43 | set "PATH=%CUDA_PATH_V10_2%\bin;%PATH%" 44 | ) 45 | 46 | :optcheck 47 | 48 | IF "%BUILD_CSPRNG%" == "" ( 49 | call internal\check_opts.bat 50 | IF ERRORLEVEL 1 goto eof 51 | 52 | call internal\copy.bat 53 | IF ERRORLEVEL 1 goto eof 54 | ) 55 | 56 | call internal\setup.bat 57 | IF ERRORLEVEL 1 goto eof 58 | 59 | :eof 60 | -------------------------------------------------------------------------------- /packaging/windows/cuda92.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | IF NOT "%BUILD_CSPRNG%" == "" ( 4 | set MODULE_NAME=csprng 5 | ) ELSE ( 6 | set MODULE_NAME=pytorch 7 | ) 8 | 9 | IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" ( 10 | call internal\clone.bat 11 | cd .. 12 | IF ERRORLEVEL 1 goto eof 13 | ) ELSE ( 14 | call internal\clean.bat 15 | ) 16 | 17 | call internal\check_deps.bat 18 | IF ERRORLEVEL 1 goto eof 19 | 20 | REM Check for optional components 21 | 22 | set USE_CUDA= 23 | set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 24 | 25 | IF "%NVTOOLSEXT_PATH%"=="" ( 26 | echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing 27 | exit /b 1 28 | goto optcheck 29 | ) 30 | 31 | IF "%CUDA_PATH_V9_2%"=="" ( 32 | echo CUDA 9.2 not found, failing 33 | exit /b 1 34 | ) ELSE ( 35 | IF "%BUILD_CSPRNG%" == "" ( 36 | set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0 37 | set TORCH_NVCC_FLAGS=-Xfatbin -compress-all 38 | ) ELSE ( 39 | set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50 40 | ) 41 | 42 | set "CUDA_PATH=%CUDA_PATH_V9_2%" 43 | set "PATH=%CUDA_PATH_V9_2%\bin;%PATH%" 44 | ) 45 | 46 | :optcheck 47 | 48 | IF "%BUILD_CSPRNG%" == "" ( 49 | call internal\check_opts.bat 50 | IF ERRORLEVEL 1 goto eof 51 | 52 | call internal\copy.bat 53 | IF ERRORLEVEL 1 goto eof 54 | ) 55 | 56 | call internal\setup.bat 57 | IF ERRORLEVEL 1 goto eof 58 | 59 | :eof 60 | -------------------------------------------------------------------------------- /packaging/windows/internal/auth.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | : From the following doc, the build won't be triggered if the users don't sign in daily. 4 | : https://docs.microsoft.com/en-us/azure/devops/pipelines/build/triggers?tabs=yaml&view=vsts#my-build-didnt-run-what-happened 5 | : To avoid this problem, we can just go through the sign in process using the following command. 6 | 7 | :auth_start 8 | 9 | if "%RETRY_TIMES%" == "" ( 10 | set /a RETRY_TIMES=10 11 | set /a SLEEP_TIME=2 12 | ) else ( 13 | set /a RETRY_TIMES=%RETRY_TIMES%-1 14 | set /a SLEEP_TIME=%SLEEP_TIME%*2 15 | ) 16 | 17 | for /f "usebackq tokens=*" %%i in (`curl -so NUL -w "%%{http_code}" -u %VSTS_AUTH% https://dev.azure.com/pytorch`) do ( 18 | set STATUS_CODE=%%i 19 | ) 20 | 21 | IF NOT "%STATUS_CODE%" == "200" ( 22 | echo Auth retry times remaining: %RETRY_TIMES% 23 | echo Sleep time: %SLEEP_TIME% seconds 24 | IF %RETRY_TIMES% EQU 0 ( 25 | echo Auth failed 26 | goto err 27 | ) 28 | waitfor SomethingThatIsNeverHappening /t %SLEEP_TIME% 2>nul || ver >nul 29 | goto auth_start 30 | ) ELSE ( 31 | echo Login Attempt Succeeded 32 | goto auth_end 33 | ) 34 | 35 | :err 36 | 37 | : Throw a warning if it fails 38 | powershell -c "Write-Warning 'Login Attempt Failed'" 39 | 40 | :auth_end 41 | 42 | set RETRY_TIMES= 43 | set SLEEP_TIME= 44 | set STATUS_CODE= 45 | 46 | exit /b 0 47 | -------------------------------------------------------------------------------- /packaging/windows/internal/build_conda.bat: -------------------------------------------------------------------------------- 1 | if "%VC_YEAR%" == "2017" set VSDEVCMD_ARGS=-vcvars_ver=14.13 2 | if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs2017_install.ps1 3 | if errorlevel 1 exit /b 1 4 | 5 | call packaging/windows/internal/cuda_install.bat 6 | if errorlevel 1 exit /b 1 7 | 8 | call packaging/windows/internal/nightly_defaults.bat Conda 9 | if errorlevel 1 exit /b 1 10 | 11 | set PYTORCH_FINAL_PACKAGE_DIR=%CD%\packaging\windows\output 12 | if not exist "%PYTORCH_FINAL_PACKAGE_DIR%" mkdir %PYTORCH_FINAL_PACKAGE_DIR% 13 | 14 | bash ./packaging/conda/build_csprng.sh %CUDA_VERSION% %TORCHCSPRNG_BUILD_VERSION% %TORCHCSPRNG_BUILD_NUMBER% 15 | if errorlevel 1 exit /b 1 16 | -------------------------------------------------------------------------------- /packaging/windows/internal/build_wheels.bat: -------------------------------------------------------------------------------- 1 | if "%VC_YEAR%" == "2017" set VSDEVCMD_ARGS=-vcvars_ver=14.13 2 | if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs2017_install.ps1 3 | if errorlevel 1 exit /b 1 4 | 5 | call packaging/windows/internal/cuda_install.bat 6 | if errorlevel 1 exit /b 1 7 | 8 | call packaging/windows/internal/nightly_defaults.bat Wheels 9 | if errorlevel 1 exit /b 1 10 | 11 | call packaging/windows/build_csprng.bat %CUDA_VERSION% %TORCHCSPRNG_BUILD_VERSION% %TORCHCSPRNG_BUILD_NUMBER% 12 | if errorlevel 1 exit /b 1 13 | -------------------------------------------------------------------------------- /packaging/windows/internal/check_deps.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | REM Check for necessary components 4 | 5 | IF NOT "%PROCESSOR_ARCHITECTURE%"=="AMD64" ( 6 | echo You should use 64 bits Windows to build and run PyTorch 7 | exit /b 1 8 | ) 9 | 10 | IF "%BUILD_CSPRNG%" == "" ( 11 | where /q cmake.exe 12 | 13 | IF ERRORLEVEL 1 ( 14 | echo CMake is required to compile PyTorch on Windows 15 | exit /b 1 16 | ) 17 | ) 18 | 19 | IF NOT EXIST "%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" ( 20 | echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows 21 | exit /b 1 22 | ) 23 | 24 | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do ( 25 | if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( 26 | set "VS15INSTALLDIR=%%i" 27 | set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" 28 | goto vswhere 29 | ) 30 | ) 31 | 32 | :vswhere 33 | IF "%VS15VCVARSALL%"=="" ( 34 | echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows 35 | exit /b 1 36 | ) 37 | 38 | set MSSdk=1 39 | set DISTUTILS_USE_SDK=1 40 | 41 | where /q python.exe 42 | 43 | IF ERRORLEVEL 1 ( 44 | echo Python x64 3.5 or up is required to compile PyTorch on Windows 45 | exit /b 1 46 | ) 47 | 48 | for /F "usebackq delims=" %%i in (`python -c "import sys; print('{0[0]}{0[1]}'.format(sys.version_info))"`) do ( 49 | set /a PYVER=%%i 50 | ) 51 | 52 | if %PYVER% LSS 35 ( 53 | echo Warning: PyTorch for Python 2 under Windows is experimental. 54 | echo Python x64 3.5 or up is recommended to compile PyTorch on Windows 55 | echo Maybe you can create a virual environment if you have conda installed: 56 | echo ^> conda create -n test python=3.6 pyyaml mkl numpy 57 | echo ^> activate test 58 | ) 59 | 60 | for /F "usebackq delims=" %%i in (`python -c "import struct;print( 8 * struct.calcsize('P'))"`) do ( 61 | set /a PYSIZE=%%i 62 | ) 63 | 64 | if %PYSIZE% NEQ 64 ( 65 | echo Python x64 3.5 or up is required to compile PyTorch on Windows 66 | exit /b 1 67 | ) 68 | -------------------------------------------------------------------------------- /packaging/windows/internal/check_opts.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | REM Check for optional components 4 | 5 | where /q ninja.exe 6 | 7 | IF NOT ERRORLEVEL 1 ( 8 | echo Ninja found, using it to speed up builds 9 | set CMAKE_GENERATOR=Ninja 10 | ) 11 | 12 | where /q clcache.exe 13 | 14 | IF NOT ERRORLEVEL 1 ( 15 | echo clcache found, using it to speed up builds 16 | set CC=clcache 17 | set CXX=clcache 18 | ) 19 | 20 | where /q sccache.exe 21 | 22 | IF NOT ERRORLEVEL 1 ( 23 | echo sccache found, using it to speed up builds 24 | set CC=sccache cl 25 | set CXX=sccache cl 26 | ) 27 | 28 | IF exist "%MKLProductDir%\mkl\lib\intel64_win" ( 29 | echo MKL found, adding it to build 30 | set "LIB=%MKLProductDir%\mkl\lib\intel64_win;%MKLProductDir%\compiler\lib\intel64_win;%LIB%"; 31 | ) 32 | 33 | exit /b 0 34 | -------------------------------------------------------------------------------- /packaging/windows/internal/clean.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | cd %MODULE_NAME% 4 | python setup.py clean 5 | cd .. 6 | -------------------------------------------------------------------------------- /packaging/windows/internal/clone.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | :: The conda and wheels jobs are seperated on Windows, so we don't need to clone again. 4 | IF "%BUILD_CSPRNG%" == "" ( 5 | if exist "%NIGHTLIES_PYTORCH_ROOT%" ( 6 | xcopy /E /Y /Q "%NIGHTLIES_PYTORCH_ROOT%" pytorch\ 7 | cd pytorch 8 | goto submodule 9 | ) 10 | ) 11 | 12 | git clone https://github.com/%PYTORCH_REPO%/%MODULE_NAME% 13 | 14 | cd %MODULE_NAME% 15 | 16 | IF NOT "%BUILD_CSPRNG%" == "" goto latest_end 17 | 18 | IF "%PYTORCH_BRANCH%" == "latest" ( goto latest_start ) else ( goto latest_end ) 19 | 20 | :latest_start 21 | 22 | if "%NIGHTLIES_DATE%" == "" ( goto date_start ) else ( goto date_end ) 23 | 24 | :date_start 25 | 26 | set "DATE_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyy_MM_dd'" 27 | set "DATE_COMPACT_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyyMMdd'" 28 | 29 | FOR /F "delims=" %%i IN ('powershell -c "%DATE_CMD%"') DO set NIGHTLIES_DATE=%%i 30 | FOR /F "delims=" %%i IN ('powershell -c "%DATE_COMPACT_CMD%"') DO set NIGHTLIES_DATE_COMPACT=%%i 31 | 32 | :date_end 33 | 34 | if "%NIGHTLIES_DATE_COMPACT%" == "" set NIGHTLIES_DATE_COMPACT=%NIGHTLIES_DATE:~0,4%%NIGHTLIES_DATE:~5,2%%NIGHTLIES_DATE:~8,2% 35 | 36 | :: Switch to the latest commit by 11:59 yesterday 37 | echo PYTORCH_BRANCH is set to latest so I will find the last commit 38 | echo before 0:00 midnight on %NIGHTLIES_DATE% 39 | set git_date=%NIGHTLIES_DATE:_=-% 40 | FOR /F "delims=" %%i IN ('git log --before %git_date% -n 1 "--pretty=%%H"') DO set last_commit=%%i 41 | echo Setting PYTORCH_BRANCH to %last_commit% since that was the last 42 | echo commit before %NIGHTLIES_DATE% 43 | set PYTORCH_BRANCH=%last_commit% 44 | 45 | :latest_end 46 | 47 | IF "%PYTORCH_BRANCH%" == "" ( 48 | set PYTORCH_BRANCH=v%TORCHCSPRNG_BUILD_VERSION% 49 | ) 50 | git checkout %PYTORCH_BRANCH% 51 | IF ERRORLEVEL 1 git checkout tags/%PYTORCH_BRANCH% 52 | 53 | :submodule 54 | 55 | git submodule update --init --recursive 56 | IF ERRORLEVEL 1 exit /b 1 57 | -------------------------------------------------------------------------------- /packaging/windows/internal/copy.bat: -------------------------------------------------------------------------------- 1 | copy "%CUDA_PATH%\bin\cusparse64_%CUDA_VERSION%.dll*" pytorch\torch\lib 2 | copy "%CUDA_PATH%\bin\cublas64_%CUDA_VERSION%.dll*" pytorch\torch\lib 3 | copy "%CUDA_PATH%\bin\cudart64_%CUDA_VERSION%.dll*" pytorch\torch\lib 4 | copy "%CUDA_PATH%\bin\curand64_%CUDA_VERSION%.dll*" pytorch\torch\lib 5 | copy "%CUDA_PATH%\bin\cufft64_%CUDA_VERSION%.dll*" pytorch\torch\lib 6 | copy "%CUDA_PATH%\bin\cufftw64_%CUDA_VERSION%.dll*" pytorch\torch\lib 7 | 8 | copy "%CUDA_PATH%\bin\cudnn64_%CUDNN_VERSION%.dll*" pytorch\torch\lib 9 | copy "%CUDA_PATH%\bin\nvrtc64_%CUDA_VERSION%*.dll*" pytorch\torch\lib 10 | copy "%CUDA_PATH%\bin\nvrtc-builtins64_%CUDA_VERSION%.dll*" pytorch\torch\lib 11 | 12 | copy "C:\Program Files\NVIDIA Corporation\NvToolsExt\bin\x64\nvToolsExt64_1.dll*" pytorch\torch\lib 13 | copy "%CONDA_LIB_PATH%\libiomp*5md.dll" pytorch\torch\lib 14 | -------------------------------------------------------------------------------- /packaging/windows/internal/copy_cpu.bat: -------------------------------------------------------------------------------- 1 | copy "%CONDA_LIB_PATH%\libiomp*5md.dll" pytorch\torch\lib 2 | -------------------------------------------------------------------------------- /packaging/windows/internal/cuda_install.bat: -------------------------------------------------------------------------------- 1 | @echo on 2 | 3 | if "%CU_VERSION%" == "cpu" ( 4 | echo Skipping for CPU builds 5 | exit /b 0 6 | ) 7 | 8 | set SRC_DIR=%~dp0\.. 9 | 10 | if not exist "%SRC_DIR%\temp_build" mkdir "%SRC_DIR%\temp_build" 11 | 12 | set /a CUDA_VER=%CU_VERSION:cu=% 13 | set CUDA_VER_MAJOR=%CUDA_VER:~0,-1% 14 | set CUDA_VER_MINOR=%CUDA_VER:~-1,1% 15 | set CUDA_VERSION_STR=%CUDA_VER_MAJOR%.%CUDA_VER_MINOR% 16 | 17 | if %CUDA_VER% EQU 92 goto cuda92 18 | if %CUDA_VER% EQU 100 goto cuda100 19 | if %CUDA_VER% EQU 101 goto cuda101 20 | if %CUDA_VER% EQU 102 goto cuda102 21 | if %CUDA_VER% EQU 110 goto cuda110 22 | if %CUDA_VER% EQU 111 goto cuda111 23 | if %CUDA_VER% EQU 112 goto cuda112 24 | 25 | echo CUDA %CUDA_VERSION_STR% is not supported 26 | exit /b 1 27 | 28 | :cuda92 29 | if not exist "%SRC_DIR%\temp_build\cuda_9.2.148_win10.exe" ( 30 | curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cuda_9.2.148_win10.exe --output "%SRC_DIR%\temp_build\cuda_9.2.148_win10.exe" 31 | if errorlevel 1 exit /b 1 32 | set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_9.2.148_win10.exe" 33 | set "ARGS=nvcc_9.2 cuobjdump_9.2 nvprune_9.2 cupti_9.2 cublas_9.2 cublas_dev_9.2 cudart_9.2 cufft_9.2 cufft_dev_9.2 curand_9.2 curand_dev_9.2 cusolver_9.2 cusolver_dev_9.2 cusparse_9.2 cusparse_dev_9.2 nvgraph_9.2 nvgraph_dev_9.2 npp_9.2 npp_dev_9.2 nvrtc_9.2 nvrtc_dev_9.2 nvml_dev_9.2" 34 | ) 35 | 36 | if not exist "%SRC_DIR%\temp_build\cudnn-9.2-windows10-x64-v7.2.1.38.zip" ( 37 | curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cudnn-9.2-windows10-x64-v7.2.1.38.zip --output "%SRC_DIR%\temp_build\cudnn-9.2-windows10-x64-v7.2.1.38.zip" 38 | if errorlevel 1 exit /b 1 39 | set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-9.2-windows10-x64-v7.2.1.38.zip" 40 | ) 41 | 42 | goto cuda_common 43 | 44 | :cuda100 45 | 46 | if not exist "%SRC_DIR%\temp_build\cuda_10.0.130_411.31_win10.exe" ( 47 | curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cuda_10.0.130_411.31_win10.exe --output "%SRC_DIR%\temp_build\cuda_10.0.130_411.31_win10.exe" 48 | if errorlevel 1 exit /b 1 49 | set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_10.0.130_411.31_win10.exe" 50 | set "ARGS=nvcc_10.0 cuobjdump_10.0 nvprune_10.0 cupti_10.0 cublas_10.0 cublas_dev_10.0 cudart_10.0 cufft_10.0 cufft_dev_10.0 curand_10.0 curand_dev_10.0 cusolver_10.0 cusolver_dev_10.0 cusparse_10.0 cusparse_dev_10.0 nvgraph_10.0 nvgraph_dev_10.0 npp_10.0 npp_dev_10.0 nvrtc_10.0 nvrtc_dev_10.0 nvml_dev_10.0" 51 | ) 52 | 53 | if not exist "%SRC_DIR%\temp_build\cudnn-10.0-windows10-x64-v7.4.1.5.zip" ( 54 | curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cudnn-10.0-windows10-x64-v7.4.1.5.zip --output "%SRC_DIR%\temp_build\cudnn-10.0-windows10-x64-v7.4.1.5.zip" 55 | if errorlevel 1 exit /b 1 56 | set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-10.0-windows10-x64-v7.4.1.5.zip" 57 | ) 58 | 59 | goto cuda_common 60 | 61 | :cuda101 62 | 63 | if not exist "%SRC_DIR%\temp_build\cuda_10.1.243_426.00_win10.exe" ( 64 | curl -k -L https://ossci-windows.s3.amazonaws.com/cuda_10.1.243_426.00_win10.exe --output "%SRC_DIR%\temp_build\cuda_10.1.243_426.00_win10.exe" 65 | if errorlevel 1 exit /b 1 66 | set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_10.1.243_426.00_win10.exe" 67 | set "ARGS=nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 cublas_10.1 cublas_dev_10.1 cudart_10.1 cufft_10.1 cufft_dev_10.1 curand_10.1 curand_dev_10.1 cusolver_10.1 cusolver_dev_10.1 cusparse_10.1 cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1" 68 | ) 69 | 70 | if not exist "%SRC_DIR%\temp_build\cudnn-10.1-windows10-x64-v7.6.4.38.zip" ( 71 | curl -k -L https://ossci-windows.s3.amazonaws.com/cudnn-10.1-windows10-x64-v7.6.4.38.zip --output "%SRC_DIR%\temp_build\cudnn-10.1-windows10-x64-v7.6.4.38.zip" 72 | if errorlevel 1 exit /b 1 73 | set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-10.1-windows10-x64-v7.6.4.38.zip" 74 | ) 75 | 76 | goto cuda_common 77 | 78 | :cuda102 79 | 80 | if not exist "%SRC_DIR%\temp_build\cuda_10.2.89_441.22_win10.exe" ( 81 | curl -k -L https://ossci-windows.s3.amazonaws.com/cuda_10.2.89_441.22_win10.exe --output "%SRC_DIR%\temp_build\cuda_10.2.89_441.22_win10.exe" 82 | if errorlevel 1 exit /b 1 83 | set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_10.2.89_441.22_win10.exe" 84 | set "ARGS=nvcc_10.2 cuobjdump_10.2 nvprune_10.2 cupti_10.2 cublas_10.2 cublas_dev_10.2 cudart_10.2 cufft_10.2 cufft_dev_10.2 curand_10.2 curand_dev_10.2 cusolver_10.2 cusolver_dev_10.2 cusparse_10.2 cusparse_dev_10.2 nvgraph_10.2 nvgraph_dev_10.2 npp_10.2 npp_dev_10.2 nvrtc_10.2 nvrtc_dev_10.2 nvml_dev_10.2" 85 | ) 86 | 87 | if not exist "%SRC_DIR%\temp_build\cudnn-10.2-windows10-x64-v7.6.5.32.zip" ( 88 | curl -k -L https://ossci-windows.s3.amazonaws.com/cudnn-10.2-windows10-x64-v7.6.5.32.zip --output "%SRC_DIR%\temp_build\cudnn-10.2-windows10-x64-v7.6.5.32.zip" 89 | if errorlevel 1 exit /b 1 90 | set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-10.2-windows10-x64-v7.6.5.32.zip" 91 | ) 92 | 93 | goto cuda_common 94 | 95 | :cuda110 96 | 97 | if not exist "%SRC_DIR%\temp_build\cuda_11.0.2_451.48_win10.exe" ( 98 | curl -k -L https://ossci-windows.s3.amazonaws.com/cuda_11.0.2_451.48_win10.exe --output "%SRC_DIR%\temp_build\cuda_11.0.2_451.48_win10.exe" 99 | if errorlevel 1 exit /b 1 100 | set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_11.0.2_451.48_win10.exe" 101 | set "ARGS=nvcc_11.0 cuobjdump_11.0 nvprune_11.0 nvprof_11.0 cupti_11.0 cublas_11.0 cublas_dev_11.0 cudart_11.0 cufft_11.0 cufft_dev_11.0 curand_11.0 curand_dev_11.0 cusolver_11.0 cusolver_dev_11.0 cusparse_11.0 cusparse_dev_11.0 npp_11.0 npp_dev_11.0 nvrtc_11.0 nvrtc_dev_11.0 nvml_dev_11.0" 102 | ) 103 | 104 | if not exist "%SRC_DIR%\temp_build\cudnn-11.0-windows-x64-v8.0.4.30.zip" ( 105 | curl -k -L https://ossci-windows.s3.amazonaws.com/cudnn-11.0-windows-x64-v8.0.4.30.zip --output "%SRC_DIR%\temp_build\cudnn-11.0-windows-x64-v8.0.4.30.zip" 106 | if errorlevel 1 exit /b 1 107 | set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-11.0-windows-x64-v8.0.4.30.zip" 108 | ) 109 | 110 | goto cuda_common 111 | 112 | :cuda111 113 | 114 | if not exist "%SRC_DIR%\temp_build\cuda_11.1.0_456.43_win10.exe" ( 115 | curl -k -L https://ossci-windows.s3.amazonaws.com/cuda_11.1.0_456.43_win10.exe --output "%SRC_DIR%\temp_build\cuda_11.1.0_456.43_win10.exe" 116 | if errorlevel 1 exit /b 1 117 | set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_11.1.0_456.43_win10.exe" 118 | set "ARGS=nvcc_11.1 cuobjdump_11.1 nvprune_11.1 nvprof_11.1 cupti_11.1 cublas_11.1 cublas_dev_11.1 cudart_11.1 cufft_11.1 cufft_dev_11.1 curand_11.1 curand_dev_11.1 cusolver_11.1 cusolver_dev_11.1 cusparse_11.1 cusparse_dev_11.1 npp_11.1 npp_dev_11.1 nvrtc_11.1 nvrtc_dev_11.1 nvml_dev_11.1" 119 | ) 120 | 121 | @REM There is no downloadable driver for Tesla on CUDA 11.1 yet. We will use 122 | @REM the driver inside CUDA 123 | if "%JOB_EXECUTOR%" == "windows-with-nvidia-gpu" set "ARGS=%ARGS% Display.Driver" 124 | 125 | if not exist "%SRC_DIR%\temp_build\cudnn-11.1-windows-x64-v8.0.5.39.zip" ( 126 | curl -k -L https://ossci-windows.s3.amazonaws.com/cudnn-11.1-windows-x64-v8.0.5.39.zip --output "%SRC_DIR%\temp_build\cudnn-11.1-windows-x64-v8.0.5.39.zip" 127 | if errorlevel 1 exit /b 1 128 | set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-11.1-windows-x64-v8.0.5.39.zip" 129 | ) 130 | 131 | goto cuda_common 132 | 133 | :cuda112 134 | 135 | if not exist "%SRC_DIR%\temp_build\cuda_11.2.0_460.89_win10.exe" ( 136 | curl -k -L https://ossci-windows.s3.amazonaws.com/cuda_11.2.0_460.89_win10.exe --output "%SRC_DIR%\temp_build\cuda_11.2.0_460.89_win10.exe" 137 | if errorlevel 1 exit /b 1 138 | set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_11.2.0_460.89_win10.exe" 139 | set "ARGS=nvcc_11.2 cuobjdump_11.2 nvprune_11.2 nvprof_11.2 cupti_11.2 cublas_11.2 cublas_dev_11.2 cudart_11.2 cufft_11.2 cufft_dev_11.2 curand_11.2 curand_dev_11.2 cusolver_11.2 cusolver_dev_11.2 cusparse_11.2 cusparse_dev_11.2 npp_11.2 npp_dev_11.2 nvrtc_11.2 nvrtc_dev_11.2 nvml_dev_11.2" 140 | ) 141 | 142 | if not exist "%SRC_DIR%\temp_build\cudnn-11.2-windows-x64-v8.1.0.77.zip" ( 143 | curl -k -L http://s3.amazonaws.com/ossci-windows/cudnn-11.2-windows-x64-v8.1.0.77.zip --output "%SRC_DIR%\temp_build\cudnn-11.2-windows-x64-v8.1.0.77.zip" 144 | if errorlevel 1 exit /b 1 145 | set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-11.2-windows-x64-v8.1.0.77.zip" 146 | ) 147 | 148 | goto cuda_common 149 | 150 | :cuda_common 151 | 152 | if not exist "%SRC_DIR%\temp_build\NvToolsExt.7z" ( 153 | curl -k -L https://www.dropbox.com/s/9mcolalfdj4n979/NvToolsExt.7z?dl=1 --output "%SRC_DIR%\temp_build\NvToolsExt.7z" 154 | if errorlevel 1 exit /b 1 155 | ) 156 | 157 | if not exist "%SRC_DIR%\temp_build\gpu_driver_dlls.7z" ( 158 | curl -k -L "https://drive.google.com/u/0/uc?id=1injUyo3lnarMgWyRcXqKg4UGnN0ysmuq&export=download" --output "%SRC_DIR%\temp_build\gpu_driver_dlls.zip" 159 | if errorlevel 1 exit /b 1 160 | ) 161 | 162 | echo Installing CUDA toolkit... 163 | 7z x %CUDA_SETUP_FILE% -o"%SRC_DIR%\temp_build\cuda" 164 | pushd "%SRC_DIR%\temp_build\cuda" 165 | start /wait setup.exe -s %ARGS% 166 | popd 167 | 168 | echo Installing VS integration... 169 | xcopy /Y "%SRC_DIR%\temp_build\cuda\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\IDE\VC\VCTargets\BuildCustomizations" 170 | 171 | echo Installing NvToolsExt... 172 | 7z x %SRC_DIR%\temp_build\NvToolsExt.7z -o"%SRC_DIR%\temp_build\NvToolsExt" 173 | mkdir "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\bin\x64" 174 | mkdir "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\include" 175 | mkdir "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\lib\x64" 176 | xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\bin\x64\*.*" "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\bin\x64" 177 | xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\include\*.*" "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\include" 178 | xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\lib\x64\*.*" "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\lib\x64" 179 | 180 | echo Setting up environment... 181 | set "PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\libnvvp;%PATH%" 182 | set "CUDA_PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%" 183 | set "CUDA_PATH_V%CUDA_VER_MAJOR%_%CUDA_VER_MINOR%=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%" 184 | set "NVTOOLSEXT_PATH=%ProgramFiles%\NVIDIA Corporation\NvToolsExt\bin\x64" 185 | 186 | if not exist "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin\nvcc.exe" ( 187 | echo CUDA %CUDA_VERSION_STR% installed failed. 188 | exit /b 1 189 | ) 190 | 191 | echo Installing cuDNN... 192 | 7z x %CUDNN_SETUP_FILE% -o"%SRC_DIR%\temp_build\cudnn" 193 | xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\bin\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin" 194 | xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\lib\x64\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\lib\x64" 195 | xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\include\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\include" 196 | 197 | echo Installing GPU driver DLLs 198 | 7z x %SRC_DIR%\temp_build\gpu_driver_dlls.zip -o"C:\Windows\System32" 199 | 200 | echo Cleaning temp files 201 | rd /s /q "%SRC_DIR%\temp_build" || ver > nul 202 | -------------------------------------------------------------------------------- /packaging/windows/internal/dep_install.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | REM curl -k https://www.7-zip.org/a/7z1805-x64.exe -O 4 | REM if errorlevel 1 exit /b 1 5 | 6 | REM start /wait 7z1805-x64.exe /S 7 | REM if errorlevel 1 exit /b 1 8 | 9 | REM set "PATH=%ProgramFiles%\7-Zip;%PATH%" 10 | 11 | choco feature disable --name showDownloadProgress 12 | choco feature enable --name allowGlobalConfirmation 13 | 14 | choco install curl 7zip 15 | -------------------------------------------------------------------------------- /packaging/windows/internal/env_fix.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | :: Caution: Please don't use this script locally 4 | :: It may destroy your build environment. 5 | 6 | setlocal 7 | 8 | IF NOT EXIST "%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" ( 9 | echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows 10 | exit /b 1 11 | ) 12 | 13 | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do ( 14 | if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( 15 | set "VS15INSTALLDIR=%%i" 16 | set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" 17 | goto vswhere 18 | ) 19 | ) 20 | 21 | :vswhere 22 | 23 | IF "%VS15VCVARSALL%"=="" ( 24 | echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows 25 | exit /b 1 26 | ) 27 | 28 | call "%VS15VCVARSALL%" x86_amd64 29 | for /f "usebackq tokens=*" %%i in (`where link.exe`) do move "%%i" "%%i.bak" 30 | 31 | endlocal 32 | -------------------------------------------------------------------------------- /packaging/windows/internal/nightly_defaults.bat: -------------------------------------------------------------------------------- 1 | @echo on 2 | 3 | if "%~1"=="" goto arg_error 4 | if NOT "%~2"=="" goto arg_error 5 | goto arg_end 6 | 7 | :arg_error 8 | 9 | echo Illegal number of parameters. Pass packge type `Conda` or `Wheels`. 10 | exit /b 1 11 | 12 | :arg_end 13 | 14 | echo "nightly_defaults.bat at %CD% starting at %DATE%" 15 | 16 | set SRC_DIR=%~dp0\.. 17 | 18 | :: NIGHTLIES_FOLDER 19 | :: N.B. this is also defined in cron_start.sh 20 | :: An arbitrary root folder to store all nightlies folders, each of which is a 21 | :: parent level date folder with separate subdirs for logs, wheels, conda 22 | :: packages, etc. This should be kept the same across all scripts called in a 23 | :: cron job, so it only has a default value in the top-most script 24 | :: build_cron.sh to avoid the default values from diverging. 25 | if "%NIGHTLIES_FOLDER%" == "" set "NIGHTLIES_FOLDER=%SRC_DIR%" 26 | 27 | :: NIGHTLIES_DATE 28 | :: N.B. this is also defined in cron_start.sh 29 | :: The date in YYYY_mm_dd format that we are building for. If this is not 30 | :: already set, then this will first try to find the date of the nightlies 31 | :: folder that this builder repo exists in; e.g. if this script exists in 32 | :: some_dir/2019_09_04/builder/cron/ then this will be set to 2019_09_04 (must 33 | :: match YYYY_mm_dd). This is for convenience when debugging/uploading past 34 | :: dates, so that you don't have to set NIGHTLIES_DATE yourself. If a date 35 | :: folder cannot be found in that exact location, then this will default to 36 | :: the current date. 37 | 38 | 39 | if "%NIGHTLIES_DATE%" == "" ( goto date_start ) else ( goto date_end ) 40 | 41 | :date_start 42 | 43 | set "DATE_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyy_MM_dd'" 44 | set "DATE_COMPACT_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyyMMdd'" 45 | 46 | FOR /F "delims=" %%i IN ('powershell -c "%DATE_CMD%"') DO set NIGHTLIES_DATE=%%i 47 | FOR /F "delims=" %%i IN ('powershell -c "%DATE_COMPACT_CMD%"') DO set NIGHTLIES_DATE_COMPACT=%%i 48 | 49 | :date_end 50 | 51 | if "%NIGHTLIES_DATE_COMPACT%" == "" set NIGHTLIES_DATE_COMPACT=%NIGHTLIES_DATE:~0,4%%NIGHTLIES_DATE:~5,2%%NIGHTLIES_DATE:~8,2% 52 | 53 | :: Used in lots of places as the root dir to store all conda/wheel/manywheel 54 | :: packages as well as logs for the day 55 | set today=%NIGHTLIES_FOLDER%\%NIGHTLIES_DATE% 56 | mkdir "%today%" || ver >nul 57 | 58 | 59 | ::############################################################################# 60 | :: Add new configuration variables below this line. 'today' should always be 61 | :: defined ASAP to avoid weird errors 62 | ::############################################################################# 63 | 64 | 65 | :: List of people to email when things go wrong. This is passed directly to 66 | :: `mail -t` 67 | :: TODO: Not supported yet 68 | if "%NIGHTLIES_EMAIL_LIST%" == "" set NIGHTLIES_EMAIL_LIST=peterghost86@gmail.com 69 | 70 | :: PYTORCH_CREDENTIALS_FILE 71 | :: A bash file that exports credentials needed to upload to aws and anaconda. 72 | :: Needed variables are PYTORCH_ANACONDA_USERNAME, PYTORCH_ANACONDA_PASSWORD, 73 | :: AWS_ACCESS_KEY_ID, and AWS_SECRET_ACCESS_KEY. Or it can just export the AWS 74 | :: keys and then prepend a logged-in conda installation to the path. 75 | :: TODO: Not supported yet 76 | if "%PYTORCH_CREDENTIALS_FILE%" == "" set PYTORCH_CREDENTIALS_FILE=/c/Users/administrator/nightlies/credentials.sh 77 | 78 | :: Location of the temporary miniconda that is downloaded to install conda-build 79 | :: and aws to upload finished packages TODO this is messy to install this in 80 | :: upload.sh and later use it in upload_logs.sh 81 | if "%CONDA_UPLOADER_INSTALLATION%" == "" set "CONDA_UPLOADER_INSTALLATION=%today%\miniconda" 82 | 83 | :: N.B. BUILDER_REPO and BUILDER_BRANCH are both set in cron_start.sh, as that 84 | :: is the script that actually clones the builder repo that /this/ script is 85 | :: running from. 86 | pushd "%SRC_DIR%\.." 87 | set NIGHTLIES_BUILDER_ROOT=%CD% 88 | popd 89 | 90 | :: The shared pytorch repo to be used by all builds 91 | if "%NIGHTLIES_PYTORCH_ROOT%" == "" set "NIGHTLIES_PYTORCH_ROOT=%today%\csprng" 92 | 93 | :: PYTORCH_REPO 94 | :: The Github org/user whose fork of Pytorch to check out (git clone 95 | :: https://github.com//pytorch.git). This will always be cloned 96 | :: fresh to build with. Default is 'pytorch' 97 | if "%PYTORCH_REPO%" == "" set PYTORCH_REPO=pytorch 98 | 99 | :: PYTORCH_BRANCH 100 | :: The branch of Pytorch to checkout for building (git checkout ). 101 | :: This can either be the name of the branch (e.g. git checkout 102 | :: my_branch_name) or can be a git commit (git checkout 4b2674n...). Default 103 | :: is 'latest', which is a special term that signals to pull the last commit 104 | :: before 0:00 midnight on the NIGHTLIES_DATE 105 | if "%PYTORCH_BRANCH%" == "" set PYTORCH_BRANCH=nightly 106 | 107 | :: Clone the requested pytorch checkout 108 | if exist "%NIGHTLIES_PYTORCH_ROOT%" ( goto clone_end ) else ( goto clone_start ) 109 | 110 | :clone_start 111 | 112 | git clone --recursive "https://github.com/%PYTORCH_REPO%/csprng.git" "%NIGHTLIES_PYTORCH_ROOT%" 113 | pushd "%NIGHTLIES_PYTORCH_ROOT%" 114 | 115 | if "%PYTORCH_BRANCH%" == "latest" ( goto latest_start ) else ( goto latest_end ) 116 | 117 | :latest_start 118 | 119 | :: Switch to the latest commit by 11:59 yesterday 120 | echo PYTORCH_BRANCH is set to latest so I will find the last commit 121 | echo before 0:00 midnight on %NIGHTLIES_DATE% 122 | set git_date=%NIGHTLIES_DATE:_=-% 123 | FOR /F "delims=" %%i IN ('git log --before %git_date% -n 1 "--pretty=%%H"') DO set last_commit=%%i 124 | echo Setting PYTORCH_BRANCH to %last_commit% since that was the last 125 | echo commit before %NIGHTLIES_DATE% 126 | set PYTORCH_BRANCH=%last_commit% 127 | 128 | :latest_end 129 | 130 | git checkout "%PYTORCH_BRANCH%" 131 | git submodule update 132 | popd 133 | 134 | :clone_end 135 | 136 | if "%CUDA_VERSION%" == "cpu" ( 137 | set _DESIRED_CUDA=cpu 138 | ) else ( 139 | set _DESIRED_CUDA=cu%CUDA_VERSION% 140 | ) 141 | 142 | :: PYTORCH_BUILD_VERSION 143 | :: The actual version string. Used in conda like 144 | :: pytorch-nightly==1.0.0.dev20180908 145 | :: or in manylinux like 146 | :: torch_nightly-1.0.0.dev20180908-cp27-cp27m-linux_x86_64.whl 147 | if "%TORCHCSPRNG_BUILD_VERSION%" == "" set TORCHCSPRNG_BUILD_VERSION=0.9.0.dev%NIGHTLIES_DATE_COMPACT% 148 | 149 | if "%~1" == "Wheels" ( 150 | if not "%CUDA_VERSION%" == "102" ( 151 | set TORCHCSPRNG_BUILD_VERSION=%TORCHCSPRNG_BUILD_VERSION%+%_DESIRED_CUDA% 152 | ) 153 | ) 154 | 155 | :: PYTORCH_BUILD_NUMBER 156 | :: This is usually the number 1. If more than one build is uploaded for the 157 | :: same version/date, then this can be incremented to 2,3 etc in which case 158 | :: '.post2' will be appended to the version string of the package. This can 159 | :: be set to '0' only if OVERRIDE_PACKAGE_VERSION is being used to bypass 160 | :: all the version string logic in downstream scripts. Since we use the 161 | :: override below, exporting this shouldn't actually matter. 162 | if "%TORCHCSPRNG_BUILD_NUMBER%" == "" set /a TORCHCSPRNG_BUILD_NUMBER=1 163 | if %TORCHCSPRNG_BUILD_NUMBER% GTR 1 set TORCHCSPRNG_BUILD_VERSION=%TORCHCSPRNG_BUILD_VERSION%%TORCHCSPRNG_BUILD_NUMBER% 164 | 165 | :: The nightly builds use their own versioning logic, so we override whatever 166 | :: logic is in setup.py or other scripts 167 | :: TODO: Not supported yet 168 | set OVERRIDE_PACKAGE_VERSION=%TORCHCSPRNG_BUILD_VERSION% 169 | set BUILD_VERSION=%TORCHCSPRNG_BUILD_VERSION% 170 | 171 | :: Build folder for conda builds to use 172 | if "%TORCH_CONDA_BUILD_FOLDER%" == "" set TORCH_CONDA_BUILD_FOLDER=torchcsprng 173 | 174 | :: TORCH_PACKAGE_NAME 175 | :: The name of the package to upload. This should probably be pytorch or 176 | :: pytorch-nightly. N.B. that pip will change all '-' to '_' but conda will 177 | :: not. This is dealt with in downstream scripts. 178 | :: TODO: Not supported yet 179 | if "%TORCH_PACKAGE_NAME%" == "" set TORCH_PACKAGE_NAME=torchcsprng 180 | 181 | :: PIP_UPLOAD_FOLDER should end in a slash. This is to handle it being empty 182 | :: (when uploading to e.g. whl/cpu/) and also to handle nightlies (when 183 | :: uploading to e.g. /whl/nightly/cpu) 184 | :: TODO: Not supported yet 185 | if "%PIP_UPLOAD_FOLDER%" == "" set "PIP_UPLOAD_FOLDER=nightly\" 186 | 187 | :: The location of the binary_sizes dir in s3 is hardcoded into 188 | :: upload_binary_sizes.sh 189 | 190 | :: DAYS_TO_KEEP 191 | :: How many days to keep around for clean.sh. Build folders older than this 192 | :: will be purged at the end of cron jobs. '1' means to keep only the current 193 | :: day. Values less than 1 are not allowed. The default is 5. 194 | :: TODO: Not supported yet 195 | if "%DAYS_TO_KEEP%" == "" set /a DAYS_TO_KEEP=5 196 | if %DAYS_TO_KEEP% LSS 1 ( 197 | echo DAYS_TO_KEEP cannot be less than 1. 198 | echo A value of 1 means to only keep the build for today 199 | exit /b 1 200 | ) 201 | -------------------------------------------------------------------------------- /packaging/windows/internal/publish.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | set SRC_DIR=%~dp0 4 | pushd %SRC_DIR% 5 | 6 | if NOT "%CUDA_VERSION%" == "cpu" ( 7 | set PACKAGE_SUFFIX=_cuda%CUDA_VERSION% 8 | ) else ( 9 | set PACKAGE_SUFFIX= 10 | ) 11 | 12 | if "%PACKAGEFULLNAME%" == "Conda" ( 13 | set PACKAGE=conda 14 | ) else ( 15 | set PACKAGE=wheels 16 | ) 17 | 18 | if not defined PACKAGE_SUFFIX ( 19 | set PUBLISH_BRANCH=csprng_%PACKAGE%_%DESIRED_PYTHON% 20 | ) else ( 21 | set PUBLISH_BRANCH=csprng_%PACKAGE%_%DESIRED_PYTHON%%PACKAGE_SUFFIX% 22 | ) 23 | 24 | git clone %ARTIFACT_REPO_URL% -b %PUBLISH_BRANCH% --single-branch >nul 2>&1 25 | 26 | IF ERRORLEVEL 1 ( 27 | echo Branch %PUBLISH_BRANCH% not exist, falling back to master 28 | set NO_BRANCH=1 29 | git clone %ARTIFACT_REPO_URL% -b master --single-branch >nul 2>&1 30 | ) 31 | 32 | IF ERRORLEVEL 1 ( 33 | echo Clone failed 34 | goto err 35 | ) 36 | 37 | cd pytorch_builder 38 | attrib -s -h -r . /s /d 39 | 40 | :: Empty repo 41 | rd /s /q . || ver >nul 42 | 43 | IF NOT EXIST %PACKAGE% mkdir %PACKAGE% 44 | 45 | xcopy /S /E /Y ..\..\output\*.* %PACKAGE%\ 46 | 47 | git config --global user.name "Azure DevOps" 48 | git config --global user.email peterghost86@gmail.com 49 | git init 50 | git checkout --orphan %PUBLISH_BRANCH% 51 | git remote add origin %ARTIFACT_REPO_URL% 52 | git add . 53 | git commit -m "Update artifacts" 54 | 55 | :push 56 | 57 | if "%RETRY_TIMES%" == "" ( 58 | set /a RETRY_TIMES=10 59 | set /a SLEEP_TIME=2 60 | ) else ( 61 | set /a RETRY_TIMES=%RETRY_TIMES%-1 62 | set /a SLEEP_TIME=%SLEEP_TIME%*2 63 | ) 64 | 65 | git push origin %PUBLISH_BRANCH% -f > nul 2>&1 66 | 67 | IF ERRORLEVEL 1 ( 68 | echo Git push retry times remaining: %RETRY_TIMES% 69 | echo Sleep time: %SLEEP_TIME% seconds 70 | IF %RETRY_TIMES% EQU 0 ( 71 | echo Push failed 72 | goto err 73 | ) 74 | waitfor SomethingThatIsNeverHappening /t %SLEEP_TIME% 2>nul || ver >nul 75 | goto push 76 | ) ELSE ( 77 | set RETRY_TIMES= 78 | set SLEEP_TIME= 79 | ) 80 | 81 | popd 82 | 83 | exit /b 0 84 | 85 | :err 86 | 87 | popd 88 | 89 | exit /b 1 90 | -------------------------------------------------------------------------------- /packaging/windows/internal/setup.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | echo The flags after configuring: 4 | echo NO_CUDA=%NO_CUDA% 5 | echo CMAKE_GENERATOR=%CMAKE_GENERATOR% 6 | if "%NO_CUDA%"=="" echo CUDA_PATH=%CUDA_PATH% 7 | if NOT "%CC%"=="" echo CC=%CC% 8 | if NOT "%CXX%"=="" echo CXX=%CXX% 9 | if NOT "%DISTUTILS_USE_SDK%"=="" echo DISTUTILS_USE_SDK=%DISTUTILS_USE_SDK% 10 | 11 | set SRC_DIR=%~dp0\.. 12 | 13 | IF "%VSDEVCMD_ARGS%" == "" ( 14 | call "%VS15VCVARSALL%" x64 15 | ) ELSE ( 16 | call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% 17 | ) 18 | 19 | pushd %SRC_DIR% 20 | 21 | IF NOT exist "setup.py" ( 22 | cd %MODULE_NAME% 23 | ) 24 | 25 | if "%CXX%"=="sccache cl" ( 26 | sccache --stop-server 27 | sccache --start-server 28 | sccache --zero-stats 29 | ) 30 | 31 | :pytorch 32 | :: This stores in e.g. D:/_work/1/s/windows/output/cpu 33 | pip wheel -e . --no-deps --wheel-dir ../output 34 | 35 | :build_end 36 | IF ERRORLEVEL 1 exit /b 1 37 | IF NOT ERRORLEVEL 0 exit /b 1 38 | 39 | if "%CXX%"=="sccache cl" ( 40 | taskkill /im sccache.exe /f /t || ver > nul 41 | taskkill /im nvcc.exe /f /t || ver > nul 42 | ) 43 | 44 | cd .. 45 | -------------------------------------------------------------------------------- /packaging/windows/internal/test.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | set SRC_DIR=%~dp0\.. 4 | pushd %SRC_DIR% 5 | 6 | set PYTHON_VERSION=%PYTHON_PREFIX:py=cp% 7 | 8 | if "%BUILD_CSPRNG%" == "" ( 9 | pip install future pytest coverage hypothesis protobuf 10 | ) ELSE ( 11 | pip install future pytest "pillow>=4.1.1" 12 | ) 13 | 14 | for /F "delims=" %%i in ('where /R %SRC_DIR%\output *%MODULE_NAME%*%PYTHON_VERSION%*.whl') do pip install "%%i" 15 | 16 | if ERRORLEVEL 1 exit /b 1 17 | 18 | if NOT "%BUILD_CSPRNG%" == "" ( 19 | echo Smoke testing imports 20 | python -c "import torchcsprng" 21 | if ERRORLEVEL 1 exit /b 1 22 | goto smoke_test_end 23 | ) 24 | 25 | echo Smoke testing imports 26 | python -c "import torch" 27 | if ERRORLEVEL 1 exit /b 1 28 | 29 | python -c "from caffe2.python import core" 30 | if ERRORLEVEL 1 exit /b 1 31 | 32 | echo Checking that MKL is available 33 | python -c "import torch; exit(0 if torch.backends.mkl.is_available() else 1)" 34 | if ERRORLEVEL 1 exit /b 1 35 | 36 | setlocal EnableDelayedExpansion 37 | set NVIDIA_GPU_EXISTS=0 38 | for /F "delims=" %%i in ('wmic path win32_VideoController get name') do ( 39 | set GPUS=%%i 40 | if not "x!GPUS:NVIDIA=!" == "x!GPUS!" ( 41 | SET NVIDIA_GPU_EXISTS=1 42 | goto gpu_check_end 43 | ) 44 | ) 45 | :gpu_check_end 46 | endlocal & set NVIDIA_GPU_EXISTS=%NVIDIA_GPU_EXISTS% 47 | 48 | if NOT "%CUDA_PREFIX%" == "cpu" if "%NVIDIA_GPU_EXISTS%" == "1" ( 49 | echo Checking that CUDA archs are setup correctly 50 | python -c "import torch; torch.randn([3,5]).cuda()" 51 | if ERRORLEVEL 1 exit /b 1 52 | 53 | echo Checking that magma is available 54 | python -c "import torch; torch.rand(1).cuda(); exit(0 if torch.cuda.has_magma else 1)" 55 | if ERRORLEVEL 1 exit /b 1 56 | 57 | echo Checking that CuDNN is available 58 | python -c "import torch; exit(0 if torch.backends.cudnn.is_available() else 1)" 59 | if ERRORLEVEL 1 exit /b 1 60 | ) 61 | :smoke_test_end 62 | 63 | echo Not running unit tests. Hopefully these problems are caught by CI 64 | goto test_end 65 | 66 | if "%BUILD_CSPRNG%" == "" ( 67 | cd pytorch\test 68 | python run_test.py -v 69 | ) else ( 70 | cd csprng 71 | pytest . 72 | ) 73 | 74 | if ERRORLEVEL 1 exit /b 1 75 | 76 | :test_end 77 | 78 | popd 79 | exit /b 0 80 | -------------------------------------------------------------------------------- /packaging/windows/internal/upload.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | IF "%CONDA_UPLOADER_INSTALLATION%" == "" goto precheck_fail 4 | IF "%PYTORCH_FINAL_PACKAGE_DIR%" == "" goto precheck_fail 5 | IF "%today%" == "" goto precheck_fail 6 | IF "%PYTORCH_ANACONDA_USERNAME%" == "" goto precheck_fail 7 | IF "%PYTORCH_ANACONDA_PASSWORD%" == "" goto precheck_fail 8 | 9 | goto precheck_pass 10 | 11 | :precheck_fail 12 | 13 | echo Please run nightly_defaults.bat first. 14 | echo And remember to set `PYTORCH_FINAL_PACKAGE_DIR` 15 | echo Finally, don't forget to set anaconda tokens 16 | exit /b 1 17 | 18 | :precheck_pass 19 | 20 | pushd %today% 21 | 22 | :: Install anaconda client 23 | set "CONDA_HOME=%CONDA_UPLOADER_INSTALLATION%" 24 | set "tmp_conda=%CONDA_HOME%" 25 | set "miniconda_exe=%CD%\miniconda.exe" 26 | rmdir /s /q "%CONDA_HOME%" 27 | del miniconda.exe 28 | curl -k https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "%miniconda_exe%" 29 | popd 30 | 31 | IF ERRORLEVEL 1 ( 32 | echo Conda download failed 33 | exit /b 1 34 | ) 35 | 36 | call %~dp0\..\..\conda\install_conda.bat 37 | 38 | IF ERRORLEVEL 1 ( 39 | echo Conda installation failed 40 | exit /b 1 41 | ) 42 | 43 | set "ORIG_PATH=%PATH%" 44 | set "PATH=%CONDA_HOME%;%CONDA_HOME%\scripts;%CONDA_HOME%\Library\bin;%PATH%" 45 | 46 | REM conda install -y anaconda-client 47 | pip install git+https://github.com/peterjc123/anaconda-client.git@log_more_meaningfull_errors 48 | IF ERRORLEVEL 1 ( 49 | echo Anaconda client installation failed 50 | exit /b 1 51 | ) 52 | 53 | set PYTORCH_FINAL_PACKAGE= 54 | :: Upload all the packages under `PYTORCH_FINAL_PACKAGE_DIR` 55 | FOR /F "delims=" %%i IN ('where /R %PYTORCH_FINAL_PACKAGE_DIR% *csprng*.tar.bz2') DO ( 56 | set "PYTORCH_FINAL_PACKAGE=%%i" 57 | ) 58 | 59 | IF "%PYTORCH_FINAL_PACKAGE%" == "" ( 60 | echo No package to upload 61 | exit /b 0 62 | ) 63 | 64 | :upload 65 | 66 | if "%RETRY_TIMES%" == "" ( 67 | set /a RETRY_TIMES=10 68 | set /a SLEEP_TIME=2 69 | ) else ( 70 | set /a RETRY_TIMES=%RETRY_TIMES%-1 71 | set /a SLEEP_TIME=%SLEEP_TIME%*2 72 | ) 73 | 74 | REM bash -c "yes | anaconda login --username "%PYTORCH_ANACONDA_USERNAME%" --password "%PYTORCH_ANACONDA_PASSWORD%"" 75 | anaconda login --username "%PYTORCH_ANACONDA_USERNAME%" --password "%PYTORCH_ANACONDA_PASSWORD%" 76 | IF ERRORLEVEL 1 ( 77 | echo Anaconda client login failed 78 | exit /b 1 79 | ) 80 | 81 | echo Uploading %PYTORCH_FINAL_PACKAGE% to Anaconda Cloud 82 | anaconda upload "%PYTORCH_FINAL_PACKAGE%" -u pytorch-nightly --label main --force --no-progress 83 | 84 | IF ERRORLEVEL 1 ( 85 | echo Anaconda upload retry times remaining: %RETRY_TIMES% 86 | echo Sleep time: %SLEEP_TIME% seconds 87 | IF %RETRY_TIMES% EQU 0 ( 88 | echo Upload failed 89 | exit /b 1 90 | ) 91 | waitfor SomethingThatIsNeverHappening /t %SLEEP_TIME% 2>nul || ver >nul 92 | goto upload 93 | ) ELSE ( 94 | set RETRY_TIMES= 95 | set SLEEP_TIME= 96 | ) 97 | -------------------------------------------------------------------------------- /packaging/windows/internal/vc_env_helper.bat: -------------------------------------------------------------------------------- 1 | @echo on 2 | 3 | set VC_VERSION_LOWER=16 4 | set VC_VERSION_UPPER=17 5 | if "%VC_YEAR%" == "2017" ( 6 | set VC_VERSION_LOWER=15 7 | set VC_VERSION_UPPER=16 8 | ) 9 | 10 | for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do ( 11 | if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( 12 | set "VS15INSTALLDIR=%%i" 13 | set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" 14 | goto vswhere 15 | ) 16 | ) 17 | 18 | :vswhere 19 | if "%VSDEVCMD_ARGS%" == "" ( 20 | call "%VS15VCVARSALL%" x64 || exit /b 1 21 | ) else ( 22 | call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1 23 | ) 24 | 25 | @echo on 26 | 27 | set DISTUTILS_USE_SDK=1 28 | 29 | set args=%1 30 | shift 31 | :start 32 | if [%1] == [] goto done 33 | set args=%args% %1 34 | shift 35 | goto start 36 | 37 | :done 38 | if "%args%" == "" ( 39 | echo Usage: vc_env_helper.bat [command] [args] 40 | echo e.g. vc_env_helper.bat cl /c test.cpp 41 | ) 42 | 43 | %args% || exit /b 1 44 | -------------------------------------------------------------------------------- /packaging/windows/internal/vc_install_helper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | if [[ "$CU_VERSION" == "cu92" ]]; then 6 | export VC_YEAR=2017 7 | export VSDEVCMD_ARGS="-vcvars_ver=14.13" 8 | powershell packaging/windows/internal/vs2017_install.ps1 9 | elif [[ "$CU_VERSION" == "cu100" ]]; then 10 | export VC_YEAR=2017 11 | export VSDEVCMD_ARGS="" 12 | powershell packaging/windows/internal/vs2017_install.ps1 13 | else 14 | export VC_YEAR=2019 15 | export VSDEVCMD_ARGS="" 16 | fi 17 | -------------------------------------------------------------------------------- /packaging/windows/internal/vs2017_install.ps1: -------------------------------------------------------------------------------- 1 | $VS_DOWNLOAD_LINK = "https://aka.ms/vs/15/release/vs_buildtools.exe" 2 | $VS_INSTALL_ARGS = @("--nocache","--quiet","--wait", "--add Microsoft.VisualStudio.Workload.VCTools", 3 | "--add Microsoft.VisualStudio.Component.VC.Tools.14.13", 4 | "--add Microsoft.Component.MSBuild", 5 | "--add Microsoft.VisualStudio.Component.Roslyn.Compiler", 6 | "--add Microsoft.VisualStudio.Component.TextTemplating", 7 | "--add Microsoft.VisualStudio.Component.VC.CoreIde", 8 | "--add Microsoft.VisualStudio.Component.VC.Redist.14.Latest", 9 | "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core", 10 | "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64", 11 | "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81") 12 | 13 | curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe 14 | if ($LASTEXITCODE -ne 0) { 15 | echo "Download of the VS 2017 installer failed" 16 | exit 1 17 | } 18 | 19 | $process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru 20 | Remove-Item -Path vs_installer.exe -Force 21 | $exitCode = $process.ExitCode 22 | if (($exitCode -ne 0) -and ($exitCode -ne 3010)) { 23 | echo "VS 2017 installer exited with code $exitCode, which should be one of [0, 3010]." 24 | exit 1 25 | } 26 | -------------------------------------------------------------------------------- /packaging/windows/internal/vs2019_install.ps1: -------------------------------------------------------------------------------- 1 | $VS_DOWNLOAD_LINK = "https://aka.ms/vs/16/release/vs_buildtools.exe" 2 | $VS_INSTALL_ARGS = @("--nocache","--quiet","--wait", "--add Microsoft.VisualStudio.Workload.VCTools", 3 | "--add Microsoft.Component.MSBuild", 4 | "--add Microsoft.VisualStudio.Component.Roslyn.Compiler", 5 | "--add Microsoft.VisualStudio.Component.VC.CoreBuildTools", 6 | "--add Microsoft.VisualStudio.Component.VC.Redist.14.Latest", 7 | "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64") 8 | 9 | curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe 10 | if ($LASTEXITCODE -ne 0) { 11 | echo "Download of the VS 2019 installer failed" 12 | exit 1 13 | } 14 | 15 | $process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru 16 | Remove-Item -Path vs_installer.exe -Force 17 | $exitCode = $process.ExitCode 18 | if (($exitCode -ne 0) -and ($exitCode -ne 3010)) { 19 | echo "VS 2019 installer exited with code $exitCode, which should be one of [0, 3010]." 20 | exit 1 21 | } 22 | -------------------------------------------------------------------------------- /packaging/windows/internal/vs_install.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | set VS_DOWNLOAD_LINK=https://aka.ms/vs/15/release/vs_enterprise.exe 4 | set VS_INSTALL_PATH=C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise 5 | set VS_INSTALL_ARGS=--nocache --quiet --wait --add Microsoft.VisualStudio.Component.VC.Tools.14.11 6 | set VSDEVCMD_ARGS=-vcvars_ver=14.11 7 | 8 | curl -k -L %VS_DOWNLOAD_LINK% --output vs_installer.exe 9 | if errorlevel 1 exit /b 1 10 | 11 | start /wait vs_installer.exe modify --installPath "%VS_INSTALL_PATH%" %VS_INSTALL_ARGS% 12 | if not errorlevel 0 exit /b 1 13 | if errorlevel 1 if not errorlevel 3010 exit /b 1 14 | if errorlevel 3011 exit /b 1 15 | -------------------------------------------------------------------------------- /packaging/windows/old/cuda100.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | IF NOT "%BUILD_CSPRNG%" == "" ( 4 | set MODULE_NAME=csprng 5 | ) ELSE ( 6 | set MODULE_NAME=pytorch 7 | ) 8 | 9 | IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" ( 10 | call internal\clone.bat 11 | cd .. 12 | IF ERRORLEVEL 1 goto eof 13 | ) ELSE ( 14 | call internal\clean.bat 15 | ) 16 | 17 | call internal\check_deps.bat 18 | IF ERRORLEVEL 1 goto eof 19 | 20 | REM Check for optional components 21 | 22 | set NO_CUDA= 23 | set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 24 | 25 | IF "%NVTOOLSEXT_PATH%"=="" ( 26 | echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing 27 | exit /b 1 28 | goto optcheck 29 | ) 30 | 31 | IF "%CUDA_PATH_V10_0%"=="" ( 32 | echo CUDA 10.0 not found, failing 33 | exit /b 1 34 | ) ELSE ( 35 | IF "%BUILD_CSPRNG%" == "" ( 36 | set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0;7.5 37 | set TORCH_NVCC_FLAGS=-Xfatbin -compress-all 38 | ) ELSE ( 39 | set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50 40 | ) 41 | 42 | set "CUDA_PATH=%CUDA_PATH_V10_0%" 43 | set "PATH=%CUDA_PATH_V10_0%\bin;%PATH%" 44 | ) 45 | 46 | :optcheck 47 | 48 | IF "%BUILD_CSPRNG%" == "" ( 49 | call internal\check_opts.bat 50 | IF ERRORLEVEL 1 goto eof 51 | 52 | call internal\copy.bat 53 | IF ERRORLEVEL 1 goto eof 54 | ) 55 | 56 | call internal\setup.bat 57 | IF ERRORLEVEL 1 goto eof 58 | 59 | :eof 60 | -------------------------------------------------------------------------------- /packaging/windows/old/cuda90.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | IF NOT "%BUILD_CSPRNG%" == "" ( 4 | set MODULE_NAME=csprng 5 | ) ELSE ( 6 | set MODULE_NAME=pytorch 7 | ) 8 | 9 | IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" ( 10 | call internal\clone.bat 11 | cd .. 12 | IF ERRORLEVEL 1 goto eof 13 | ) ELSE ( 14 | call internal\clean.bat 15 | ) 16 | 17 | call internal\check_deps.bat 18 | IF ERRORLEVEL 1 goto eof 19 | 20 | REM Check for optional components 21 | 22 | set NO_CUDA= 23 | set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 24 | 25 | IF "%NVTOOLSEXT_PATH%"=="" ( 26 | echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing 27 | exit /b 1 28 | goto optcheck 29 | ) 30 | 31 | IF "%CUDA_PATH_V9_0%"=="" ( 32 | echo CUDA 9 not found, failing 33 | exit /b 1 34 | ) ELSE ( 35 | IF "%BUILD_CSPRNG%" == "" ( 36 | set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;7.0 37 | set TORCH_NVCC_FLAGS=-Xfatbin -compress-all 38 | ) ELSE ( 39 | set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50 40 | ) 41 | 42 | set "CUDA_PATH=%CUDA_PATH_V9_0%" 43 | set "PATH=%CUDA_PATH_V9_0%\bin;%PATH%" 44 | ) 45 | 46 | :optcheck 47 | 48 | IF "%BUILD_CSPRNG%" == "" ( 49 | call internal\check_opts.bat 50 | IF ERRORLEVEL 1 goto eof 51 | 52 | call internal\copy.bat 53 | IF ERRORLEVEL 1 goto eof 54 | ) 55 | 56 | call internal\setup.bat 57 | IF ERRORLEVEL 1 goto eof 58 | 59 | :eof 60 | -------------------------------------------------------------------------------- /packaging/windows/templates/auth_task.yml: -------------------------------------------------------------------------------- 1 | jobs: 2 | - job: 'VSTS_Auth_Task' 3 | timeoutInMinutes: 5 4 | cancelTimeoutInMinutes: 5 5 | variables: 6 | - group: 'peterjc-vsts-token' 7 | 8 | pool: 9 | vmImage: 'vs2017-win2016' 10 | 11 | steps: 12 | - checkout: self 13 | clean: true 14 | 15 | - template: vsts_auth.yml 16 | parameters: 17 | auth: $(vsts_auth) 18 | -------------------------------------------------------------------------------- /packaging/windows/templates/build_conda.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | msagent: false 3 | 4 | steps: 5 | - bash: 'find . -name "*.sh" -exec dos2unix {} +' 6 | displayName: Replace file endings 7 | 8 | - script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%' 9 | displayName: 'Create final package directory' 10 | 11 | - bash: './packaging/conda/build_csprng.sh $CUDA_VERSION $TORCHCSPRNG_BUILD_VERSION $TORCHCSPRNG_BUILD_NUMBER' 12 | displayName: Build 13 | env: 14 | ${{ if eq(parameters.msagent, 'true') }}: 15 | MAX_JOBS: 2 16 | -------------------------------------------------------------------------------- /packaging/windows/templates/build_task.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | package: '' 3 | spec: '' 4 | jobDesc: '' 5 | packageDesc: '' 6 | msagent: true 7 | cpuEnabled: true 8 | cudaEnabled: true 9 | condaEnabled: true 10 | wheelsEnabled: true 11 | override: false 12 | 13 | jobs: 14 | - job: 'Windows_${{ parameters.spec }}_${{ parameters.package }}_Build' 15 | timeoutInMinutes: 60 16 | cancelTimeoutInMinutes: 5 17 | condition: > 18 | or(and(eq('${{ parameters.package }}', 'Conda'), eq('${{ parameters.spec }}', 'CPU'), 19 | eq('${{ parameters.condaEnabled }}', 'true'), eq('${{ parameters.cpuEnabled }}', 'true')), 20 | and(eq('${{ parameters.package }}', 'Wheels'), eq('${{ parameters.spec }}', 'CPU'), 21 | eq('${{ parameters.wheelsEnabled }}', 'true'), eq('${{ parameters.cpuEnabled }}', 'true')), 22 | and(eq('${{ parameters.package }}', 'Conda'), eq('${{ parameters.spec }}', 'CUDA'), 23 | eq('${{ parameters.condaEnabled }}', 'true'), eq('${{ parameters.cudaEnabled }}', 'true')), 24 | and(eq('${{ parameters.package }}', 'Wheels'), eq('${{ parameters.spec }}', 'CUDA'), 25 | eq('${{ parameters.wheelsEnabled }}', 'true'), eq('${{ parameters.cudaEnabled }}', 'true'))) 26 | variables: 27 | - ${{ if eq(parameters.override, 'true') }}: 28 | - name: TORCHCSPRNG_BUILD_NUMBER 29 | value: 1 30 | - name: PYTORCH_REPO 31 | value: 'pytorch' 32 | - name: PYTORCH_BRANCH 33 | value: 'v0.4.0' 34 | - ${{ if eq(parameters.msagent, 'true') }}: 35 | - name: USE_SCCACHE 36 | value: 0 37 | - ${{ if eq(parameters.msagent, 'false') }}: 38 | - name: USE_SCCACHE 39 | value: 1 40 | - ${{ if eq(parameters.package, 'Conda') }}: 41 | - group: peterjc_anaconda_token 42 | - name: PYTORCH_FINAL_PACKAGE_DIR 43 | value: '$(Build.Repository.LocalPath)\packaging\windows\output' 44 | 45 | strategy: 46 | maxParallel: 10 47 | matrix: 48 | ${{ if eq(parameters.spec, 'CPU') }}: 49 | PY3.5: 50 | DESIRED_PYTHON: 3.5 51 | CUDA_VERSION: cpu 52 | PY3.6: 53 | DESIRED_PYTHON: 3.6 54 | CUDA_VERSION: cpu 55 | PY3.7: 56 | DESIRED_PYTHON: 3.7 57 | CUDA_VERSION: cpu 58 | PY3.8: 59 | DESIRED_PYTHON: 3.8 60 | CUDA_VERSION: cpu 61 | PY3.9: 62 | DESIRED_PYTHON: 3.9 63 | CUDA_VERSION: cpu 64 | ${{ if ne(parameters.spec, 'CPU') }}: 65 | PY3.5_92: 66 | DESIRED_PYTHON: 3.5 67 | CUDA_VERSION: 92 68 | PY3.6_92: 69 | DESIRED_PYTHON: 3.6 70 | CUDA_VERSION: 92 71 | PY3.7_92: 72 | DESIRED_PYTHON: 3.7 73 | CUDA_VERSION: 92 74 | PY3.8_92: 75 | DESIRED_PYTHON: 3.8 76 | CUDA_VERSION: 92 77 | PY3.9_92: 78 | DESIRED_PYTHON: 3.9 79 | CUDA_VERSION: 92 80 | PY3.5_101: 81 | DESIRED_PYTHON: 3.5 82 | CUDA_VERSION: 101 83 | PY3.6_101: 84 | DESIRED_PYTHON: 3.6 85 | CUDA_VERSION: 101 86 | PY3.7_101: 87 | DESIRED_PYTHON: 3.7 88 | CUDA_VERSION: 101 89 | PY3.8_101: 90 | DESIRED_PYTHON: 3.8 91 | CUDA_VERSION: 101 92 | PY3.9_101: 93 | DESIRED_PYTHON: 3.9 94 | CUDA_VERSION: 101 95 | PY3.5_102: 96 | DESIRED_PYTHON: 3.5 97 | CUDA_VERSION: 102 98 | PY3.6_102: 99 | DESIRED_PYTHON: 3.6 100 | CUDA_VERSION: 102 101 | PY3.7_102: 102 | DESIRED_PYTHON: 3.7 103 | CUDA_VERSION: 102 104 | PY3.8_102: 105 | DESIRED_PYTHON: 3.8 106 | CUDA_VERSION: 102 107 | PY3.9_102: 108 | DESIRED_PYTHON: 3.9 109 | CUDA_VERSION: 102 110 | 111 | pool: 112 | ${{ if eq(parameters.msagent, 'true') }}: 113 | vmImage: 'vs2017-win2016' 114 | ${{ if eq(parameters.msagent, 'false') }}: 115 | name: 'release' 116 | 117 | steps: 118 | - checkout: self 119 | clean: true 120 | 121 | - template: setup_env_for_msagent.yml 122 | parameters: 123 | msagent: ${{ parameters.msagent }} 124 | 125 | # - ${{ if and(eq(parameters.override, 'true'), eq(parameters.package, 'Wheels')) }}: 126 | # - template: override_pytorch_version.yml 127 | 128 | - template: setup_nightly_variables.yml 129 | parameters: 130 | package: ${{ parameters.package }} 131 | 132 | - ${{ if eq(parameters.package, 'Wheels') }}: 133 | - template: build_wheels.yml 134 | parameters: 135 | msagent: ${{ parameters.msagent }} 136 | 137 | - ${{ if eq(parameters.package, 'Conda') }}: 138 | - template: build_conda.yml 139 | parameters: 140 | msagent: ${{ parameters.msagent }} 141 | 142 | - ${{ if or(eq(parameters.package, 'Wheels'), eq(parameters.package, 'Conda')) }}: 143 | - template: publish_test_results.yml 144 | parameters: 145 | msagent: ${{ parameters.msagent }} 146 | 147 | # If you want to upload binaries to S3 & Anaconda Cloud, please uncomment this section. 148 | - ${{ if and(eq(parameters.package, 'Wheels'), eq(parameters.spec, 'CPU')) }}: 149 | - template: upload_to_s3.yml 150 | parameters: 151 | cuVer: '$(CUDA_VERSION)' 152 | cudaVer: '$(CUDA_VERSION)' 153 | 154 | - ${{ if and(eq(parameters.package, 'Wheels'), ne(parameters.spec, 'CPU')) }}: 155 | - template: upload_to_s3.yml 156 | parameters: 157 | cuVer: 'cu$(CUDA_VERSION)' 158 | cudaVer: 'cuda$(CUDA_VERSION)' 159 | 160 | - ${{ if eq(parameters.package, 'Conda') }}: 161 | - template: upload_to_conda.yml 162 | parameters: 163 | user: $(peterjc_conda_username) 164 | pass: $(peterjc_conda_password) 165 | 166 | # If you want to upload binaries to Azure Git, please uncomment this section. 167 | # - ${{ if or(eq(parameters.package, 'Wheels'), eq(parameters.package, 'Conda')) }}: 168 | # - template: publish_test_results.yml 169 | # parameters: 170 | # msagent: ${{ parameters.msagent }} 171 | # - template: publish_packages.yml 172 | # parameters: 173 | # package: ${{ parameters.package }} 174 | -------------------------------------------------------------------------------- /packaging/windows/templates/build_wheels.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | msagent: false 3 | 4 | steps: 5 | - script: 'call packaging/windows/build_csprng.bat %CUDA_VERSION% %TORCHCSPRNG_BUILD_VERSION% %TORCHCSPRNG_BUILD_NUMBER%' 6 | displayName: Build 7 | env: 8 | ${{ if eq(parameters.msagent, 'true') }}: 9 | MAX_JOBS: 2 10 | -------------------------------------------------------------------------------- /packaging/windows/templates/linux_build_task.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | msagent: true 3 | enabled: false 4 | 5 | jobs: 6 | - job: 'Linux_CPU_Conda_Build' 7 | timeoutInMinutes: 0 8 | cancelTimeoutInMinutes: 5 9 | condition: ${{ eq(parameters.enabled, 'true') }} 10 | variables: 11 | CUDA_VERSION: cpu 12 | TORCH_CONDA_BUILD_FOLDER: pytorch-nightly 13 | PYTORCH_FINAL_PACKAGE_DIR: '$(Build.Repository.LocalPath)/output' 14 | 15 | strategy: 16 | maxParallel: 10 17 | matrix: 18 | PY3.5: 19 | DESIRED_PYTHON: 3.5 20 | 21 | pool: 22 | vmImage: 'ubuntu-16.04' 23 | 24 | steps: 25 | - checkout: self 26 | clean: true 27 | 28 | - script: 'sudo apt-get install p7zip-full' 29 | displayName: 'Install 7Zip' 30 | 31 | - task: CondaEnvironment@1 32 | displayName: 'Install conda-build' 33 | inputs: 34 | packageSpecs: 'conda-build' 35 | 36 | - template: build_conda.yml 37 | parameters: 38 | msagent: ${{ parameters.msagent }} 39 | -------------------------------------------------------------------------------- /packaging/windows/templates/override_pytorch_version.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - script: 'windows/internal/override_pytorch_version.bat' 3 | displayName: 'Override PyTorch Build Version for Wheels' 4 | 5 | - script: 'echo $(PYTORCH_BUILD_VERSION)' 6 | displayName: 'Show PyTorch Build Version' 7 | -------------------------------------------------------------------------------- /packaging/windows/templates/publish_packages.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | package: '' 3 | 4 | steps: 5 | - script: 'packaging/windows/internal/publish.bat' 6 | displayName: 'Upload packages to Azure DevOps Repo' 7 | env: 8 | PACKAGEFULLNAME: ${{ parameters.package }} 9 | -------------------------------------------------------------------------------- /packaging/windows/templates/publish_test_results.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - task: PublishTestResults@2 # No test results to publish 3 | inputs: 4 | testResultsFiles: 'windows/pytorch/test/**/*.xml' 5 | testRunTitle: 'Publish test results' 6 | enabled: false 7 | -------------------------------------------------------------------------------- /packaging/windows/templates/setup_env_for_msagent.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | msagent: false 3 | 4 | steps: 5 | - ${{ if eq(parameters.msagent, 'true') }}: 6 | - task: BatchScript@1 7 | displayName: 'Install 7Zip & cURL' 8 | inputs: 9 | filename: 'packaging/windows/internal/dep_install.bat' 10 | 11 | modifyEnvironment: true 12 | 13 | - task: BatchScript@1 14 | displayName: 'Install Visual Studio 2017' 15 | inputs: 16 | filename: 'packaging/windows/internal/vs_install.bat' 17 | 18 | modifyEnvironment: true 19 | 20 | - task: BatchScript@1 21 | displayName: 'Install CUDA' 22 | inputs: 23 | filename: 'packaging/windows/internal/cuda_install.bat' 24 | 25 | modifyEnvironment: true 26 | -------------------------------------------------------------------------------- /packaging/windows/templates/setup_nightly_variables.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | package: '' 3 | 4 | steps: 5 | - task: BatchScript@1 6 | displayName: 'Setup nightly variables' 7 | inputs: 8 | filename: 'packaging/windows/internal/nightly_defaults.bat' 9 | arguments: ${{ parameters.package }} 10 | 11 | modifyEnvironment: true 12 | -------------------------------------------------------------------------------- /packaging/windows/templates/upload_to_conda.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | user: '' 3 | pass: '' 4 | 5 | steps: 6 | - script: 'call packaging/windows/internal/upload.bat' 7 | displayName: 'Upload packages to Anaconda Cloud' 8 | env: 9 | PYTORCH_ANACONDA_USERNAME: ${{ parameters.user }} 10 | PYTORCH_ANACONDA_PASSWORD: ${{ parameters.pass }} 11 | -------------------------------------------------------------------------------- /packaging/windows/templates/upload_to_s3.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | cuVer: '' 3 | cudaVer: '' 4 | 5 | steps: 6 | - task: AmazonWebServices.aws-vsts-tools.S3Upload.S3Upload@1 7 | displayName: 'Upload ${{ parameters.cuVer }} wheel to S3' 8 | inputs: 9 | awsCredentials: 'Pytorch S3 bucket' 10 | bucketName: 'pytorch' 11 | sourceFolder: 'packaging/windows/output' 12 | globExpressions: '*.whl' 13 | targetFolder: 'whl/nightly/${{ parameters.cuVer }}/' 14 | filesAcl: 'public-read' 15 | flattenFolders: 'true' 16 | -------------------------------------------------------------------------------- /packaging/windows/templates/vsts_auth.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | auth: '' 3 | 4 | steps: 5 | - script: 'call packaging/windows/internal/auth.bat' 6 | displayName: 'Sign in to Azure Pipelines' 7 | env: 8 | VSTS_AUTH: ${{ parameters.auth }} 9 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import distutils.command.clean 2 | import glob 3 | import os 4 | import shutil 5 | import subprocess 6 | import sys 7 | 8 | import torch 9 | from setuptools import find_packages, setup 10 | from torch.utils.cpp_extension import BuildExtension, CppExtension, CUDAExtension 11 | 12 | version = open("version.txt", "r").read().strip() 13 | sha = "Unknown" 14 | package_name = "torchcsprng" 15 | 16 | cwd = os.path.dirname(os.path.abspath(__file__)) 17 | 18 | try: 19 | sha = ( 20 | subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=cwd) 21 | .decode("ascii") 22 | .strip() 23 | ) 24 | except Exception: 25 | pass 26 | 27 | if os.getenv("BUILD_VERSION"): 28 | version = os.getenv("BUILD_VERSION") 29 | elif sha != "Unknown": 30 | version += "+" + sha[:7] 31 | print("Building wheel {}-{}".format(package_name, version)) 32 | 33 | 34 | def write_version_file(): 35 | version_path = os.path.join(cwd, "torchcsprng", "version.py") 36 | with open(version_path, "w") as f: 37 | f.write("__version__ = '{}'\n".format(version)) 38 | f.write("git_version = {}\n".format(repr(sha))) 39 | # f.write("from torchcsprng.extension import _check_cuda_version\n") 40 | # f.write("if _check_cuda_version() > 0:\n") 41 | # f.write(" cuda = _check_cuda_version()\n") 42 | 43 | 44 | write_version_file() 45 | 46 | with open("README.md", "r") as fh: 47 | long_description = fh.read() 48 | 49 | 50 | requirements = [ 51 | "torch", 52 | ] 53 | 54 | 55 | def append_flags(flags, flags_to_append): 56 | for flag in flags_to_append: 57 | if not flag in flags: 58 | flags.append(flag) 59 | return flags 60 | 61 | 62 | def get_extensions(): 63 | build_cuda = torch.cuda.is_available() or os.getenv("FORCE_CUDA", "0") == "1" 64 | 65 | module_name = "torchcsprng" 66 | 67 | extensions_dir = os.path.join(cwd, module_name, "csrc") 68 | 69 | openmp = "ATen parallel backend: OpenMP" in torch.__config__.parallel_info() 70 | 71 | main_file = glob.glob(os.path.join(extensions_dir, "*.cpp")) 72 | source_cpu = glob.glob(os.path.join(extensions_dir, "cpu", "*.cpp")) 73 | 74 | sources = main_file + source_cpu 75 | extension = CppExtension 76 | 77 | define_macros = [] 78 | 79 | cxx_flags = os.getenv("CXX_FLAGS", "") 80 | if cxx_flags == "": 81 | cxx_flags = [] 82 | else: 83 | cxx_flags = cxx_flags.split(" ") 84 | if openmp: 85 | if sys.platform == "linux": 86 | cxx_flags = append_flags(cxx_flags, ["-fopenmp"]) 87 | elif sys.platform == "win32": 88 | cxx_flags = append_flags(cxx_flags, ["/openmp"]) 89 | # elif sys.platform == 'darwin': 90 | # cxx_flags = append_flags(cxx_flags, ['-Xpreprocessor', '-fopenmp']) 91 | 92 | if build_cuda: 93 | extension = CUDAExtension 94 | source_cuda = glob.glob(os.path.join(extensions_dir, "cuda", "*.cu")) 95 | sources += source_cuda 96 | 97 | define_macros += [("WITH_CUDA", None)] 98 | 99 | nvcc_flags = os.getenv("NVCC_FLAGS", "") 100 | if nvcc_flags == "": 101 | nvcc_flags = [] 102 | else: 103 | nvcc_flags = nvcc_flags.split(" ") 104 | nvcc_flags = append_flags(nvcc_flags, ["--expt-extended-lambda", "-Xcompiler"]) 105 | extra_compile_args = { 106 | "cxx": cxx_flags, 107 | "nvcc": nvcc_flags, 108 | } 109 | else: 110 | extra_compile_args = { 111 | "cxx": cxx_flags, 112 | } 113 | 114 | ext_modules = [ 115 | extension( 116 | module_name + "._C", 117 | sources, 118 | define_macros=define_macros, 119 | extra_compile_args=extra_compile_args, 120 | ) 121 | ] 122 | 123 | return ext_modules 124 | 125 | 126 | class clean(distutils.command.clean.clean): 127 | def run(self): 128 | with open(".gitignore", "r") as f: 129 | ignores = f.read() 130 | start_deleting = False 131 | for wildcard in filter(None, ignores.split("\n")): 132 | if ( 133 | wildcard 134 | == "# do not change or delete this comment - `python setup.py clean` deletes everything after this line" 135 | ): 136 | start_deleting = True 137 | if not start_deleting: 138 | continue 139 | for filename in glob.glob(wildcard): 140 | try: 141 | os.remove(filename) 142 | except OSError: 143 | shutil.rmtree(filename, ignore_errors=True) 144 | 145 | # It's an old-style class in Python 2.7... 146 | distutils.command.clean.clean.run(self) 147 | 148 | 149 | setup( 150 | # Metadata 151 | name=package_name, 152 | version=version, 153 | author="Pavel Belevich", 154 | author_email="pbelevich@fb.com", 155 | url="https://github.com/pytorch/csprng", 156 | description="Cryptographically secure pseudorandom number generators for PyTorch", 157 | long_description=long_description, 158 | long_description_content_type="text/markdown", 159 | license="BSD-3", 160 | # Package info 161 | packages=find_packages(exclude=("test",)), 162 | classifiers=[ 163 | "Intended Audience :: Developers", 164 | "Intended Audience :: Education", 165 | "Intended Audience :: Science/Research", 166 | "License :: OSI Approved :: BSD License", 167 | "Programming Language :: C++", 168 | "Programming Language :: Python :: 3", 169 | "Programming Language :: Python :: 3.6", 170 | "Programming Language :: Python :: 3.7", 171 | "Programming Language :: Python :: 3.8", 172 | "Programming Language :: Python :: 3.9", 173 | "Topic :: Scientific/Engineering", 174 | "Topic :: Scientific/Engineering :: Mathematics", 175 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 176 | "Topic :: Software Development", 177 | "Topic :: Software Development :: Libraries", 178 | "Topic :: Software Development :: Libraries :: Python Modules", 179 | ], 180 | python_requires=">=3.6", 181 | install_requires=requirements, 182 | ext_modules=get_extensions(), 183 | test_suite="test", 184 | cmdclass={ 185 | "build_ext": BuildExtension, 186 | "clean": clean, 187 | }, 188 | ) 189 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved. 2 | # 3 | # This source code is licensed under the BSD-style license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /torchcsprng/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved. 2 | # 3 | # This source code is licensed under the BSD-style license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | from torchcsprng._C import * 8 | 9 | 10 | try: 11 | from .version import __version__, git_version # noqa: F401 12 | except ImportError: 13 | pass 14 | -------------------------------------------------------------------------------- /torchcsprng/__init__.pyi: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved. 2 | # 3 | # This source code is licensed under the BSD-style license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from torch import Generator, Tensor 7 | 8 | def supports_cuda() -> bool: ... 9 | def create_random_device_generator(token: str = "") -> Generator: ... 10 | def create_mt19937_generator(seed: int = 0): ... 11 | def encrypt(input: Tensor, output: Tensor, key: Tensor, cipher, mode): ... 12 | def decrypt(input: Tensor, output: Tensor, key: Tensor, cipher, mode): ... 13 | def __version__() -> str: ... 14 | def git_version() -> str: ... 15 | -------------------------------------------------------------------------------- /torchcsprng/csrc/OffsetCalculator.cuh: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved. 3 | * 4 | * This source code is licensed under the BSD-style license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "THCIntegerDivider.cuh" 16 | 17 | /// OffsetCalculator calculates the offset in bytes of a linear index for NARGS 18 | /// operands that share the same shape, but may have different strides. 19 | 20 | #ifdef __HIP_PLATFORM_HCC__ 21 | constexpr int MAX_DIMS = 16; 22 | #else 23 | constexpr int MAX_DIMS = 25; 24 | #endif 25 | 26 | template 27 | struct OffsetCalculator { 28 | // The offset for each argument. Wrapper around fixed-size array. 29 | // On CUDA, zero sized array is not allowed, so when we are handling nullary 30 | // operators, we need to create a size 1 offset to avoid compiler failure. 31 | // This size 1 offset is just a placeholder, and we will not use it. 32 | using offset_type = at::detail::Array(NARGS, 1)>; 33 | 34 | // if element_sizes is nullptr, then the strides will be in bytes, otherwise 35 | // the strides will be in # of elements. 36 | OffsetCalculator( 37 | int dims, 38 | const int64_t* sizes, 39 | const int64_t* const* strides, 40 | const int64_t* element_sizes = nullptr) 41 | : dims(dims) { 42 | TORCH_CHECK(dims <= MAX_DIMS, "tensor has too many (>", MAX_DIMS, ") dims"); 43 | for (int i = 0; i < MAX_DIMS; ++i) { 44 | if (i < dims) { 45 | sizes_[i] = IntDivider(sizes[i]); 46 | } else { 47 | sizes_[i] = IntDivider(1); 48 | } 49 | for (int arg = 0; arg < NARGS; arg++) { 50 | int64_t element_size = 51 | (element_sizes == nullptr ? 1LL : element_sizes[arg]); 52 | strides_[i][arg] = i < dims ? strides[arg][i] / element_size : 0; 53 | } 54 | } 55 | } 56 | 57 | C10_HOST_DEVICE offset_type get(index_t linear_idx) const { 58 | offset_type offsets; 59 | #pragma unroll 60 | for (int arg = 0; arg < NARGS; arg++) { 61 | offsets[arg] = 0; 62 | } 63 | 64 | #pragma unroll 65 | for (int dim = 0; dim < MAX_DIMS; ++dim) { 66 | if (dim == dims) { 67 | break; 68 | } 69 | auto divmod = sizes_[dim].divmod(linear_idx); 70 | linear_idx = divmod.div; 71 | 72 | #pragma unroll 73 | for (int arg = 0; arg < NARGS; arg++) { 74 | offsets[arg] += divmod.mod * strides_[dim][arg]; 75 | } 76 | } 77 | return offsets; 78 | } 79 | 80 | int dims; 81 | IntDivider sizes_[MAX_DIMS]; 82 | index_t strides_[MAX_DIMS][std::max(NARGS, 1)]; 83 | }; 84 | 85 | template 86 | struct TrivialOffsetCalculator { 87 | // The offset for each argument. Wrapper around fixed-size array. 88 | // The offsets are in # of elements, not in bytes. 89 | // On CUDA, zero sized array is not allowed, so when we are handling nullary 90 | // operators, we need to create a size 1 offset to avoid compiler failure. 91 | // This size 1 offset is just a placeholder, and we will not use it. 92 | using offset_type = at::detail::Array(NARGS, 1)>; 93 | 94 | C10_HOST_DEVICE offset_type get(index_t linear_idx) const { 95 | offset_type offsets; 96 | #pragma unroll 97 | for (int arg = 0; arg < NARGS; arg++) { 98 | offsets[arg] = linear_idx; 99 | } 100 | return offsets; 101 | } 102 | }; 103 | 104 | template 105 | static OffsetCalculator make_offset_calculator( 106 | const at::TensorIterator& iter) { 107 | AT_ASSERT(N <= iter.ntensors()); 108 | std::array strides; 109 | for (int i = 0; i < N; i++) { 110 | strides[i] = iter.strides(i).data(); 111 | } 112 | return OffsetCalculator(iter.ndim(), iter.shape().data(), strides.data()); 113 | } 114 | -------------------------------------------------------------------------------- /torchcsprng/csrc/THCIntegerDivider.cuh: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved. 3 | * 4 | * This source code is licensed under the BSD-style license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | #ifndef THC_INTEGER_DIVIDER_INC 9 | #define THC_INTEGER_DIVIDER_INC 10 | 11 | #include 12 | #if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__) 13 | #include 14 | #endif 15 | 16 | // A utility class to implement integer division by multiplication, given a 17 | // fixed divisor. 18 | // 19 | // WARNING: The fast divider algorithm is only implemented for unsigned int; 20 | // otherwise we default to plain integer division. For unsigned int, 21 | // we further assume that the dividend is at most INT32_MAX. Thus, 22 | // IntDivider must NOT be used for general integer division. 23 | // 24 | // This reduced range is enough for our purpose, and it allows us to 25 | // slightly simplify the computation. 26 | // 27 | // (NOTE: Below, "2^k" denotes exponentiation, i.e., 1< 0), we can find a "magic number" m (2^N 30 | // <= m < 2^(N+1)) and shift s such that: 31 | // 32 | // \floor(n / d) = \floor((m * n) / 2^(N+s)). 33 | // 34 | // Given such m and s, the integer division can be then implemented as: 35 | // 36 | // let m' = m - 2^N // 0 <= m' < 2^N 37 | // 38 | // fast_integer_division(n): 39 | // // Multiply two N-bit unsigned integers: the result is a 2N-bit unsigned 40 | // // integer. Then take the higher N bits. 41 | // t = (m' * n) >> N 42 | // 43 | // // Here we use the fact that n is less than 2^(N-1): otherwise the value 44 | // // of (t + n) may not fit in an N-bit integer. 45 | // return (t + n) >> s 46 | // 47 | // Finding such a magic number is surprisingly easy: 48 | // 49 | // s = \ceil(\log_2 d) 50 | // m' = \floor(2^N * (2^s - d) / d) + 1 // Need 2N-bit integer arithmetic. 51 | // 52 | // See also: 53 | // - Division by Invariant Integers Using Multiplication, 54 | // Torbjörn Granlund and Peter L. Montgomery, 1994. 55 | // 56 | // - http://www.hackersdelight.org/magic.htm 57 | // 58 | // - http://ridiculousfish.com/blog/posts/labor-of-division-episode-i.html 59 | 60 | // Result of div/mod operation stored together. 61 | template 62 | struct DivMod { 63 | Value div, mod; 64 | 65 | C10_HOST_DEVICE DivMod(Value div, Value mod) : div(div), mod(mod) {} 66 | }; 67 | 68 | // Base case: we only have an implementation for uint32_t for now. For 69 | // everything else, we use plain division. 70 | template 71 | struct IntDivider { 72 | IntDivider() {} // Dummy constructor for arrays. 73 | IntDivider(Value d) : divisor(d) {} 74 | 75 | C10_HOST_DEVICE inline Value div(Value n) const { 76 | return n / divisor; 77 | } 78 | C10_HOST_DEVICE inline Value mod(Value n) const { 79 | return n % divisor; 80 | } 81 | C10_HOST_DEVICE inline DivMod divmod(Value n) const { 82 | return DivMod(n / divisor, n % divisor); 83 | } 84 | 85 | Value divisor; 86 | }; 87 | 88 | // Implement fast integer division. 89 | template <> 90 | struct IntDivider { 91 | static_assert(sizeof(unsigned int) == 4, "Assumes 32-bit unsigned int."); 92 | 93 | IntDivider() {} // Dummy constructor for arrays. 94 | 95 | IntDivider(unsigned int d) : divisor(d) { 96 | assert(divisor >= 1 && divisor <= INT32_MAX); 97 | 98 | // TODO: gcc/clang has __builtin_clz() but it's not portable. 99 | for (shift = 0; shift < 32; shift++) 100 | if ((1U << shift) >= divisor) 101 | break; 102 | 103 | uint64_t one = 1; 104 | uint64_t magic = ((one << 32) * ((one << shift) - divisor)) / divisor + 1; 105 | m1 = magic; 106 | assert(m1 > 0 && m1 == magic); // m1 must fit in 32 bits. 107 | } 108 | 109 | C10_HOST_DEVICE inline unsigned int div(unsigned int n) const { 110 | #if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__) 111 | // 't' is the higher 32-bits of unsigned 32-bit multiplication of 'n' and 112 | // 'm1'. 113 | unsigned int t = __umulhi(n, m1); 114 | return (t + n) >> shift; 115 | #else 116 | // Using uint64_t so that the addition does not overflow. 117 | uint64_t t = ((uint64_t)n * m1) >> 32; 118 | return (t + n) >> shift; 119 | #endif 120 | } 121 | 122 | C10_HOST_DEVICE inline unsigned int mod(unsigned int n) const { 123 | return n - div(n) * divisor; 124 | } 125 | 126 | C10_HOST_DEVICE inline DivMod divmod(unsigned int n) const { 127 | unsigned int q = div(n); 128 | return DivMod(q, n - q * divisor); 129 | } 130 | 131 | unsigned int divisor; // d above. 132 | unsigned int m1; // Magic number: m' above. 133 | unsigned int shift; // Shift amounts. 134 | }; 135 | 136 | #endif // THC_INTEGER_DIVIDER_INC 137 | -------------------------------------------------------------------------------- /torchcsprng/csrc/aes.inc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved. 3 | * 4 | * This source code is licensed under the BSD-style license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | namespace aes { 9 | 10 | // This AES implementation is based on 11 | // https://github.com/kokke/tiny-AES-c/blob/master/aes.c 12 | // authored by kokke and et al. and distributed under public domain license. 13 | // 14 | // This is free and unencumbered software released into the public domain. 15 | // 16 | // Anyone is free to copy, modify, publish, use, compile, sell, or 17 | // distribute this software, either in source code form or as a compiled 18 | // binary, for any purpose, commercial or non-commercial, and by any 19 | // means. 20 | // 21 | // In jurisdictions that recognize copyright laws, the author or authors 22 | // of this software dedicate any and all copyright interest in the 23 | // software to the public domain. We make this dedication for the benefit 24 | // of the public at large and to the detriment of our heirs and 25 | // successors. We intend this dedication to be an overt act of 26 | // relinquishment in perpetuity of all present and future rights to this 27 | // software under copyright law. 28 | // 29 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 30 | // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 31 | // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 32 | // IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 33 | // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 34 | // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 35 | // OTHER DEALINGS IN THE SOFTWARE. 36 | // 37 | // For more information, please refer to 38 | // 39 | // Adapted for CUDA by Pavel Belevich 40 | 41 | /*****************************************************************************/ 42 | /* Defines: */ 43 | /*****************************************************************************/ 44 | // The number of columns comprising a state in AES. This is a constant in AES. Value=4 45 | #define Nb 4 46 | 47 | #if defined(AES256) && (AES256 == 1) 48 | #define Nk 8 49 | #define Nr 14 50 | #elif defined(AES192) && (AES192 == 1) 51 | #define Nk 6 52 | #define Nr 12 53 | #else 54 | #define Nk 4 // The number of 32 bit words in a key. 55 | #define Nr 10 // The number of rounds in AES Cipher. 56 | #endif 57 | 58 | constexpr size_t block_t_size = 16; 59 | 60 | typedef uint8_t state_t[4][4]; 61 | 62 | // The lookup-tables are marked const so they can be placed in read-only storage instead of RAM 63 | // The numbers below can be computed dynamically trading ROM for RAM - 64 | // This can be useful in (embedded) bootloader applications, where ROM is often limited. 65 | TORCH_CSPRNG_CONSTANT const uint8_t sbox[256] = { 66 | //0 1 2 3 4 5 6 7 8 9 A B C D E F 67 | 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 68 | 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 69 | 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, 70 | 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, 71 | 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 72 | 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, 73 | 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, 74 | 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 75 | 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, 76 | 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, 77 | 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 78 | 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, 79 | 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 80 | 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 81 | 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 82 | 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 }; 83 | 84 | TORCH_CSPRNG_CONSTANT const uint8_t rsbox[256] = { 85 | 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, 86 | 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 87 | 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, 88 | 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, 89 | 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 90 | 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, 91 | 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, 92 | 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 93 | 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, 94 | 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, 95 | 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 96 | 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, 97 | 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, 98 | 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 99 | 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, 100 | 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d }; 101 | 102 | // The round constant word array, Rcon[i], contains the values given by 103 | // x to the power (i-1) being powers of x (x is denoted as {02}) in the field GF(2^8) 104 | TORCH_CSPRNG_CONSTANT const uint8_t Rcon[11] = { 105 | 0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36 }; 106 | 107 | #define getSBoxValue(num) (sbox[(num)]) 108 | 109 | #define getSBoxInvert(num) (rsbox[(num)]) 110 | 111 | // This function produces Nb(Nr+1) round keys. The round keys are used in each round to decrypt the states. 112 | TORCH_CSPRNG_HOST_DEVICE void KeyExpansion(uint8_t* RoundKey, const uint8_t* Key){ 113 | unsigned int i, j, k; 114 | uint8_t tempa[4]; // Used for the column/row operations 115 | 116 | // The first round key is the key itself. 117 | for (i = 0; i < Nk; ++i) 118 | { 119 | RoundKey[(i * 4) + 0] = Key[(i * 4) + 0]; 120 | RoundKey[(i * 4) + 1] = Key[(i * 4) + 1]; 121 | RoundKey[(i * 4) + 2] = Key[(i * 4) + 2]; 122 | RoundKey[(i * 4) + 3] = Key[(i * 4) + 3]; 123 | } 124 | 125 | // All other round keys are found from the previous round keys. 126 | for (i = Nk; i < Nb * (Nr + 1); ++i) 127 | { 128 | { 129 | k = (i - 1) * 4; 130 | tempa[0]=RoundKey[k + 0]; 131 | tempa[1]=RoundKey[k + 1]; 132 | tempa[2]=RoundKey[k + 2]; 133 | tempa[3]=RoundKey[k + 3]; 134 | 135 | } 136 | 137 | if (i % Nk == 0) 138 | { 139 | // This function shifts the 4 bytes in a word to the left once. 140 | // [a0,a1,a2,a3] becomes [a1,a2,a3,a0] 141 | 142 | // Function RotWord() 143 | { 144 | const uint8_t u8tmp = tempa[0]; 145 | tempa[0] = tempa[1]; 146 | tempa[1] = tempa[2]; 147 | tempa[2] = tempa[3]; 148 | tempa[3] = u8tmp; 149 | } 150 | 151 | // SubWord() is a function that takes a four-byte input word and 152 | // applies the S-box to each of the four bytes to produce an output word. 153 | 154 | // Function Subword() 155 | { 156 | tempa[0] = getSBoxValue(tempa[0]); 157 | tempa[1] = getSBoxValue(tempa[1]); 158 | tempa[2] = getSBoxValue(tempa[2]); 159 | tempa[3] = getSBoxValue(tempa[3]); 160 | } 161 | 162 | tempa[0] = tempa[0] ^ Rcon[i/Nk]; 163 | } 164 | #if defined(AES256) && (AES256 == 1) 165 | if (i % Nk == 4) 166 | { 167 | // Function Subword() 168 | { 169 | tempa[0] = getSBoxValue(tempa[0]); 170 | tempa[1] = getSBoxValue(tempa[1]); 171 | tempa[2] = getSBoxValue(tempa[2]); 172 | tempa[3] = getSBoxValue(tempa[3]); 173 | } 174 | } 175 | #endif 176 | j = i * 4; k=(i - Nk) * 4; 177 | RoundKey[j + 0] = RoundKey[k + 0] ^ tempa[0]; 178 | RoundKey[j + 1] = RoundKey[k + 1] ^ tempa[1]; 179 | RoundKey[j + 2] = RoundKey[k + 2] ^ tempa[2]; 180 | RoundKey[j + 3] = RoundKey[k + 3] ^ tempa[3]; 181 | } 182 | } 183 | 184 | // This function adds the round key to state. 185 | // The round key is added to the state by an XOR function. 186 | TORCH_CSPRNG_HOST_DEVICE void AddRoundKey(uint8_t round, state_t* state, const uint8_t* RoundKey) 187 | { 188 | uint8_t i,j; 189 | for (i = 0; i < 4; ++i) 190 | { 191 | for (j = 0; j < 4; ++j) 192 | { 193 | (*state)[i][j] ^= RoundKey[(round * Nb * 4) + (i * Nb) + j]; 194 | } 195 | } 196 | } 197 | 198 | // The SubBytes Function Substitutes the values in the 199 | // state matrix with values in an S-box. 200 | TORCH_CSPRNG_HOST_DEVICE void SubBytes(state_t* state) 201 | { 202 | uint8_t i, j; 203 | for (i = 0; i < 4; ++i) 204 | { 205 | for (j = 0; j < 4; ++j) 206 | { 207 | (*state)[j][i] = getSBoxValue((*state)[j][i]); 208 | } 209 | } 210 | } 211 | 212 | // The ShiftRows() function shifts the rows in the state to the left. 213 | // Each row is shifted with different offset. 214 | // Offset = Row number. So the first row is not shifted. 215 | TORCH_CSPRNG_HOST_DEVICE void ShiftRows(state_t* state) 216 | { 217 | uint8_t temp; 218 | 219 | // Rotate first row 1 columns to left 220 | temp = (*state)[0][1]; 221 | (*state)[0][1] = (*state)[1][1]; 222 | (*state)[1][1] = (*state)[2][1]; 223 | (*state)[2][1] = (*state)[3][1]; 224 | (*state)[3][1] = temp; 225 | 226 | // Rotate second row 2 columns to left 227 | temp = (*state)[0][2]; 228 | (*state)[0][2] = (*state)[2][2]; 229 | (*state)[2][2] = temp; 230 | 231 | temp = (*state)[1][2]; 232 | (*state)[1][2] = (*state)[3][2]; 233 | (*state)[3][2] = temp; 234 | 235 | // Rotate third row 3 columns to left 236 | temp = (*state)[0][3]; 237 | (*state)[0][3] = (*state)[3][3]; 238 | (*state)[3][3] = (*state)[2][3]; 239 | (*state)[2][3] = (*state)[1][3]; 240 | (*state)[1][3] = temp; 241 | } 242 | 243 | TORCH_CSPRNG_HOST_DEVICE uint8_t xtime(uint8_t x) 244 | { 245 | return ((x<<1) ^ (((x>>7) & 1) * 0x1b)); 246 | } 247 | 248 | // MixColumns function mixes the columns of the state matrix 249 | TORCH_CSPRNG_HOST_DEVICE void MixColumns(state_t* state) 250 | { 251 | uint8_t i; 252 | uint8_t Tmp, Tm, t; 253 | for (i = 0; i < 4; ++i) 254 | { 255 | t = (*state)[i][0]; 256 | Tmp = (*state)[i][0] ^ (*state)[i][1] ^ (*state)[i][2] ^ (*state)[i][3] ; 257 | Tm = (*state)[i][0] ^ (*state)[i][1] ; Tm = xtime(Tm); (*state)[i][0] ^= Tm ^ Tmp ; 258 | Tm = (*state)[i][1] ^ (*state)[i][2] ; Tm = xtime(Tm); (*state)[i][1] ^= Tm ^ Tmp ; 259 | Tm = (*state)[i][2] ^ (*state)[i][3] ; Tm = xtime(Tm); (*state)[i][2] ^= Tm ^ Tmp ; 260 | Tm = (*state)[i][3] ^ t ; Tm = xtime(Tm); (*state)[i][3] ^= Tm ^ Tmp ; 261 | } 262 | } 263 | 264 | TORCH_CSPRNG_HOST_DEVICE uint8_t Multiply(uint8_t x, uint8_t y) 265 | { 266 | return (((y & 1) * x) ^ 267 | ((y>>1 & 1) * xtime(x)) ^ 268 | ((y>>2 & 1) * xtime(xtime(x))) ^ 269 | ((y>>3 & 1) * xtime(xtime(xtime(x)))) ^ 270 | ((y>>4 & 1) * xtime(xtime(xtime(xtime(x)))))); /* this last call to xtime() can be omitted */ 271 | } 272 | 273 | // MixColumns function mixes the columns of the state matrix. 274 | // The method used to multiply may be difficult to understand for the inexperienced. 275 | // Please use the references to gain more information. 276 | TORCH_CSPRNG_HOST_DEVICE void InvMixColumns(state_t* state) 277 | { 278 | int i; 279 | uint8_t a, b, c, d; 280 | for (i = 0; i < 4; ++i) 281 | { 282 | a = (*state)[i][0]; 283 | b = (*state)[i][1]; 284 | c = (*state)[i][2]; 285 | d = (*state)[i][3]; 286 | 287 | (*state)[i][0] = Multiply(a, 0x0e) ^ Multiply(b, 0x0b) ^ Multiply(c, 0x0d) ^ Multiply(d, 0x09); 288 | (*state)[i][1] = Multiply(a, 0x09) ^ Multiply(b, 0x0e) ^ Multiply(c, 0x0b) ^ Multiply(d, 0x0d); 289 | (*state)[i][2] = Multiply(a, 0x0d) ^ Multiply(b, 0x09) ^ Multiply(c, 0x0e) ^ Multiply(d, 0x0b); 290 | (*state)[i][3] = Multiply(a, 0x0b) ^ Multiply(b, 0x0d) ^ Multiply(c, 0x09) ^ Multiply(d, 0x0e); 291 | } 292 | } 293 | 294 | // The SubBytes Function Substitutes the values in the 295 | // state matrix with values in an S-box. 296 | TORCH_CSPRNG_HOST_DEVICE void InvSubBytes(state_t* state) 297 | { 298 | uint8_t i, j; 299 | for (i = 0; i < 4; ++i) 300 | { 301 | for (j = 0; j < 4; ++j) 302 | { 303 | (*state)[j][i] = getSBoxInvert((*state)[j][i]); 304 | } 305 | } 306 | } 307 | 308 | TORCH_CSPRNG_HOST_DEVICE void InvShiftRows(state_t* state) 309 | { 310 | uint8_t temp; 311 | 312 | // Rotate first row 1 columns to right 313 | temp = (*state)[3][1]; 314 | (*state)[3][1] = (*state)[2][1]; 315 | (*state)[2][1] = (*state)[1][1]; 316 | (*state)[1][1] = (*state)[0][1]; 317 | (*state)[0][1] = temp; 318 | 319 | // Rotate second row 2 columns to right 320 | temp = (*state)[0][2]; 321 | (*state)[0][2] = (*state)[2][2]; 322 | (*state)[2][2] = temp; 323 | 324 | temp = (*state)[1][2]; 325 | (*state)[1][2] = (*state)[3][2]; 326 | (*state)[3][2] = temp; 327 | 328 | // Rotate third row 3 columns to right 329 | temp = (*state)[0][3]; 330 | (*state)[0][3] = (*state)[1][3]; 331 | (*state)[1][3] = (*state)[2][3]; 332 | (*state)[2][3] = (*state)[3][3]; 333 | (*state)[3][3] = temp; 334 | } 335 | 336 | TORCH_CSPRNG_HOST_DEVICE void encrypt(uint8_t* state, const uint8_t* key) { 337 | uint8_t RoundKey[176]; 338 | KeyExpansion(RoundKey, key); 339 | 340 | uint8_t round = 0; 341 | 342 | // Add the First round key to the state before starting the rounds. 343 | AddRoundKey(0, (state_t*)state, RoundKey); 344 | 345 | // There will be Nr rounds. 346 | // The first Nr-1 rounds are identical. 347 | // These Nr rounds are executed in the loop below. 348 | // Last one without MixColumns() 349 | for (round = 1; ; ++round) 350 | { 351 | SubBytes((state_t*)state); 352 | ShiftRows((state_t*)state); 353 | if (round == Nr) { 354 | break; 355 | } 356 | MixColumns((state_t*)state); 357 | AddRoundKey(round, (state_t*)state, RoundKey); 358 | } 359 | // Add round key to last round 360 | AddRoundKey(Nr, (state_t*)state, RoundKey); 361 | } 362 | 363 | TORCH_CSPRNG_HOST_DEVICE void decrypt(uint8_t* state, const uint8_t* key) { 364 | uint8_t RoundKey[176]; 365 | KeyExpansion(RoundKey, key); 366 | 367 | uint8_t round = 0; 368 | 369 | // Add the First round key to the state before starting the rounds. 370 | AddRoundKey(Nr, (state_t*)state, RoundKey); 371 | 372 | // There will be Nr rounds. 373 | // The first Nr-1 rounds are identical. 374 | // These Nr rounds are executed in the loop below. 375 | // Last one without InvMixColumn() 376 | for (round = (Nr - 1); ; --round) 377 | { 378 | InvShiftRows((state_t*)state); 379 | InvSubBytes((state_t*)state); 380 | AddRoundKey(round, (state_t*)state, RoundKey); 381 | if (round == 0) { 382 | break; 383 | } 384 | InvMixColumns((state_t*)state); 385 | } 386 | } 387 | 388 | } 389 | -------------------------------------------------------------------------------- /torchcsprng/csrc/block_cipher.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved. 3 | * 4 | * This source code is licensed under the BSD-style license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "OffsetCalculator.cuh" 16 | #include "macros.h" 17 | 18 | #if defined(__CUDACC__) || defined(__HIPCC__) 19 | #include 20 | #include 21 | #endif 22 | 23 | #if defined(__CUDACC__) || defined(__HIPCC__) 24 | #define UNROLL_IF_CUDA #pragma unroll 25 | #else 26 | #define UNROLL_IF_CUDA 27 | #endif 28 | 29 | namespace torch { 30 | namespace csprng { 31 | 32 | template 33 | TORCH_CSPRNG_HOST_DEVICE static void copy_input_to_block( 34 | int64_t idx, 35 | uint8_t* block, 36 | int block_size, 37 | void* input_ptr, 38 | int64_t input_numel, 39 | int input_type_size, 40 | input_index_calc_t input_index_calc) { 41 | for (auto i = 0; i < block_size / input_type_size; ++i) { 42 | const auto linear_index = idx * (block_size / input_type_size) + i; 43 | if (linear_index < input_numel) { 44 | std::memcpy( 45 | block + i * input_type_size, 46 | &(reinterpret_cast( 47 | input_ptr)[input_index_calc(linear_index)]), 48 | input_type_size); 49 | } 50 | } 51 | } 52 | 53 | template 54 | TORCH_CSPRNG_HOST_DEVICE static void copy_block_to_output( 55 | int64_t idx, 56 | uint8_t* block, 57 | int output_elem_per_block, 58 | void* output_ptr, 59 | int64_t output_numel, 60 | int output_type_size, 61 | output_index_calc_t output_index_calc) { 62 | for (auto i = 0; i < output_elem_per_block; ++i) { 63 | const auto linear_index = idx * output_elem_per_block + i; 64 | if (linear_index < output_numel) { 65 | std::memcpy( 66 | &(reinterpret_cast( 67 | output_ptr)[output_index_calc(linear_index)]), 68 | block + i * output_type_size, 69 | output_type_size); 70 | } 71 | } 72 | } 73 | 74 | template < 75 | int block_size, 76 | typename cipher_t, 77 | typename input_index_calc_t, 78 | typename output_index_calc_t, 79 | typename transform_t> 80 | TORCH_CSPRNG_HOST_DEVICE static void block_cipher_kernel_helper( 81 | int64_t idx, 82 | cipher_t cipher, 83 | int output_elem_per_block, 84 | void* input_ptr, 85 | int64_t input_numel, 86 | int input_type_size, 87 | input_index_calc_t input_index_calc, 88 | void* output_ptr, 89 | int64_t output_numel, 90 | int output_type_size, 91 | output_index_calc_t output_index_calc, 92 | transform_t transform) { 93 | uint8_t block[block_size]; 94 | std::memset(&block, 0, block_size); // is it ok to use zeros as padding? 95 | if (input_ptr != nullptr) { 96 | copy_input_to_block( 97 | idx, 98 | block, 99 | block_size, 100 | input_ptr, 101 | input_numel, 102 | input_type_size, 103 | input_index_calc); 104 | } 105 | cipher(idx, block); 106 | transform(block); 107 | copy_block_to_output( 108 | idx, 109 | block, 110 | output_elem_per_block, 111 | output_ptr, 112 | output_numel, 113 | output_type_size, 114 | output_index_calc); 115 | } 116 | 117 | #if defined(__CUDACC__) || defined(__HIPCC__) 118 | template < 119 | int block_size, 120 | typename cipher_t, 121 | typename input_index_calc_t, 122 | typename output_index_calc_t, 123 | typename transform_t> 124 | __global__ static void block_cipher_kernel_cuda( 125 | cipher_t cipher, 126 | int output_elem_per_block, 127 | void* input_ptr, 128 | int64_t input_numel, 129 | int input_type_size, 130 | input_index_calc_t input_index_calc, 131 | void* output_ptr, 132 | int64_t output_numel, 133 | int output_type_size, 134 | output_index_calc_t output_index_calc, 135 | transform_t transform) { 136 | const auto idx = blockIdx.x * blockDim.x + threadIdx.x; 137 | block_cipher_kernel_helper( 138 | idx, 139 | cipher, 140 | output_elem_per_block, 141 | input_ptr, 142 | input_numel, 143 | input_type_size, 144 | input_index_calc, 145 | output_ptr, 146 | output_numel, 147 | output_type_size, 148 | output_index_calc, 149 | transform); 150 | } 151 | #endif 152 | 153 | template < 154 | int block_size, 155 | typename cipher_t, 156 | typename input_index_calc_t, 157 | typename output_index_calc_t, 158 | typename transform_t> 159 | static void block_cipher_kernel_cpu_serial( 160 | int64_t begin, 161 | int64_t end, 162 | cipher_t cipher, 163 | int output_elem_per_block, 164 | void* input_ptr, 165 | int64_t input_numel, 166 | int input_type_size, 167 | input_index_calc_t input_index_calc, 168 | void* output_ptr, 169 | int64_t output_numel, 170 | int output_type_size, 171 | output_index_calc_t output_index_calc, 172 | transform_t transform) { 173 | for (auto idx = begin; idx < end; ++idx) { 174 | block_cipher_kernel_helper( 175 | idx, 176 | cipher, 177 | output_elem_per_block, 178 | input_ptr, 179 | input_numel, 180 | input_type_size, 181 | input_index_calc, 182 | output_ptr, 183 | output_numel, 184 | output_type_size, 185 | output_index_calc, 186 | transform); 187 | } 188 | } 189 | 190 | template < 191 | int block_size, 192 | typename cipher_t, 193 | typename input_index_calc_t, 194 | typename output_index_calc_t, 195 | typename transform_t> 196 | static void block_cipher_kernel_cpu( 197 | int64_t total, 198 | cipher_t cipher, 199 | int output_elem_per_block, 200 | void* input_ptr, 201 | int64_t input_numel, 202 | int input_type_size, 203 | input_index_calc_t input_index_calc, 204 | void* output_ptr, 205 | int64_t output_numel, 206 | int output_type_size, 207 | output_index_calc_t output_index_calc, 208 | transform_t transform_func) { 209 | if (total < at::internal::GRAIN_SIZE || at::get_num_threads() == 1) { 210 | block_cipher_kernel_cpu_serial( 211 | 0, 212 | total, 213 | cipher, 214 | output_elem_per_block, 215 | input_ptr, 216 | input_numel, 217 | input_type_size, 218 | input_index_calc, 219 | output_ptr, 220 | output_numel, 221 | output_type_size, 222 | output_index_calc, 223 | transform_func); 224 | } else { 225 | at::parallel_for( 226 | 0, total, at::internal::GRAIN_SIZE, [&](int64_t begin, int64_t end) { 227 | block_cipher_kernel_cpu_serial( 228 | begin, 229 | end, 230 | cipher, 231 | output_elem_per_block, 232 | input_ptr, 233 | input_numel, 234 | input_type_size, 235 | input_index_calc, 236 | output_ptr, 237 | output_numel, 238 | output_type_size, 239 | output_index_calc, 240 | transform_func); 241 | }); 242 | } 243 | } 244 | 245 | template < 246 | int block_size, 247 | typename cipher_t, 248 | typename input_index_calc_t, 249 | typename output_index_calc_t, 250 | typename transform_t> 251 | void block_cipher( 252 | void* input_ptr, 253 | int64_t input_numel, 254 | int input_type_size, 255 | input_index_calc_t input_index_calc, 256 | void* output_ptr, 257 | int64_t output_numel, 258 | int output_type_size, 259 | output_index_calc_t output_index_calc, 260 | at::Device device, 261 | cipher_t cipher, 262 | int output_elem_per_block, 263 | transform_t transform_func) { 264 | if (output_ptr == nullptr || output_numel == 0) { 265 | return; 266 | } 267 | 268 | if (device.type() == at::kCPU) { 269 | const auto total = 270 | (output_numel + output_elem_per_block - 1) / output_elem_per_block; 271 | block_cipher_kernel_cpu( 272 | total, 273 | cipher, 274 | output_elem_per_block, 275 | input_ptr, 276 | input_numel, 277 | input_type_size, 278 | input_index_calc, 279 | output_ptr, 280 | output_numel, 281 | output_type_size, 282 | output_index_calc, 283 | transform_func); 284 | } else if (device.type() == at::kCUDA) { 285 | #if defined(__CUDACC__) || defined(__HIPCC__) 286 | const auto threads = 256; 287 | const auto grid = (output_numel + (threads * output_elem_per_block) - 1) / 288 | (threads * output_elem_per_block); 289 | auto stream = at::cuda::getCurrentCUDAStream(); 290 | block_cipher_kernel_cuda<<>>( 291 | cipher, 292 | output_elem_per_block, 293 | input_ptr, 294 | input_numel, 295 | input_type_size, 296 | input_index_calc, 297 | output_ptr, 298 | output_numel, 299 | output_type_size, 300 | output_index_calc, 301 | transform_func); 302 | AT_CUDA_CHECK(cudaGetLastError()); 303 | #else 304 | TORCH_CHECK(false, "torchcsprng was compiled without CUDA support"); 305 | #endif 306 | } else { 307 | TORCH_CHECK(false, "block_cipher supports only CPU and CUDA devices"); 308 | } 309 | } 310 | 311 | template 312 | void block_cipher(at::Tensor input, at::Tensor output, cipher_t cipher) { 313 | const auto input_ptr = input.data_ptr(); 314 | const auto input_numel = input.numel(); 315 | 316 | // Otherwise OffsetCalculator/IntDivider crashes with integer division by zero 317 | if (input_ptr == nullptr || input_numel == 0) { 318 | return; 319 | } 320 | 321 | const auto input_type_size = input.element_size(); 322 | const auto input_offset_calc = 323 | make_offset_calculator<1>(at::TensorIterator::nullary_op(input)); 324 | const auto input_index_calc = 325 | [input_offset_calc] TORCH_CSPRNG_HOST_DEVICE(uint32_t li) -> uint32_t { 326 | return input_offset_calc.get(li)[0]; 327 | }; 328 | 329 | const auto output_ptr = output.data_ptr(); 330 | const auto output_numel = output.numel(); 331 | 332 | // Otherwise OffsetCalculator/IntDivider crashes with integer division by zero 333 | if (output_ptr == nullptr || output_numel == 0) { 334 | return; 335 | } 336 | 337 | const auto output_type_size = output.element_size(); 338 | const auto output_offset_calc = 339 | make_offset_calculator<1>(at::TensorIterator::nullary_op(output)); 340 | const auto output_index_calc = 341 | [output_offset_calc] TORCH_CSPRNG_HOST_DEVICE(uint32_t li) -> uint32_t { 342 | return output_offset_calc.get(li)[0]; 343 | }; 344 | 345 | const auto device = output.device(); 346 | 347 | torch::csprng::block_cipher( 348 | input_ptr, 349 | input_numel, 350 | input_type_size, 351 | input_index_calc, 352 | output_ptr, 353 | output_numel, 354 | output_type_size, 355 | output_index_calc, 356 | device, 357 | cipher, 358 | block_size / output_type_size, 359 | [] TORCH_CSPRNG_HOST_DEVICE(uint8_t * x) {}); 360 | } 361 | 362 | } // namespace csprng 363 | } // namespace torch 364 | -------------------------------------------------------------------------------- /torchcsprng/csrc/cpu/kernels.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved. 3 | * 4 | * This source code is licensed under the BSD-style license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | #include "../kernels_commons.h" 9 | 10 | namespace torch { 11 | namespace csprng { 12 | namespace cpu { 13 | 14 | #include "../kernels_body.inc" 15 | 16 | } 17 | } // namespace csprng 18 | } // namespace torch 19 | -------------------------------------------------------------------------------- /torchcsprng/csrc/cpu/kernels.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved. 3 | * 4 | * This source code is licensed under the BSD-style license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | 13 | namespace torch { 14 | namespace csprng { 15 | namespace cpu { 16 | 17 | #include "../kernels_decls.inc" 18 | 19 | } 20 | } // namespace csprng 21 | } // namespace torch 22 | -------------------------------------------------------------------------------- /torchcsprng/csrc/cuda/kernels.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved. 3 | * 4 | * This source code is licensed under the BSD-style license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | #include "../kernels_commons.h" 9 | 10 | namespace torch { 11 | namespace csprng { 12 | namespace cuda { 13 | 14 | #include "../kernels_body.inc" 15 | 16 | } 17 | } // namespace csprng 18 | } // namespace torch 19 | -------------------------------------------------------------------------------- /torchcsprng/csrc/cuda/kernels.cuh: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved. 3 | * 4 | * This source code is licensed under the BSD-style license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | 13 | namespace torch { 14 | namespace csprng { 15 | namespace cuda { 16 | 17 | #include "../kernels_decls.inc" 18 | 19 | } 20 | } // namespace csprng 21 | } // namespace torch 22 | -------------------------------------------------------------------------------- /torchcsprng/csrc/kernels_commons.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved. 3 | * 4 | * This source code is licensed under the BSD-style license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include "block_cipher.h" 17 | #include "macros.h" 18 | 19 | inline uint64_t make64BitsFrom32Bits(uint32_t hi, uint32_t lo) { 20 | return (static_cast(hi) << 32) | lo; 21 | } 22 | 23 | // CUDA CSPRNG is actually CPU generator which is used only to generate a random 24 | // key on CPU for AES running in a block mode on CUDA 25 | struct CSPRNGGeneratorImpl : public c10::GeneratorImpl { 26 | CSPRNGGeneratorImpl(bool use_rd) 27 | : c10:: 28 | GeneratorImpl{at::Device(at::DeviceType::CPU), at::DispatchKeySet(at::DispatchKey::CustomRNGKeyId)}, 29 | use_rd_{use_rd} {} 30 | CSPRNGGeneratorImpl(const std::string& token) 31 | : c10:: 32 | GeneratorImpl{at::Device(at::DeviceType::CPU), at::DispatchKeySet(at::DispatchKey::CustomRNGKeyId)}, 33 | use_rd_{true}, 34 | rd_{token} {} 35 | CSPRNGGeneratorImpl(uint64_t seed) 36 | : c10:: 37 | GeneratorImpl{at::Device(at::DeviceType::CPU), at::DispatchKeySet(at::DispatchKey::CustomRNGKeyId)}, 38 | use_rd_{false}, 39 | mt_{static_cast(seed)} {} 40 | ~CSPRNGGeneratorImpl() = default; 41 | uint32_t random() { 42 | return use_rd_ ? rd_() : mt_(); 43 | } 44 | uint64_t random64() { 45 | return use_rd_ ? make64BitsFrom32Bits(rd_(), rd_()) 46 | : make64BitsFrom32Bits(mt_(), mt_()); 47 | } 48 | 49 | void set_current_seed(uint64_t seed) override { 50 | throw std::runtime_error("not implemented"); 51 | } 52 | uint64_t current_seed() const override { 53 | throw std::runtime_error("not implemented"); 54 | } 55 | uint64_t seed() override { 56 | throw std::runtime_error("not implemented"); 57 | } 58 | CSPRNGGeneratorImpl* clone_impl() const override { 59 | throw std::runtime_error("not implemented"); 60 | } 61 | 62 | static at::DeviceType device_type() { 63 | return at::DeviceType::CPU; 64 | } 65 | 66 | void set_state(const c10::TensorImpl& new_state) override { 67 | throw std::runtime_error("not implemented"); 68 | } 69 | c10::intrusive_ptr get_state() const override { 70 | throw std::runtime_error("not implemented"); 71 | } 72 | 73 | void set_offset(uint64_t offset) override { 74 | throw std::runtime_error("not implemented"); 75 | } 76 | uint64_t get_offset() const override { 77 | throw std::runtime_error("not implenented"); 78 | } 79 | bool use_rd_; 80 | std::random_device rd_; 81 | std::mt19937 mt_; 82 | }; 83 | -------------------------------------------------------------------------------- /torchcsprng/csrc/kernels_decls.inc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved. 3 | * 4 | * This source code is licensed under the BSD-style license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | // ==================================================== Random ======================================================== 9 | 10 | at::Tensor& random_(at::Tensor& self, std::optional generator); 11 | 12 | at::Tensor& random_from_to(at::Tensor& self, int64_t from, std::optional to, std::optional generator); 13 | 14 | at::Tensor& random_to(at::Tensor& self, int64_t to, std::optional generator); 15 | 16 | // ==================================================== Uniform ======================================================= 17 | 18 | at::Tensor& uniform_(at::Tensor& self, double from, double to, std::optional generator); 19 | 20 | // ==================================================== Normal ======================================================== 21 | 22 | at::Tensor& normal_(at::Tensor& self, double mean, double std, std::optional generator); 23 | 24 | at::Tensor& normal_Tensor_float_out(at::Tensor& output, const at::Tensor& mean, double std, std::optional gen); 25 | 26 | at::Tensor& normal_float_Tensor_out(at::Tensor& output, double mean, const at::Tensor& std, std::optional gen); 27 | 28 | at::Tensor& normal_Tensor_Tensor_out(at::Tensor& output, const at::Tensor& mean, const at::Tensor& std, std::optional gen); 29 | 30 | at::Tensor normal_Tensor_float(const at::Tensor& mean, double std, std::optional gen); 31 | 32 | at::Tensor normal_float_Tensor(double mean, const at::Tensor& std, std::optional gen); 33 | 34 | at::Tensor normal_Tensor_Tensor(const at::Tensor& mean, const at::Tensor& std, std::optional gen); 35 | 36 | // ==================================================== Cauchy ======================================================== 37 | 38 | at::Tensor& cauchy_(at::Tensor& self, double median, double sigma, std::optional generator); 39 | 40 | // ================================================== LogNormal ======================================================= 41 | 42 | at::Tensor& log_normal_(at::Tensor& self, double mean, double std, std::optional gen); 43 | 44 | // ================================================== Geometric ======================================================= 45 | 46 | at::Tensor& geometric_(at::Tensor& self, double p, std::optional gen); 47 | 48 | // ================================================== Exponential ===================================================== 49 | 50 | at::Tensor& exponential_(at::Tensor& self, double lambda, std::optional gen); 51 | 52 | // ================================================Encrypt/Decrypt===================================================== 53 | 54 | Tensor encrypt(Tensor input, Tensor output, Tensor key, const std::string& cipher, const std::string& mode); 55 | 56 | Tensor decrypt(Tensor input, Tensor output, Tensor key, const std::string& cipher, const std::string& mode); 57 | -------------------------------------------------------------------------------- /torchcsprng/csrc/macros.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved. 3 | * 4 | * This source code is licensed under the BSD-style license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | #pragma once 9 | 10 | #if defined(__CUDACC__) || defined(__HIPCC__) 11 | #define TORCH_CSPRNG_HOST_DEVICE __host__ __device__ 12 | #define TORCH_CSPRNG_CONSTANT __constant__ 13 | #else 14 | #define TORCH_CSPRNG_HOST_DEVICE 15 | #define TORCH_CSPRNG_CONSTANT 16 | #endif 17 | -------------------------------------------------------------------------------- /version.txt: -------------------------------------------------------------------------------- 1 | 0.3.0a0 2 | --------------------------------------------------------------------------------