├── .gitignore ├── .gitlab-ci.yml ├── CHANGELOG.md ├── CONTRIBUTING.rst ├── LICENSE.md ├── MANIFEST.in ├── Makefile ├── README.md ├── benchmarks └── benchmark_block_sizes.py ├── conda ├── README.org ├── build.sh ├── conda_build_config.yaml ├── cuda-10.2.yaml ├── cuda-11.1.yaml └── meta.yaml ├── conda_docker_build ├── Dockerfile └── download_cuda_environments.sh ├── conda_docker_release.sh ├── doc_sources ├── CHANGELOG.md ├── LICENSE.md ├── Makefile ├── README.md ├── conf.py ├── examples.rst ├── index.rst ├── make.bat ├── modules.rst └── msd_pytorch.rst ├── docs ├── .buildinfo ├── .nojekyll ├── CHANGELOG.html ├── LICENSE.html ├── README.html ├── _modules │ ├── index.html │ └── msd_pytorch │ │ ├── bench.html │ │ ├── conv.html │ │ ├── conv_relu.html │ │ ├── errors.html │ │ ├── image_dataset.html │ │ ├── main.html │ │ ├── msd_block.html │ │ ├── msd_model.html │ │ ├── msd_module.html │ │ ├── msd_regression_model.html │ │ ├── msd_segmentation_model.html │ │ ├── relu_inplace.html │ │ └── stitch.html ├── _sources │ ├── CHANGELOG.md.txt │ ├── LICENSE.md.txt │ ├── README.md.txt │ ├── examples.rst.txt │ ├── index.rst.txt │ ├── modules.rst.txt │ └── msd_pytorch.rst.txt ├── _static │ ├── basic.css │ ├── css │ │ ├── badge_only.css │ │ └── theme.css │ ├── doctools.js │ ├── documentation_options.js │ ├── file.png │ ├── fonts │ │ ├── Inconsolata-Bold.ttf │ │ ├── Inconsolata-Regular.ttf │ │ ├── Inconsolata.ttf │ │ ├── Lato-Bold.ttf │ │ ├── Lato-Regular.ttf │ │ ├── Lato │ │ │ ├── lato-bold.eot │ │ │ ├── lato-bold.ttf │ │ │ ├── lato-bold.woff │ │ │ ├── lato-bold.woff2 │ │ │ ├── lato-bolditalic.eot │ │ │ ├── lato-bolditalic.ttf │ │ │ ├── lato-bolditalic.woff │ │ │ ├── lato-bolditalic.woff2 │ │ │ ├── lato-italic.eot │ │ │ ├── lato-italic.ttf │ │ │ ├── lato-italic.woff │ │ │ ├── lato-italic.woff2 │ │ │ ├── lato-regular.eot │ │ │ ├── lato-regular.ttf │ │ │ ├── lato-regular.woff │ │ │ └── lato-regular.woff2 │ │ ├── RobotoSlab-Bold.ttf │ │ ├── RobotoSlab-Regular.ttf │ │ ├── RobotoSlab │ │ │ ├── roboto-slab-v7-bold.eot │ │ │ ├── roboto-slab-v7-bold.ttf │ │ │ ├── roboto-slab-v7-bold.woff │ │ │ ├── roboto-slab-v7-bold.woff2 │ │ │ ├── roboto-slab-v7-regular.eot │ │ │ ├── roboto-slab-v7-regular.ttf │ │ │ ├── roboto-slab-v7-regular.woff │ │ │ └── roboto-slab-v7-regular.woff2 │ │ ├── fontawesome-webfont.eot │ │ ├── fontawesome-webfont.svg │ │ ├── fontawesome-webfont.ttf │ │ ├── fontawesome-webfont.woff │ │ └── fontawesome-webfont.woff2 │ ├── jquery-3.4.1.js │ ├── jquery.js │ ├── js │ │ ├── modernizr.min.js │ │ └── theme.js │ ├── language_data.js │ ├── minus.png │ ├── plus.png │ ├── pygments.css │ ├── searchtools.js │ ├── underscore-1.3.1.js │ └── underscore.js ├── examples.html ├── genindex.html ├── index.html ├── modules.html ├── msd_pytorch.html ├── objects.inv ├── py-modindex.html ├── search.html └── searchindex.js ├── examples └── getting_started.py ├── msd_pytorch ├── VERSION ├── __init__.py ├── bench.py ├── conv2d.py ├── conv3d.py ├── errors.py ├── image_dataset.py ├── msd_block.py ├── msd_custom_convolutions.cpp ├── msd_custom_convolutions │ ├── conv2d_backward_bias.cu │ ├── conv2d_backward_k.cu │ ├── conv2d_backward_x.cu │ ├── conv2d_forward.cu │ ├── conv3d_backward_bias.cu │ ├── conv3d_backward_k.cu │ ├── conv3d_backward_x.cu │ ├── conv3d_forward.cu │ ├── device_tensor.h │ ├── kernel_utils.cuh │ ├── kernels.cuh │ ├── torch_cuda_dispatch.cu │ └── torch_cuda_dispatch.h ├── msd_model.py ├── msd_module.py ├── msd_regression_model.py ├── msd_segmentation_model.py └── tests │ ├── __init__.py │ ├── test_conv2d.py │ ├── test_conv3d.py │ ├── test_image_dataset.py │ ├── test_msd_block.py │ ├── test_msd_model.py │ ├── test_msd_module.py │ ├── test_msd_pytorch.py │ ├── test_msd_regression_model.py │ ├── test_msd_segmentation_model.py │ └── utils.py ├── setup.cfg └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | /*.so 2 | /*.egg-info 3 | *~ 4 | /build 5 | /.eggs 6 | __pycache__ 7 | /msd_*.torch 8 | /htmlcov/ 9 | /.coverage 10 | *.doctree 11 | /docs/doctrees/*.pickle 12 | /ninja_msd_custom_convolutions 13 | /conda_docker_build/cuda_*_linux* 14 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | # This fixes issues with conda-build and versions of git < 1.9. Cloning from a 3 | # shallow clone is not possible. 4 | GIT_DEPTH: 0 5 | GIT_STRATEGY: clone 6 | 7 | stages: 8 | - build 9 | - publish 10 | 11 | build-9.2: 12 | stage: build 13 | image: pytorch/manylinux-cuda92 14 | variables: 15 | cudatoolkit: "9.2" 16 | # See: 17 | # pytorch commit cd207737017db8c81584763207df20bc6110ed75 18 | # https://en.wikipedia.org/wiki/CUDA#GPUs_supported 19 | # https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#options-for-steering-gpu-code-generation-gpu-architecture 20 | TORCH_CUDA_ARCH_LIST: "3.5 5.2 6.0 6.1" 21 | script: 22 | - wget -nv https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O install_conda.sh 23 | - bash install_conda.sh -b 24 | - export PATH="$HOME/miniconda3/bin:$PATH" 25 | - mkdir -p /cache/pkgs 26 | - rsync -a $HOME/miniconda3/pkgs /cache/pkgs 27 | - rm -rf $HOME/miniconda3/pkgs 28 | - ln -s /cache/pkgs $HOME/miniconda3/ 29 | - conda install -y -c pytorch pytorch=1.4 python=3.7 30 | - conda install -y conda-build conda-verify 31 | - conda build conda/ -c aahendriksen -c pytorch -c defaults -c conda-forge -m conda/cuda-${cudatoolkit}.yaml 32 | - mkdir -p artifacts 33 | - mv /root/miniconda3/conda-bld/linux-64/msd_pytorch*.bz2 artifacts/ 34 | artifacts: 35 | paths: 36 | - artifacts/ 37 | expire_in: 7 days 38 | 39 | build-10.0: 40 | stage: build 41 | image: pytorch/manylinux-cuda100 42 | variables: 43 | cudatoolkit: "10.0" 44 | # See: 45 | # pytorch commit cd207737017db8c81584763207df20bc6110ed75 46 | # https://en.wikipedia.org/wiki/CUDA#GPUs_supported 47 | # https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#options-for-steering-gpu-code-generation-gpu-architecture 48 | TORCH_CUDA_ARCH_LIST: "3.5 5.2 6.0 6.1 7.0+PTX" 49 | script: 50 | - wget -nv https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O install_conda.sh 51 | - bash install_conda.sh -b 52 | - export PATH="$HOME/miniconda3/bin:$PATH" 53 | - conda install -y -c pytorch pytorch=1.4 python=3.7 54 | - conda install -y conda-build conda-verify 55 | - conda build conda/ -c aahendriksen -c pytorch -c defaults -c conda-forge -m conda/cuda-${cudatoolkit}.yaml 56 | - mkdir -p artifacts 57 | - mv /root/miniconda3/conda-bld/linux-64/msd_pytorch*.bz2 artifacts/ 58 | artifacts: 59 | paths: 60 | - artifacts/ 61 | expire_in: 7 days 62 | 63 | 64 | build-10.1: 65 | stage: build 66 | image: pytorch/manylinux-cuda101 67 | variables: 68 | cudatoolkit: "10.1" 69 | # See: 70 | # pytorch commit cd207737017db8c81584763207df20bc6110ed75 71 | # https://en.wikipedia.org/wiki/CUDA#GPUs_supported 72 | # https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#options-for-steering-gpu-code-generation-gpu-architecture 73 | TORCH_CUDA_ARCH_LIST: "3.5 5.2 6.0 6.1 7.0+PTX" 74 | script: 75 | - wget -nv https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O install_conda.sh 76 | - bash install_conda.sh -b 77 | - export PATH="$HOME/miniconda3/bin:$PATH" 78 | - conda install -y -c pytorch pytorch=1.4 python=3.7 79 | - conda install -y conda-build conda-verify 80 | - conda build conda/ -c aahendriksen -c pytorch -c defaults -c conda-forge -m conda/cuda-${cudatoolkit}.yaml 81 | - mkdir -p artifacts 82 | - mv /root/miniconda3/conda-bld/linux-64/msd_pytorch*.bz2 artifacts/ 83 | artifacts: 84 | paths: 85 | - artifacts/ 86 | expire_in: 7 days 87 | 88 | publish-dev: 89 | stage: publish 90 | image: continuumio/miniconda3 91 | only: 92 | - dev 93 | variables: 94 | ANACONDA_USERNAME: $ANACONDA_USERNAME 95 | ANACONDA_PASSWORD: $ANACONDA_PASSWORD 96 | script: 97 | - conda install -yq anaconda-client 98 | - set +x 99 | - anaconda login --username "$ANACONDA_USERNAME" --password "$ANACONDA_PASSWORD" 100 | - set -x 101 | - anaconda upload --label dev artifacts/*.bz2 102 | - anaconda logout 103 | 104 | publish-release: 105 | stage: publish 106 | image: continuumio/miniconda3 107 | only: 108 | - tags 109 | variables: 110 | ANACONDA_USERNAME: $ANACONDA_USERNAME 111 | ANACONDA_PASSWORD: $ANACONDA_PASSWORD 112 | script: 113 | - conda install -yq anaconda-client 114 | - set +x 115 | - anaconda login --username "$ANACONDA_USERNAME" --password "$ANACONDA_PASSWORD" 116 | - set -x 117 | - anaconda upload artifacts/*.bz2 118 | - anaconda logout 119 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [Unreleased] 8 | ### Added 9 | ### Fixed 10 | ### Removed 11 | 12 | 13 | ## [0.10.1] - 2021-05-26 14 | ### Fixed 15 | - Replace outdated import from setup.py 16 | 17 | ## [0.10.0] - 2021-05-26 18 | ### Added 19 | - 3D convolution support. 20 | - Support for pytorch 1.8.1 21 | ### Removed 22 | - Removed support for pytorch versions < 1.8 23 | 24 | 25 | ## [0.9.1] - 2020-10-27 26 | ### Fixed 27 | - Fixed the conda build for pytorch versions < 1.5 28 | 29 | ## [0.9.0] - 2020-10-26 30 | ### Added 31 | - Support for Python 3.8 32 | - Suport for PyTorch 1.5 33 | ### Fixed 34 | - Fixed normalization for multi-channel input and output. 35 | ### Removed 36 | - Support for PyTorch 1.1 37 | 38 | ## [0.8.0] - 2020-03.10 39 | ### Added 40 | ### Fixed 41 | - Weights access when pruning in pytorch 1.4 42 | ### Removed 43 | - torchvision dependency 44 | - sacred dependency 45 | - `msd_pytorch.relu_inplace` 46 | - command-line interface 47 | - old MSDModule 48 | - stitch functions and modules 49 | 50 | ## [0.7.3] - 2020-03-10 51 | ### Fixed 52 | - Bug in relabeling code in ImageDataset. 53 | 54 | ## [0.7.2] - 2019-07-30 55 | ### Added 56 | - Support for multi-gpu execution. Use `parallel=True` when 57 | constructing a `MSDRegressionModel` or `MSDSegmentationModel`. 58 | ### Fixed 59 | - Make `model.forward()` more memory-efficient. 60 | ### Removed 61 | 62 | ## [0.7.1] - 2019-05-27 63 | ### Added 64 | - Add `weights_path` command-line argument to msd executable to indicate 65 | where to store final weights. 66 | - Add `MSDBlock2d`: this is a faster and slightly more memory efficient 67 | implementation of the same MSD component. Many thanks to Jonas 68 | Adler for suggesting this way of structuring the code! 69 | ### Changed 70 | - The MSD models use `MSDBlock2d` implementation by default now. 71 | 72 | ## 0.6.2 - 2019-05-23 73 | ### Added 74 | - Initial release. 75 | 76 | [Unreleased]: https://www.github.com/ahendriksen/msd_pytorch/compare/v0.10.1...master 77 | [0.10.1]: https://www.github.com/ahendriksen/msd_pytorch/compare/v0.10.0...v0.10.1 78 | [0.10.0]: https://www.github.com/ahendriksen/msd_pytorch/compare/v0.9.1...v0.10.0 79 | [0.9.1]: https://www.github.com/ahendriksen/msd_pytorch/compare/v0.9.0...v0.9.1 80 | [0.9.0]: https://www.github.com/ahendriksen/msd_pytorch/compare/v0.8.0...v0.9.0 81 | [0.8.0]: https://www.github.com/ahendriksen/msd_pytorch/compare/v0.7.3...v0.8.0 82 | [0.7.2]: https://www.github.com/ahendriksen/msd_pytorch/compare/v0.7.2...v0.7.3 83 | [0.7.2]: https://www.github.com/ahendriksen/msd_pytorch/compare/v0.7.1...v0.7.2 84 | [0.7.1]: https://www.github.com/ahendriksen/msd_pytorch/compare/v0.6.2...v0.7.1 85 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | ============ 4 | Contributing 5 | ============ 6 | 7 | Contributions are welcome, and they are greatly appreciated! Every little bit 8 | helps, and credit will always be given. 9 | 10 | You can contribute in many ways: 11 | 12 | Types of Contributions 13 | ---------------------- 14 | 15 | Report Bugs 16 | ~~~~~~~~~~~ 17 | 18 | Report bugs at https://github.com/ahendriksen/msd_pytorch/issues. 19 | 20 | If you are reporting a bug, please include: 21 | 22 | * Your operating system name and version. 23 | * Any details about your local setup that might be helpful in troubleshooting. 24 | * Detailed steps to reproduce the bug. 25 | 26 | 27 | Submit Feedback 28 | ~~~~~~~~~~~~~~~ 29 | 30 | The best way to send feedback is to file an issue at https://github.com/ahendriksen/msd_pytorch/issues. 31 | 32 | If you are proposing a feature: 33 | 34 | * Explain in detail how it would work. 35 | * Keep the scope as narrow as possible, to make it easier to implement. 36 | * Remember that this is a volunteer-driven project, and that contributions 37 | are welcome :) 38 | 39 | Building from source 40 | -------------------- 41 | 42 | Pytorch versions >= 1.7 43 | ~~~~~~~~~~~~~~~~~~~~~~ 44 | 45 | Use the following: 46 | 47 | $ CC=/opt/sw/gcc-7.3.0/bin/gcc CXX=/opt/sw/gcc-7.3.0/bin/g++ python setup.py emit_ninja 48 | 49 | 50 | 51 | Pytorch versions < 1.7 52 | ~~~~~~~~~~~~~~~~~~~~~~ 53 | 54 | Use the following: 55 | 56 | $ CC=/opt/sw/gcc-7.3.0/bin/gcc GXX=/opt/sw/gcc-7.3.0/bin/ CXX=/opt/sw/gcc-7.3.0/bin/g++ python setup.py emit_ninja 57 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | An implementation of Mixed-Scale Dense networks in PyTorch. 5 | Copyright (C) 2019 Allard Hendriksen 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | 20 | Also add information on how to contact you by electronic and paper mail. 21 | 22 | You should also get your employer (if you work as a programmer) or school, 23 | if any, to sign a "copyright disclaimer" for the program, if necessary. 24 | For more information on this, and how to apply and follow the GNU GPL, see 25 | . 26 | 27 | The GNU General Public License does not permit incorporating your program 28 | into proprietary programs. If your program is a subroutine library, you 29 | may consider it more useful to permit linking proprietary applications with 30 | the library. If this is what you want to do, use the GNU Lesser General 31 | Public License instead of this License. But first, please read 32 | . 33 | 34 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include msd_pytorch/VERSION -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean clean-test clean-pyc clean-build docs help install_dev 2 | .DEFAULT_GOAL := help 3 | 4 | define BROWSER_PYSCRIPT 5 | import os, webbrowser, sys 6 | 7 | try: 8 | from urllib import pathname2url 9 | except: 10 | from urllib.request import pathname2url 11 | 12 | webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) 13 | endef 14 | export BROWSER_PYSCRIPT 15 | 16 | define PRINT_HELP_PYSCRIPT 17 | import re, sys 18 | 19 | for line in sys.stdin: 20 | match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) 21 | if match: 22 | target, help = match.groups() 23 | print("%-20s %s" % (target, help)) 24 | endef 25 | export PRINT_HELP_PYSCRIPT 26 | 27 | BROWSER := python -c "$$BROWSER_PYSCRIPT" 28 | 29 | help: 30 | @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) 31 | 32 | clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts 33 | 34 | clean-build: ## remove build artifacts 35 | rm -fr build/ 36 | rm -fr dist/ 37 | rm -fr .eggs/ 38 | find . -name '*.egg-info' -exec rm -fr {} + 39 | find . -name '*.egg' -exec rm -f {} + 40 | 41 | clean-pyc: ## remove Python file artifacts 42 | find . -name '*.pyc' -exec rm -f {} + 43 | find . -name '*.pyo' -exec rm -f {} + 44 | find . -name '*~' -exec rm -f {} + 45 | find . -name '__pycache__' -exec rm -fr {} + 46 | 47 | clean-test: ## remove test and coverage artifacts 48 | rm -fr .tox/ 49 | rm -f .coverage 50 | rm -fr htmlcov/ 51 | rm -fr .pytest_cache 52 | 53 | lint: ## check style with flake8 54 | flake8 msd_pytorch tests 55 | 56 | test: ## run tests quickly with the default Python 57 | py.test 58 | 59 | test-all: ## run tests on every Python version with tox 60 | tox 61 | 62 | coverage: ## check code coverage quickly with the default Python 63 | coverage run setup.py test 64 | coverage report -m 65 | coverage html 66 | $(BROWSER) htmlcov/index.html 67 | 68 | docs/.nojekyll: 69 | mkdir -p docs 70 | touch docs/.nojekyll 71 | 72 | docs: docs/.nojekyll install_dev ## generate Sphinx HTML documentation, including API docs 73 | rm -f doc_sources/msd_pytorch.rst 74 | rm -f doc_sources/modules.rst 75 | sphinx-apidoc -o doc_sources/ msd_pytorch '*/tests' 76 | make -C doc_sources clean 77 | make -C doc_sources html 78 | $(BROWSER) docs/index.html 79 | 80 | servedocs: docs ## compile the docs watching for changes 81 | watchmedo shell-command -p '*.rst' -c '$(MAKE) -C doc_sources html' -R -D . 82 | 83 | install: clean ## install the package to the active Python's site-packages 84 | python setup.py install 85 | 86 | install_dev: 87 | # conda install pytorch=1.0.1 cudatoolkit=10.0 -c pytorch 88 | # https://stackoverflow.com/a/28842733 89 | pip install -e .[dev] 90 | 91 | conda_package: 92 | conda install conda-build conda-verify -y 93 | conda build conda/ -c defaults -c pytorch -c conda-forge -c aahendriksen 94 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3560114.svg)](https://doi.org/10.5281/zenodo.3560114) 2 | 3 | # Mixed-scale Dense Networks for PyTorch 4 | 5 | An implementation of Mixed-Scale Dense networks in PyTorch. 6 | 7 | * Free software: GNU General Public License v3 8 | * Documentation: [https://ahendriksen.github.io/msd_pytorch] 9 | 10 | ## Getting Started 11 | 12 | It takes a few steps to setup Mixed-scale Dense Networks for PyTorch 13 | on your machine. We recommend installing [Anaconda package 14 | manager](https://www.anaconda.com/download/) for Python 3. 15 | 16 | ### Requirements 17 | 18 | This package requires 19 | 20 | - Linux x64 21 | - CUDA 10.0 and/or 11.0 compatible graphics card 22 | - [Anaconda package manager](https://www.anaconda.com/download/) 23 | 24 | This package is compatible with python 3.7, 3.8, and 3.9. 25 | 26 | ### Installing with Conda 27 | 28 | The following instructions install msd_pytorch with pytorch version 1.8.1: 29 | ``` 30 | conda install msd_pytorch=0.10.1 cudatoolkit=11.1 -c aahendriksen -c pytorch -c defaults -c conda-forge 31 | conda install msd_pytorch=0.10.1 cudatoolkit=10.2 -c aahendriksen -c pytorch -c defaults -c conda-forge 32 | ``` 33 | 34 | **Note**: The order of the channels is important. If you install pytorch from 35 | the default conda channel or from conda-forge, installation might fail. 36 | 37 | ### Installing from source 38 | 39 | To install msd_pytorch from source, you need to have the CUDA toolkit 40 | installed. Specifically, you need `nvcc` and a compatible C++ 41 | compiler. Moreover, you need to have a working installation of 42 | PyTorch. 43 | 44 | To get the source code, simply clone this GitHub project. 45 | ``` shell 46 | git clone https://github.com/ahendriksen/msd_pytorch.git 47 | cd msd_pytorch 48 | ``` 49 | 50 | Using pip to install the package automatically triggers the 51 | compilation of the native C++ and CUDA code. So you need to direct the 52 | installer to a CUDA-compatible C++ compiler in this way: 53 | ``` shell 54 | CC=/path/to/compatible/cpp/compiler pip install -e .[dev] 55 | ``` 56 | Or, if the standard C++ compiler is compatible with CUDA: 57 | ``` shell 58 | pip install -e .[dev] 59 | ``` 60 | 61 | ### Running the examples 62 | 63 | To learn more about the functionality of the package check out our 64 | examples folder. 65 | 66 | 67 | ## Cite 68 | If you find our work useful, please cite as: 69 | 70 | ``` 71 | @software{hendriksen-2019-msd-pytor, 72 | author = {Hendriksen, Allard A.}, 73 | title = {ahendriksenh/msd\_pytorch: v0.7.2}, 74 | month = dec, 75 | year = 2019, 76 | publisher = {Zenodo}, 77 | version = {v0.7.2}, 78 | doi = {10.5281/zenodo.3560114}, 79 | url = {https://doi.org/10.5281/zenodo.3560114} 80 | } 81 | ``` 82 | 83 | ## Authors and contributors 84 | 85 | * **Allard Hendriksen** - *Initial work* 86 | * **Ryan Pollitt** - *Port CUDA convolution code from 2D to 3D!* 87 | * **Jonas Adler** - *Discussions and code* 88 | * **Richard Schoonhoven** - *Testing and patches* 89 | 90 | See also the list of [contributors](https://github.com/ahendriksen/msd_pytorch/contributors) who participated in this project. 91 | 92 | ## How to contribute 93 | 94 | Contributions are always welcome. Please submit pull requests against the `dev` branch. 95 | 96 | If you have any issues, questions, or remarks, then please open an issue on GitHub. 97 | 98 | ## License 99 | 100 | This project is licensed under the GNU General Public License v3 - see the [LICENSE.md](LICENSE.md) file for details. 101 | -------------------------------------------------------------------------------- /benchmarks/benchmark_block_sizes.py: -------------------------------------------------------------------------------- 1 | from time import perf_counter as timer 2 | import torch 3 | import msd_custom_convolutions as cc 4 | import numpy as np 5 | from functools import partial 6 | import argparse 7 | import json 8 | from tqdm import tqdm 9 | 10 | 11 | def timeit(f, it): 12 | torch.cuda.synchronize() 13 | start = timer() 14 | f(it) 15 | torch.cuda.synchronize() 16 | return timer() - start 17 | 18 | 19 | def measure_small_execution_time(f, 20 | num_iters=[0,1,2,3], 21 | num_trials=2, 22 | ci=True): 23 | """Measures the execution time of a function f 24 | 25 | `f` must take an int `it` and compute `it` iterations. 26 | 27 | This function estimates how long a single iteration works using the 28 | methodology proposed in 29 | 30 | Moreno, C., & Fischmeister, S. (2017). Accurate measurement of small 31 | execution times—getting around measurement errors. IEEE Embedded Systems 32 | Letters, 9(1), 17–20. http://dx.doi.org/10.1109/les.2017.2654160 33 | 34 | The function returns: 35 | 1. an estimate for the execution time of a single iteration, and 36 | 2. a 90% confidence interval for the estimate (if `ci=True`). 37 | 38 | """ 39 | try: 40 | # Warmup 41 | f(max(num_iters)) 42 | # Measure 43 | num_iters = np.array(list(num_iters) * num_trials) 44 | timings = np.array([timeit(f, it) for it in tqdm(num_iters, leave=False)]) 45 | 46 | slope, intercept = np.polyfit(num_iters, timings, deg=1) 47 | 48 | if not ci: 49 | return slope 50 | 51 | # Follows exposition in: 52 | # https://en.wikipedia.org/wiki/Simple_linear_regression#Confidence_intervals 53 | n = len(timings) 54 | timings_hat = slope * num_iters + intercept 55 | error = timings_hat - timings 56 | s_beta_hat = np.sqrt( 57 | 1 / (n - 2) * np.sum(error ** 2) / 58 | np.sum((num_iters - num_iters.mean()) ** 2) 59 | ) 60 | # Sample a million elements form a standard_t distribution for 90% 61 | # confidence interval 62 | N = 1_000_000 63 | t = np.sort(np.random.standard_t(n - 2, N)) 64 | ci_5, ci_95 = t[5 * N // 100], t[95 * N // 100] 65 | 66 | ci = (slope + ci_5 * s_beta_hat, slope + ci_95 * s_beta_hat) 67 | return slope, ci 68 | 69 | except RuntimeError: 70 | return np.inf, (np.inf, np.inf) 71 | 72 | 73 | def conv3d_forward(input, weight, bias, output, grad_output, grad_input, grad_weight, dilation, block_size=(8, 8, 8)): 74 | cc.conv_forward(input, weight, bias, output, dilation, block_size=block_size) 75 | 76 | 77 | def conv3d_backward_x(input, weight, bias, output, grad_output, grad_input, grad_weight, dilation, block_size=(8, 8, 8)): 78 | cc.conv_backward_x(grad_output, weight, grad_input, dilation, block_size=tuple(block_size)) 79 | 80 | 81 | def conv3d_backward_k(input, weight, bias, output, grad_output, grad_input, grad_weight, dilation, block_size=(8, 8, 8)): 82 | cc.conv_backward_k(grad_output, input, grad_weight, dilation, block_size=block_size) 83 | 84 | 85 | def conv3d_relu_forward(input, weight, bias, output, grad_output, grad_input, grad_weight, dilation, block_size=(8, 8, 8)): 86 | cc.conv_relu_forward(input, weight, bias, output, dilation, block_size=block_size) 87 | 88 | 89 | def conv3d_relu_backward_x(input, weight, bias, output, grad_output, grad_input, grad_weight, dilation, block_size=(8, 8, 8)): 90 | cc.conv_relu_backward_x(output, grad_output, weight, grad_input, dilation, block_size=tuple(block_size)) 91 | 92 | 93 | def conv3d_relu_backward_k(input, weight, bias, output, grad_output, grad_input, grad_weight, dilation, block_size=(8, 8, 8)): 94 | cc.conv_relu_backward_k(output, grad_output, input, grad_weight, dilation, block_size=block_size) 95 | 96 | 97 | # def conv3d_backward_bias(grad_output, grad_bias, block_size=(8, 8, 8)): 98 | # cc.conv_backward_bias(grad_output, grad_bias, block_size=block_size) 99 | 100 | # def conv3d_relu_backward_bias(output, grad_output, grad_bias, block_size=(8, 8, 8)): 101 | # cc.conv_relu_backward_bias(output, grad_output, grad_bias, block_size=block_size) 102 | 103 | functions = { 104 | 'conv3d_forward': (512, conv3d_forward), 105 | 'conv3d_backward_x': (512, conv3d_backward_x), 106 | 'conv3d_backward_k': (512, conv3d_backward_k), 107 | 'conv3d_relu_forward': (512, conv3d_relu_forward), 108 | 'conv3d_relu_backward_x': (512, conv3d_relu_backward_x), 109 | 'conv3d_relu_backward_k': (512, conv3d_relu_backward_k), 110 | } 111 | 112 | 113 | def iter_conv(iterations, c_in=1, c_out=1, N=32, dilation=1, block_size=(4, 4, 4), fun="conv_backward_x"): 114 | # Determine convolution function to use 115 | _, f = functions[fun] 116 | 117 | # Create data (constant time) 118 | x = torch.randn(1, c_in, N, N, N).cuda() 119 | grad_x = torch.randn(1, c_in, N, N, N).cuda() 120 | w = torch.randn(c_out, c_in, 3, 3, 3).cuda() 121 | grad_w = torch.randn(c_out, c_in, 3, 3, 3).cuda() 122 | bias = torch.randn(c_out).cuda() 123 | y = torch.randn(1, c_out, N, N, N).cuda() 124 | grad_y = torch.randn(1, c_out, N, N, N).cuda() 125 | 126 | # Execute function (variable time) 127 | for _ in range(iterations): 128 | f(x, w, bias, y, grad_y, grad_x, grad_w, dilation, block_size) 129 | torch.cuda.synchronize() 130 | 131 | 132 | def main(args): 133 | if args.csv: 134 | print("function, N, time, time_ci_min, time_ci_max, block_size") 135 | else: 136 | print(cc) 137 | 138 | num_trials = args.num_trials 139 | for N in args.N: 140 | if "all" in args.functions: 141 | arg_functions = list(functions.keys()) 142 | else: 143 | arg_functions = args.functions 144 | 145 | for fun in arg_functions: 146 | if not args.csv: 147 | print() 148 | print("------------------------------------------------------------") 149 | print(f"-- {fun}") 150 | print("------------------------------------------------------------") 151 | 152 | max_threads, _ = functions[fun] 153 | pows_of_2 = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048] 154 | 155 | if args.block_sizes is None: 156 | block_sizes = [] 157 | for block_z in [b for b in pows_of_2 if b <= max_threads]: 158 | for block_y in [b for b in pows_of_2 if block_z * b <= max_threads]: 159 | for block_x in [b for b in pows_of_2 if block_z * block_y * b <= max_threads]: 160 | block_sizes.append((block_z, block_y, block_x)) 161 | else: 162 | block_sizes = list(map(tuple, args.block_sizes)) 163 | 164 | for block_size in block_sizes: 165 | f = partial(iter_conv, N=N, c_in=10, block_size=block_size, fun=fun) 166 | slope, (s, S) = measure_small_execution_time(f, num_trials=num_trials) 167 | if args.csv: 168 | print(f"{fun}, {N:04d}, {slope:0.3e}, {s:0.3e}, {S:0.3e}, {repr(block_size).replace(', ', '_')}") 169 | else: 170 | print(f"{N:04d}: {slope:0.3e} in [{s:0.3e} -- {S:0.3e}] ({block_size})") 171 | 172 | 173 | if __name__ == '__main__': 174 | parser = argparse.ArgumentParser() 175 | parser.add_argument("--num_trials", default=30, type=int) 176 | parser.add_argument("--N", default=[256], nargs="+", type=int) 177 | parser.add_argument("--block_sizes", default=None, type=json.loads) 178 | parser.add_argument("--csv", default=False, action='store_const', const='True') 179 | parser.add_argument( 180 | "--functions", 181 | default=["conv3d_backward_x", "conv3d_relu_backward_x"], 182 | nargs="+", 183 | type=str, 184 | ) 185 | args = parser.parse_args() 186 | main(args) 187 | -------------------------------------------------------------------------------- /conda/README.org: -------------------------------------------------------------------------------- 1 | #+TITLE: Readme 2 | 3 | Pytorch has a complex compatibility table: 4 | 5 | | Pytorch version | Compatible CUDA versions | Python versions | 6 | |-----------------+--------------------------+--------------------| 7 | | 1.4 | 9.2, 10.1 | 2.7, 3.5 -- 3.8 | 8 | | 1.5 | 9.2, 10.1, 10.2 | 2.7, 3.5 -- 3.8 | 9 | | 1.6 | 9.2, 10.1, 10.2 | 3.6, 3.7, 3.8 | 10 | | 1.7 | 9.2, 10.1, 10.2, 11.0 | 3.6, 3.7, 3.8 | 11 | | 1.8 | 10.1, 10.2, 11.1 | 3.6, 3.7, 3.8, 3.9 | 12 | 13 | | Python versions | CUDA version | Pytorch versions | 14 | |-----------------+--------------+------------------| 15 | | 3.6 -- 3.8 | 9.2 | 1.4 -- 1.7 | 16 | | 3.6 -- 3.8 | 10.1 | 1.4 -- 1.8 | 17 | | 3.6 -- 3.8 | 10.2 | 1.5 -- 1.8 | 18 | | 3.6 -- 3.8 | 11.0 | 1.7 | 19 | | 3.6 -- 3.8 | 11.1 | 1.8 | 20 | | 3.9 | 10.1 | 1.8 | 21 | | 3.9 | 10.2 | 1.8 | 22 | | 3.9 | 11.1 | 1.8 | 23 | 24 | Check https://download.pytorch.com/whl/torch_stable.html to see all installable 25 | versions. 26 | 27 | 28 | * How to build using docker 29 | 30 | From the project root directory execute: 31 | 32 | #+begin_src bash 33 | mkdir -p build/out 34 | 35 | MOUNTS="-v ${PWD}/build/out/:/opt/conda/conda-bld/linux-64/ -v ${PWD}/:/msd_build_dir/ -v pkgs:/opt/conda/pkgs" 36 | export CUDA_HOME=/usr/local/cuda-9.2/ 37 | export TORCH_CUDA_ARCH_LIST="3.5 5.2 6.0 6.1 7.0+PTX" 38 | export cudatoolkit=9.2 39 | 40 | sudo docker run -e CUDA_HOME=${CUDA_HOME} -e TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST}" -e cudatoolkit=${cudatoolkit} $MOUNTS -w /msd_build_dir/ msd-build-env /opt/conda/bin/conda mambabuild conda/ -c aahendriksen -c pytorch -c defaults -c conda-forge -m conda/cuda-9.2.yaml 41 | #+end_src 42 | ** For testing 43 | Using ~test.yaml~ 44 | #+begin_src bash 45 | mkdir -p conda_docker_out 46 | 47 | export cudatoolkit=10.2 48 | export CUDA_HOME=/usr/local/cuda-${cudatoolkit}/ 49 | export TORCH_CUDA_ARCH_LIST="3.5 5.2 6.0 6.1 7.0+PTX" 50 | export MAX_JOBS=4 51 | 52 | MOUNTS="-v ${PWD}/conda_docker_out/:/opt/conda/conda-bld/linux-64/ -v ${PWD}/:/msd_build_dir/ -v pkgs:/opt/conda/pkgs" 53 | 54 | sudo docker run -e CUDA_HOME=${CUDA_HOME} -e TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST}" -e cudatoolkit=${cudatoolkit} -e MAX_JOBS=${MAX_JOBS} $MOUNTS -w /msd_build_dir/ msd-build-env /opt/conda/bin/conda mambabuild conda/ -c aahendriksen -c pytorch -c defaults -c conda-forge -m conda/test.yaml 55 | #+end_src 56 | 57 | 58 | 59 | #+begin_src bash 60 | mkdir -p conda_docker_out 61 | 62 | MOUNTS="-v ${PWD}/conda_docker_out/:/opt/conda/conda-bld/linux-64/ -v ${PWD}/:/msd_build_dir/ -v pkgs:/opt/conda/pkgs" 63 | export cudatoolkit=9.2 64 | export CUDA_HOME=/usr/local/cuda-${cudatoolkit}/ 65 | export TORCH_CUDA_ARCH_LIST="3.5 5.2 6.0 6.1 7.0+PTX" 66 | 67 | sudo docker run -e CUDA_HOME=${CUDA_HOME} -e TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST}" -e cudatoolkit=${cudatoolkit} $MOUNTS -w /msd_build_dir/ msd-build-env /opt/conda/bin/conda mambabuild conda/ -c aahendriksen -c pytorch -c defaults -c conda-forge -m conda/cuda-9.2.yaml 68 | #+end_src 69 | 70 | 71 | 72 | * How to create the docker container 73 | First, download all versions of the cuda toolkit: 74 | 75 | #+begin_src bash 76 | cd conda/build_env 77 | bash ./download_cuda_environments.sh 78 | #+end_src 79 | 80 | Then, build the docker container. This can take a while.. The final image will 81 | be roughly 25GB. 82 | #+begin_src bash 83 | cd conda/build_env 84 | sudo DOCKER_BUILDKIT=1 docker build -t msd-build-env ./ 85 | #+end_src 86 | 87 | If you are on a recent version of Docker (> 18.9), then you can use the buildkit 88 | backend which is considerably more efficient. See 89 | - [[https://www.cloudsavvyit.com/10271/understanding-the-docker-build-context-why-you-should-use-dockerignore/][Understanding the Docker Build Context (Why You Should Use Dockerignore) – CloudSavvy IT]] 90 | - [[https://docs.docker.com/engine/reference/builder/#buildkit][Dockerfile reference | Docker Documentation]] 91 | -------------------------------------------------------------------------------- /conda/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "------------------------------------------------------------" 4 | echo "Build msd-pytorch Environment: " 5 | echo "------------------------------------------------------------" 6 | 7 | env 8 | 9 | echo "------------------------------------------------------------" 10 | 11 | if [ -f $CUDA_HOME/bin/nvcc ] ; then 12 | echo "-- CUDA already installed"; 13 | else 14 | echo "Using version ${cudatoolkit} " 15 | export CUDA_HOME=/usr/local/cuda-${cudatoolkit} 16 | echo " set CUDA_HOME=${CUDA_HOME}" 17 | fi 18 | 19 | 20 | $PYTHON setup.py clean 21 | $PYTHON setup.py install --single-version-externally-managed --record record.txt || exit 1 22 | -------------------------------------------------------------------------------- /conda/conda_build_config.yaml: -------------------------------------------------------------------------------- 1 | c_compiler_version: # [linux] 2 | - 7.3 # [linux] 3 | cxx_compiler_version: # [linux] 4 | - 7.3 # [linux] 5 | pytorch: 6 | - 1.8 7 | python: 8 | - 3.7 9 | - 3.8 10 | - 3.9 11 | ccache_method: "mklink" 12 | -------------------------------------------------------------------------------- /conda/cuda-10.2.yaml: -------------------------------------------------------------------------------- 1 | cudatoolkit: 2 | - 10.2 3 | -------------------------------------------------------------------------------- /conda/cuda-11.1.yaml: -------------------------------------------------------------------------------- 1 | cudatoolkit: 2 | - 11.1 3 | -------------------------------------------------------------------------------- /conda/meta.yaml: -------------------------------------------------------------------------------- 1 | 2 | {% set data = load_setup_py_data() %} 3 | 4 | package: 5 | name: msd_pytorch 6 | # If this release is tagged, use bare version number. Otherwise, 7 | # append 'dev' to version number. 8 | # This scheme complies with PEP-386: 9 | # https://www.python.org/dev/peps/pep-0386/ 10 | {% if environ.get('GIT_DESCRIBE_NUMBER', '0') == '0' %} 11 | version: {{ data['version'] }} 12 | {% else %} 13 | version: {{ data['version'] + "dev" }} 14 | {% endif %} 15 | 16 | source: 17 | # 18 | git_url: ../ 19 | 20 | build: 21 | number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }} 22 | string: py_{{ python }}_cuda_{{ cudatoolkit }}_pytorch_{{ pytorch }} 23 | script_env: 24 | # Used to store multiple versions of the CUDA architecture; 25 | # Is used by torch from version 1.3 onwards.. 26 | - TORCH_CUDA_ARCH_LIST 27 | # Used to select the right version of nvcc 28 | - CUDA_HOME 29 | # Used to increase the number of concurrent compilations in the ninja build 30 | # step. 31 | - MAX_JOBS 32 | 33 | requirements: 34 | build: 35 | - {{ compiler('c') }} 36 | - {{ compiler('cxx') }} 37 | # https://github.com/conda/conda-build/issues/4047 38 | - ccache 39 | host: 40 | - python 41 | - setuptools 42 | - pytorch {{pytorch}} 43 | - cudatoolkit {{cudatoolkit}} 44 | # Force downloading gpu packages for pytorch: 45 | - _pytorch_select=0.2 46 | {% for dep in data['setup_requires'] %} 47 | - {{ dep.lower() }} 48 | {% endfor %} 49 | run: 50 | - python 51 | - {{ pin_compatible('pytorch', max_pin='x.x') }} 52 | - {{ pin_compatible('cudatoolkit', max_pin='x.x') }} 53 | # dependencies are defined in setup.py 54 | {% for dep in data['install_requires'] %} 55 | - {{ dep.lower() }} 56 | {% endfor %} 57 | 58 | about: 59 | home: https://github.com/ahendriksen/msd_pytorch 60 | license_file: LICENSE.md 61 | summary: "An implementation of Mixed-Scale Dense networks in PyTorch. " 62 | 63 | extra: 64 | maintainers: 65 | - Allard Hendriksen 66 | -------------------------------------------------------------------------------- /conda_docker_build/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:buster-slim AS build-base 2 | ENV DEBIAN_FRONTEND noninteractive 3 | RUN apt-get update -q && apt-get install -q -y libxml2 build-essential 4 | 5 | FROM build-base AS cuda-102 6 | RUN touch /root/cuda102 7 | COPY cuda_10.2.89_440.33.01_linux.run /root 8 | RUN /bin/bash /root/cuda_10.2.89_440.33.01_linux.run --toolkit --silent --installpath=/usr/local/cuda-10.2 && \ 9 | rm -f /root/cuda_10.2.89_440.33.01_linux.run 10 | 11 | FROM build-base AS cuda-111 12 | RUN touch /root/cuda111 13 | COPY cuda_11.1.1_455.32.00_linux.run /root 14 | RUN /bin/bash /root/cuda_11.1.1_455.32.00_linux.run --toolkit --silent --installpath=/usr/local/cuda-11.1 && \ 15 | rm -f /root/cuda_*_linux.run 16 | 17 | FROM build-base AS cuda-113 18 | RUN touch /root/cuda111 19 | COPY cuda_11.3.1_465.19.01_linux.run /root 20 | RUN /bin/bash /root/cuda_11.3.1_465.19.01_linux.run --toolkit --silent --installpath=/usr/local/cuda-11.3 && \ 21 | rm -f /root/cuda_*_linux.run 22 | 23 | 24 | FROM build-base AS cuda-base 25 | RUN touch /root/cuda 26 | COPY --from=cuda-102 /usr/local/cuda-10.2 /usr/local/cuda-10.2 27 | COPY --from=cuda-111 /usr/local/cuda-11.1 /usr/local/cuda-11.1 28 | COPY --from=cuda-113 /usr/local/cuda-11.3 /usr/local/cuda-11.3 29 | 30 | FROM debian:buster-slim AS mamba-base 31 | ARG VERSION=0.11.3 32 | RUN apt-get -q update && apt-get install -q -y \ 33 | bzip2 \ 34 | ca-certificates \ 35 | curl \ 36 | && rm -rf /var/lib/{apt,dpkg,cache,log} 37 | RUN curl -L https://micromamba.snakepit.net/api/micromamba/linux-64/$VERSION | \ 38 | tar -xj -C /tmp bin/micromamba 39 | 40 | 41 | FROM cuda-base 42 | ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 43 | ENV MAMBA_ROOT_PREFIX=/opt/conda 44 | 45 | RUN apt-get update -q && apt-get install git -yq 46 | 47 | # Use bash in Dockerfile RUN commands and make sure bashrc is sourced when 48 | # executing commands with /bin/bash -c 49 | # Needed to have the micromamba activate command configured etc. 50 | ENV BASH_ENV /root/.bashrc 51 | SHELL ["/bin/bash", "-c"] 52 | 53 | # Setting $BASH_ENV and the SHELL command will not result in .bashrc being sourced when 54 | # you supply the program to run as an argument to the "docker run" command. 55 | # Manually add directory for micromamba installed executables to PATH as a workaround. 56 | ENV PATH "$MAMBA_ROOT_PREFIX/bin:$PATH" 57 | 58 | COPY --from=mamba-base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 59 | COPY --from=mamba-base /tmp/bin/micromamba /bin/micromamba 60 | 61 | RUN ln -s /bin/micromamba /bin/mamba && \ 62 | ln -s /bin/micromamba /bin/conda && \ 63 | ln -s /bin/micromamba /bin/miniconda && \ 64 | /bin/micromamba shell init -s bash -p $MAMBA_ROOT_PREFIX && \ 65 | echo "micromamba activate base" >> /root/.bashrc 66 | 67 | RUN mamba install -y -n base boa conda-build conda-verify -c defaults -c conda-forge 68 | RUN mamba install -y -n base pytorch=1.8 -c pytorch 69 | CMD ["/bin/bash"] 70 | -------------------------------------------------------------------------------- /conda_docker_build/download_cuda_environments.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | # https://developer.nvidia.com/cuda-toolkit-archive 5 | 6 | # https://developer.nvidia.com/compute/cuda/9.2/Prod2/local_installers/cuda_9.2.148_396.37_linux 7 | # https://developer.nvidia.com/compute/cuda/10.0/Prod/local_installers/cuda_10.0.130_410.48_linux 8 | # http://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_418.87.00_linux.run 9 | # http://developer.download.nvidia.com/compute/cuda/10.2/Prod/local_installers/cuda_10.2.89_440.33.01_linux.run 10 | # https://developer.download.nvidia.com/compute/cuda/11.0.3/local_installers/cuda_11.0.3_450.51.06_linux.run 11 | 12 | urls="http://developer.download.nvidia.com/compute/cuda/10.2/Prod/local_installers/cuda_10.2.89_440.33.01_linux.run 13 | https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda_11.1.1_455.32.00_linux.run 14 | https://developer.download.nvidia.com/compute/cuda/11.3.1/local_installers/cuda_11.3.1_465.19.01_linux.run 15 | " 16 | 17 | for F in $urls; do 18 | [ -f `basename $F` ] || (wget ${F} ) 19 | done 20 | -------------------------------------------------------------------------------- /conda_docker_release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | mkdir -p conda_docker_out 5 | mkdir -p release_out 6 | 7 | export MAX_JOBS=4 8 | 9 | MOUNTS="-v ${PWD}/conda_docker_out/:/opt/conda/conda-bld/linux-64/ -v ${PWD}/:/msd_build_dir/ -v pkgs:/opt/conda/pkgs" 10 | 11 | 12 | export cudatoolkit=10.2 13 | export CUDA_HOME=/usr/local/cuda-${cudatoolkit}/ 14 | # For more information about TORCH_CUDA_ARCH_LIST, see: 15 | # - https://github.com/pytorch/pytorch/commit/cd207737017db8c81584763207df20bc6110ed75 16 | # - https://en.wikipedia.org/wiki/CUDA#GPUs_supported 17 | # - https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#options-for-steering-gpu-code-generation-gpu-architecture 18 | 19 | export TORCH_CUDA_ARCH_LIST="3.5+PTX 6.0+PTX 7.0+PTX" 20 | docker run --rm -e CUDA_HOME=${CUDA_HOME} -e TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST}" -e cudatoolkit=${cudatoolkit} -e MAX_JOBS=${MAX_JOBS} $MOUNTS -w /msd_build_dir/ msd-build-env /opt/conda/bin/conda mambabuild conda/ -c aahendriksen -c pytorch -c defaults -c conda-forge -m conda/cuda-${cudatoolkit}.yaml 21 | cp conda_docker_out/*tar.bz2 release_out/ 22 | 23 | export cudatoolkit=11.1 24 | export CUDA_HOME=/usr/local/cuda-${cudatoolkit}/ 25 | export TORCH_CUDA_ARCH_LIST="3.5+PTX 6.0+PTX 7.0+PTX 8.0+PTX" 26 | docker run --rm -e CUDA_HOME=${CUDA_HOME} -e TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST}" -e cudatoolkit=${cudatoolkit} -e MAX_JOBS=${MAX_JOBS} $MOUNTS -w /msd_build_dir/ msd-build-env /opt/conda/bin/conda mambabuild conda/ -c aahendriksen -c pytorch -c defaults -c conda-forge -m conda/cuda-${cudatoolkit}.yaml 27 | cp conda_docker_out/*tar.bz2 release_out/ 28 | -------------------------------------------------------------------------------- /doc_sources/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ../CHANGELOG.md -------------------------------------------------------------------------------- /doc_sources/LICENSE.md: -------------------------------------------------------------------------------- 1 | ../LICENSE.md -------------------------------------------------------------------------------- /doc_sources/README.md: -------------------------------------------------------------------------------- 1 | ../README.md -------------------------------------------------------------------------------- /doc_sources/examples.rst: -------------------------------------------------------------------------------- 1 | Examples 2 | ======== 3 | 4 | This page is intended to give you a simple introduction to the 5 | msd_pytorch package and how to work with it. The following code 6 | example trains an MSD regression or segmentation network on a dataset. 7 | 8 | .. literalinclude:: ../examples/getting_started.py 9 | :language: python 10 | -------------------------------------------------------------------------------- /doc_sources/index.rst: -------------------------------------------------------------------------------- 1 | .. You can adapt this file completely to your liking, but it should at least 2 | contain the root `toctree` directive. 3 | 4 | Welcome to the documentation of Mixed-scale Dense Networks for PyTorch! 5 | ======================================================================= 6 | 7 | Contents: 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | 13 | README 14 | examples 15 | modules 16 | CHANGELOG 17 | 18 | 19 | Indices and tables 20 | ================== 21 | 22 | * :ref:`genindex` 23 | * :ref:`modindex` 24 | * :ref:`search` 25 | -------------------------------------------------------------------------------- /doc_sources/modules.rst: -------------------------------------------------------------------------------- 1 | msd_pytorch 2 | =========== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | msd_pytorch 8 | -------------------------------------------------------------------------------- /doc_sources/msd_pytorch.rst: -------------------------------------------------------------------------------- 1 | msd\_pytorch package 2 | ==================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | msd\_pytorch.bench module 8 | ------------------------- 9 | 10 | .. automodule:: msd_pytorch.bench 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | msd\_pytorch.conv module 16 | ------------------------ 17 | 18 | .. automodule:: msd_pytorch.conv 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | msd\_pytorch.conv\_relu module 24 | ------------------------------ 25 | 26 | .. automodule:: msd_pytorch.conv_relu 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | msd\_pytorch.errors module 32 | -------------------------- 33 | 34 | .. automodule:: msd_pytorch.errors 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | msd\_pytorch.image\_dataset module 40 | ---------------------------------- 41 | 42 | .. automodule:: msd_pytorch.image_dataset 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | msd\_pytorch.main module 48 | ------------------------ 49 | 50 | .. automodule:: msd_pytorch.main 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | msd\_pytorch.msd\_block module 56 | ------------------------------ 57 | 58 | .. automodule:: msd_pytorch.msd_block 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | msd\_pytorch.msd\_model module 64 | ------------------------------ 65 | 66 | .. automodule:: msd_pytorch.msd_model 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | msd\_pytorch.msd\_module module 72 | ------------------------------- 73 | 74 | .. automodule:: msd_pytorch.msd_module 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | msd\_pytorch.msd\_regression\_model module 80 | ------------------------------------------ 81 | 82 | .. automodule:: msd_pytorch.msd_regression_model 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | msd\_pytorch.msd\_segmentation\_model module 88 | -------------------------------------------- 89 | 90 | .. automodule:: msd_pytorch.msd_segmentation_model 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | msd\_pytorch.relu\_inplace module 96 | --------------------------------- 97 | 98 | .. automodule:: msd_pytorch.relu_inplace 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | msd\_pytorch.stitch module 104 | -------------------------- 105 | 106 | .. automodule:: msd_pytorch.stitch 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | 112 | Module contents 113 | --------------- 114 | 115 | .. automodule:: msd_pytorch 116 | :members: 117 | :undoc-members: 118 | :show-inheritance: 119 | -------------------------------------------------------------------------------- /docs/.buildinfo: -------------------------------------------------------------------------------- 1 | # Sphinx build info version 1 2 | # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. 3 | config: 5fff3eb39da73fea8e8b561f1e26f147 4 | tags: 645f666f9bcd5a90fca523b33c5a78b7 5 | -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/.nojekyll -------------------------------------------------------------------------------- /docs/_modules/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Overview: module code — Mixed-scale Dense Networks for PyTorch documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 |
44 | 45 | 93 | 94 |
95 | 96 | 97 | 103 | 104 | 105 |
106 | 107 |
108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 |
126 | 127 |
    128 | 129 |
  • Docs »
    • 130 | 131 |
  • Overview: module code
    • 132 | 133 | 134 |
  • 135 | 136 |
    • 137 | 138 |
139 | 140 | 141 |
142 |
143 |
144 | 163 | 164 |
165 |
166 | 167 | 168 |
169 | 170 |
171 |

172 | © Copyright 2018, Allard Hendriksen 173 | 174 |

175 |
176 | Built with Sphinx using a theme provided by Read the Docs. 177 | 178 |
179 | 180 |
181 |
182 | 183 |
184 | 185 |
186 | 187 | 188 | 189 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | -------------------------------------------------------------------------------- /docs/_modules/msd_pytorch/errors.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | msd_pytorch.errors — Mixed-scale Dense Networks for PyTorch documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 |
44 | 45 | 93 | 94 |
95 | 96 | 97 | 103 | 104 | 105 |
106 | 107 |
108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 |
126 | 127 |
    128 | 129 |
  • Docs »
    • 130 | 131 |
  • Module code »
    • 132 | 133 |
  • msd_pytorch.errors
    • 134 | 135 | 136 |
  • 137 | 138 |
    • 139 | 140 |
141 | 142 | 143 |
144 |
145 |
146 |
147 | 148 |

Source code for msd_pytorch.errors

149 | 
[docs]class Error(Exception):
150 |     """Base class for exceptions in msd_pytorch."""
151 | 
152 |     pass

153 | 
154 | 
155 | 
[docs]class InputError(Error):
156 |     """Exception raised for errors in the input.
157 | 
158 |     Attributes:
159 |         message -- explanation of the error
160 |     """
161 | 
162 | 
[docs]    def __init__(self, message):
163 |         self.message = message

164 |
165 | 166 |
167 | 168 |
169 |
170 | 171 | 172 |
173 | 174 |
175 |

176 | © Copyright 2018, Allard Hendriksen 177 | 178 |

179 |
180 | Built with Sphinx using a theme provided by Read the Docs. 181 | 182 |
183 | 184 |
185 |
186 | 187 |
188 | 189 |
190 | 191 | 192 | 193 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | -------------------------------------------------------------------------------- /docs/_sources/CHANGELOG.md.txt: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [Unreleased] 8 | ### Added 9 | ### Fixed 10 | ### Removed 11 | 12 | ## [0.7.2] - 2019-07-30 13 | ### Added 14 | - Support for multi-gpu execution. Use `parallel=True` when 15 | constructing a `MSDRegressionModel` or `MSDSegmentationModel`. 16 | ### Fixed 17 | - Make `model.forward()` more memory-efficient. 18 | ### Removed 19 | 20 | ## [0.7.1] - 2019-05-27 21 | ### Added 22 | - Add `weights_path` command-line argument to msd executable to indicate 23 | where to store final weights. 24 | - Add `MSDBlock2d`: this is a faster and slightly more memory efficient 25 | implementation of the same MSD component. Many thanks to Jonas 26 | Adler for suggesting this way of structuring the code! 27 | ### Changed 28 | - The MSD models use `MSDBlock2d` implementation by default now. 29 | 30 | ## 0.6.2 - 2019-05-23 31 | ### Added 32 | - Initial release. 33 | 34 | [Unreleased]: https://www.github.com/ahendriksen/msd_pytorch/compare/v0.7.2...master 35 | [0.7.2]: https://www.github.com/ahendriksen/msd_pytorch/compare/v0.7.1...v0.7.2 36 | [0.7.1]: https://www.github.com/ahendriksen/msd_pytorch/compare/v0.6.2...v0.7.1 37 | -------------------------------------------------------------------------------- /docs/_sources/LICENSE.md.txt: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | An implementation of Mixed-Scale Dense networks in PyTorch. 5 | Copyright (C) 2019 Allard Hendriksen 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | 20 | Also add information on how to contact you by electronic and paper mail. 21 | 22 | You should also get your employer (if you work as a programmer) or school, 23 | if any, to sign a "copyright disclaimer" for the program, if necessary. 24 | For more information on this, and how to apply and follow the GNU GPL, see 25 | . 26 | 27 | The GNU General Public License does not permit incorporating your program 28 | into proprietary programs. If your program is a subroutine library, you 29 | may consider it more useful to permit linking proprietary applications with 30 | the library. If this is what you want to do, use the GNU Lesser General 31 | Public License instead of this License. But first, please read 32 | . 33 | 34 | -------------------------------------------------------------------------------- /docs/_sources/README.md.txt: -------------------------------------------------------------------------------- 1 | # Mixed-scale Dense Networks for PyTorch 2 | 3 | An implementation of Mixed-Scale Dense networks in PyTorch. 4 | 5 | This paragraph should contain a high-level description of the package, with a 6 | brief overview of its features and limitations. 7 | 8 | 9 | * Free software: GNU General Public License v3 10 | * Documentation: [https://ahendriksen.github.io/msd_pytorch] 11 | 12 | ## Getting Started 13 | 14 | It takes a few steps to setup Mixed-scale Dense Networks for PyTorch 15 | on your machine. We recommend installing [Anaconda package 16 | manager](https://www.anaconda.com/download/) for Python 3. 17 | 18 | ### Requirements 19 | 20 | This package requires 21 | 22 | - Linux x64 23 | - CUDA 9.0 and/or 10.0 compatible graphics card 24 | - [Anaconda package manager](https://www.anaconda.com/download/) 25 | 26 | This package is compatible with python 3.6 and 3.7. 27 | 28 | ### Installing with Conda 29 | 30 | Simply install with either cuda version 9.0 or version 10.0: 31 | ``` 32 | conda install -c conda-forge -c aahendriksen -c pytorch msd_pytorch cudatoolkit=9.0 33 | # or 34 | conda install -c conda-forge -c aahendriksen -c pytorch msd_pytorch cudatoolkit=10.0 35 | ``` 36 | 37 | ### Installing from source 38 | 39 | To install msd_pytorch from source, you need to have the CUDA toolkit 40 | installed. Specifically, you need `nvcc` and a compatible C++ 41 | compiler. Moreover, you need to have a working installation of 42 | PyTorch. 43 | 44 | To install PyTorch, we recommend using conda. Install PyTorch with 45 | either of these versions of cudatoolkit: 46 | ``` shell 47 | conda install pytorch=1.1.0 torchvision cudatoolkit=10.0 -c pytorch 48 | conda install pytorch=1.1.0 torchvision cudatoolkit=9.0 -c pytorch 49 | ``` 50 | 51 | To get the source code, simply clone this GitHub project. 52 | ``` shell 53 | git clone https://github.com/ahendriksen/msd_pytorch.git 54 | cd msd_pytorch 55 | ``` 56 | 57 | Using pip to install the package automatically triggers the 58 | compilation of the native C++ and CUDA code. So you need to direct the 59 | installer to a CUDA-compatible C++ compiler in this way: 60 | ``` shell 61 | GXX=/path/to/compatible/cpp/compiler pip install -e .[dev] 62 | ``` 63 | Or, if the standard C++ compiler is compatible with CUDA: 64 | ``` shell 65 | pip install -e .[dev] 66 | ``` 67 | 68 | ### Using the tools 69 | 70 | The msd_pytorch package ships with some command-line tools to make 71 | your life easier. If you have input and target images in directories 72 | `./train/input/` and `./train/target/`, then you can train a network 73 | to do regression with the following command in your terminal: 74 | 75 | ``` shell 76 | msd regression -p with train_input_glob='./train/input/*' train_target_glob='./train/target/*' epochs=10 msd.depth=30 77 | ``` 78 | 79 | Similarly, segmentation is possible using the following command: 80 | ``` shell 81 | msd segmentation -p with train_input_glob='./train/input/*' train_target_glob='./train/target/*' epochs=10 msd.depth=30 labels=[0,1,2,3] 82 | ``` 83 | 84 | More command-line arguments are available 85 | 86 | ``` yaml 87 | epochs = 1 # The number of epochs to train for 88 | labels = [0, 1] # The labels that you expect in your segmentation targets (if you are doing segmentation) 89 | train_input_glob = '' # The glob pattern for the training set input data 90 | train_target_glob = '' # The glob pattern for the training set target data 91 | val_input_glob = '' # The glob pattern for the validation set input data 92 | val_target_glob = '' # The glob pattern for the validation set input data 93 | msd: 94 | c_in = 1 # Number of input channels 95 | c_out = 1 # Number of output channels (for regression; see `labels` for segmentation) 96 | depth = 10 # The depth of the network 97 | width = 1 # The width of the network 98 | dilations = [1, 2, ..., 10] # The dilation-scheme that is used in the network 99 | loss = 'L2' # Which loss to use for regression (options: "L1" or "L2") 100 | ``` 101 | 102 | The path specification for the images is a path with optional glob 103 | pattern describing the image file paths. Tildes and other HOME 104 | directory specifications are expanded with `os.path.expanduser` and 105 | symlinks are resolved. 106 | 107 | If the path points to a directory, then all files in the directory are 108 | included. If the path points to file, then that single file is 109 | included. 110 | 111 | Alternatively, one may specify a "glob pattern" to match 112 | specific files in the directory. 113 | 114 | Examples: 115 | 116 | * `"~/train_images/"` 117 | * `"~/train_images/cats*.png"` 118 | * `"~/train_images/*.tif"` 119 | * `"~/train_images/scan*"` 120 | * `"~/train_images/just_one_image.jpeg"` 121 | 122 | ### Running the examples 123 | 124 | To learn more about the functionality of the package check out our 125 | examples folder. 126 | 127 | ## Authors and contributors 128 | 129 | * **Allard Hendriksen** - *Initial work* 130 | * **Jonas Adler** - *Discussions and code* 131 | 132 | See also the list of [contributors](https://github.com/ahendriksen/msd_pytorch/contributors) who participated in this project. 133 | 134 | ## How to contribute 135 | 136 | Contributions are always welcome. Please submit pull requests against the `master` branch. 137 | 138 | If you have any issues, questions, or remarks, then please open an issue on GitHub. 139 | 140 | ## License 141 | 142 | This project is licensed under the GNU General Public License v3 - see the [LICENSE.md](LICENSE.md) file for details. 143 | -------------------------------------------------------------------------------- /docs/_sources/examples.rst.txt: -------------------------------------------------------------------------------- 1 | Examples 2 | ======== 3 | 4 | This page is intended to give you a simple introduction to the 5 | msd_pytorch package and how to work with it. The following code 6 | example trains an MSD regression or segmentation network on a dataset. 7 | 8 | .. literalinclude:: ../examples/getting_started.py 9 | :language: python 10 | -------------------------------------------------------------------------------- /docs/_sources/index.rst.txt: -------------------------------------------------------------------------------- 1 | .. You can adapt this file completely to your liking, but it should at least 2 | contain the root `toctree` directive. 3 | 4 | Welcome to the documentation of Mixed-scale Dense Networks for PyTorch! 5 | ======================================================================= 6 | 7 | Contents: 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | 13 | README 14 | examples 15 | modules 16 | CHANGELOG 17 | 18 | 19 | Indices and tables 20 | ================== 21 | 22 | * :ref:`genindex` 23 | * :ref:`modindex` 24 | * :ref:`search` 25 | -------------------------------------------------------------------------------- /docs/_sources/modules.rst.txt: -------------------------------------------------------------------------------- 1 | msd_pytorch 2 | =========== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | msd_pytorch 8 | -------------------------------------------------------------------------------- /docs/_sources/msd_pytorch.rst.txt: -------------------------------------------------------------------------------- 1 | msd\_pytorch package 2 | ==================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | msd\_pytorch.bench module 8 | ------------------------- 9 | 10 | .. automodule:: msd_pytorch.bench 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | msd\_pytorch.conv module 16 | ------------------------ 17 | 18 | .. automodule:: msd_pytorch.conv 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | msd\_pytorch.conv\_relu module 24 | ------------------------------ 25 | 26 | .. automodule:: msd_pytorch.conv_relu 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | msd\_pytorch.errors module 32 | -------------------------- 33 | 34 | .. automodule:: msd_pytorch.errors 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | msd\_pytorch.image\_dataset module 40 | ---------------------------------- 41 | 42 | .. automodule:: msd_pytorch.image_dataset 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | msd\_pytorch.main module 48 | ------------------------ 49 | 50 | .. automodule:: msd_pytorch.main 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | msd\_pytorch.msd\_block module 56 | ------------------------------ 57 | 58 | .. automodule:: msd_pytorch.msd_block 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | msd\_pytorch.msd\_model module 64 | ------------------------------ 65 | 66 | .. automodule:: msd_pytorch.msd_model 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | msd\_pytorch.msd\_module module 72 | ------------------------------- 73 | 74 | .. automodule:: msd_pytorch.msd_module 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | msd\_pytorch.msd\_regression\_model module 80 | ------------------------------------------ 81 | 82 | .. automodule:: msd_pytorch.msd_regression_model 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | msd\_pytorch.msd\_segmentation\_model module 88 | -------------------------------------------- 89 | 90 | .. automodule:: msd_pytorch.msd_segmentation_model 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | msd\_pytorch.relu\_inplace module 96 | --------------------------------- 97 | 98 | .. automodule:: msd_pytorch.relu_inplace 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | msd\_pytorch.stitch module 104 | -------------------------- 105 | 106 | .. automodule:: msd_pytorch.stitch 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | 112 | Module contents 113 | --------------- 114 | 115 | .. automodule:: msd_pytorch 116 | :members: 117 | :undoc-members: 118 | :show-inheritance: 119 | -------------------------------------------------------------------------------- /docs/_static/css/badge_only.css: -------------------------------------------------------------------------------- 1 | .fa:before{-webkit-font-smoothing:antialiased}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-weight:normal;font-style:normal;src:url("../fonts/fontawesome-webfont.eot");src:url("../fonts/fontawesome-webfont.eot?#iefix") format("embedded-opentype"),url("../fonts/fontawesome-webfont.woff") format("woff"),url("../fonts/fontawesome-webfont.ttf") format("truetype"),url("../fonts/fontawesome-webfont.svg#FontAwesome") format("svg")}.fa:before{display:inline-block;font-family:FontAwesome;font-style:normal;font-weight:normal;line-height:1;text-decoration:inherit}a .fa{display:inline-block;text-decoration:inherit}li .fa{display:inline-block}li .fa-large:before,li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-0.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before,ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before{content:""}.icon-book:before{content:""}.fa-caret-down:before{content:""}.icon-caret-down:before{content:""}.fa-caret-up:before{content:""}.icon-caret-up:before{content:""}.fa-caret-left:before{content:""}.icon-caret-left:before{content:""}.fa-caret-right:before{content:""}.icon-caret-right:before{content:""}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:"Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;z-index:400}.rst-versions a{color:#2980B9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27AE60;*zoom:1}.rst-versions .rst-current-version:before,.rst-versions .rst-current-version:after{display:table;content:""}.rst-versions .rst-current-version:after{clear:both}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book{float:left}.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#E74C3C;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#F1C40F;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:gray;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:solid 1px #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .icon-book{float:none}.rst-versions.rst-badge .fa-book{float:none}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book{float:left}.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge .rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width: 768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}} 2 | -------------------------------------------------------------------------------- /docs/_static/documentation_options.js: -------------------------------------------------------------------------------- 1 | var DOCUMENTATION_OPTIONS = { 2 | URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'), 3 | VERSION: '', 4 | LANGUAGE: 'en', 5 | COLLAPSE_INDEX: false, 6 | BUILDER: 'html', 7 | FILE_SUFFIX: '.html', 8 | HAS_SOURCE: true, 9 | SOURCELINK_SUFFIX: '.txt', 10 | NAVIGATION_WITH_KEYS: false 11 | }; -------------------------------------------------------------------------------- /docs/_static/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/file.png -------------------------------------------------------------------------------- /docs/_static/fonts/Inconsolata-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Inconsolata-Bold.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Inconsolata-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Inconsolata-Regular.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Inconsolata.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Inconsolata.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato-Bold.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato-Regular.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bold.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato/lato-bold.eot -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato/lato-bold.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato/lato-bold.woff -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato/lato-bold.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bolditalic.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato/lato-bolditalic.eot -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bolditalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato/lato-bolditalic.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bolditalic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato/lato-bolditalic.woff -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bolditalic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato/lato-bolditalic.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-italic.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato/lato-italic.eot -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-italic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato/lato-italic.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato/lato-italic.woff -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato/lato-italic.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato/lato-regular.eot -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato/lato-regular.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato/lato-regular.woff -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/Lato/lato-regular.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/RobotoSlab-Bold.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/RobotoSlab-Regular.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.eot -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.eot -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /docs/_static/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /docs/_static/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/_static/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /docs/_static/js/theme.js: -------------------------------------------------------------------------------- 1 | /* sphinx_rtd_theme version 0.4.3 | MIT license */ 2 | /* Built 20190212 16:02 */ 3 | require=function r(s,a,l){function c(e,n){if(!a[e]){if(!s[e]){var i="function"==typeof require&&require;if(!n&&i)return i(e,!0);if(u)return u(e,!0);var t=new Error("Cannot find module '"+e+"'");throw t.code="MODULE_NOT_FOUND",t}var o=a[e]={exports:{}};s[e][0].call(o.exports,function(n){return c(s[e][1][n]||n)},o,o.exports,r,s,a,l)}return a[e].exports}for(var u="function"==typeof require&&require,n=0;n"),i("table.docutils.footnote").wrap("
"),i("table.docutils.citation").wrap("
"),i(".wy-menu-vertical ul").not(".simple").siblings("a").each(function(){var e=i(this);expand=i(''),expand.on("click",function(n){return t.toggleCurrent(e),n.stopPropagation(),!1}),e.prepend(expand)})},reset:function(){var n=encodeURI(window.location.hash)||"#";try{var e=$(".wy-menu-vertical"),i=e.find('[href="'+n+'"]');if(0===i.length){var t=$('.document [id="'+n.substring(1)+'"]').closest("div.section");0===(i=e.find('[href="#'+t.attr("id")+'"]')).length&&(i=e.find('[href="#"]'))}0this.docHeight||(this.navBar.scrollTop(i),this.winPosition=n)},onResize:function(){this.winResize=!1,this.winHeight=this.win.height(),this.docHeight=$(document).height()},hashChange:function(){this.linkScroll=!0,this.win.one("hashchange",function(){this.linkScroll=!1})},toggleCurrent:function(n){var e=n.closest("li");e.siblings("li.current").removeClass("current"),e.siblings().find("li.current").removeClass("current"),e.find("> ul li.current").removeClass("current"),e.toggleClass("current")}},"undefined"!=typeof window&&(window.SphinxRtdTheme={Navigation:e.exports.ThemeNav,StickyNav:e.exports.ThemeNav}),function(){for(var r=0,n=["ms","moz","webkit","o"],e=0;e 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Welcome to the documentation of Mixed-scale Dense Networks for PyTorch! — Mixed-scale Dense Networks for PyTorch documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 |
45 | 46 | 94 | 95 |
96 | 97 | 98 | 104 | 105 | 106 |
107 | 108 |
109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 |
127 | 128 |
    129 | 130 |
  • Docs »
    • 131 | 132 |
  • Welcome to the documentation of Mixed-scale Dense Networks for PyTorch!
    • 133 | 134 | 135 |
  • 136 | 137 | 138 | View page source 139 | 140 | 141 |
    • 142 | 143 |
144 | 145 | 146 |
147 |
148 |
149 |
150 | 151 |
152 |

Welcome to the documentation of Mixed-scale Dense Networks for PyTorch!

153 |

Contents:

154 |
155 | 176 |
177 |
178 |
179 |

Indices and tables

180 | 185 |
186 | 187 | 188 |
189 | 190 |
191 |
192 | 193 | 199 | 200 | 201 |
202 | 203 |
204 |

205 | © Copyright 2018, Allard Hendriksen 206 | 207 |

208 |
209 | Built with Sphinx using a theme provided by Read the Docs. 210 | 211 |
212 | 213 |
214 |
215 | 216 |
217 | 218 |
219 | 220 | 221 | 222 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | -------------------------------------------------------------------------------- /docs/modules.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | msd_pytorch — Mixed-scale Dense Networks for PyTorch documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 |
46 | 47 | 98 | 99 |
100 | 101 | 102 | 108 | 109 | 110 |
111 | 112 |
113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 |
131 | 132 |
    133 | 134 |
  • Docs »
    • 135 | 136 |
  • msd_pytorch
    • 137 | 138 | 139 |
  • 140 | 141 | 142 | View page source 143 | 144 | 145 |
    • 146 | 147 |
148 | 149 | 150 |
151 |
152 |
153 | 183 | 184 |
185 |
186 | 187 | 195 | 196 | 197 |
198 | 199 |
200 |

201 | © Copyright 2018, Allard Hendriksen 202 | 203 |

204 |
205 | Built with Sphinx using a theme provided by Read the Docs. 206 | 207 |
208 | 209 |
210 |
211 | 212 |
213 | 214 |
215 | 216 | 217 | 218 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | -------------------------------------------------------------------------------- /docs/objects.inv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ahendriksen/msd_pytorch/2f11e1caf1dc9becc0e11b20f256d73548bb7a88/docs/objects.inv -------------------------------------------------------------------------------- /docs/search.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Search — Mixed-scale Dense Networks for PyTorch documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 |
45 | 46 | 94 | 95 |
96 | 97 | 98 | 104 | 105 | 106 |
107 | 108 |
109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 |
127 | 128 |
    129 | 130 |
  • Docs »
    • 131 | 132 |
  • Search
    • 133 | 134 | 135 |
  • 136 | 137 | 138 | 139 |
    • 140 | 141 |
142 | 143 | 144 |
145 |
146 |
147 |
148 | 149 | 157 | 158 | 159 |
160 | 161 |
162 | 163 |
164 | 165 |
166 |
167 | 168 | 169 |
170 | 171 |
172 |

173 | © Copyright 2018, Allard Hendriksen 174 | 175 |

176 |
177 | Built with Sphinx using a theme provided by Read the Docs. 178 | 179 |
180 | 181 |
182 |
183 | 184 |
185 | 186 |
187 | 188 | 189 | 190 | 195 | 196 | 197 | 198 | 199 | 200 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | -------------------------------------------------------------------------------- /examples/getting_started.py: -------------------------------------------------------------------------------- 1 | """ 2 | A short example to get you started with Mixed-scale Dense Networks for PyTorch 3 | """ 4 | import msd_pytorch as mp 5 | from torch.utils.data import DataLoader 6 | import numpy as np 7 | import torch 8 | 9 | ############################################################################### 10 | # Network parameters # 11 | ############################################################################### 12 | # The number of input channels of the MSD network 13 | c_in = 1 14 | # The depth of the MSD network. Good values range between 30 and 200. 15 | depth = 30 16 | # The width of the MSD network. A value of 1 is recommended. 17 | width = 1 18 | # The dilation scheme to use for the MSD network. The default is [1, 19 | # 2, ..., 10], but [1, 2, 4, 8] is good too. 20 | dilations = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] 21 | 22 | # If using the MSD network for regression, set these parameters too. 23 | 24 | # The loss function to use. Can be "L1" or "L2". 25 | loss = "L2" 26 | # The number of output channels of the network. 27 | c_out = 1 28 | 29 | ############################################################################### 30 | # Dataset parameters # 31 | ############################################################################### 32 | 33 | # The glob pattern for the training set input data. For instance: "~/train/input*.tif". 34 | train_input_glob = "~/datasets/MLTestData/train/noisy/*1.tiff" 35 | # The glob pattern for the training set target data. For instance: "~/train/target/img*.png" 36 | train_target_glob = "~/datasets/MLTestData/train/label/*1.tiff" 37 | # The glob pattern for the validation set input data. Can be left 38 | # empty if there is no validation data. 39 | val_input_glob = "" 40 | # The glob pattern for the validation set input data. Can be left 41 | # empty if there is no validation data. 42 | val_target_glob = "" 43 | 44 | # If you are doing segmentation, set the values of the label you are 45 | # expecting in the target dataset. You can either set this value to an 46 | # integer `k`, indicating that the label set is {0, 1, ..., k-1}, or 47 | # explicitly set the expected label values using a list, as is done 48 | # below. 49 | labels = [0, 1, 2, 3, 4] 50 | 51 | ############################################################################### 52 | # Training parameters # 53 | ############################################################################### 54 | # The number of epochs to train for 55 | epochs = 10 56 | # Which task to perform. Can be "regression" or "segmentation" 57 | task = "segmentation" 58 | # The mini-batch size used in training. 59 | batch_size = 3 60 | 61 | ############################################################################### 62 | # Loading data # 63 | ############################################################################### 64 | 65 | print("Load training dataset") 66 | if task == "regression": 67 | train_ds = mp.ImageDataset(train_input_glob, train_target_glob) 68 | else: 69 | train_ds = mp.ImageDataset(train_input_glob, train_target_glob, labels=labels) 70 | 71 | train_dl = DataLoader(train_ds, batch_size, shuffle=True) 72 | 73 | # Load Validation dataset (if specified) 74 | if val_input_glob: 75 | print("Load validation set") 76 | if task == "regression": 77 | val_ds = mp.ImageDataset(val_input_glob, val_target_glob) 78 | else: 79 | val_ds = mp.ImageDataset(val_input_glob, val_target_glob, labels=labels) 80 | 81 | val_dl = DataLoader(val_ds, batch_size, shuffle=False) 82 | else: 83 | print("No validation set loaded") 84 | val_dl = None 85 | 86 | 87 | print(f"Create {task} network model") 88 | if task == "regression": 89 | model = mp.MSDRegressionModel( 90 | c_in, c_out, depth, width, dilations=dilations, loss=loss 91 | ) 92 | else: 93 | model = mp.MSDSegmentationModel( 94 | c_in, train_ds.num_labels, depth, width, dilations=dilations 95 | ) 96 | 97 | # The network works best if the input data has mean zero and has a 98 | # standard deviation of 1. To achieve this, we get a rough estimate of 99 | # correction parameters from the training data. These parameters are 100 | # not updated after this step and are stored in the network, so that 101 | # they are not lost when the network is saved to and loaded from disk. 102 | print("Start estimating normalization parameters") 103 | model.set_normalization(train_dl) 104 | print("Done estimating normalization parameters") 105 | 106 | print("Starting training...") 107 | best_validation_error = np.inf 108 | validation_error = 0.0 109 | 110 | for epoch in range(epochs): 111 | # Train 112 | model.train(train_dl, 1) 113 | # Compute training error 114 | train_error = model.validate(train_dl) 115 | print(f"{epoch:05} Training error: {train_error: 0.6f}") 116 | # Compute validation error 117 | if val_dl is not None: 118 | validation_error = model.validate(val_dl) 119 | print(f"{epoch:05} Validation error: {validation_error: 0.6f}") 120 | # Save network if worthwile 121 | if validation_error < best_validation_error or val_dl is None: 122 | best_validation_error = validation_error 123 | model.save(f"msd_network_epoch_{epoch}.torch", epoch) 124 | 125 | # Save final network parameters 126 | model.save(f"msd_network_epoch_{epoch}.torch", epoch) 127 | 128 | ############################################################################### 129 | # Applying the trained network to new data # 130 | ############################################################################### 131 | 132 | # 1) Recreate the model and load the saved parameters. 133 | print(f"Create {task} network model") 134 | if task == "regression": 135 | model = mp.MSDRegressionModel( 136 | c_in, c_out, depth, width, dilations=dilations, loss=loss 137 | ) 138 | else: 139 | model = mp.MSDSegmentationModel( 140 | c_in, train_ds.num_labels, depth, width, dilations=dilations 141 | ) 142 | 143 | # The parameters can be reloaded again: 144 | epoch = model.load(f"msd_network_epoch_{epoch}.torch") 145 | 146 | # 2) Load new data. 147 | # The glob pattern for new input data. 148 | new_data_input_glob = "~/datasets/new-data/noisy/*.tiff" 149 | 150 | print("Load dataset") 151 | # Since usually no target data is available for new images, just put 152 | # the input images as target as well. They will be ignored in the rest 153 | # of the script. 154 | ds = mp.ImageDataset(new_data_input_glob, new_data_input_glob) 155 | 156 | # Do not shuffle when applying to new data: 157 | dl = DataLoader(ds, batch_size, shuffle=False) 158 | 159 | # 3) apply to new data: 160 | for i, data in enumerate(dl): 161 | # data contains a tuple of input and target data. Ignore the 162 | # target data. 163 | inp, _ = data 164 | # Move input to GPU and apply the network: 165 | output = model.net(inp.cuda()) 166 | # Output has dimensions BxCxHxW (batch size, # channels, height, width). 167 | 168 | if task == "segmentation": 169 | # For each label, a separate output channel is created. The 170 | # output channel with the highest value is the classification 171 | # of the network. 172 | prediction = torch.max(output.data, 1).indices 173 | if task == "regression": 174 | # No processing has to be performed for regression: 175 | prediction = output 176 | 177 | # To convert to a numpy array, detach the output tensor from the 178 | # pytorch computation graph, move it to cpu, and convert it to a 179 | # numpy array. 180 | prediction_numpy = prediction.detach().cpu().numpy() 181 | 182 | # Optionally, if the prediction has a single output channel, you 183 | # may want to remove the channel dimension as follows: 184 | prediction_numpy = prediction_numpy.squeeze() 185 | 186 | # Save to tiff file or do other processing: 187 | for b in range(batch_size): 188 | output_index = i * batch_size + b 189 | if output_index < len(ds): 190 | print(f"At iteration {i}: saving slice {output_index} with shape {prediction_numpy[b].shape}") 191 | # Save CxHxW dimensioned numpy array into tif file: 192 | # tifffile.imsave(f"output_{output_index:05d}.tif", prediction_numpy[b]) 193 | -------------------------------------------------------------------------------- /msd_pytorch/VERSION: -------------------------------------------------------------------------------- 1 | 0.10.1 2 | -------------------------------------------------------------------------------- /msd_pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Top-level package for Mixed-scale Dense Networks for PyTorch.""" 4 | 5 | __author__ = """Allard Hendriksen""" 6 | __email__ = "allard.hendriksen@cwi.nl" 7 | 8 | 9 | def __get_version(): 10 | import os.path 11 | 12 | version_filename = os.path.join(os.path.dirname(__file__), "VERSION") 13 | with open(version_filename) as version_file: 14 | version = version_file.read().strip() 15 | return version 16 | 17 | 18 | __version__ = __get_version() 19 | 20 | 21 | import msd_pytorch.image_dataset 22 | import msd_pytorch.errors 23 | from .image_dataset import ImageDataset 24 | from .msd_regression_model import MSDRegressionModel 25 | from .msd_segmentation_model import MSDSegmentationModel 26 | -------------------------------------------------------------------------------- /msd_pytorch/bench.py: -------------------------------------------------------------------------------- 1 | import timeit 2 | import sys 3 | import math 4 | 5 | 6 | def _format_time(timespan, precision=3): 7 | """Formats the timespan in a human readable form""" 8 | 9 | if timespan >= 60.0: 10 | # we have more than a minute, format that in a human readable form 11 | # Idea from http://snipplr.com/view/5713/ 12 | parts = [("d", 60 * 60 * 24), ("h", 60 * 60), ("min", 60), ("s", 1)] 13 | time = [] 14 | leftover = timespan 15 | for suffix, length in parts: 16 | value = int(leftover / length) 17 | if value > 0: 18 | leftover = leftover % length 19 | time.append(u"%s%s" % (str(value), suffix)) 20 | if leftover < 1: 21 | break 22 | return " ".join(time) 23 | 24 | # Unfortunately the unicode 'micro' symbol can cause problems in 25 | # certain terminals. 26 | # See bug: https://bugs.launchpad.net/ipython/+bug/348466 27 | # Try to prevent crashes by being more secure than it needs to 28 | # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set. 29 | units = [u"s", u"ms", u"us", "ns"] # the save value 30 | if hasattr(sys.stdout, "encoding") and sys.stdout.encoding: 31 | try: 32 | u"\xb5".encode(sys.stdout.encoding) 33 | units = [u"s", u"ms", u"\xb5s", "ns"] 34 | except Exception: 35 | pass 36 | scaling = [1, 1e3, 1e6, 1e9] 37 | 38 | if timespan > 0.0: 39 | order = min(-int(math.floor(math.log10(timespan)) // 3), 3) 40 | else: 41 | order = 3 42 | return u"%.*g %s" % (precision, timespan * scaling[order], units[order]) 43 | 44 | 45 | class TimeitResult(object): 46 | """ 47 | Object returned by the timeit magic with info about the run. 48 | 49 | Contains the following attributes : 50 | 51 | loops: (int) number of loops done per measurement 52 | repeat: (int) number of times the measurement has been repeated 53 | best: (float) best execution time / number 54 | all_runs: (list of float) execution time of each run (in s) 55 | 56 | """ 57 | 58 | def __init__(self, name, loops, repeat, best, worst, all_runs, precision=3): 59 | self.name = name 60 | self.loops = loops 61 | self.repeat = repeat 62 | self.best = best 63 | self.worst = worst 64 | self.all_runs = all_runs 65 | self._precision = precision 66 | self.timings = [dt / self.loops for dt in all_runs] 67 | 68 | @property 69 | def average(self): 70 | return math.fsum(self.timings) / len(self.timings) 71 | 72 | @property 73 | def stdev(self): 74 | mean = self.average 75 | return ( 76 | math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings) 77 | ) ** 0.5 78 | 79 | def __str__(self): 80 | pm = "+-" 81 | if hasattr(sys.stdout, "encoding") and sys.stdout.encoding: 82 | try: 83 | u"\xb1".encode(sys.stdout.encoding) 84 | pm = u"\xb1" 85 | except Exception: 86 | pass 87 | return u"{name:<30}{mean:<7} {pm} {std:<7} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops} loop{loop_plural} each)".format( 88 | name=self.name + ":", 89 | pm=pm, 90 | runs=self.repeat, 91 | loops=self.loops, 92 | loop_plural="" if self.loops == 1 else "s", 93 | run_plural="" if self.repeat == 1 else "s", 94 | mean=_format_time(self.average, self._precision), 95 | std=_format_time(self.stdev, self._precision), 96 | ) 97 | 98 | def _repr_pretty_(self, p, cycle): 99 | unic = self.__str__() 100 | p.text(u"") 101 | 102 | 103 | def bench(name, timer, repeat=timeit.default_repeat): 104 | number = 0 105 | # determine number so that 0.2 <= total time < 2.0 106 | for index in range(0, 10): 107 | number = 10 ** index 108 | time_number = timer.timeit(number) 109 | if time_number >= 0.2: 110 | break 111 | 112 | all_runs = timer.repeat(repeat, number) 113 | best = min(all_runs) / number 114 | worst = max(all_runs) / number 115 | timeit_result = TimeitResult(name, number, repeat, best, worst, all_runs) 116 | return timeit_result 117 | -------------------------------------------------------------------------------- /msd_pytorch/conv2d.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch as t 3 | from torch.autograd import Function 4 | from torch.nn import Parameter 5 | import msd_custom_convolutions as cc 6 | 7 | 8 | def conv2d_forward(input, weight, bias, output, dilation, block_size=(0, 16, 16)): 9 | cc.conv_forward(input, weight, bias, output, dilation, block_size=block_size) 10 | 11 | 12 | def conv2d_backward_x(grad_output, weight, grad_input, dilation, block_size=(0, 16, 16)): 13 | cc.conv_backward_x(grad_output, weight, grad_input, dilation, block_size=block_size) 14 | 15 | 16 | def conv2d_backward_k(grad_output, input, grad_weight, dilation, block_size=(0, 16, 16)): 17 | cc.conv_backward_k(grad_output, input, grad_weight, dilation, block_size=block_size) 18 | 19 | 20 | def conv2d_backward_bias(grad_output, grad_bias, block_size=(0, 16, 16)): 21 | cc.conv_backward_bias(grad_output, grad_bias, block_size=block_size) 22 | 23 | 24 | def conv2d_relu_forward(input, weight, bias, output, dilation, block_size=(0, 16, 16)): 25 | cc.conv_relu_forward(input, weight, bias, output, dilation, block_size=block_size) 26 | 27 | 28 | def conv2d_relu_backward_x(output, grad_output, weight, grad_input, dilation, block_size=(0, 16, 16)): 29 | cc.conv_relu_backward_x(output, grad_output, weight, grad_input, dilation, block_size=block_size) 30 | 31 | 32 | def conv2d_relu_backward_k(output, grad_output, input, grad_weight, dilation, block_size=(0, 16, 16)): 33 | cc.conv_relu_backward_k(output, grad_output, input, grad_weight, dilation, block_size=block_size) 34 | 35 | 36 | def conv2d_relu_backward_bias(output, grad_output, grad_bias, block_size=(0, 16, 16)): 37 | cc.conv_relu_backward_bias(output, grad_output, grad_bias, block_size=block_size) 38 | 39 | 40 | class Conv2dFunction(Function): 41 | @staticmethod 42 | def forward(ctx, input, weight, bias, output, stride, dilation): 43 | ctx.save_for_backward(input, weight, bias) 44 | ctx.dilation = dilation 45 | conv2d_forward(input, weight, bias, output.data, dilation) 46 | return output 47 | 48 | @staticmethod 49 | def backward(ctx, grad_output): 50 | # restore variables 51 | input, weight, bias = ctx.saved_tensors 52 | dilation = ctx.dilation 53 | 54 | # ensure that grad_output is contiguous 55 | grad_output = grad_output.contiguous() 56 | 57 | # Input 58 | grad_input = t.zeros_like(input) 59 | conv2d_backward_x(grad_output, weight, grad_input, dilation) 60 | # Weight 61 | grad_weight = t.zeros_like(weight) 62 | conv2d_backward_k(grad_output, input, grad_weight, dilation) 63 | # Bias 64 | grad_bias = t.zeros_like(bias) 65 | conv2d_backward_bias(grad_output, grad_bias) 66 | 67 | return grad_input, grad_weight, grad_bias, None, None, None, None 68 | 69 | 70 | conv2d = Conv2dFunction.apply 71 | 72 | 73 | class Conv2dModule(nn.Module): 74 | def __init__(self, output, in_channels, out_channels, kernel_size=3, dilation=1): 75 | super(Conv2dModule, self).__init__() 76 | 77 | w = t.Tensor(out_channels, in_channels, kernel_size, kernel_size) 78 | self.weight = Parameter(w.cuda()) 79 | self.bias = Parameter(t.Tensor(out_channels).cuda()) 80 | self.output = output 81 | 82 | self.dilation = dilation 83 | self.stride = 1 84 | 85 | def forward(self, input): 86 | assert self.output.is_cuda 87 | return conv2d( 88 | input, self.weight, self.bias, self.output, self.stride, self.dilation 89 | ) 90 | 91 | 92 | class Conv2dReluFunction(Function): 93 | @staticmethod 94 | def forward(ctx, input, weight, bias, output, stride, dilation): 95 | ctx.dilation = dilation 96 | conv2d_relu_forward(input, weight, bias, output.data, dilation) 97 | ctx.save_for_backward(input, output, weight, bias) 98 | return output 99 | 100 | @staticmethod 101 | def backward(ctx, grad_output): 102 | # restore variables 103 | input, output, weight, bias = ctx.saved_tensors 104 | dilation = ctx.dilation 105 | 106 | # ensure that grad_output is contiguous 107 | grad_output = grad_output.contiguous() 108 | 109 | # Input 110 | grad_input = t.zeros_like(input) 111 | conv2d_relu_backward_x( 112 | output, grad_output, weight, grad_input, dilation 113 | ) 114 | # Weight 115 | grad_weight = t.zeros_like(weight) 116 | conv2d_relu_backward_k( 117 | output, grad_output, input, grad_weight, dilation 118 | ) 119 | # Bias 120 | grad_bias = t.zeros_like(bias) 121 | conv2d_relu_backward_bias(output, grad_output, grad_bias) 122 | 123 | return grad_input, grad_weight, grad_bias, None, None, None, None 124 | 125 | 126 | conv2d_relu = Conv2dReluFunction.apply 127 | 128 | 129 | class Conv2dReluModule(nn.Module): 130 | def __init__(self, output, in_channels, out_channels, kernel_size=3, dilation=1): 131 | super().__init__() 132 | 133 | w = t.Tensor(out_channels, in_channels, kernel_size, kernel_size) 134 | self.weight = Parameter(w.cuda()) 135 | self.bias = Parameter(t.Tensor(out_channels).cuda()) 136 | self.output = output 137 | 138 | self.dilation = dilation 139 | self.stride = 1 140 | 141 | def forward(self, input): 142 | assert self.output.is_cuda 143 | return conv2d_relu( 144 | input, self.weight, self.bias, self.output, self.stride, self.dilation 145 | ) 146 | -------------------------------------------------------------------------------- /msd_pytorch/conv3d.py: -------------------------------------------------------------------------------- 1 | import torch as t 2 | from torch import nn 3 | from torch.nn import Parameter 4 | from torch.autograd import Function 5 | import msd_custom_convolutions as cc 6 | 7 | 8 | def conv3d_forward(input, weight, bias, output, dilation, block_size=(8, 8, 8)): 9 | cc.conv_forward(input, weight, bias, output, dilation, block_size=block_size) 10 | 11 | 12 | def conv3d_backward_x(grad_output, weight, grad_input, dilation, block_size=(4, 4, 4)): 13 | cc.conv_backward_x(grad_output, weight, grad_input, dilation, block_size=tuple(block_size)) 14 | 15 | 16 | def conv3d_backward_k(grad_output, input, grad_weight, dilation, block_size=(8, 8, 8)): 17 | cc.conv_backward_k(grad_output, input, grad_weight, dilation, block_size=block_size) 18 | 19 | 20 | def conv3d_backward_bias(grad_output, grad_bias, block_size=(8, 8, 8)): 21 | cc.conv_backward_bias(grad_output, grad_bias, block_size=block_size) 22 | 23 | 24 | def conv3d_relu_forward(input, weight, bias, output, dilation, block_size=(2, 2, 32)): 25 | cc.conv_relu_forward(input, weight, bias, output, dilation, block_size=block_size) 26 | 27 | 28 | def conv3d_relu_backward_x(output, grad_output, weight, grad_input, dilation, block_size=(2, 4, 16)): 29 | cc.conv_relu_backward_x(output, grad_output, weight, grad_input, dilation, block_size=tuple(block_size)) 30 | 31 | 32 | def conv3d_relu_backward_k(output, grad_output, input, grad_weight, dilation, block_size=(2, 8, 16)): 33 | cc.conv_relu_backward_k(output, grad_output, input, grad_weight, dilation, block_size=block_size) 34 | 35 | 36 | def conv3d_relu_backward_bias(output, grad_output, grad_bias, block_size=(8, 8, 8)): 37 | cc.conv_relu_backward_bias(output, grad_output, grad_bias, block_size=block_size) 38 | 39 | 40 | class Conv3DFunction(Function): 41 | @staticmethod 42 | def forward(ctx, input, weight, bias, output, stride, dilation): 43 | ctx.dilation = dilation 44 | ctx.save_for_backward(input, weight, bias) 45 | conv3d_forward(input, weight, bias, output.data, dilation) 46 | return output 47 | 48 | @staticmethod 49 | def backward(ctx, grad_output): 50 | # restore variables 51 | input, weight, bias = ctx.saved_tensors 52 | dilation = ctx.dilation 53 | 54 | # ensure that grad_output is contiguous 55 | grad_output = grad_output.contiguous() 56 | 57 | # Input 58 | grad_input = t.zeros_like(input) 59 | conv3d_backward_x( 60 | grad_output, weight, grad_input, dilation 61 | ) 62 | # Weight 63 | grad_weight = t.zeros_like(weight) 64 | conv3d_backward_k( 65 | grad_output, input, grad_weight, dilation 66 | ) 67 | # Bias 68 | grad_bias = t.zeros_like(bias) 69 | conv3d_backward_bias(grad_output, grad_bias) 70 | 71 | return grad_input, grad_weight, grad_bias, None, None, None, None 72 | 73 | 74 | conv3d = Conv3DFunction.apply 75 | 76 | 77 | class Conv3DModule(nn.Module): 78 | def __init__(self, output, in_channels, out_channels, kernel_size=3, dilation=1): 79 | super().__init__() 80 | 81 | w = t.Tensor(out_channels, in_channels, kernel_size, kernel_size, kernel_size) 82 | self.weight = Parameter(w.cuda()) 83 | self.bias = Parameter(t.Tensor(out_channels).cuda()) 84 | self.output = output 85 | 86 | self.dilation = dilation 87 | self.stride = 1 88 | 89 | def forward(self, input): 90 | assert self.output.is_cuda 91 | return conv3d( 92 | input, self.weight, self.bias, self.output, self.stride, self.dilation 93 | ) 94 | 95 | 96 | class Conv3DReluFunction(Function): 97 | @staticmethod 98 | def forward(ctx, input, weight, bias, output, stride, dilation): 99 | ctx.dilation = dilation 100 | conv3d_relu_forward(input, weight, bias, output.data, dilation) 101 | ctx.save_for_backward(input, output, weight, bias) 102 | return output 103 | 104 | @staticmethod 105 | def backward(ctx, grad_output): 106 | # restore variables 107 | input, output, weight, bias = ctx.saved_tensors 108 | dilation = ctx.dilation 109 | 110 | # ensure that grad_output is contiguous 111 | grad_output = grad_output.contiguous() 112 | 113 | # Input 114 | grad_input = t.zeros_like(input) 115 | conv3d_relu_backward_x( 116 | output, grad_output, weight, grad_input, dilation 117 | ) 118 | # Weight 119 | grad_weight = t.zeros_like(weight) 120 | conv3d_relu_backward_k( 121 | output, grad_output, input, grad_weight, dilation 122 | ) 123 | # Bias 124 | grad_bias = t.zeros_like(bias) 125 | conv3d_relu_backward_bias(output, grad_output, grad_bias) 126 | 127 | return grad_input, grad_weight, grad_bias, None, None, None, None 128 | 129 | 130 | conv3d_relu = Conv3DReluFunction.apply 131 | 132 | 133 | class Conv3DReluModule(nn.Module): 134 | def __init__(self, output, in_channels, out_channels, kernel_size=3, dilation=1): 135 | super().__init__() 136 | 137 | w = t.Tensor(out_channels, in_channels, kernel_size, kernel_size, kernel_size) 138 | self.weight = Parameter(w.cuda()) 139 | self.bias = Parameter(t.Tensor(out_channels).cuda()) 140 | self.output = output 141 | 142 | self.dilation = dilation 143 | self.stride = 1 144 | 145 | def forward(self, input): 146 | assert self.output.is_cuda 147 | return conv3d_relu( 148 | input, self.weight, self.bias, self.output, self.stride, self.dilation 149 | ) 150 | -------------------------------------------------------------------------------- /msd_pytorch/errors.py: -------------------------------------------------------------------------------- 1 | class Error(Exception): 2 | """Base class for exceptions in msd_pytorch.""" 3 | 4 | pass 5 | 6 | 7 | class InputError(Error): 8 | """Exception raised for errors in the input. 9 | 10 | Attributes: 11 | message -- explanation of the error 12 | """ 13 | 14 | def __init__(self, message): 15 | self.message = message 16 | -------------------------------------------------------------------------------- /msd_pytorch/msd_custom_convolutions/conv2d_backward_bias.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "device_tensor.h" 5 | #include "kernel_utils.cuh" 6 | 7 | 8 | /////////////////////////////////////////////////////////////////////////////// 9 | // Convolution: Backward Bias // 10 | /////////////////////////////////////////////////////////////////////////////// 11 | 12 | template 13 | __global__ void 14 | conv_relu_backward_bias(dTensor4R output, 15 | dTensor4R grad_output, 16 | dTensor1R grad_bias) 17 | { 18 | // This implementation of the backward pass wrt the convolution 19 | // bias uses warp reduction. 20 | 21 | int B = grad_output.size(0); 22 | int C_OUT = grad_output.size(1); 23 | int H = grad_output.size(2); 24 | int W = grad_output.size(3); 25 | 26 | int h = threadIdx.y + blockDim.y * blockIdx.y; 27 | int w = threadIdx.x + blockDim.x * blockIdx.x; 28 | int pId = threadIdx.y * blockDim.x + threadIdx.x; 29 | 30 | // Used for warpReduceSum: 31 | bool leader = 0 == pId % 32; 32 | int mask = __activemask(); 33 | 34 | if (W <= w || H <= h) { 35 | return; 36 | } 37 | for (int c_out=0; c_out < C_OUT; c_out++) { 38 | scalar_t g = 0; 39 | for (int b=0; b < B; b++) { 40 | if (0.0 < output[b][c_out][h][w]) { 41 | g += grad_output[b][c_out][h][w]; 42 | } 43 | } 44 | g = warpReduceSum(mask, g); 45 | if (leader) { 46 | atomicAdd(&grad_bias[c_out], g); 47 | } 48 | } 49 | } 50 | 51 | // Specializations: 52 | template 53 | __global__ void 54 | conv_relu_backward_bias(dTensor4Rfloat output, 55 | dTensor4Rfloat grad_output, 56 | dTensor1Rfloat grad_bias); 57 | template 58 | __global__ void 59 | conv_relu_backward_bias(dTensor4Rdouble output, 60 | dTensor4Rdouble grad_output, 61 | dTensor1Rdouble grad_bias); -------------------------------------------------------------------------------- /msd_pytorch/msd_custom_convolutions/conv2d_backward_k.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "device_tensor.h" 5 | #include "kernel_utils.cuh" 6 | 7 | /////////////////////////////////////////////////////////////////////////////// 8 | // Convolution: Backward Kernel // 9 | /////////////////////////////////////////////////////////////////////////////// 10 | 11 | 12 | template 13 | __global__ void 14 | conv_backward_k(dTensor4R grad_output, 15 | dTensor4R input, 16 | dTensor4R grad_kernel, 17 | int dilation) 18 | { 19 | // A less naive implementation of the backward pass wrt the 20 | // convolution kernel weights. Here, the gradient sums are reduced 21 | // accross the warp before being written to global memory. This 22 | // implementation does not perform too badly. 23 | int B = grad_output.size(0); 24 | int C_OUT = grad_output.size(1); 25 | int C_IN = input.size(1); 26 | int H = grad_output.size(2); 27 | int W = grad_output.size(3); 28 | 29 | int h = threadIdx.y + blockDim.y * blockIdx.y; 30 | int w = threadIdx.x + blockDim.x * blockIdx.x; 31 | int pId = threadIdx.y * blockDim.x + threadIdx.x; 32 | 33 | if (W <= w || H <= h) { 34 | return; 35 | } 36 | 37 | bool leader = 0 == pId % 32; 38 | int mask = __activemask(); 39 | 40 | for (int b=0; b < B; b++) { 41 | for (int c_in=0; c_in < C_IN; c_in++) { 42 | for (int c_out=0; c_out < C_OUT; c_out++) { 43 | for (int p=-1; p <= 1; p++) { 44 | for (int q=-1; q <= 1; q++) { 45 | int h_ = reflect(h + p * dilation, (int) H); 46 | int w_ = reflect(w + q * dilation, (int) W); 47 | scalar_t g = 48 | input[b][c_in][h_][w_] * 49 | grad_output[b][c_out][h][w]; 50 | g = warpReduceSum(mask, g); 51 | if (leader) { 52 | atomicAdd(&grad_kernel[c_out][c_in][p+1][q+1], g); 53 | } 54 | } 55 | } 56 | } 57 | } 58 | } 59 | } 60 | 61 | template 62 | __global__ void 63 | conv_relu_backward_k(dTensor4R output, 64 | dTensor4R grad_output, 65 | dTensor4R input, 66 | dTensor4R grad_kernel, 67 | int dilation) 68 | { 69 | // In this approach, the gradient sums are reduced accross the 70 | // warp before being written to global memory. This 71 | // implementation does not perform too badly. 72 | int B = grad_output.size(0); 73 | int C_OUT = grad_output.size(1); 74 | int C_IN = input.size(1); 75 | int H = grad_output.size(2); 76 | int W = grad_output.size(3); 77 | 78 | int h = threadIdx.y + blockDim.y * blockIdx.y; 79 | int w = threadIdx.x + blockDim.x * blockIdx.x; 80 | int pId = threadIdx.y * blockDim.x + threadIdx.x; 81 | 82 | if (W <= w || H <= h) { 83 | return; 84 | } 85 | 86 | bool leader = 0 == pId % 32; 87 | int mask = __activemask(); 88 | 89 | for (int b=0; b < B; b++) { 90 | for (int c_in=0; c_in < C_IN; c_in++) { 91 | for (int c_out=0; c_out < C_OUT; c_out++) { 92 | for (int p=-1; p <= 1; p++) { 93 | for (int q=-1; q <= 1; q++) { 94 | int h_ = reflect(h + p * dilation, (int) H); 95 | int w_ = reflect(w + q * dilation, (int) W); 96 | // Only propagate gradient if relu is positive. 97 | scalar_t propagate = (0.0 < output[b][c_out][h][w]) ? 1.0 : 0.0; 98 | 99 | scalar_t g = 100 | input[b][c_in][h_][w_] * 101 | grad_output[b][c_out][h][w] * 102 | propagate; 103 | g = warpReduceSum(mask, g); 104 | if (leader) { 105 | atomicAdd(&grad_kernel[c_out][c_in][p+1][q+1], g); 106 | } 107 | } 108 | } 109 | } 110 | } 111 | } 112 | } 113 | 114 | 115 | // Specialize: 116 | 117 | template 118 | __global__ void 119 | conv_backward_k(dTensor4Rfloat grad_output, 120 | dTensor4Rfloat input, 121 | dTensor4Rfloat grad_kernel, 122 | int dilation); 123 | 124 | template 125 | __global__ void 126 | conv_backward_k(dTensor4Rdouble grad_output, 127 | dTensor4Rdouble input, 128 | dTensor4Rdouble grad_kernel, 129 | int dilation); 130 | 131 | template 132 | __global__ void 133 | conv_relu_backward_k(dTensor4Rfloat output, 134 | dTensor4Rfloat grad_output, 135 | dTensor4Rfloat input, 136 | dTensor4Rfloat grad_kernel, 137 | int dilation); 138 | 139 | template 140 | __global__ void 141 | conv_relu_backward_k(dTensor4Rdouble output, 142 | dTensor4Rdouble grad_output, 143 | dTensor4Rdouble input, 144 | dTensor4Rdouble grad_kernel, 145 | int dilation); -------------------------------------------------------------------------------- /msd_pytorch/msd_custom_convolutions/conv2d_forward.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "device_tensor.h" 5 | #include "kernel_utils.cuh" 6 | 7 | 8 | template 9 | static __inline__ __device__ void pack_shared_mem(scalar_t* dest, scalar_t* src, int n){ 10 | int pId = threadIdx.x + blockDim.x * threadIdx.y; 11 | int num_threads = blockDim.x * blockDim.y; 12 | 13 | for (int i=pId; i < n; i+=num_threads) { 14 | dest[i] = src[i]; 15 | } 16 | __syncthreads(); 17 | } 18 | 19 | /////////////////////////////////////////////////////////////////////////////// 20 | // Convolution:Forward // 21 | /////////////////////////////////////////////////////////////////////////////// 22 | template 23 | __global__ void 24 | conv_forward(dTensor4R input, 25 | dTensor4R kernel, 26 | dTensor1R bias, 27 | dTensor4R output, 28 | int dilation) 29 | { 30 | // The following has been done to improve performance: 31 | // 1) This implementation caches the kernel weights. 32 | // 2) This implementation precomputes data offsets in x and y 33 | // direction instead of pointers. 34 | 35 | // LIMITS: 36 | // 49152 bytes of shared memory per block 37 | // 12288 floats of shared memory per block 38 | // +- 1300 kernels can be stored in shared mem 39 | // So we must have: 40 | // C_IN * C_OUT < 1300 41 | extern __shared__ int shared_memory[]; 42 | scalar_t* buf = (scalar_t*) shared_memory; 43 | 44 | int B = output.size(0); 45 | int C_OUT = output.size(1); 46 | int C_IN = input.size(1); 47 | int H = input.size(2); 48 | int W = input.size(3); 49 | 50 | int h = threadIdx.y + blockDim.y * blockIdx.y; 51 | int w = threadIdx.x + blockDim.x * blockIdx.x; 52 | 53 | // // Load kernels into shared memory 54 | int num_kernel_elems = kernel.size(0) * kernel.size(1) * kernel.size(2) * kernel.size(3); 55 | pack_shared_mem(buf, kernel.data(), num_kernel_elems); 56 | mcc::TensorAccessor kernel_buffer = kernel.unpack_from(buf); 57 | 58 | if (W <= w || H <= h) { 59 | return; 60 | } 61 | 62 | // Precompute data offsets: 63 | int hs[3] = {0}; 64 | int ws[3] = {0}; 65 | 66 | for (int i=-1; i <= 1; i++) { 67 | hs[i + 1] = reflect(h + dilation * i, (int) H); 68 | ws[i + 1] = reflect(w + dilation * i, (int) W); 69 | } 70 | 71 | // Actually compute the convolution 72 | for (int b=0; b < B; b++) { 73 | for (int c_out=0; c_out < C_OUT; c_out++) { 74 | scalar_t o = bias[c_out]; 75 | for (int c_in=0; c_in < C_IN; c_in++) { 76 | scalar_t *kernel0 = &kernel_buffer[c_out][c_in][0][0]; 77 | #pragma unroll 78 | for (int p=-1; p <= 1; p++) { 79 | #pragma unroll 80 | for (int q=-1; q <= 1; q++) { 81 | o += input[b][c_in][hs[p + 1]][ws[q + 1]] * (*kernel0); 82 | // Incrementing the kernel pointer works because 83 | // the kernel weights are contiguous and the 84 | // data_offsets are prepared to be in the same 85 | // order as the kernel weights. 86 | kernel0++; 87 | } 88 | } 89 | } 90 | output[b][c_out][h][w] = o; 91 | } 92 | } 93 | } 94 | 95 | template 96 | __global__ void 97 | conv_relu_forward(dTensor4R input, 98 | dTensor4R kernel, 99 | dTensor1R bias, 100 | dTensor4R output, 101 | int dilation) 102 | { 103 | // The following has been done to improve performance: 104 | // 1) This implementation caches the kernel weights. 105 | // 2) This implementation precomputes data offsets in x and y 106 | // direction instead of pointers. 107 | 108 | // LIMITS: 109 | // 49152 bytes of shared memory per block 110 | // 12288 floats of shared memory per block 111 | // +- 1300 kernels can be stored in shared mem 112 | // So we must have: 113 | // C_IN * C_OUT < 1300 114 | extern __shared__ int shared_memory[]; 115 | 116 | int B = output.size(0); 117 | int C_OUT = output.size(1); 118 | int C_IN = input.size(1); 119 | int H = input.size(2); 120 | int W = input.size(3); 121 | 122 | int h = threadIdx.y + blockDim.y * blockIdx.y; 123 | int w = threadIdx.x + blockDim.x * blockIdx.x; 124 | int pId = threadIdx.x + blockDim.x * threadIdx.y; 125 | int num_threads = blockDim.x * blockDim.y; 126 | 127 | // Load kernels into shared memory 128 | scalar_t* kernel_buf = (scalar_t*) shared_memory; 129 | int num_kernel_elems = kernel.size(0) * kernel.size(1) * kernel.size(2) * kernel.size(3); 130 | 131 | for (int i=pId; i < num_kernel_elems; i+=num_threads) { 132 | kernel_buf[i] = kernel.data()[i]; 133 | } 134 | // We can index kernel_buffer like a 4d tensor. 135 | mcc::TensorAccessor kernel_buffer = kernel.unpack_from(kernel_buf); 136 | 137 | __syncthreads(); 138 | 139 | if (W <= w || H <= h) { 140 | return; 141 | } 142 | 143 | // Precompute data offsets: 144 | int hs[3] = {0}; 145 | int ws[3] = {0}; 146 | 147 | for (int i=-1; i <= 1; i++) { 148 | hs[i + 1] = reflect(h + dilation * i, (int) H); 149 | ws[i + 1] = reflect(w + dilation * i, (int) W); 150 | } 151 | 152 | // Actually compute the convolution 153 | for (int b=0; b < B; b++) { 154 | for (int c_out=0; c_out < C_OUT; c_out++) { 155 | scalar_t o = bias[c_out]; 156 | for (int c_in=0; c_in < C_IN; c_in++) { 157 | scalar_t *kernel0 = &kernel_buffer[c_out][c_in][0][0]; 158 | #pragma unroll 159 | for (int p=-1; p <= 1; p++) { 160 | #pragma unroll 161 | for (int q=-1; q <= 1; q++) { 162 | o += input[b][c_in][hs[p + 1]][ws[q + 1]] * (*kernel0); 163 | // Incrementing the kernel pointer works because 164 | // the kernel weights are contiguous and the 165 | // data_offsets are prepared to be in the same 166 | // order as the kernel weights. 167 | kernel0++; 168 | } 169 | } 170 | } 171 | output[b][c_out][h][w] = max(0.0, o); 172 | } 173 | } 174 | } 175 | 176 | 177 | template 178 | __global__ void 179 | conv_forward(dTensor4Rfloat input, 180 | dTensor4Rfloat kernel, 181 | dTensor1Rfloat bias, 182 | dTensor4Rfloat output, 183 | int dilation); 184 | 185 | template 186 | __global__ void 187 | conv_forward(dTensor4Rdouble input, 188 | dTensor4Rdouble kernel, 189 | dTensor1Rdouble bias, 190 | dTensor4Rdouble output, 191 | int dilation); 192 | 193 | template 194 | __global__ void 195 | conv_relu_forward(dTensor4Rfloat input, 196 | dTensor4Rfloat kernel, 197 | dTensor1Rfloat bias, 198 | dTensor4Rfloat output, 199 | int dilation); 200 | 201 | template 202 | __global__ void 203 | conv_relu_forward(dTensor4Rdouble input, 204 | dTensor4Rdouble kernel, 205 | dTensor1Rdouble bias, 206 | dTensor4Rdouble output, 207 | int dilation); -------------------------------------------------------------------------------- /msd_pytorch/msd_custom_convolutions/conv3d_backward_bias.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "device_tensor.h" 5 | #include "kernel_utils.cuh" 6 | 7 | 8 | /////////////////////////////////////////////////////////////////////////////// 9 | // Convolution: Backward Bias // 10 | /////////////////////////////////////////////////////////////////////////////// 11 | 12 | template 13 | __global__ void 14 | conv3d_relu_backward_bias(dTensor5R output, 15 | dTensor5R grad_output, 16 | dTensor1R grad_bias) 17 | { 18 | // This implementation of the backward pass wrt the convolution 19 | // bias uses warp reduction. 20 | 21 | int B = grad_output.size(0); 22 | int C_OUT = grad_output.size(1); 23 | int D = grad_output.size(2); 24 | int H = grad_output.size(3); 25 | int W = grad_output.size(4); 26 | 27 | int d = threadIdx.z + blockDim.z * blockIdx.z; 28 | int h = threadIdx.y + blockDim.y * blockIdx.y; 29 | int w = threadIdx.x + blockDim.x * blockIdx.x; 30 | int pId = threadIdx.x + blockDim.x * threadIdx.y + blockDim.x * blockDim.y * threadIdx.z; 31 | 32 | // Used for warpReduceSum: 33 | bool leader = 0 == pId % 32; 34 | int mask = __activemask(); 35 | 36 | if (W <= w || H <= h || D <= d) { 37 | return; 38 | } 39 | for (int c_out=0; c_out < C_OUT; c_out++) { 40 | scalar_t g = 0; 41 | for (int b=0; b < B; b++) { 42 | if (0.0 < output[b][c_out][d][h][w]) { 43 | g += grad_output[b][c_out][d][h][w]; 44 | } 45 | } 46 | g = warpReduceSum(mask, g); 47 | if (leader) { 48 | atomicAdd(&grad_bias[c_out], g); 49 | } 50 | } 51 | } 52 | 53 | // Specializations: 54 | template 55 | __global__ void 56 | conv3d_relu_backward_bias(dTensor5Rfloat output, 57 | dTensor5Rfloat grad_output, 58 | dTensor1Rfloat grad_bias); 59 | template 60 | __global__ void 61 | conv3d_relu_backward_bias(dTensor5Rdouble output, 62 | dTensor5Rdouble grad_output, 63 | dTensor1Rdouble grad_bias); -------------------------------------------------------------------------------- /msd_pytorch/msd_custom_convolutions/conv3d_backward_k.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "device_tensor.h" 5 | #include "kernel_utils.cuh" 6 | 7 | /////////////////////////////////////////////////////////////////////////////// 8 | // Convolution: Backward Kernel // 9 | /////////////////////////////////////////////////////////////////////////////// 10 | 11 | template 12 | __global__ void 13 | conv3d_backward_k(dTensor5R grad_output, 14 | dTensor5R input, 15 | dTensor5R grad_kernel, 16 | int dilation) 17 | { 18 | // In this approach, the gradient sums are reduced accross the 19 | // warp before being written to global memory. This 20 | // implementation does not perform too badly. 21 | int B = grad_output.size(0); 22 | int C_OUT = grad_output.size(1); 23 | int C_IN = input.size(1); 24 | int D = grad_output.size(2); 25 | int H = grad_output.size(3); 26 | int W = grad_output.size(4); 27 | 28 | int d = threadIdx.z + blockDim.z * blockIdx.z; 29 | int h = threadIdx.y + blockDim.y * blockIdx.y; 30 | int w = threadIdx.x + blockDim.x * blockIdx.x; 31 | int pId = threadIdx.x + blockDim.x * threadIdx.y + blockDim.x * blockDim.y * threadIdx.z; 32 | 33 | if (W <= w || H <= h || D <= d) { 34 | return; 35 | } 36 | 37 | bool leader = 0 == pId % 32; 38 | int mask = __activemask(); 39 | 40 | for (int b=0; b < B; b++) { 41 | for (int c_in=0; c_in < C_IN; c_in++) { 42 | for (int c_out=0; c_out < C_OUT; c_out++) { 43 | for (int p=-1; p <= 1; p++) { 44 | for (int q=-1; q <= 1; q++) { 45 | for (int r=-1; r <= 1; r++) { 46 | int d_ = reflect(d + p * dilation, (int) D); 47 | int h_ = reflect(h + q * dilation, (int) H); 48 | int w_ = reflect(w + r * dilation, (int) W); 49 | 50 | scalar_t g = 51 | input[b][c_in][d_][h_][w_] * 52 | grad_output[b][c_out][d][h][w]; 53 | 54 | g = warpReduceSum(mask, g); 55 | if (leader) { 56 | atomicAdd(&grad_kernel[c_out][c_in][p+1][q+1][r+1], g); 57 | } 58 | } 59 | } 60 | } 61 | } 62 | } 63 | } 64 | } 65 | 66 | template 67 | __global__ void 68 | conv3d_relu_backward_k(dTensor5R output, 69 | dTensor5R grad_output, 70 | dTensor5R input, 71 | dTensor5R grad_kernel, 72 | int dilation) 73 | { 74 | // In this approach, the gradient sums are reduced accross the 75 | // warp before being written to global memory. This 76 | // implementation does not perform too badly. 77 | int B = grad_output.size(0); 78 | int C_OUT = grad_output.size(1); 79 | int C_IN = input.size(1); 80 | int D = grad_output.size(2); 81 | int H = grad_output.size(3); 82 | int W = grad_output.size(4); 83 | 84 | int d = threadIdx.z + blockDim.z * blockIdx.z; 85 | int h = threadIdx.y + blockDim.y * blockIdx.y; 86 | int w = threadIdx.x + blockDim.x * blockIdx.x; 87 | int pId = threadIdx.x + blockDim.x * threadIdx.y + blockDim.x * blockDim.y * threadIdx.z; 88 | 89 | if (W <= w || H <= h || D <= d) { 90 | return; 91 | } 92 | 93 | bool leader = 0 == pId % 32; 94 | int mask = __activemask(); 95 | 96 | for (int b=0; b < B; b++) { 97 | for (int c_in=0; c_in < C_IN; c_in++) { 98 | for (int c_out=0; c_out < C_OUT; c_out++) { 99 | for (int p=-1; p <= 1; p++) { 100 | for (int q=-1; q <= 1; q++) { 101 | for (int r=-1; r <= 1; r++) { 102 | int d_ = reflect(d + p * dilation, (int) D); 103 | int h_ = reflect(h + q * dilation, (int) H); 104 | int w_ = reflect(w + r * dilation, (int) W); 105 | // Only propagate gradient if relu is positive. 106 | scalar_t propagate = (0.0 < output[b][c_out][d][h][w]) ? 1.0 : 0.0; 107 | 108 | scalar_t g = 109 | input[b][c_in][d_][h_][w_] * 110 | grad_output[b][c_out][d][h][w] * 111 | propagate; 112 | g = warpReduceSum(mask, g); 113 | if (leader) { 114 | atomicAdd(&grad_kernel[c_out][c_in][p+1][q+1][r+1], g); 115 | } 116 | } 117 | } 118 | } 119 | } 120 | } 121 | } 122 | } 123 | 124 | template 125 | __global__ void 126 | conv3d_backward_k(dTensor5Rfloat grad_output, 127 | dTensor5Rfloat input, 128 | dTensor5Rfloat grad_kernel, 129 | int dilation); 130 | 131 | template 132 | __global__ void 133 | conv3d_backward_k(dTensor5Rdouble grad_output, 134 | dTensor5Rdouble input, 135 | dTensor5Rdouble grad_kernel, 136 | int dilation); 137 | 138 | template 139 | __global__ void 140 | conv3d_relu_backward_k(dTensor5Rfloat output, 141 | dTensor5Rfloat grad_output, 142 | dTensor5Rfloat input, 143 | dTensor5Rfloat grad_kernel, 144 | int dilation); 145 | 146 | template 147 | __global__ void 148 | conv3d_relu_backward_k(dTensor5Rdouble output, 149 | dTensor5Rdouble grad_output, 150 | dTensor5Rdouble input, 151 | dTensor5Rdouble grad_kernel, 152 | int dilation); -------------------------------------------------------------------------------- /msd_pytorch/msd_custom_convolutions/kernel_utils.cuh: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | // In the MSD pytorch source code, we sometimes need atomicAdd for 64bit floats. 6 | // This is not supported for compute capability < 6.0 (pre-GTX 10XX series). So 7 | // Nvidia proposes the following fix: 8 | // https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#atomic-functions 9 | #if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 600 10 | #else 11 | static __inline__ __device__ double atomicAdd(double* address, double val) 12 | { 13 | unsigned long long int* address_as_ull = 14 | (unsigned long long int*)address; 15 | unsigned long long int old = *address_as_ull, assumed; 16 | 17 | do { 18 | assumed = old; 19 | old = atomicCAS(address_as_ull, assumed, 20 | __double_as_longlong(val + 21 | __longlong_as_double(assumed))); 22 | 23 | // Note: uses integer comparison to avoid hang in case of NaN (since NaN != NaN) 24 | } while (assumed != old); 25 | 26 | return __longlong_as_double(old); 27 | } 28 | #endif 29 | 30 | __device__ __forceinline__ int 31 | reflect(int i, int dimi) { 32 | if (i < 0) { 33 | i = -i - 1; 34 | } else if (i >= dimi) { 35 | i = 2 * dimi - i - 1; 36 | } 37 | return i; 38 | } 39 | 40 | template 41 | __inline__ __device__ 42 | scalar_t warpReduceSum(int mask, scalar_t val) { 43 | for (int offset = warpSize/2; offset > 0; offset /= 2) 44 | val += __shfl_down_sync(mask, val, offset); 45 | return val; 46 | } 47 | -------------------------------------------------------------------------------- /msd_pytorch/msd_custom_convolutions/kernels.cuh: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "device_tensor.h" 3 | 4 | /////////////////////////////////////////////////////////////////////////////// 5 | // Convolution: Backward Bias // 6 | /////////////////////////////////////////////////////////////////////////////// 7 | 8 | template 9 | __global__ void 10 | conv_relu_backward_bias(dTensor4R output, 11 | dTensor4R grad_output, 12 | dTensor1R grad_bias); 13 | 14 | template 15 | __global__ void 16 | conv3d_relu_backward_bias(dTensor5R output, 17 | dTensor5R grad_output, 18 | dTensor1R grad_bias); 19 | 20 | /////////////////////////////////////////////////////////////////////////////// 21 | // Convolution: Backward Kernel // 22 | /////////////////////////////////////////////////////////////////////////////// 23 | 24 | template 25 | __global__ void 26 | conv_backward_k(dTensor4R grad_output, 27 | dTensor4R input, 28 | dTensor4R grad_kernel, 29 | int dilation); 30 | 31 | template 32 | __global__ void 33 | conv_relu_backward_k(dTensor4R output, 34 | dTensor4R grad_output, 35 | dTensor4R input, 36 | dTensor4R grad_kernel, 37 | int dilation); 38 | 39 | template 40 | __global__ void 41 | conv3d_backward_k(dTensor5R grad_output, 42 | dTensor5R input, 43 | dTensor5R grad_kernel, 44 | int dilation); 45 | 46 | template 47 | __global__ void 48 | conv3d_relu_backward_k(dTensor5R output, 49 | dTensor5R grad_output, 50 | dTensor5R input, 51 | dTensor5R grad_kernel, 52 | int dilation); 53 | 54 | /////////////////////////////////////////////////////////////////////////////// 55 | // Convolution: Backward Input // 56 | /////////////////////////////////////////////////////////////////////////////// 57 | 58 | template 59 | __global__ void 60 | conv_backward_x(dTensor4R grad_output, 61 | dTensor4R kernel, 62 | dTensor4R grad_input, 63 | int dilation); 64 | 65 | template 66 | __global__ void 67 | conv_relu_backward_x(dTensor4R output, 68 | dTensor4R grad_output, 69 | dTensor4R kernel, 70 | dTensor4R grad_input, 71 | int dilation); 72 | 73 | template 74 | __global__ void 75 | conv3d_backward_x(dTensor5R grad_output, 76 | dTensor5R kernel, 77 | dTensor5R grad_input, 78 | int dilation); 79 | 80 | template 81 | __global__ void 82 | conv3d_relu_backward_x(dTensor5R output, 83 | dTensor5R grad_output, 84 | dTensor5R kernel, 85 | dTensor5R grad_input, 86 | int dilation); 87 | 88 | /////////////////////////////////////////////////////////////////////////////// 89 | // Convolution:Forward // 90 | /////////////////////////////////////////////////////////////////////////////// 91 | 92 | template 93 | __global__ void 94 | conv_forward(dTensor4R input, 95 | dTensor4R kernel, 96 | dTensor1R bias, 97 | dTensor4R output, 98 | int dilation); 99 | 100 | template 101 | __global__ void 102 | conv_relu_forward(dTensor4R input, 103 | dTensor4R kernel, 104 | dTensor1R bias, 105 | dTensor4R output, 106 | int dilation); 107 | 108 | template 109 | __global__ void 110 | conv3d_forward(dTensor5R input, 111 | dTensor5R kernel, 112 | dTensor1R bias, 113 | dTensor5R output, 114 | int dilation); 115 | 116 | template 117 | __global__ void 118 | conv3d_relu_forward(dTensor5R input, 119 | dTensor5R kernel, 120 | dTensor1R bias, 121 | dTensor5R output, 122 | int dilation); 123 | -------------------------------------------------------------------------------- /msd_pytorch/msd_custom_convolutions/torch_cuda_dispatch.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | void conv_cuda_forward(torch::Tensor input_t, 5 | torch::Tensor kernel_t, 6 | torch::Tensor bias_t, 7 | torch::Tensor out_t, 8 | int dilation, 9 | std::tuple block_size); 10 | 11 | void conv_cuda_backward_x(torch::Tensor grad_output_t, 12 | torch::Tensor kernel_t, 13 | torch::Tensor grad_input_t, 14 | int dilation, 15 | std::tuple block_size); 16 | 17 | 18 | void conv_cuda_backward_k(torch::Tensor grad_output, 19 | torch::Tensor input, 20 | torch::Tensor grad_kernel, 21 | int dilation, 22 | std::tuple block_size); 23 | 24 | 25 | void conv_relu_cuda_forward(torch::Tensor input_t, 26 | torch::Tensor kernel_t, 27 | torch::Tensor bias_t, 28 | torch::Tensor out_t, 29 | int dilation, 30 | std::tuple block_size); 31 | 32 | void conv_relu_cuda_backward_x(torch::Tensor output_t, 33 | torch::Tensor grad_output_t, 34 | torch::Tensor kernel_t, 35 | torch::Tensor grad_input_t, 36 | int dilation, 37 | std::tuple block_size); 38 | 39 | void conv_relu_cuda_backward_k(torch::Tensor output, 40 | torch::Tensor grad_output, 41 | torch::Tensor input, 42 | torch::Tensor grad_kernel, 43 | int dilation, 44 | std::tuple block_size); 45 | 46 | void conv_relu_cuda_backward_bias(torch::Tensor output, 47 | torch::Tensor grad_output, 48 | torch::Tensor grad_bias, 49 | std::tuple block_size); 50 | -------------------------------------------------------------------------------- /msd_pytorch/msd_module.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from math import sqrt 3 | import numpy as np 4 | from msd_pytorch.msd_block import MSDBlock 5 | 6 | 7 | def units_in_front(c_in, width, layer_depth): 8 | """Calculate how many intermediate images are in front of current layer 9 | 10 | * The input channels count as intermediate images 11 | * The `layer_depth` index is zero-based: the first hidden layer 12 | has index zero. 13 | 14 | :param c_in: The number of input channels of the MSD module 15 | :param width: The width of the MSD module 16 | :param layer_depth: 17 | The depth of the layer for which we are calculating the units 18 | in front. This index is zero-based: the first hidden layer 19 | has index zero. 20 | :returns: 21 | :rtype: 22 | 23 | """ 24 | return c_in + width * layer_depth 25 | 26 | 27 | def init_convolution_weights(conv_weight, c_in, c_out, width, depth): 28 | """Initialize MSD convolution kernel weights 29 | 30 | Based on: 31 | 32 | Pelt, Daniel M., & Sethian, J. A. (2017). A mixed-scale dense 33 | convolutional neural network for image analysis. Proceedings of 34 | the National Academy of Sciences, 115(2), 35 | 254–259. http://dx.doi.org/10.1073/pnas.1715832114 36 | 37 | :param conv_weight: 38 | The kernel weight data 39 | :param c_in: 40 | Number of input channels of the MSD module 41 | :param c_out: 42 | Number of output channels of the MSD module 43 | :param width: 44 | The width of the MSD module 45 | :param depth: 46 | The depth of the MSD module. This is the number of hidden layers. 47 | :returns: Nothing 48 | :rtype: 49 | 50 | """ 51 | # The number of parameters in the convolution depends on whether 52 | # the convolution is 2D or 3D. We multiply all non-channel 53 | # dimensions of the weight here to get the right answer. 54 | num_params = np.product(conv_weight.shape[2:]) 55 | num_channels = num_params * (c_in + width * (depth - 1)) + c_out 56 | std_dev = sqrt(2 / num_channels) 57 | conv_weight.normal_(0, std_dev) 58 | 59 | 60 | class MSDFinalLayer(torch.nn.Module): 61 | """Documentation for MSDFinalLayer 62 | 63 | Implements the final 1x1 multiplication and bias addition for all 64 | intermediate layers to get to the output layer. 65 | 66 | Initializes the weight and bias to zero. 67 | """ 68 | 69 | def __init__(self, c_in, c_out): 70 | super(MSDFinalLayer, self).__init__() 71 | self.c_in = c_in 72 | self.c_out = c_out 73 | self.linear = torch.nn.Conv1d(c_in, c_out, 1) 74 | self.reset_parameters() 75 | 76 | def forward(self, input): 77 | b, c_in, *size = input.shape 78 | tmp_size = input[0, 0, ...].numel() 79 | 80 | # Put channels in last place in input shape 81 | output = input.view(b, c_in, tmp_size) 82 | output = self.linear(output) 83 | output = output.view(b, self.c_out, *size) 84 | return output 85 | 86 | def reset_parameters(self): 87 | self.linear.weight.data.zero_() 88 | self.linear.bias.data.zero_() 89 | 90 | 91 | class MSDModule(torch.nn.Module): 92 | def __init__( 93 | self, c_in, c_out, depth, width, dilations=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], ndim=2 94 | ): 95 | """Create a msd module 96 | 97 | :param c_in: # of input channels 98 | :param c_out: # of output channels 99 | :param depth: # of layers 100 | :param width: # the width of the module 101 | :param dilations: `list(int)` 102 | 103 | A list of dilations to use. Default is ``[1, 2, ..., 10]``. A 104 | good alternative is ``[1, 2, 4, 8]``. The dilations are 105 | repeated when the depth of the module exceeds the length of 106 | the list. 107 | 108 | :param ndim: `int` 109 | 110 | The dimension of the convolutions. 2D convolutions are used by 111 | default. 3D is also possible. 112 | 113 | :returns: an MSD module 114 | :rtype: MSDModule 115 | 116 | """ 117 | super(MSDModule, self).__init__() 118 | self.c_in = c_in 119 | self.c_out = c_out 120 | self.depth = depth 121 | self.width = width 122 | self.dilations = [dilations[i % len(dilations)] for i in range(depth)] 123 | 124 | if depth < 1: 125 | raise ValueError(f"Depth must be at least 1. Got: {depth}.") 126 | if width < 1: 127 | raise ValueError(f"Width must be at least 1. Got: {width}.") 128 | 129 | self.msd_block = MSDBlock(self.c_in, self.dilations, self.width, ndim=ndim) 130 | self.final_layer = MSDFinalLayer(c_in=c_in + width * depth, c_out=c_out) 131 | 132 | self.reset_parameters() 133 | 134 | def reset_parameters(self): 135 | # Initialize weights for hidden layers: 136 | for w in self.msd_block.weights: 137 | init_convolution_weights( 138 | w.data, self.c_in, self.c_out, self.width, self.depth 139 | ) 140 | 141 | self.msd_block.bias.data.zero_() 142 | self.final_layer.reset_parameters() 143 | 144 | def forward(self, input): 145 | output = self.msd_block(input) 146 | output = self.final_layer(output) 147 | return output 148 | -------------------------------------------------------------------------------- /msd_pytorch/msd_regression_model.py: -------------------------------------------------------------------------------- 1 | from msd_pytorch.msd_model import MSDModel 2 | import torch.nn as nn 3 | 4 | 5 | loss_functions = {"L1": nn.L1Loss(), "L2": nn.MSELoss()} 6 | 7 | 8 | class MSDRegressionModel(MSDModel): 9 | """An MSD network for regression. 10 | 11 | This class provides helper methods for using the MSD network 12 | module for regression. 13 | 14 | Refer to the documentation of 15 | :class:`~msd_pytorch.msd_model.MSDModel` for more information on 16 | the helper methods and attributes. 17 | 18 | """ 19 | 20 | def __init__( 21 | self, 22 | c_in, 23 | c_out, 24 | depth, 25 | width, 26 | *, 27 | dilations=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 28 | loss="L2", 29 | parallel=False, 30 | ndim=2, 31 | ): 32 | """Create a new MSD network for regression. 33 | 34 | :param c_in: The number of input channels. 35 | :param c_out: The number of output channels. 36 | :param depth: The depth of the MSD network. 37 | :param width: The width of the MSD network. 38 | :param dilations: `list(int)` 39 | 40 | A list of dilations to use. Default is ``[1, 2, ..., 10]``. A 41 | good alternative is ``[1, 2, 4, 8]``. The dilations are 42 | repeated when there are more layers than supplied dilations. 43 | 44 | :param loss: `string` 45 | 46 | A string describing the loss function that should be 47 | used. Currently, the following losses are supported: 48 | 49 | * "L1" - ``nn.L1Loss()`` 50 | * "L2" - ``nn.MSELoss()`` 51 | 52 | :param parallel: `bool` 53 | 54 | Whether or not to execute the model on multiple GPUs. Note 55 | that the batch size must be a multiple of the number of 56 | available GPUs. 57 | 58 | :param ndim: `int` 59 | 60 | The dimension of the convolutions. 2D convolutions are used by 61 | default. 3D is also possible. 62 | 63 | :returns: 64 | :rtype: 65 | 66 | """ 67 | super().__init__(c_in, c_out, depth, width, dilations, ndim=ndim) 68 | 69 | self.criterion = loss_functions.get(loss) 70 | if self.criterion is None: 71 | raise ValueError( 72 | "The loss must be one of {list(loss_functions.keys())}. Supplied {loss_function}. " 73 | ) 74 | 75 | # Define the whole network: 76 | self.net = nn.Sequential(self.scale_in, self.msd, self.scale_out) 77 | self.net.cuda() 78 | if parallel: 79 | self.net = nn.DataParallel(self.net) 80 | 81 | # Train only MSD parameters: 82 | self.init_optimizer(self.msd) 83 | -------------------------------------------------------------------------------- /msd_pytorch/msd_segmentation_model.py: -------------------------------------------------------------------------------- 1 | from msd_pytorch.msd_model import ( 2 | MSDModel, 3 | scaling_module_set_bias, 4 | scaling_module_set_scale 5 | ) 6 | import numpy as np 7 | import torch.nn as nn 8 | 9 | 10 | class MSDSegmentationModel(MSDModel): 11 | """An MSD network for segmentation. 12 | 13 | This class provides helper methods for using the MSD network 14 | module for segmentation. 15 | 16 | Refer to the documentation of 17 | :class:`~msd_pytorch.msd_model.MSDModel` for more information on 18 | the helper methods and attributes. 19 | 20 | """ 21 | 22 | def __init__( 23 | self, 24 | c_in, 25 | num_labels, 26 | depth, 27 | width, 28 | *, 29 | dilations=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 30 | parallel=False, 31 | ndim=2, 32 | ): 33 | """Create a new MSD network for segmentation. 34 | 35 | :param c_in: The number of input channels. 36 | :param num_labels: The number of labels to divide the segmentation into. 37 | :param depth: The depth of the MSD network 38 | :param width: The width of the MSD network 39 | :param dilations: `list(int)` 40 | 41 | A list of dilations to use. Default is ``[1, 2, ..., 10]``. A 42 | good alternative is ``[1, 2, 4, 8]``. The dilations are 43 | repeated when there are more layers than supplied dilations. 44 | 45 | :param parallel: `bool` 46 | 47 | Whether or not to execute the model on multiple GPUs. Note 48 | that the batch size must be a multiple of the number of 49 | available GPUs. 50 | 51 | :param ndim: `int` 52 | 53 | The dimension of the convolutions. 2D convolutions are used by 54 | default. 3D is also possible. 55 | 56 | :returns: 57 | :rtype: 58 | 59 | """ 60 | self.num_labels = num_labels 61 | # Initialize msd network. 62 | super().__init__(c_in, num_labels, depth, width, dilations, ndim=ndim) 63 | 64 | self.criterion = nn.NLLLoss() 65 | 66 | # Initialize network 67 | net_trained = nn.Sequential(self.msd, nn.LogSoftmax(dim=1)) 68 | self.net = nn.Sequential(self.scale_in, net_trained) 69 | self.net.cuda() 70 | 71 | if parallel: 72 | self.net = nn.DataParallel(self.net) 73 | 74 | # Train all parameters apart from self.scale_in. 75 | self.init_optimizer(net_trained) 76 | 77 | def set_normalization(self, dataloader): 78 | """Normalize input data. 79 | 80 | This function goes through all the training data to compute 81 | the mean and std of the training data. It modifies the network 82 | so that all future invocations of the network first normalize 83 | input data. The normalization parameters are saved. 84 | 85 | :param dataloader: The dataloader associated to the training data. 86 | :returns: 87 | :rtype: 88 | 89 | """ 90 | mean = 0 91 | square = 0 92 | for (data_in, _) in dataloader: 93 | mean += data_in.mean() 94 | square += data_in.pow(2).mean() 95 | 96 | mean /= len(dataloader) 97 | square /= len(dataloader) 98 | std = np.sqrt(square - mean ** 2) 99 | 100 | # The input data should be roughly normally distributed after 101 | # passing through net_fixed. 102 | scaling_module_set_scale(self.scale_in, 1 / std) 103 | scaling_module_set_bias(self.scale_in, -mean / std) 104 | 105 | def set_target(self, target): 106 | # The class labels must be of long data type 107 | target = target.long() 108 | 109 | min, max = target.min(), target.max() 110 | if min < 0 or self.num_labels <= max: 111 | raise ValueError( 112 | f"Target invalid: expected values in range {[0, self.num_labels - 1]}, but got {[min, max]}" 113 | ) 114 | # The NLLLoss does not accept a channel dimension. So we 115 | # squeeze the target. 116 | target = target.squeeze(1) 117 | # The class labels must reside on the GPU 118 | target = target.cuda() 119 | self.target = target 120 | -------------------------------------------------------------------------------- /msd_pytorch/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Unit test package for msd_pytorch.""" 4 | 5 | from .utils import torch_equal 6 | -------------------------------------------------------------------------------- /msd_pytorch/tests/test_image_dataset.py: -------------------------------------------------------------------------------- 1 | import msd_pytorch.image_dataset as img_ds 2 | import logging 3 | import tifffile 4 | import imageio 5 | import numpy as np 6 | import pytest 7 | from msd_pytorch.errors import InputError 8 | from . import torch_equal 9 | 10 | 11 | def test_convert_to_integral(): 12 | img = np.ones(10) 13 | 14 | assert np.mean(img_ds._convert_to_integral(img.astype(np.int8))) == 1 15 | assert np.mean(img_ds._convert_to_integral(img.astype(np.bool))) == 1 16 | np.mean(img_ds._convert_to_integral((-img).astype(np.int8))) 17 | 18 | for t in [np.float16, np.float32, np.float64]: 19 | with pytest.raises(InputError): 20 | img_ds._convert_to_integral(img.astype(t)) 21 | 22 | 23 | def test_relabel_image(): 24 | img0 = np.ones(10, dtype=np.uint8) 25 | assert img_ds._relabel_image(img0, 2).mean() == 1 26 | 27 | img = img0 - 2 28 | with pytest.raises(InputError): 29 | img_ds._relabel_image(img, 2) 30 | 31 | img = img0 + 2 32 | with pytest.raises(InputError): 33 | img_ds._relabel_image(img, 2) 34 | with pytest.raises(InputError): 35 | img_ds._relabel_image(img, [2]).mean() 36 | 37 | assert img_ds._relabel_image(img, [3]).mean() == 0 38 | assert img_ds._relabel_image(img, 4).mean() == 3 39 | 40 | 41 | def test_empty_stack(tmp_path, caplog): 42 | """Test that image stack with non existing path logs a warning. 43 | """ 44 | 45 | img_ds.ImageStack(tmp_path / "non_existing.tif") 46 | img_ds.ImageStack(tmp_path / "non_existing*.tif") 47 | 48 | assert len(caplog.record_tuples) == 2 49 | assert np.all( 50 | [ 51 | level == logging.WARNING and "empty" in msg 52 | for (_, level, msg) in caplog.record_tuples 53 | ] 54 | ) 55 | 56 | 57 | def test_stack_no_glob(tmp_path): 58 | tifffile.imsave(tmp_path / "image.tif", np.ones((10, 10))) 59 | stack = img_ds.ImageStack(tmp_path) 60 | assert len(stack) == 1 61 | 62 | 63 | def test_image_stack(tmp_path): 64 | num_channels = 3 65 | for i in range(10): 66 | img = np.random.randn(num_channels, 10, 10) 67 | imageio.imsave(tmp_path / f"imio_{i}.png", img.swapaxes(0, 2)) 68 | imageio.imsave(tmp_path / f"imio_{i}.jpeg", img.swapaxes(0, 2)) 69 | tifffile.imsave(tmp_path / f"tiff_{i}.tif", img) 70 | tifffile.imsave(tmp_path / f"tiff2d_{i}.tif", img[0]) 71 | tifffile.imsave(tmp_path / f"tiff0_{i:05}.tif", img) 72 | 73 | pngs = img_ds.ImageStack(tmp_path / "imio_*.png") 74 | pngs_c = img_ds.ImageStack(tmp_path / "imio_*.png", collapse_channels=True) 75 | jpegs = img_ds.ImageStack(tmp_path / "imio_*.jpeg") 76 | jpegs_c = img_ds.ImageStack(tmp_path / "imio_*.jpeg", collapse_channels=True) 77 | utiffs = img_ds.ImageStack(tmp_path / "tiff_*.tif") 78 | utiffs_2d = img_ds.ImageStack(tmp_path / "tiff2d_*.tif") 79 | otiffs = img_ds.ImageStack(tmp_path / "tiff0_*.tif") 80 | 81 | # This image stack should have no labels: 82 | with pytest.raises(RuntimeError): 83 | pngs.num_labels 84 | 85 | n = len(pngs) 86 | assert n == len(pngs_c) 87 | assert n == len(jpegs) 88 | assert n == len(jpegs_c) 89 | assert n == len(utiffs) 90 | assert n == len(otiffs) 91 | for i in range(n): 92 | assert pngs[i].dim() == 3 93 | assert pngs[i].shape[0] == num_channels 94 | assert pngs_c[i].shape[0] == 1 95 | assert jpegs[i].dim() == 3 96 | assert jpegs[i].shape[0] == num_channels 97 | assert jpegs_c[i].shape[0] == 1 98 | 99 | assert utiffs_2d[i].dim() == 3 100 | assert torch_equal(otiffs[i], utiffs[i]) 101 | 102 | 103 | def test_image_stack_labels(tmp_path): 104 | num_channels = 1 105 | num_labels = 10 106 | for i in range(10): 107 | img = np.random.randint(num_labels, size=(num_channels, 10, 10), dtype=np.uint8) 108 | tifffile.imsave(tmp_path / f"tiff_{i}.tif", img) 109 | tifffile.imsave(tmp_path / f"tiff2d_{i}.tif", img[0]) 110 | tifffile.imsave(tmp_path / f"tiff0_{i:05}.tif", img) 111 | 112 | # Check for errors when there are missing labels: 113 | with pytest.raises(InputError): 114 | list(img_ds.ImageStack(tmp_path / "tiff0_*.tif", labels=range(9))) 115 | with pytest.raises(InputError): 116 | list(img_ds.ImageStack(tmp_path / "tiff0_*.tif", labels=9)) 117 | 118 | # Check that all stacks have exactly the same values 119 | utiffs = img_ds.ImageStack(tmp_path / "tiff_*.tif", labels=num_labels) 120 | utiffs_2d = img_ds.ImageStack(tmp_path / "tiff2d_*.tif", labels=num_labels) 121 | otiffs = img_ds.ImageStack(tmp_path / "tiff0_*.tif", labels=range(num_labels)) 122 | stacks = [utiffs, utiffs_2d, otiffs] 123 | 124 | n = len(utiffs) 125 | for s in stacks: 126 | assert n == len(s) 127 | 128 | for i in range(n): 129 | for s in stacks: 130 | assert s[i].dim() == 3 131 | assert s[i].shape[0] == num_channels 132 | assert torch_equal(otiffs[i], s[i]) 133 | assert s.num_labels == num_labels 134 | 135 | 136 | def test_image_dataset(tmp_path): 137 | num_channels = 3 138 | for i in range(10): 139 | for purpose in ["train", "val"]: 140 | img = np.random.randn(num_channels, 10, 10) 141 | imageio.imsave(tmp_path / f"{purpose}_imio_{i}.png", img.swapaxes(0, 2)) 142 | imageio.imsave(tmp_path / f"{purpose}_imio_{i}.jpeg", img.swapaxes(0, 2)) 143 | tifffile.imsave(tmp_path / f"{purpose}_tiff_{i}.tif", img) 144 | tifffile.imsave(tmp_path / f"{purpose}_tiff2d_{i}.tif", img[0]) 145 | tifffile.imsave(tmp_path / f"{purpose}_tiff0_{i:05}.tif", img) 146 | 147 | pngs = img_ds.ImageDataset( 148 | tmp_path / "train_imio_*.png", tmp_path / "val_imio_*.png" 149 | ) 150 | pngs_c = img_ds.ImageDataset( 151 | tmp_path / "train_imio_*.png", 152 | tmp_path / "val_imio_*.png", 153 | collapse_channels=True, 154 | ) 155 | jpegs = img_ds.ImageDataset( 156 | tmp_path / "train_imio_*.jpeg", tmp_path / "val_imio_*.jpeg" 157 | ) 158 | jpegs_c = img_ds.ImageDataset( 159 | tmp_path / "train_imio_*.jpeg", 160 | tmp_path / "val_imio_*.jpeg", 161 | collapse_channels=True, 162 | ) 163 | utiffs = img_ds.ImageDataset( 164 | tmp_path / "train_tiff_*.tif", tmp_path / "val_tiff_*.tif" 165 | ) 166 | utiffs_2d = img_ds.ImageDataset( 167 | tmp_path / "train_tiff2d_*.tif", tmp_path / "val_tiff2d_*.tif" 168 | ) 169 | otiffs = img_ds.ImageDataset( 170 | tmp_path / "train_tiff0_*.tif", tmp_path / "val_tiff0_*.tif" 171 | ) 172 | 173 | # check that image stack has no labels: 174 | with pytest.raises(RuntimeError): 175 | pngs.num_labels 176 | 177 | n = len(pngs) 178 | assert n == len(pngs_c) 179 | assert n == len(jpegs) 180 | assert n == len(jpegs_c) 181 | assert n == len(utiffs) 182 | assert n == len(otiffs) 183 | for i in range(n): 184 | for j in [0, 1]: 185 | assert pngs[i][j].dim() == 3 186 | assert pngs[i][j].shape[0] == num_channels 187 | assert pngs_c[i][j].shape[0] == 1 188 | assert jpegs[i][j].dim() == 3 189 | assert jpegs[i][j].shape[0] == num_channels 190 | assert jpegs_c[i][j].shape[0] == 1 191 | 192 | assert utiffs_2d[i][j].dim() == 3 193 | assert torch_equal(otiffs[i][j], utiffs[i][j]) 194 | -------------------------------------------------------------------------------- /msd_pytorch/tests/test_msd_block.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from . import torch_equal 3 | import torch 4 | import msd_pytorch.msd_block as msd_block 5 | import msd_pytorch.msd_module as msd_module 6 | from torch.autograd import gradcheck 7 | 8 | 9 | def test_msd_block(): 10 | # Test forward evaluation 11 | img = torch.nn.Parameter(torch.ones((1, 1, 1, 5))).cuda() 12 | 13 | model = msd_block.MSDBlock2d(1, [1, 1, 1], width=1).cuda() 14 | result = model(img) 15 | 16 | print(result) 17 | 18 | # Backward evaluation 19 | loss = torch.mean(result) 20 | loss.backward() 21 | 22 | for weight in model.weights: 23 | print('grad', weight.grad) 24 | 25 | 26 | @pytest.mark.slow 27 | def test_msd_block_2d_grad_check(): 28 | """Test using gradcheck provided by Torch. 29 | 30 | Because there is no reference implementation to compare to, we 31 | check if the gradient calculations are consistent with numerical 32 | calculations. 33 | 34 | """ 35 | dtype = torch.double 36 | B = 2 # Batch size 37 | C_IN = 3 # Input channels 38 | H = 13 # Height 39 | W = 21 # Width 40 | dilations = [1, 2, 3] # Dilation 41 | 42 | net = msd_block.MSDBlock2d(C_IN, dilations).cuda().to(dtype=dtype) 43 | x = torch.randn(B, C_IN, H, W, dtype=dtype, requires_grad=True).cuda() 44 | 45 | gradcheck(net, [x], raise_exception=True) 46 | 47 | 48 | @pytest.mark.slow 49 | def test_msd_block_3d_grad_check(): 50 | """Test using gradcheck provided by Torch. 51 | 52 | Because there is no reference implementation to compare to, we 53 | check if the gradient calculations are consistent with numerical 54 | calculations. 55 | 56 | """ 57 | dtype = torch.double 58 | B = 2 # Batch size 59 | C_IN = 3 # Input channels 60 | D, H, W = (5, 7, 11) 61 | dilations = [1, 2, 3] # Dilation 62 | 63 | net = msd_block.MSDBlock3d(C_IN, dilations).cuda().to(dtype=dtype) 64 | x = torch.randn(B, C_IN, D, H, W, dtype=dtype, requires_grad=True).cuda() 65 | 66 | gradcheck(net, [x], raise_exception=True) 67 | -------------------------------------------------------------------------------- /msd_pytorch/tests/test_msd_model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from msd_pytorch.msd_model import MSDModel 3 | from torch.utils.data import TensorDataset, DataLoader 4 | 5 | 6 | def test_normalization(): 7 | torch.manual_seed(1) 8 | 9 | means = [-10, 0, 10] 10 | stds = [1, 10] 11 | 12 | for mean in means: 13 | for std in stds: 14 | # Network 15 | width = 1 16 | depth = 10 17 | sample_size, num_channels, shape = 1, 3, (100, 100) 18 | model = MSDModel(num_channels, num_channels, depth, width) 19 | 20 | # Data 21 | # We make the input and target dataset equal, which makes 22 | # their mean and std equal. This should ensure that the 23 | # scale_in and scale_out module are each others' inverse. 24 | d_in = mean + torch.randn(sample_size, num_channels, *shape) * std 25 | d_out = d_in.clone() 26 | ds = TensorDataset(d_in, d_out) 27 | dl = DataLoader(ds, sample_size) 28 | 29 | # Calculate normalization: 30 | model.set_normalization(dl) 31 | 32 | # Ensure that scale_out is the inverse of scale_in: 33 | x = torch.randn_like(d_in) 34 | y = model.scale_out(model.scale_in(x)) 35 | assert abs(x - y).mean() < 1e-4 36 | -------------------------------------------------------------------------------- /msd_pytorch/tests/test_msd_module.py: -------------------------------------------------------------------------------- 1 | from pytest import approx 2 | import pytest 3 | from msd_pytorch.msd_module import MSDModule, MSDFinalLayer 4 | from torch.autograd import Variable 5 | import torch as t 6 | import torch.nn as nn 7 | import torch.optim as optim 8 | from torch.autograd import gradcheck 9 | 10 | 11 | def test_msd_module_3D(): 12 | net = MSDModule(2, 3, 5, 1, ndim=3).cuda() 13 | x = t.ones(1, 2, 7, 7, 7).cuda() 14 | # The final layer is initialized with zeros. Therefore the output 15 | # of an untrained network must always be zero. 16 | assert net(x).sum().item() == 0.0 17 | 18 | 19 | def test_msd_gradients(): 20 | t.manual_seed(1) 21 | 22 | dtype = t.double 23 | size = (11, 13) 24 | batch_sz = 2 25 | 26 | for depth in [9]: 27 | print(f"Depth: {depth}") 28 | width = c_in = c_out = batch_sz 29 | x = Variable(t.randn(batch_sz, c_in, *size, dtype=dtype)).cuda() 30 | x.requires_grad = True 31 | 32 | net = MSDModule(c_in, c_out, depth, width).cuda() 33 | net.double() 34 | 35 | # The weights of the final layer are initialized to zero by 36 | # default. This makes it trivial to pass gradcheck. Therefore, 37 | # we reinitialize all weights randomly. 38 | for p in net.parameters(): 39 | p.data = t.randn_like(p.data) 40 | 41 | gradcheck(net, [x], raise_exception=True, atol=1e-4, rtol=1e-3) 42 | 43 | 44 | def test_final_layer(): 45 | """Test MSDFinalLayer module 46 | 47 | We check that the msd_final module does exactly the same as a 48 | 1x1 convolution. 49 | 50 | """ 51 | for conv3d in [False, True]: 52 | 53 | shape = (25,) * (3 if conv3d else 2) 54 | k_shape = (1,) * (3 if conv3d else 2) 55 | device = t.device("cuda:0") 56 | dtype = t.double 57 | 58 | batch_size = 3 59 | c_in = 10 60 | c_out = 2 61 | 62 | input = t.randn(batch_size, c_in, *shape, dtype=dtype, device=device) 63 | bias = t.randn(c_out, dtype=dtype, device=device) 64 | weight = t.randn(c_out, c_in, 1, dtype=dtype, device=device) 65 | 66 | msd_final = MSDFinalLayer(c_in, c_out) 67 | msd_final.linear.bias.data = bias 68 | msd_final.linear.weight.data = weight 69 | 70 | if conv3d: 71 | conv = nn.Conv3d(c_in, c_out, 1) 72 | else: 73 | conv = nn.Conv2d(c_in, c_out, 1) 74 | conv.bias.data = bias 75 | conv.weight.data = weight.view(c_out, c_in, *k_shape) 76 | 77 | # Check that outputs have the same shape 78 | output1 = conv(input) 79 | output2 = msd_final(input) 80 | assert output1.shape == output2.shape 81 | 82 | # And have the same values. 83 | diff = (output1 - output2).abs().sum().item() 84 | assert diff == approx(0) 85 | 86 | 87 | def test_parameters_change(): 88 | # This test ensures that all parameters are updated after an 89 | # update step. 90 | t.manual_seed(1) 91 | 92 | size = (30, 30) 93 | for batch_sz in [1]: 94 | for depth in range(1, 20, 6): 95 | width = c_in = c_out = batch_sz 96 | x = Variable(t.randn(batch_sz, c_in, *size)).cuda() 97 | target = Variable(t.randn(batch_sz, c_out, *size)).cuda() 98 | assert x.data.is_cuda 99 | 100 | net = MSDModule(c_in, c_out, depth, width).cuda() 101 | 102 | params0 = dict((n, p.data.clone()) for n, p in net.named_parameters()) 103 | # Train for two iterations. The convolution weights in 104 | # the MSD layers are not updated after the first 105 | # training step because the final 1x1 convolution 106 | # weights are zero. 107 | optimizer = optim.Adam(net.parameters()) 108 | optimizer.zero_grad() 109 | for _ in range(2): 110 | y = net(x) 111 | assert y is not None 112 | criterion = nn.L1Loss() 113 | loss = criterion(y, target) 114 | loss.backward() 115 | optimizer.step() 116 | 117 | params1 = dict(net.named_parameters()) 118 | 119 | for name in params1.keys(): 120 | p0, p1 = params0[name], params1[name] 121 | d = abs(p0 - p1.data.clone()).sum().item() 122 | assert 0.0 < d, ( 123 | f"Parameter {name} left unchanged: \n" 124 | f"Initial value: {p0}\n" 125 | f"Current value: {p1}\n" 126 | f"Gradient: {p1.grad}\n" 127 | ) 128 | 129 | # Check that the loss is not zero 130 | assert loss.abs().item() != approx(0.0) 131 | 132 | 133 | def test_zero_depth_network(): 134 | with pytest.raises(ValueError): 135 | MSDModule(1, 1, depth=0, width=1) 136 | with pytest.raises(ValueError): 137 | MSDModule(1, 1, depth=1, width=0) 138 | -------------------------------------------------------------------------------- /msd_pytorch/tests/test_msd_pytorch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Tests for `msd_pytorch` package.""" 5 | 6 | from msd_pytorch import msd_pytorch 7 | -------------------------------------------------------------------------------- /msd_pytorch/tests/test_msd_regression_model.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import torch 3 | from torch.utils.data import TensorDataset, DataLoader 4 | from msd_pytorch.msd_regression_model import MSDRegressionModel 5 | from . import torch_equal 6 | 7 | 8 | def test_params_change_3d(): 9 | """Ensure that learning updates all parameters. 10 | """ 11 | # make test repeatable 12 | torch.manual_seed(1) 13 | 14 | c_in, c_out, depth, width = 1, 1, 11, 1 15 | model = MSDRegressionModel( 16 | c_in, c_out, depth, width, loss="L1", dilations=[1, 2, 4, 8], ndim=3, 17 | ) 18 | shape = (17, 11, 13) 19 | 20 | input = torch.randn(1, c_in, *shape) # batch size is one. 21 | target = torch.randn(1, c_out, *shape) 22 | 23 | params0 = [p.data.clone() for p in model.optimizer.param_groups[0]["params"]] 24 | 25 | model.set_input(input) 26 | model.set_target(target) 27 | for i in range(10): 28 | model.learn(input, target) 29 | 30 | params1 = [p.data.clone() for p in model.optimizer.param_groups[0]["params"]] 31 | 32 | for p0, p1 in zip(params0, params1): 33 | assert not torch_equal(p0, p1) 34 | 35 | 36 | def test_params_change(): 37 | """Ensure that learning updates all parameters. 38 | """ 39 | # make test repeatable 40 | torch.manual_seed(1) 41 | 42 | c_in, c_out, depth, width = 1, 1, 11, 1 43 | model = MSDRegressionModel( 44 | c_in, c_out, depth, width, loss="L1", dilations=[1, 2, 4, 8] 45 | ) 46 | shape = (13, 14) 47 | 48 | input = torch.randn(1, c_in, *shape) # batch size is one. 49 | target = torch.randn(1, c_out, *shape) 50 | 51 | params0 = [p.data.clone() for p in model.optimizer.param_groups[0]["params"]] 52 | 53 | model.set_input(input) 54 | model.set_target(target) 55 | for i in range(10): 56 | model.learn(input, target) 57 | 58 | params1 = [p.data.clone() for p in model.optimizer.param_groups[0]["params"]] 59 | 60 | for p0, p1 in zip(params0, params1): 61 | assert not torch_equal(p0, p1) 62 | 63 | 64 | def test_data_parallel(): 65 | """Check that msd_model is compatible with multi-GPU approaches 66 | 67 | Specifically, `torch.nn.DataParallel`. 68 | """ 69 | 70 | shape = (100, 100) 71 | inp = torch.zeros(4, 1, *shape, dtype=torch.float32, device=torch.device("cuda:0")) 72 | tgt = torch.zeros(4, 1, *shape, dtype=torch.float32, device=torch.device("cuda:0")) 73 | 74 | model = MSDRegressionModel(1, 1, 11, 1, parallel=True) 75 | model.forward(inp, tgt) 76 | model.learn(inp, tgt) 77 | 78 | 79 | def test_api_surface(tmp_path): 80 | ########################################################################### 81 | # Create network # 82 | ########################################################################### 83 | batch_size = 1 84 | c_in, c_out, depth, width = 1, 1, 11, 1 85 | model = MSDRegressionModel( 86 | c_in, c_out, depth, width, dilations=[1, 2, 3], loss="L1" 87 | ) 88 | 89 | with pytest.raises(ValueError): 90 | model = MSDRegressionModel( 91 | c_in, c_out, depth, width, dilations=[1, 2, 3], loss="invalid" 92 | ) 93 | 94 | # Data 95 | N, shape = 20, (30, 30) 96 | d_in = torch.randn(N, c_in, *shape) 97 | d_out = torch.randn(N, c_out, *shape) 98 | ds = TensorDataset(d_in, d_out) 99 | dl = DataLoader(ds, batch_size) 100 | 101 | ########################################################################### 102 | # low-level methods # 103 | ########################################################################### 104 | (inp, tgt), *_ = dl 105 | model.set_input(inp) 106 | model.set_target(tgt) 107 | model.forward(inp, tgt) 108 | model.learn(inp, tgt) 109 | 110 | ########################################################################### 111 | # Methods with dataloader # 112 | ########################################################################### 113 | # High-level methods using the dataloader: 114 | model.set_normalization(dl) 115 | model.train(dl, 1) 116 | 117 | ########################################################################### 118 | # Get floating point losses # 119 | ########################################################################### 120 | assert isinstance(model.validate(dl), float) 121 | assert isinstance(model.get_loss(), float) 122 | 123 | ########################################################################### 124 | # Test saving and loading # 125 | ########################################################################### 126 | params0 = dict((n, p.data.clone()) for n, p in model.net.named_parameters()) 127 | epoch_saved = 93 128 | model.save(tmp_path / "network.torch", epoch_saved) 129 | 130 | for p in model.net.parameters(): 131 | p.data.fill_(18.74) 132 | 133 | model = MSDRegressionModel( 134 | c_in, c_out, depth, width, dilations=[1, 2, 3], loss="L2" 135 | ) 136 | epoch_loaded = model.load(tmp_path / "network.torch") 137 | params1 = dict((n, p.data.clone()) for n, p in model.net.named_parameters()) 138 | 139 | for n in params0.keys(): 140 | p0, p1 = params0[n], params1[n] 141 | assert torch_equal(p0, p1) 142 | 143 | assert epoch_saved == epoch_loaded 144 | -------------------------------------------------------------------------------- /msd_pytorch/tests/test_msd_segmentation_model.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import torch 3 | from torch.utils.data import TensorDataset, DataLoader 4 | from msd_pytorch.msd_segmentation_model import MSDSegmentationModel 5 | from . import torch_equal 6 | 7 | 8 | def test_params_change_3d(): 9 | """Ensure that learning updates all parameters. 10 | """ 11 | # make test repeatable 12 | torch.manual_seed(1) 13 | 14 | c_in, num_labels, depth, width = 1, 3, 11, 1 15 | model = MSDSegmentationModel(c_in, num_labels, depth, width, dilations=[1, 2, 3], ndim=3) 16 | shape = (5, 13, 7) 17 | 18 | input = torch.randn(1, c_in, *shape) # batch size is one. 19 | target = torch.randint(low=0, high=num_labels, size=(1, 1, *shape)) 20 | 21 | params0 = [p.data.clone() for p in model.optimizer.param_groups[0]["params"]] 22 | 23 | model.set_input(input) 24 | model.set_target(target) 25 | for i in range(10): 26 | model.learn(input, target) 27 | 28 | params1 = [p.data.clone() for p in model.optimizer.param_groups[0]["params"]] 29 | 30 | for p0, p1 in zip(params0, params1): 31 | assert not torch_equal(p0, p1) 32 | 33 | 34 | def test_params_change(): 35 | """Ensure that learning updates all parameters. 36 | """ 37 | # make test repeatable 38 | torch.manual_seed(1) 39 | 40 | c_in, num_labels, depth, width = 1, 3, 11, 1 41 | model = MSDSegmentationModel(c_in, num_labels, depth, width, dilations=[1, 2, 3]) 42 | shape = (13, 14) 43 | 44 | input = torch.randn(1, c_in, *shape) # batch size is one. 45 | target = torch.randint(low=0, high=num_labels, size=(1, 1, *shape)) 46 | 47 | params0 = [p.data.clone() for p in model.optimizer.param_groups[0]["params"]] 48 | 49 | model.set_input(input) 50 | model.set_target(target) 51 | for i in range(10): 52 | model.learn(input, target) 53 | 54 | params1 = [p.data.clone() for p in model.optimizer.param_groups[0]["params"]] 55 | 56 | for p0, p1 in zip(params0, params1): 57 | assert not torch_equal(p0, p1) 58 | 59 | 60 | def test_data_parallel(): 61 | """Check that msd_model is compatible with multi-GPU approaches 62 | 63 | Specifically, `torch.nn.DataParallel`. 64 | """ 65 | 66 | shape = (100, 100) 67 | num_labels = 3 68 | inp = torch.zeros(4, 1, *shape, dtype=torch.float32, device=torch.device("cuda:0")) 69 | tgt = torch.randint( 70 | low=0, high=num_labels, size=(4, 1, *shape), device=torch.device("cuda:0") 71 | ) 72 | 73 | model = MSDSegmentationModel(1, num_labels, 11, 1, parallel=True) 74 | model.forward(inp, tgt) 75 | model.learn(inp, tgt) 76 | 77 | 78 | def test_api_surface(tmp_path): 79 | ########################################################################### 80 | # Create network # 81 | ########################################################################### 82 | batch_size = 1 83 | c_in, depth, width = 1, 11, 1 84 | num_labels = 4 85 | model = MSDSegmentationModel(c_in, num_labels, depth, width) 86 | 87 | # Data 88 | N, shape = 20, (30, 30) 89 | d_in = torch.randn(N, c_in, *shape) 90 | d_out = torch.randint(low=0, high=num_labels, size=(N, 1, *shape)) 91 | ds = TensorDataset(d_in, d_out) 92 | dl = DataLoader(ds, batch_size) 93 | 94 | ########################################################################### 95 | # low-level methods # 96 | ########################################################################### 97 | (inp, tgt), *_ = dl 98 | model.set_input(inp) 99 | model.set_target(tgt) 100 | with pytest.raises(ValueError): 101 | model.set_target(tgt + 1) 102 | with pytest.raises(ValueError): 103 | model.set_target(tgt * -1) 104 | 105 | model.forward(inp, tgt) 106 | model.learn(inp, tgt) 107 | 108 | ########################################################################### 109 | # Methods with dataloader # 110 | ########################################################################### 111 | # High-level methods using the dataloader: 112 | model.set_normalization(dl) 113 | model.train(dl, 1) 114 | 115 | ########################################################################### 116 | # Get floating point losses # 117 | ########################################################################### 118 | assert isinstance(model.validate(dl), float) 119 | assert isinstance(model.get_loss(), float) 120 | 121 | ########################################################################### 122 | # Test saving and loading # 123 | ########################################################################### 124 | params0 = dict((n, p.data.clone()) for n, p in model.net.named_parameters()) 125 | epoch_saved = 94 126 | model.save(tmp_path / "network.torch", epoch_saved) 127 | 128 | for p in model.net.parameters(): 129 | p.data.fill_(18.74) 130 | 131 | model = MSDSegmentationModel(c_in, num_labels, depth, width) 132 | epoch_loaded = model.load(tmp_path / "network.torch") 133 | params1 = dict((n, p.data.clone()) for n, p in model.net.named_parameters()) 134 | 135 | for n in params0.keys(): 136 | p0, p1 = params0[n], params1[n] 137 | assert torch_equal(p0, p1) 138 | 139 | assert epoch_saved == epoch_loaded 140 | -------------------------------------------------------------------------------- /msd_pytorch/tests/utils.py: -------------------------------------------------------------------------------- 1 | from pytest import approx 2 | 3 | 4 | def torch_equal(x, y): 5 | return abs(to_numpy(x)).sum() == approx(abs(to_numpy(y)).sum()) 6 | 7 | 8 | def to_numpy(x): 9 | return x.detach().cpu().numpy() 10 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | [flake8] 5 | exclude = docs, doc_sources 6 | max-line-length = 80 7 | select = C,E,F,W,B,B950 8 | ignore = E501 9 | 10 | [aliases] 11 | # Define setup.py command aliases here 12 | test = pytest 13 | 14 | [tool:pytest] 15 | collect_ignore = ['setup.py'] 16 | markers = 17 | slow: marks tests as slow (deselect with '-m "not slow"') 18 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """The setup script.""" 5 | 6 | from distutils.core import Command 7 | from setuptools import setup, find_packages 8 | from distutils import version 9 | import torch 10 | from torch.utils.cpp_extension import BuildExtension, CUDAExtension 11 | import os.path 12 | import os 13 | 14 | with open('README.md') as readme_file: 15 | readme = readme_file.read() 16 | 17 | with open('CHANGELOG.md') as history_file: 18 | history = history_file.read() 19 | 20 | with open(os.path.join('msd_pytorch', 'VERSION')) as version_file: 21 | version = version_file.read().strip() 22 | 23 | requirements = [ 24 | # Add your project's requirements here, e.g., 25 | "tifffile", 26 | "imageio" 27 | ] 28 | 29 | setup_requirements = [ 30 | 'pytest-runner' 31 | ] 32 | 33 | test_requirements = [ 34 | 'pytest' 35 | ] 36 | 37 | dev_requirements = [ 38 | 'autopep8', 39 | 'rope', 40 | 'jedi', 41 | 'flake8', 42 | 'importmagic', 43 | 'autopep8', 44 | 'black', 45 | 'yapf', 46 | 'snakeviz', 47 | # Documentation 48 | 'sphinx', 49 | 'sphinx_rtd_theme', 50 | 'recommonmark', 51 | # Other 52 | 'watchdog', 53 | 'coverage', 54 | 'pytest', 55 | 'pytest-runner' 56 | ] 57 | 58 | 59 | class EmitNinjaCommand(Command): 60 | """A custom command to emit Ninja build files.""" 61 | 62 | user_options = [] 63 | description = 'emit ninja build file' 64 | 65 | def initialize_options(self): 66 | """Set default values for options.""" 67 | # Each user option must be listed here with their default value. 68 | pass 69 | 70 | def finalize_options(self): 71 | """Post-process options.""" 72 | pass 73 | 74 | def run(self): 75 | """Run command.""" 76 | try: 77 | from torch.utils.cpp_extension import _write_ninja_file_and_build 78 | emit_ninja = _write_ninja_file_and_build 79 | except ImportError: 80 | from torch.utils.cpp_extension import _write_ninja_file_and_build_library 81 | emit_ninja = _write_ninja_file_and_build_library 82 | 83 | for e in self.distribution.ext_modules: 84 | output_dir = f"./ninja_{e.name}" 85 | os.makedirs(output_dir, exist_ok=True) 86 | 87 | nvcc_flags = e.extra_compile_args['nvcc'] 88 | # Using these flags, ninja can detect changes in included header files. 89 | # This makes incremental builds substantially faster. 90 | nvcc_flags = nvcc_flags + ["-MMD", "-MF", "$out.d"] 91 | 92 | emit_ninja( 93 | name=e.name, 94 | sources=e.sources, 95 | extra_cflags=None, # TODO: 96 | extra_cuda_cflags=nvcc_flags, 97 | extra_ldflags=e.extra_link_args, 98 | extra_include_paths=e.include_dirs, 99 | build_directory=output_dir, 100 | verbose=True, 101 | with_cuda=None, # auto-detects CUDA 102 | ) 103 | 104 | 105 | 106 | def __nvcc_args(): 107 | return [] 108 | gxx = os.environ.get('GXX') 109 | recent_torch_version = version.parse("1.7") <= version.parse(torch.__version__) 110 | # Old versions of pytorch did not add the -ccbin command-line option. 111 | if gxx is not None and not recent_torch_version: 112 | print("AAH: Added ccbin command-line argument") 113 | return ['-ccbin', gxx] 114 | else: 115 | return [] 116 | 117 | 118 | setup( 119 | author="Allard Hendriksen", 120 | author_email='allard.hendriksen@cwi.nl', 121 | classifiers=[ 122 | 'Development Status :: 2 - Pre-Alpha', 123 | 'Intended Audience :: Developers', 124 | 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', 125 | 'Natural Language :: English', 126 | 'Programming Language :: Python :: 3', 127 | 'Programming Language :: Python :: 3.6', 128 | 'Programming Language :: Python :: 3.7', 129 | ], 130 | description="An implementation of Mixed-Scale Dense networks in PyTorch. ", 131 | install_requires=requirements, 132 | license="GNU General Public License v3", 133 | long_description=readme + '\n\n' + history, 134 | include_package_data=True, 135 | keywords='msd_pytorch', 136 | name='msd_pytorch', 137 | packages=find_packages(include=['msd_pytorch']), 138 | setup_requires=setup_requirements, 139 | test_suite='msd_pytorch.tests', 140 | tests_require=test_requirements, 141 | extras_require={'dev': dev_requirements}, 142 | url='https://github.com/ahendriksen/msd_pytorch', 143 | version=version, 144 | zip_safe=False, 145 | ext_modules=[ 146 | CUDAExtension( 147 | name='msd_custom_convolutions', 148 | sources=[ 149 | 'msd_pytorch/msd_custom_convolutions.cpp', 150 | 'msd_pytorch/msd_custom_convolutions/torch_cuda_dispatch.cu', 151 | # 2D kernels: 152 | 'msd_pytorch/msd_custom_convolutions/conv2d_backward_k.cu', 153 | 'msd_pytorch/msd_custom_convolutions/conv2d_backward_bias.cu', 154 | 'msd_pytorch/msd_custom_convolutions/conv2d_backward_x.cu', 155 | 'msd_pytorch/msd_custom_convolutions/conv2d_forward.cu', 156 | # 3D kernels: 157 | 'msd_pytorch/msd_custom_convolutions/conv3d_backward_bias.cu', 158 | 'msd_pytorch/msd_custom_convolutions/conv3d_backward_k.cu', 159 | 'msd_pytorch/msd_custom_convolutions/conv3d_backward_x.cu', 160 | 'msd_pytorch/msd_custom_convolutions/conv3d_forward.cu', 161 | ], 162 | extra_compile_args={ 163 | 'cxx': [], 164 | 'nvcc': __nvcc_args(), 165 | }, 166 | ), 167 | ], 168 | cmdclass={ 169 | 'build_ext': BuildExtension, 170 | 'emit_ninja': EmitNinjaCommand, 171 | }, 172 | ) 173 | --------------------------------------------------------------------------------