├── opacus ├── utils │ ├── __init__.py │ ├── packed_sequences.py │ ├── module_inspection.py │ ├── tests │ │ └── module_inspection_test.py │ ├── uniform_sampler.py │ ├── module_modification.py │ └── tensor_utils.py ├── tests │ ├── __init__.py │ ├── dp_layers │ │ ├── __init__.py │ │ ├── dp_multihead_attention_test.py │ │ └── dp_rnn_test.py │ ├── grad_samples │ │ ├── __init__.py │ │ ├── sequence_bias_test.py │ │ ├── instance_norm1d_test.py │ │ ├── instance_norm2d_test.py │ │ ├── instance_norm3d_test.py │ │ ├── linear_test.py │ │ ├── embedding_test.py │ │ ├── group_norm_test.py │ │ ├── layer_norm_test.py │ │ ├── conv1d_test.py │ │ ├── conv2d_test.py │ │ ├── conv3d_test.py │ │ ├── dp_rnn_test.py │ │ └── dp_multihead_attention_test.py │ ├── poisson_test.py │ ├── dp_model_inspector_test.py │ └── multigpu_gradcheck.py ├── version.py ├── __init__.py ├── layers │ ├── __init__.py │ ├── dp_ddp.py │ └── param_rename.py ├── grad_sample │ ├── dp_multihead_attention.py │ ├── README.md │ ├── linear.py │ ├── __init__.py │ ├── group_norm.py │ ├── dp_rnn.py │ ├── layer_norm.py │ ├── instance_norm.py │ ├── embedding.py │ ├── conv.py │ └── utils.py ├── README.md └── scripts │ └── compute_dp_sgd_privacy.py ├── website ├── static │ ├── .nojekyll │ ├── BERT.png │ ├── CIFAR10.png │ ├── clippings.png │ ├── dp_not_GD.png │ ├── img │ │ ├── oss_logo.png │ │ ├── expanding_arrows.svg │ │ ├── pytorch_logo.svg │ │ ├── modular.svg │ │ ├── opacus_favicon.svg │ │ ├── opacus_logo.svg │ │ └── opacus_logo_vertical.svg │ ├── BERT_calibration.png │ ├── clippings_summary.png │ ├── CIFAR10_calibration.png │ ├── css │ │ ├── code_block_buttons.css │ │ └── custom.css │ └── pygments.css ├── sidebars.json ├── sphinx │ ├── source │ │ ├── dp_rnn.rst │ │ ├── scripts.rst │ │ ├── stats.rst │ │ ├── layers.rst │ │ ├── clipping.rst │ │ ├── tensor_utils.rst │ │ ├── privacy_analysis.rst │ │ ├── dp_model_inspector.rst │ │ ├── module_inspection.rst │ │ ├── module_modification.rst │ │ ├── compute_dp_sgd_privacy.rst │ │ ├── privacy_engine.rst │ │ ├── utils.rst │ │ ├── per_sample_gradient_clip.rst │ │ ├── dp_multihead_attention.rst │ │ ├── index.rst │ │ └── per_sample_gradients.rst │ ├── Makefile │ └── make.bat ├── tutorials.json ├── package.json ├── core │ ├── TutorialSidebar.js │ ├── Tutorial.js │ └── Footer.js ├── scripts │ ├── parse_sphinx.py │ ├── build_website.sh │ └── parse_tutorials.py ├── siteConfig.js └── pages │ └── tutorials │ └── index.js ├── MANIFEST.in ├── requirements.txt ├── tutorials ├── img │ └── BERT.png └── README ├── examples ├── __init__.py ├── README.md ├── char-lstm_README.md ├── imdb_README.md ├── scripts │ ├── make_small_imagenet_N_classes.sh │ └── make_small_imagenet_sampled.sh └── mnist_README.md ├── docs ├── img │ └── epsilon-delta-dp.png └── introduction.md ├── .github ├── ISSUE_TEMPLATE │ ├── documentation.md │ ├── questions-help-support.md │ ├── feature-request.md │ └── bug-report.md ├── workflows │ ├── python-publish.yml │ └── test-installation.yml └── PULL_REQUEST_TEMPLATE.md ├── dev_requirements.txt ├── scripts ├── pytorch_install.sh ├── pytorch_install.ps1 └── install_via_pip.sh ├── .gitignore ├── setup.py ├── .circleci └── flake8_config.ini ├── CODE_OF_CONDUCT.md ├── CHANGELOG.md └── CONTRIBUTING.md /opacus/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /website/static/.nojekyll: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy>=1.15 2 | torch>=1.3 3 | scipy>=1.2 4 | config -------------------------------------------------------------------------------- /website/sidebars.json: -------------------------------------------------------------------------------- 1 | { 2 | "docs": { 3 | "About": ["introduction", "faq"] 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /tutorials/img/BERT.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/woodyx218/opacus_global_clipping/HEAD/tutorials/img/BERT.png -------------------------------------------------------------------------------- /website/sphinx/source/dp_rnn.rst: -------------------------------------------------------------------------------- 1 | DPRNN 2 | ======= 3 | .. automodule:: opacus.layers.dp_rnn 4 | :members: 5 | -------------------------------------------------------------------------------- /website/sphinx/source/scripts.rst: -------------------------------------------------------------------------------- 1 | Scripts 2 | ======= 3 | .. toctree:: 4 | 5 | compute_dp_sgd_privacy 6 | -------------------------------------------------------------------------------- /website/sphinx/source/stats.rst: -------------------------------------------------------------------------------- 1 | Stats 2 | ===== 3 | 4 | .. automodule:: opacus.utils.stats 5 | :members: 6 | -------------------------------------------------------------------------------- /website/static/BERT.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/woodyx218/opacus_global_clipping/HEAD/website/static/BERT.png -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | -------------------------------------------------------------------------------- /website/static/CIFAR10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/woodyx218/opacus_global_clipping/HEAD/website/static/CIFAR10.png -------------------------------------------------------------------------------- /docs/img/epsilon-delta-dp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/woodyx218/opacus_global_clipping/HEAD/docs/img/epsilon-delta-dp.png -------------------------------------------------------------------------------- /opacus/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | -------------------------------------------------------------------------------- /website/sphinx/source/layers.rst: -------------------------------------------------------------------------------- 1 | DP Layers 2 | ========= 3 | .. toctree:: 4 | 5 | dp_multihead_attention 6 | dp_rnn 7 | -------------------------------------------------------------------------------- /website/static/clippings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/woodyx218/opacus_global_clipping/HEAD/website/static/clippings.png -------------------------------------------------------------------------------- /website/static/dp_not_GD.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/woodyx218/opacus_global_clipping/HEAD/website/static/dp_not_GD.png -------------------------------------------------------------------------------- /website/static/img/oss_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/woodyx218/opacus_global_clipping/HEAD/website/static/img/oss_logo.png -------------------------------------------------------------------------------- /opacus/tests/dp_layers/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | -------------------------------------------------------------------------------- /website/sphinx/source/clipping.rst: -------------------------------------------------------------------------------- 1 | Clipping Utils 2 | ============== 3 | 4 | .. automodule:: opacus.utils.clipping 5 | :members: 6 | -------------------------------------------------------------------------------- /website/static/BERT_calibration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/woodyx218/opacus_global_clipping/HEAD/website/static/BERT_calibration.png -------------------------------------------------------------------------------- /opacus/tests/grad_samples/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | -------------------------------------------------------------------------------- /website/sphinx/source/tensor_utils.rst: -------------------------------------------------------------------------------- 1 | Tensor Utils 2 | ============= 3 | 4 | .. automodule:: opacus.utils.tensor_utils 5 | :members: 6 | -------------------------------------------------------------------------------- /website/static/clippings_summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/woodyx218/opacus_global_clipping/HEAD/website/static/clippings_summary.png -------------------------------------------------------------------------------- /website/static/CIFAR10_calibration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/woodyx218/opacus_global_clipping/HEAD/website/static/CIFAR10_calibration.png -------------------------------------------------------------------------------- /opacus/version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | __version__ = "0.15.0" 5 | -------------------------------------------------------------------------------- /website/sphinx/source/privacy_analysis.rst: -------------------------------------------------------------------------------- 1 | Privacy Analysis 2 | ================ 3 | 4 | .. automodule:: opacus.privacy_analysis 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/dp_model_inspector.rst: -------------------------------------------------------------------------------- 1 | Model Validation 2 | ================ 3 | 4 | .. automodule:: opacus.dp_model_inspector 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/module_inspection.rst: -------------------------------------------------------------------------------- 1 | Module Inspection 2 | ================= 3 | 4 | .. automodule:: opacus.utils.module_inspection 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/module_modification.rst: -------------------------------------------------------------------------------- 1 | Module Modification 2 | =================== 3 | 4 | .. automodule:: opacus.utils.module_modification 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/compute_dp_sgd_privacy.rst: -------------------------------------------------------------------------------- 1 | Compute DP-SGD Privacy 2 | ====================== 3 | 4 | .. automodule:: opacus.scripts.compute_dp_sgd_privacy 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/privacy_engine.rst: -------------------------------------------------------------------------------- 1 | Privacy Engine 2 | ============== 3 | 4 | .. automodule:: opacus.privacy_engine 5 | 6 | .. autoclass:: PrivacyEngine 7 | :members: 8 | -------------------------------------------------------------------------------- /website/sphinx/source/utils.rst: -------------------------------------------------------------------------------- 1 | Utils 2 | ===== 3 | .. toctree:: 4 | 5 | tensor_utils 6 | stats 7 | module_inspection 8 | module_modification 9 | clipping 10 | -------------------------------------------------------------------------------- /website/sphinx/source/per_sample_gradient_clip.rst: -------------------------------------------------------------------------------- 1 | Gradient Clipping 2 | ================= 3 | 4 | .. automodule:: opacus.per_sample_gradient_clip 5 | 6 | .. autoclass:: PerSampleGradientClipper 7 | :members: 8 | -------------------------------------------------------------------------------- /website/sphinx/source/dp_multihead_attention.rst: -------------------------------------------------------------------------------- 1 | DPMultiheadAttention 2 | ======================== 3 | 4 | .. automodule:: opacus.layers.dp_multihead_attention 5 | 6 | .. autoclass:: SequenceBias 7 | :members: 8 | 9 | .. autoclass:: DPMultiheadAttention 10 | :members: 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/documentation.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F4DA Documentation" 3 | about: Report an issue related to https://pytorch.org/docs 4 | 5 | --- 6 | 7 | ## 📚 Documentation 8 | 9 | 10 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | This folder contains multiple examples to get you started on training differentially private models! 3 | 4 | Note that you may not have all the required packages. You can install opacus's dev version, which will 5 | bring in all the required packages in these examples: 6 | 7 | ``` 8 | pip install opacus[dev] 9 | ``` 10 | -------------------------------------------------------------------------------- /tutorials/README: -------------------------------------------------------------------------------- 1 | # Tutorials 2 | This folder contains multiple tutorials to get you started on training differentially private models! 3 | 4 | Note that you may not have all the required packages. You can install opacus's dev version, which will 5 | bring in all the required packages in these tutorials: 6 | 7 | ``` 8 | pip install opacus[dev] 9 | ``` 10 | -------------------------------------------------------------------------------- /dev_requirements.txt: -------------------------------------------------------------------------------- 1 | torch==1.8.1 2 | torchvision>=0.9.1 3 | tqdm>=4.40 4 | requests>=2.25.1 5 | black 6 | pytest 7 | flake8 8 | sphinx 9 | sphinx-autodoc-typehints 10 | mypy>=0.760 11 | isort 12 | torchcsprng 13 | hypothesis 14 | tensorboard 15 | datasets 16 | transformers 17 | scikit-learn 18 | pytorch-lightning 19 | jsonargparse[signatures]>=3.19.3 # required for Lightning CLI 20 | -------------------------------------------------------------------------------- /opacus/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | from . import utils 5 | from .per_sample_gradient_clip import PerSampleGradientClipper 6 | from .privacy_engine import PrivacyEngine 7 | from .version import __version__ 8 | 9 | 10 | __all__ = ["PrivacyEngine", "PerSampleGradientClipper", "utils", "__version__"] 11 | -------------------------------------------------------------------------------- /website/tutorials.json: -------------------------------------------------------------------------------- 1 | { 2 | "Using Opacus": [ 3 | { 4 | "id": "building_image_classifier", 5 | "title": "Building image classifier with Differential Privacy" 6 | }, 7 | { 8 | "id": "building_text_classifier", 9 | "title": "Building text classifier with Differential Privacy" 10 | }, 11 | { 12 | "id": "building_lstm_name_classifier", 13 | "title": "Training a differentially private LSTM model for name classification" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /website/sphinx/source/index.rst: -------------------------------------------------------------------------------- 1 | :github_url: https://github.com/pytorch/opacus 2 | 3 | Opacus API Reference 4 | ==================== 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | :caption: Contents: 9 | 10 | .. toctree:: 11 | :maxdepth: 2 12 | :caption: API Reference 13 | 14 | privacy_engine 15 | privacy_analysis 16 | per_sample_gradient_clip 17 | per_sample_gradients 18 | dp_model_inspector 19 | layers 20 | utils 21 | scripts 22 | 23 | Indices and Tables 24 | ================== 25 | 26 | * :ref:`genindex` 27 | * :ref:`modindex` 28 | * :ref:`search` 29 | -------------------------------------------------------------------------------- /opacus/layers/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | from .dp_ddp import DifferentiallyPrivateDistributedDataParallel 5 | from .dp_multihead_attention import DPMultiheadAttention, SequenceBias 6 | from .dp_rnn import DPGRU, DPLSTM, DPRNN 7 | from .param_rename import RenameParamsMixin 8 | 9 | 10 | __all__ = [ 11 | "DPRNN", 12 | "DPGRU", 13 | "DPLSTM", 14 | "DPMultiheadAttention", 15 | "RenameParamsMixin", 16 | "SequenceBias", 17 | "DifferentiallyPrivateDistributedDataParallel", 18 | ] 19 | -------------------------------------------------------------------------------- /website/sphinx/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = source 8 | BUILDDIR = build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 20 | -------------------------------------------------------------------------------- /website/static/img/expanding_arrows.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/questions-help-support.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "❓Questions/Help/Support" 3 | about: Do you need support? We have resources. 4 | 5 | --- 6 | 7 | ## ❓ Questions and Help 8 | 9 | ### Before asking: 10 | 1. Search the [FAQs](https://opacus.ai/docs/faq). 11 | 2. Search our [Forums](https://discuss.pytorch.org/search?q=question%20category%3A29). 12 | 3. Search the [Docs](https://opacus.ai/api/). 13 | 14 | > :warning: **This issue tracker is not a help form and this issue will be closed**. 15 | 16 | Our primary means of support is our discussion forum: 17 | 18 | - [Discussion Forum](https://discuss.pytorch.org/c/opacus/29). We actively monitor questions you post in the PyTorch forums with the category `Opacus`. 19 | -------------------------------------------------------------------------------- /website/static/css/code_block_buttons.css: -------------------------------------------------------------------------------- 1 | /* "Copy" code block button */ 2 | pre { 3 | position: relative; 4 | } 5 | 6 | pre .btnIcon { 7 | position: absolute; 8 | top: 4px; 9 | z-index: 2; 10 | cursor: pointer; 11 | border: 1px solid transparent; 12 | padding: 0; 13 | color: #000; 14 | background-color: transparent; 15 | height: 30px; 16 | transition: all .25s ease-out; 17 | } 18 | 19 | pre .btnIcon:hover { 20 | text-decoration: none; 21 | } 22 | 23 | .btnIcon__body { 24 | align-items: center; 25 | display: flex; 26 | } 27 | 28 | .btnIcon svg { 29 | fill: currentColor; 30 | margin-right: .4em; 31 | } 32 | 33 | .btnIcon__label { 34 | font-size: 11px; 35 | } 36 | 37 | .btnClipboard { 38 | right: 10px; 39 | } 40 | -------------------------------------------------------------------------------- /opacus/grad_sample/dp_multihead_attention.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | 5 | import torch 6 | from opacus.layers.dp_multihead_attention import SequenceBias 7 | 8 | from .utils import create_or_extend_grad_sample, register_grad_sampler 9 | 10 | 11 | @register_grad_sampler(SequenceBias) 12 | def compute_sequence_bias_grad_sample( 13 | layer: SequenceBias, A: torch.Tensor, B: torch.Tensor, batch_dim: int = 0 14 | ) -> None: 15 | """ 16 | Computes per sample gradients for ``SequenceBias`` layer 17 | 18 | Args: 19 | layer: Layer 20 | A: Activations 21 | B: Backpropagations 22 | batch_dim: Batch dimension position 23 | """ 24 | create_or_extend_grad_sample(layer.bias, B[:, -1], batch_dim) 25 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/sequence_bias_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | import hypothesis.strategies as st 5 | import torch 6 | from hypothesis import given, settings 7 | from opacus.layers import SequenceBias 8 | 9 | from .common import GradSampleHooks_test 10 | 11 | 12 | class SequenceBias_test(GradSampleHooks_test): 13 | @given( 14 | N=st.integers(1, 4), 15 | T=st.integers(10, 20), 16 | D=st.integers(4, 8), 17 | ) 18 | @settings(deadline=10000) 19 | def test_batch_second( 20 | self, 21 | N: int, 22 | T: int, 23 | D: int, 24 | ): 25 | 26 | seqbias = SequenceBias(D) 27 | x = torch.randn([T, N, D]) 28 | self.run_test(x, seqbias, batch_first=False) 29 | -------------------------------------------------------------------------------- /website/sphinx/source/per_sample_gradients.rst: -------------------------------------------------------------------------------- 1 | Gradient Sample Module 2 | ====================== 3 | 4 | .. automodule:: opacus.grad_sample.grad_sample_module 5 | :members: 6 | 7 | .. automodule:: opacus.grad_sample.conv 8 | :members: 9 | 10 | .. automodule:: opacus.grad_sample.dp_rnn 11 | :members: 12 | 13 | .. automodule:: opacus.grad_sample.dp_multihead_attention 14 | :members: 15 | 16 | .. automodule:: opacus.grad_sample.embedding 17 | :members: 18 | 19 | .. automodule:: opacus.grad_sample.group_norm 20 | :members: 21 | 22 | .. automodule:: opacus.grad_sample.instance_norm 23 | :members: 24 | 25 | .. automodule:: opacus.grad_sample.layer_norm 26 | :members: 27 | 28 | .. automodule:: opacus.grad_sample.linear 29 | :members: 30 | 31 | .. automodule:: opacus.grad_sample.utils 32 | :members: 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F680 Feature Request" 3 | about: Submit a proposal/request for a new feature in Opacus 4 | 5 | --- 6 | 7 | ## 🚀 Feature 8 | 9 | 10 | ## Motivation 11 | 12 | 13 | 14 | ## Pitch 15 | 16 | 17 | 18 | ## Alternatives 19 | 20 | 21 | 22 | ## Additional context 23 | 24 | 25 | -------------------------------------------------------------------------------- /website/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "scripts": { 3 | "examples": "docusaurus-examples", 4 | "start": "docusaurus-start", 5 | "build": "docusaurus-build", 6 | "publish-gh-pages": "docusaurus-publish", 7 | "write-translations": "docusaurus-write-translations", 8 | "version": "docusaurus-version", 9 | "rename-version": "docusaurus-rename-version" 10 | }, 11 | "devDependencies": { 12 | "docusaurus": "^1.9.0" 13 | }, 14 | "dependencies": { 15 | "prismjs": "^1.23.0", 16 | "bl": "^1.2.3" 17 | }, 18 | "resolutions": { 19 | "trim-newlines": "3.0.1", 20 | "normalize-url": "4.5.1", 21 | "highlight.js" : "10.5.0", 22 | "react-dev-utils": "11.0.4", 23 | "immer": "8.0.1", 24 | "prismjs": "1.23.0", 25 | "bl": "1.2.3", 26 | "glob-parent": "5.1.2", 27 | "browserslist": "4.16.5" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/instance_norm1d_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | import hypothesis.strategies as st 5 | import torch 6 | import torch.nn as nn 7 | from hypothesis import given, settings 8 | 9 | from .common import GradSampleHooks_test 10 | 11 | 12 | class InstanceNorm1d_test(GradSampleHooks_test): 13 | @given( 14 | N=st.integers(1, 4), 15 | C=st.integers(1, 3), 16 | W=st.integers(5, 10), 17 | ) 18 | @settings(deadline=10000) 19 | def test_3d_input( 20 | self, 21 | N: int, 22 | C: int, 23 | W: int, 24 | ): 25 | 26 | x = torch.randn([N, C, W]) 27 | norm = nn.InstanceNorm1d(num_features=C, affine=True, track_running_stats=False) 28 | self.run_test(x, norm, batch_first=True) 29 | -------------------------------------------------------------------------------- /docs/introduction.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: introduction 3 | title: Introduction 4 | --- 5 | 6 | Opacus is a library that enables training PyTorch models with differential privacy. It supports training with minimal code changes required on the client, has little impact on training performance and allows the client to online track the privacy budget expended at any given moment. 7 | 8 | Please refer to [this post](https://ai.facebook.com/blog/introducing-opacus-a-high-speed-library-for-training-pytorch-models-with-differential-privacy/) to read more about Opacus. 9 | 10 | ## Target audience 11 | Opacus is aimed at two target audiences: 12 | 13 | 1. ML practitioners will find this to be a gentle introduction to training a model with differential privacy as it requires minimal code changes. 14 | 2. Differential Privacy scientists will find this easy to experiment and tinker with, allowing them to focus on what matters. 15 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/instance_norm2d_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | import hypothesis.strategies as st 5 | import torch 6 | import torch.nn as nn 7 | from hypothesis import given, settings 8 | 9 | from .common import GradSampleHooks_test 10 | 11 | 12 | class InstanceNorm2d_test(GradSampleHooks_test): 13 | @given( 14 | N=st.integers(1, 4), 15 | C=st.integers(1, 3), 16 | W=st.integers(5, 10), 17 | H=st.integers(4, 8), 18 | ) 19 | @settings(deadline=10000) 20 | def test_4d_input( 21 | self, 22 | N: int, 23 | C: int, 24 | W: int, 25 | H: int, 26 | ): 27 | 28 | x = torch.randn([N, C, H, W]) 29 | norm = nn.InstanceNorm2d(num_features=C, affine=True, track_running_stats=False) 30 | self.run_test(x, norm, batch_first=True) 31 | -------------------------------------------------------------------------------- /website/static/img/pytorch_logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 8 | pytorch_logo 9 | 11 | 13 | 14 | -------------------------------------------------------------------------------- /website/sphinx/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/instance_norm3d_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | import hypothesis.strategies as st 5 | import torch 6 | import torch.nn as nn 7 | from hypothesis import given, settings 8 | 9 | from .common import GradSampleHooks_test 10 | 11 | 12 | class InstanceNorm3d_test(GradSampleHooks_test): 13 | @given( 14 | N=st.integers(1, 4), 15 | C=st.integers(1, 3), 16 | W=st.integers(5, 10), 17 | H=st.integers(4, 8), 18 | Z=st.integers(1, 4), 19 | ) 20 | @settings(deadline=10000) 21 | def test_5d_input( 22 | self, 23 | N: int, 24 | C: int, 25 | W: int, 26 | H: int, 27 | Z: int, 28 | ): 29 | x = torch.randn([N, C, Z, H, W]) 30 | norm = nn.InstanceNorm3d(num_features=C, affine=True, track_running_stats=False) 31 | self.run_test(x, norm, batch_first=True) 32 | -------------------------------------------------------------------------------- /opacus/grad_sample/README.md: -------------------------------------------------------------------------------- 1 | # Grad Samples 2 | An integral part of Opacus is to compute per-sample gradients. In order to make this computation as fast as possible, we provide vectorized code for each of the most common "basic modules" that are the building blocks of most ML models. If your model uses these building blocks, then you don't have to do anything! 3 | 4 | We always welcome PRs to add nn.Modules we don't yet support, but we also support registering custom grad_sample functions that can expand support just for your project, or even override Opacus's default implementations if they don't suit your needs. 5 | 6 | Override as following: 7 | 8 | ```python 9 | from opacus.grad_sample import register_grad_sampler 10 | 11 | @register_grad_sampler(nn.MyCustomClass) 12 | def compute_grad_sample(module, activations, backprops): 13 | pass 14 | ``` 15 | 16 | Note that you can also pass a list to the decorator, and register one function against multiple nn.Module classes. 17 | -------------------------------------------------------------------------------- /opacus/grad_sample/linear.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | import torch 5 | import torch.nn as nn 6 | 7 | from .utils import create_or_extend_grad_sample, register_grad_sampler 8 | 9 | 10 | @register_grad_sampler(nn.Linear) 11 | def compute_linear_grad_sample( 12 | layer: nn.Linear, A: torch.Tensor, B: torch.Tensor, batch_dim: int = 0 13 | ) -> None: 14 | """ 15 | Computes per sample gradients for ``nn.Linear`` layer 16 | 17 | Args: 18 | layer: Layer 19 | A: Activations 20 | B: Backpropagations 21 | batch_dim: Batch dimension position 22 | """ 23 | gs = torch.einsum("n...i,n...j->nij", B, A) 24 | create_or_extend_grad_sample(layer.weight, gs, batch_dim) 25 | if layer.bias is not None: 26 | 27 | create_or_extend_grad_sample( 28 | layer.bias, 29 | torch.einsum("n...k->nk", B), 30 | batch_dim, 31 | ) 32 | -------------------------------------------------------------------------------- /opacus/layers/dp_ddp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | import torch 5 | import torch.nn as nn 6 | 7 | 8 | def average_gradients(model): 9 | world_size = torch.distributed.get_world_size() 10 | for param in model.parameters(): 11 | if not param.requires_grad: 12 | continue 13 | torch.distributed.all_reduce(param.grad, op=torch.distributed.ReduceOp.SUM) 14 | param.grad /= world_size 15 | 16 | 17 | class DifferentiallyPrivateDistributedDataParallel(nn.Module): 18 | def __init__(self, model): 19 | super().__init__() 20 | 21 | # Synchronize the model 22 | params = list(model.parameters()) 23 | with torch.no_grad(): 24 | for p in params: 25 | torch.distributed.broadcast(p.data, 0) 26 | 27 | self.module = model 28 | 29 | def forward(self, *args, **kwargs): 30 | return self.module(*args, **kwargs) 31 | -------------------------------------------------------------------------------- /opacus/grad_sample/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | from .conv import compute_conv_grad_sample # noqa 5 | from .dp_multihead_attention import compute_sequence_bias_grad_sample # noqa 6 | from .dp_rnn import compute_rnn_linear_grad_sample # noqa 7 | from .embedding import compute_embedding_grad_sample # noqa 8 | from .grad_sample_module import GradSampleModule 9 | from .group_norm import compute_group_norm_grad_sample # noqa 10 | from .instance_norm import compute_instance_norm_grad_sample # noqa 11 | from .layer_norm import compute_layer_norm_grad_sample # noqa 12 | from .linear import compute_linear_grad_sample # noqa 13 | from .utils import ( 14 | create_or_accumulate_grad_sample, 15 | create_or_extend_grad_sample, 16 | register_grad_sampler, 17 | ) 18 | 19 | 20 | __all__ = [ 21 | "GradSampleModule", 22 | "register_grad_sampler", 23 | "create_or_accumulate_grad_sample", 24 | "create_or_extend_grad_sample", 25 | ] 26 | -------------------------------------------------------------------------------- /opacus/grad_sample/group_norm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | 9 | from .utils import create_or_extend_grad_sample, register_grad_sampler 10 | 11 | 12 | @register_grad_sampler(nn.GroupNorm) 13 | def compute_group_norm_grad_sample( 14 | layer: nn.GroupNorm, 15 | A: torch.Tensor, 16 | B: torch.Tensor, 17 | batch_dim: int = 0, 18 | ) -> None: 19 | """ 20 | Computes per sample gradients for GroupNorm 21 | 22 | Args: 23 | layer: Layer 24 | A: Activations 25 | B: Backpropagations 26 | batch_dim: Batch dimension position 27 | """ 28 | gs = F.group_norm(A, layer.num_groups, eps=layer.eps) * B 29 | create_or_extend_grad_sample(layer.weight, torch.einsum("ni...->ni", gs), batch_dim) 30 | if layer.bias is not None: 31 | create_or_extend_grad_sample( 32 | layer.bias, torch.einsum("ni...->ni", B), batch_dim 33 | ) 34 | -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflows will upload a Python Package using Twine when a release is created 2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 3 | 4 | name: Upload Opacus Python Package to PyPI 5 | 6 | on: 7 | release: 8 | types: [created] 9 | 10 | jobs: 11 | deploy: 12 | 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Set up Python 18 | uses: actions/setup-python@v2 19 | with: 20 | python-version: '3.6.9' 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install --upgrade setuptools wheel twine 25 | - name: Build and publish 26 | env: 27 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 28 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 29 | run: | 30 | rm -rf dist/ 31 | python setup.py sdist bdist_wheel 32 | twine upload dist/* 33 | -------------------------------------------------------------------------------- /opacus/grad_sample/dp_rnn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | 5 | import torch 6 | from opacus.layers.dp_rnn import RNNLinear 7 | 8 | from .utils import create_or_accumulate_grad_sample, register_grad_sampler 9 | 10 | 11 | @register_grad_sampler(RNNLinear) 12 | def compute_rnn_linear_grad_sample( 13 | layer: RNNLinear, A: torch.Tensor, B: torch.Tensor 14 | ) -> None: 15 | """ 16 | Computes per sample gradients for ``RNNLinear`` layer. The RNN-like (DPLSTM, DPGRU) models 17 | are written using this layer as its building block. 18 | 19 | class 20 | 21 | Args: 22 | layer: Layer 23 | A: Activations 24 | B: Backpropagations 25 | """ 26 | 27 | gs = torch.einsum("n...i,n...j->nij", B, A) 28 | create_or_accumulate_grad_sample(layer.weight, gs, layer) 29 | 30 | if layer.bias is not None: 31 | create_or_accumulate_grad_sample( 32 | layer.bias, 33 | torch.einsum("n...k->nk", B), 34 | layer, 35 | ) 36 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/linear_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | import hypothesis.strategies as st 5 | import torch 6 | import torch.nn as nn 7 | from hypothesis import given, settings 8 | 9 | from .common import GradSampleHooks_test 10 | 11 | 12 | class Linear_test(GradSampleHooks_test): 13 | @given( 14 | N=st.integers(1, 4), 15 | Z=st.integers(1, 4), 16 | H=st.integers(1, 3), 17 | W=st.integers(10, 17), 18 | input_dim=st.integers(2, 4), 19 | bias=st.booleans(), 20 | ) 21 | @settings(deadline=10000) 22 | def test_input_bias( 23 | self, 24 | N: int, 25 | Z: int, 26 | W: int, 27 | H: int, 28 | input_dim: int, 29 | bias: bool, 30 | ): 31 | 32 | if input_dim == 2: 33 | x_shape = [N, W] 34 | if input_dim == 3: 35 | x_shape = [N, Z, W] 36 | if input_dim == 4: 37 | x_shape = [N, Z, H, W] 38 | 39 | linear = nn.Linear(W, W + 2, bias=bias) 40 | x = torch.randn(x_shape) 41 | self.run_test(x, linear, batch_first=True) 42 | -------------------------------------------------------------------------------- /opacus/grad_sample/layer_norm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | from opacus.utils.tensor_utils import sum_over_all_but_batch_and_last_n 9 | 10 | from .utils import create_or_extend_grad_sample, register_grad_sampler 11 | 12 | 13 | @register_grad_sampler(nn.LayerNorm) 14 | def compute_layer_norm_grad_sample( 15 | layer: nn.LayerNorm, 16 | A: torch.Tensor, 17 | B: torch.Tensor, 18 | batch_dim: int = 0, 19 | ) -> None: 20 | """ 21 | Computes per sample gradients for LayerNorm 22 | 23 | Args: 24 | layer: Layer 25 | A: Activations 26 | B: Backpropagations 27 | batch_dim: Batch dimension position 28 | """ 29 | create_or_extend_grad_sample( 30 | layer.weight, 31 | sum_over_all_but_batch_and_last_n( 32 | F.layer_norm(A, layer.normalized_shape, eps=layer.eps) * B, 33 | layer.weight.dim(), 34 | ), 35 | batch_dim, 36 | ) 37 | create_or_extend_grad_sample( 38 | layer.bias, 39 | sum_over_all_but_batch_and_last_n(B, layer.bias.dim()), 40 | batch_dim, 41 | ) 42 | -------------------------------------------------------------------------------- /scripts/pytorch_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | # Borrowed from https://github.com/OpenMined/PySyft 4 | 5 | set -e 6 | TORCH_VERSION=$1 7 | 8 | if [ "$TORCH_VERSION" = "1.4.0" ] 9 | then 10 | TORCHVISION_VERSION="0.5.0" 11 | elif [ "$TORCH_VERSION" = "1.5.0" ] 12 | then 13 | TORCHVISION_VERSION="0.6.0" 14 | elif [ "$TORCH_VERSION" = "1.5.1" ] 15 | then 16 | TORCHVISION_VERSION="0.6.1" 17 | elif [ "$TORCH_VERSION" = "1.6.0" ] 18 | then 19 | TORCHVISION_VERSION="0.7" 20 | TORCHCSPRNG_VERSION="0.1.2" 21 | elif [ "$TORCH_VERSION" = "1.7.0" ] 22 | then 23 | TORCHVISION_VERSION="0.8.1" 24 | TORCHCSPRNG_VERSION="0.1.3" 25 | elif [ "$TORCH_VERSION" = "1.7.1" ] 26 | then 27 | TORCHVISION_VERSION="0.8.2" 28 | TORCHCSPRNG_VERSION="0.1.4" 29 | fi 30 | pip install torch=="${TORCH_VERSION}" 31 | pip install torchvision==${TORCHVISION_VERSION} 32 | 33 | # torchcsprng 34 | if [ "$TORCH_VERSION" = "1.4.0" ] 35 | then 36 | echo "No torchcsprng" 37 | elif [ "$TORCH_VERSION" = "1.5.0" ] 38 | then 39 | echo "No torchcsprng" 40 | elif [ "$TORCH_VERSION" = "1.5.1" ] 41 | then 42 | echo "No torchcsprng" 43 | else 44 | pip install torchcsprng==${TORCHCSPRNG_VERSION} 45 | fi 46 | -------------------------------------------------------------------------------- /scripts/pytorch_install.ps1: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | # Borrowed from https://github.com/OpenMined/PySyft 4 | 5 | [string]$TORCH_VERSION=$args[0] 6 | If ($TORCH_VERSION -eq "1.4.0") { 7 | $TORCHVISION_VERSION="0.5.0" 8 | } Elseif ( $TORCH_VERSION -eq "1.5.0" ) { 9 | $TORCHVISION_VERSION="0.6.0" 10 | } Elseif ( $TORCH_VERSION -eq "1.5.1" ) { 11 | $TORCHVISION_VERSION="0.6.1" 12 | } Elseif ($TORCH_VERSION -eq "1.6.0") { 13 | $TORCHVISION_VERSION="0.7" 14 | $TORCHCSPRNG_VERSION="0.1.2" 15 | } Elseif ($TORCH_VERSION -eq "1.7.0") { 16 | $TORCHVISION_VERSION="0.8.1" 17 | $TORCHCSPRNG_VERSION="0.1.3" 18 | } Elseif ($TORCH_VERSION -eq "1.7.1") { 19 | $TORCHVISION_VERSION="0.8.2" 20 | $TORCHCSPRNG_VERSION="0.1.4" 21 | } 22 | pip install torch==$TORCH_VERSION+cpu torchvision==$TORCHVISION_VERSION+cpu -f https://download.pytorch.org/whl/torch_stable.html 23 | 24 | If ($TORCH_VERSION -eq "1.4.0") { 25 | echo "No torchcsprng" 26 | } Elseif ($TORCH_VERSION -eq "1.5.0" ) { 27 | echo "No torchcsprng" 28 | } Elseif ($TORCH_VERSION -eq "1.5.1" ) { 29 | echo "No torchcsprng" 30 | } Else { 31 | pip install torchcsprng==$TORCHCSPRNG_VERSION+cpu -f https://download.pytorch.org/whl/torch_stable.html 32 | } 33 | -------------------------------------------------------------------------------- /examples/char-lstm_README.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | This example demonstrates how to run a privacy-preserving LSTM (with DP) on a name classification task. This is the same task as in this old PyTorch tutorial https://pytorch.org/tutorials/intermediate/char_rnn_classification_tutorial.html, with a rewrite that aligns its style to modern PyTorch. 4 | 5 | # Sample run 6 | 7 | Download the training zip from https://download.pytorch.org/tutorial/data.zip and then extract it to a local directory. In this example, we assume that the data path is /home/data_folder/names/*.txt 8 | 9 | Run with dp: 10 | 11 | ``` 12 | python char-lstm-classification.py --epochs=50 --learning-rate=2.0 --hidden-size=128 --delta=8e-5 --sample-rate=0.05 --n-lstm-layers=1 --sigma=1.0 --max-per-sample-grad-norm=1.5 --device=cuda:0 --data-root="/my/folder/data/names/" --test-every 5 13 | ``` 14 | 15 | You should get something like this: Test Accuracy: 0.739542 (ε = 11.83, δ = 8e-05) for α = 2.7 16 | 17 | Run without dp: 18 | 19 | ``` 20 | python char-lstm-classification.py --epochs=50 --learning-rate=0.5 --hidden-size=128 --sample-rate=0.05 --n-lstm-layers=1 --disable-dp --device=cuda:1 --data-root="/my/folder/data/names/" --test-every 5 21 | ``` 22 | You should get something like this: Test Accuracy: 0.760716 23 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/embedding_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | import hypothesis.strategies as st 5 | import torch 6 | import torch.nn as nn 7 | from hypothesis import given, settings 8 | 9 | from .common import GradSampleHooks_test 10 | 11 | 12 | class Embedding_test(GradSampleHooks_test): 13 | @given( 14 | N=st.integers(1, 4), 15 | T=st.integers(1, 5), 16 | Q=st.integers(1, 4), 17 | R=st.integers(1, 2), 18 | V=st.integers(2, 32), 19 | D=st.integers(10, 17), 20 | dim=st.integers(1, 4), 21 | ) 22 | @settings(deadline=10000) 23 | def test_input_across_dims( 24 | self, 25 | N: int, 26 | T: int, 27 | Q: int, 28 | R: int, 29 | V: int, 30 | D: int, 31 | dim: int, 32 | ): 33 | 34 | if dim == 1: 35 | size = [T] 36 | elif dim == 2: 37 | size = [N, T] 38 | elif dim == 3: 39 | size = [N, T, Q] 40 | elif dim == 4: 41 | size = [N, T, Q, R] 42 | 43 | emb = nn.Embedding(V, D) 44 | x = torch.randint(low=0, high=V - 1, size=size) 45 | self.run_test(x, emb, batch_first=True) 46 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Types of changes 2 | 3 | 4 | 5 | - [ ] Bug fix (non-breaking change which fixes an issue) 6 | - [ ] New feature (non-breaking change which adds functionality) 7 | - [ ] Breaking change (fix or feature that would cause existing functionality to change) 8 | - [ ] Docs change / refactoring / dependency upgrade 9 | 10 | ## Motivation and Context / Related issue 11 | 12 | 13 | 14 | 15 | 16 | ## How Has This Been Tested (if it applies) 17 | 18 | 19 | 20 | ## Checklist 21 | 22 | 23 | 24 | 25 | - [ ] The documentation is up-to-date with the changes I made. 26 | - [ ] I have read the **CONTRIBUTING** document and completed the CLA (see **CONTRIBUTING**). 27 | - [ ] All tests passed, and additional code has been covered with new tests. 28 | -------------------------------------------------------------------------------- /opacus/grad_sample/instance_norm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | from typing import Union 5 | 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | 10 | from .utils import create_or_extend_grad_sample, register_grad_sampler 11 | 12 | 13 | @register_grad_sampler( 14 | [ 15 | nn.InstanceNorm1d, 16 | nn.InstanceNorm2d, 17 | nn.InstanceNorm3d, 18 | ] 19 | ) 20 | def compute_instance_norm_grad_sample( 21 | layer: Union[ 22 | nn.InstanceNorm1d, 23 | nn.InstanceNorm2d, 24 | nn.InstanceNorm3d, 25 | ], 26 | A: torch.Tensor, 27 | B: torch.Tensor, 28 | batch_dim: int = 0, 29 | ) -> None: 30 | """ 31 | Computes per sample gradients for InstanceNorm layers 32 | 33 | Args: 34 | layer: Layer 35 | A: Activations 36 | B: Backpropagations 37 | batch_dim: Batch dimension position 38 | """ 39 | gs = F.instance_norm(A, eps=layer.eps) * B 40 | create_or_extend_grad_sample(layer.weight, torch.einsum("ni...->ni", gs), batch_dim) 41 | if layer.bias is not None: 42 | create_or_extend_grad_sample( 43 | layer.bias, torch.einsum("ni...->ni", B), batch_dim 44 | ) 45 | -------------------------------------------------------------------------------- /website/static/img/modular.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /opacus/grad_sample/embedding.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | import torch 5 | import torch.nn as nn 6 | 7 | from .utils import create_or_extend_grad_sample, register_grad_sampler 8 | 9 | 10 | @register_grad_sampler(nn.Embedding) 11 | def compute_embedding_grad_sample( 12 | layer: nn.Embedding, A: torch.Tensor, B: torch.Tensor, batch_dim: int = 0 13 | ) -> None: 14 | """ 15 | Computes per sample gradients for ``nn.Embedding`` layer. 16 | 17 | Args: 18 | layer: Layer 19 | A: Activations 20 | B: Backpropagations 21 | batch_dim: Batch dimension position 22 | """ 23 | saved = torch.backends.cudnn.deterministic 24 | torch.backends.cudnn.deterministic = True 25 | 26 | batch_size = A.shape[batch_dim] 27 | index = ( 28 | A.unsqueeze(-1) 29 | .expand(*A.shape, layer.embedding_dim) 30 | .reshape(batch_size, -1, layer.embedding_dim) 31 | ) 32 | grad_sample = torch.zeros( 33 | batch_size, *layer.weight.shape, device=layer.weight.device 34 | ) 35 | grad_sample.scatter_add_(1, index, B.reshape(batch_size, -1, layer.embedding_dim)) 36 | torch.backends.cudnn.deterministic = saved 37 | 38 | create_or_extend_grad_sample(layer.weight, grad_sample, batch_dim) 39 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/group_norm_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | from typing import Union 5 | 6 | import hypothesis.strategies as st 7 | import torch 8 | import torch.nn as nn 9 | from hypothesis import given, settings 10 | 11 | from .common import GradSampleHooks_test 12 | 13 | 14 | class GroupNorm_test(GradSampleHooks_test): 15 | """ 16 | We only test the case with ``affine=True`` here, because it is the only case that will actually 17 | compute a gradient. There is no grad_sample from this module otherwise. 18 | """ 19 | 20 | @given( 21 | N=st.integers(1, 4), 22 | C=st.integers(1, 8), 23 | H=st.integers(5, 10), 24 | W=st.integers(4, 8), 25 | num_groups=st.sampled_from([1, 4, "C"]), 26 | ) 27 | @settings(deadline=10000) 28 | def test_3d_input_groups( 29 | self, 30 | N: int, 31 | C: int, 32 | H: int, 33 | W: int, 34 | num_groups: Union[int, str], 35 | ): 36 | 37 | if num_groups == "C": 38 | num_groups = C 39 | 40 | if C % num_groups != 0: 41 | return 42 | 43 | x = torch.randn([N, C, H, W]) 44 | norm = nn.GroupNorm(num_groups=num_groups, num_channels=C, affine=True) 45 | self.run_test(x, norm, batch_first=True) 46 | -------------------------------------------------------------------------------- /opacus/tests/poisson_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | import unittest 4 | 5 | import torch 6 | from opacus import PrivacyEngine 7 | from opacus.utils.uniform_sampler import UniformWithReplacementSampler 8 | from torch import nn, optim 9 | 10 | 11 | class PoissonSamplingTest(unittest.TestCase): 12 | def test_poisson_sampling(self): 13 | B = 1 14 | N = 10 15 | d = 10 16 | dataset = [(i, torch.randn(d), torch.randn(d)) for i in range(N)] 17 | 18 | model = nn.Linear(d, d) 19 | optimizer = optim.SGD(model.parameters(), lr=0.1) 20 | engine = PrivacyEngine( 21 | model, 22 | sample_rate=B / N, 23 | target_epsilon=1.0, 24 | epochs=10, 25 | poisson=True, 26 | max_grad_norm=1, 27 | sample_size=N, 28 | ) 29 | engine.attach(optimizer) 30 | 31 | generator = torch.Generator() 32 | generator.manual_seed(7) 33 | sampler = UniformWithReplacementSampler( 34 | num_samples=N, sample_rate=B / N, generator=generator 35 | ) 36 | dataloader = torch.utils.data.DataLoader(dataset, batch_sampler=sampler) 37 | 38 | # Sampler with seed=7 should generate [], [7], [], [], [9], [0], [], [], [1], [4] 39 | for (_, x, y) in dataloader: 40 | prediction = model(x) 41 | loss = torch.mean((prediction - y) ** 2) 42 | 43 | optimizer.zero_grad() 44 | loss.backward() 45 | optimizer.step() 46 | -------------------------------------------------------------------------------- /examples/imdb_README.md: -------------------------------------------------------------------------------- 1 | # Simple run 2 | run with dp: 3 | ``` 4 | python imdb.py --device cuda:0 5 | 6 | ``` 7 | for with no dp: 8 | ``` 9 | python imdb.py --device cuda:0 --disable-dp 10 | ``` 11 | 12 | Default parameters are tweaked for optimal performance with DP 13 | 14 | # Expeted results 15 | We use the same architecture and dataset as described in https://arxiv.org/pdf/1911.11607.pdf. 16 | 17 | Reference implementation for the paper (tensorflow privacy): https://github.com/tensorflow/privacy/blob/master/research/GDP_2019/imdb_tutorial.py 18 | 19 | Unless specified otherwise, we set `batch_size=64` for a faster iteration (opacus has higher requirements in memory compared to tensorflow-privacy). We didn't use virtual batches for this experiment, and therefowe were limited to the given batch size. 20 | All metrics are calculated on test data after training for 20 epochs 21 | 22 | | Approach | Accuracy | 23 | | ----------------------------------------|:--------:| 24 | | Claimed in paper (no DP) | 84% | 25 | | tf_privacy (no DP) | 84% | 26 | | opacus (no DP) | 84% | 27 | | Claimed in paper (with DP) | 84% | 28 | | tf_privacy (with DP) | 80% | 29 | | tf_privacy (with DP, bsz=512) | 84% | 30 | | opacus (Adam, lr = 0.02) | 74% | 31 | | opacus (Adam, lr = 0.02, 100 epochs) | 78% | 32 | | opacus (Adagrad, lr = 0.02) | 59% | 33 | | opacus (Adagrad, lr = 0.1) | 65% | 34 | | opacus (SGD, lr = 0.1) | 63% | 35 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/layer_norm_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | import hypothesis.strategies as st 5 | import torch 6 | import torch.nn as nn 7 | from hypothesis import given, settings 8 | 9 | from .common import GradSampleHooks_test 10 | 11 | 12 | class LayerNorm_test(GradSampleHooks_test): 13 | @given( 14 | N=st.integers(1, 4), 15 | Z=st.integers(1, 4), 16 | H=st.integers(1, 3), 17 | W=st.integers(5, 10), 18 | input_dim=st.integers(2, 4), 19 | norm_dim=st.integers(1, 3), 20 | ) 21 | @settings(deadline=10000) 22 | def test_input_norm( 23 | self, 24 | N: int, 25 | Z: int, 26 | W: int, 27 | H: int, 28 | input_dim: int, 29 | norm_dim: int, 30 | ): 31 | 32 | if norm_dim >= input_dim: 33 | return 34 | if norm_dim == 1: 35 | normalized_shape = W 36 | if input_dim == 2: 37 | x_shape = [N, W] 38 | if input_dim == 3: 39 | x_shape = [N, Z, W] 40 | if input_dim == 4: 41 | x_shape = [N, Z, H, W] 42 | elif norm_dim == 2: 43 | if input_dim == 3: 44 | normalized_shape = [Z, W] 45 | x_shape = [N, Z, W] 46 | if input_dim == 4: 47 | normalized_shape = [H, W] 48 | x_shape = [N, Z, H, W] 49 | elif norm_dim == 3: 50 | normalized_shape = [Z, H, W] 51 | x_shape = [N, Z, H, W] 52 | 53 | norm = nn.LayerNorm(normalized_shape, elementwise_affine=True) 54 | x = torch.randn(x_shape) 55 | self.run_test(x, norm, batch_first=True) 56 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F41B Bug Report" 3 | about: Submit a bug report to help us improve Opacus 4 | 5 | --- 6 | 7 | ## 🐛 Bug 8 | 9 | 10 | 11 | 12 | ## Please reproduce using our [template Colab](https://colab.research.google.com/drive/1pCyN3_gN3WyhyQt8--eBPN5zVf2XZxiN?usp=sharing) and post here the link 13 | 14 | 15 | ## To Reproduce 16 | > :warning: **We cannot help you without you sharing reproducible code**. Do not ignore this part :) 17 | Steps to reproduce the behavior: 18 | 19 | 1. 20 | 2. 21 | 3. 22 | 23 | 24 | 25 | ## Expected behavior 26 | 27 | 28 | 29 | ## Environment 30 | 31 | Please copy and paste the output from our 32 | [environment collection script](https://raw.githubusercontent.com/pytorch/pytorch/main/torch/utils/collect_env.py) 33 | (or fill out the checklist below manually). 34 | 35 | You can get the script and run it with: 36 | 37 | ``` 38 | wget https://raw.githubusercontent.com/pytorch/pytorch/main/torch/utils/collect_env.py 39 | # For security purposes, please check the contents of collect_env.py before running it. 40 | python collect_env.py 41 | ``` 42 | 43 | - PyTorch Version (e.g., 1.0): 44 | - OS (e.g., Linux): 45 | - How you installed PyTorch (`conda`, `pip`, source): 46 | - Build command you used (if compiling from source): 47 | - Python version: 48 | - CUDA/cuDNN version: 49 | - GPU models and configuration: 50 | - Any other relevant information: 51 | 52 | ## Additional context 53 | 54 | 55 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/conv1d_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | from typing import Callable 5 | 6 | import hypothesis.strategies as st 7 | import torch 8 | import torch.nn as nn 9 | from hypothesis import given, settings 10 | 11 | from .common import GradSampleHooks_test, expander, shrinker 12 | 13 | 14 | class Conv1d_test(GradSampleHooks_test): 15 | @given( 16 | N=st.integers(1, 4), 17 | C=st.sampled_from([1, 3, 32]), 18 | W=st.integers(6, 10), 19 | out_channels_mapper=st.sampled_from([expander, shrinker]), 20 | kernel_size=st.integers(2, 3), 21 | stride=st.integers(1, 2), 22 | padding=st.integers(0, 2), 23 | dilation=st.integers(1, 2), 24 | groups=st.integers(1, 12), 25 | ) 26 | @settings(deadline=10000) 27 | def test_conv1d( 28 | self, 29 | N: int, 30 | C: int, 31 | W: int, 32 | out_channels_mapper: Callable[[int], int], 33 | kernel_size: int, 34 | stride: int, 35 | padding: int, 36 | dilation: int, 37 | groups: int, 38 | ): 39 | 40 | out_channels = out_channels_mapper(C) 41 | if ( 42 | C % groups != 0 or out_channels % groups != 0 43 | ): # since in_channels and out_channels must be divisible by groups 44 | return 45 | 46 | x = torch.randn([N, C, W]) 47 | conv = nn.Conv1d( 48 | in_channels=C, 49 | out_channels=out_channels, 50 | kernel_size=kernel_size, 51 | stride=stride, 52 | padding=padding, 53 | dilation=dilation, 54 | groups=groups, 55 | ) 56 | self.run_test(x, conv, batch_first=True, atol=10e-5, rtol=10e-4) 57 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/conv2d_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | from typing import Callable 5 | 6 | import hypothesis.strategies as st 7 | import torch 8 | import torch.nn as nn 9 | from hypothesis import given, settings 10 | 11 | from .common import GradSampleHooks_test, expander, shrinker 12 | 13 | 14 | class Conv2d_test(GradSampleHooks_test): 15 | @given( 16 | N=st.integers(1, 4), 17 | C=st.sampled_from([1, 3, 32]), 18 | H=st.integers(6, 10), 19 | W=st.integers(6, 10), 20 | out_channels_mapper=st.sampled_from([expander, shrinker]), 21 | kernel_size=st.integers(2, 3), 22 | stride=st.integers(1, 2), 23 | padding=st.sampled_from([0, 2]), 24 | dilation=st.integers(1, 2), 25 | groups=st.integers(1, 16), 26 | ) 27 | @settings(deadline=10000) 28 | def test_conv2d( 29 | self, 30 | N: int, 31 | C: int, 32 | H: int, 33 | W: int, 34 | out_channels_mapper: Callable[[int], int], 35 | kernel_size: int, 36 | stride: int, 37 | padding: int, 38 | dilation: int, 39 | groups: int, 40 | ): 41 | 42 | out_channels = out_channels_mapper(C) 43 | if ( 44 | C % groups != 0 or out_channels % groups != 0 45 | ): # since in_channels and out_channels must be divisible by groups 46 | return 47 | 48 | x = torch.randn([N, C, H, W]) 49 | conv = nn.Conv2d( 50 | in_channels=C, 51 | out_channels=out_channels, 52 | kernel_size=kernel_size, 53 | stride=stride, 54 | padding=padding, 55 | dilation=dilation, 56 | groups=groups, 57 | ) 58 | self.run_test(x, conv, batch_first=True, atol=10e-5, rtol=10e-4) 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # Downloaded data 7 | data/ 8 | 9 | # Buck Targets 10 | TARGETS 11 | 12 | # Distribution / packaging 13 | .Python 14 | env/ 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | 32 | # watchman 33 | # Watchman helper 34 | .watchmanconfig 35 | 36 | # OSX 37 | *.DS_Store 38 | 39 | # Atom plugin files and ctags 40 | .ftpconfig 41 | .ftpconfig.cson 42 | .ftpignore 43 | *.tags 44 | *.tags1 45 | 46 | # PyInstaller 47 | # Usually these files are written by a python script from a template 48 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 49 | *.manifest 50 | *.spec 51 | 52 | # Installer logs 53 | pip-log.txt 54 | pip-delete-this-directory.txt 55 | 56 | # Unit test / coverage reports 57 | htmlcov/ 58 | .tox/ 59 | .coverage 60 | .coverage.* 61 | .cache 62 | nosetests.xml 63 | coverage.xml 64 | *.cover 65 | .hypothesis/ 66 | .pytest_cache/ 67 | 68 | # PyBuilder 69 | target/ 70 | 71 | # Jupyter Notebook 72 | .ipynb_checkpoints 73 | 74 | # pyenv 75 | .python-version 76 | 77 | # dotenv 78 | .env 79 | 80 | # virtualenv 81 | .venv 82 | venv/ 83 | ENV/ 84 | 85 | # mypy 86 | .mypy_cache/ 87 | 88 | # vim 89 | *.swp 90 | 91 | # Sphinx documentation 92 | sphinx/build/ 93 | 94 | # Docusaurus 95 | website/build/ 96 | website/i18n/ 97 | website/node_modules/ 98 | 99 | ## Generated for tutorials 100 | website/_tutorials/ 101 | website/static/files/ 102 | website/pages/tutorials/* 103 | !website/pages/tutorials/index.js 104 | 105 | ## Generated for Sphinx 106 | website/pages/api/ 107 | website/static/js/* 108 | !website/static/js/mathjax.js 109 | !website/static/js/code_block_buttons.js 110 | website/static/_sphinx-sources/ 111 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/conv3d_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | from typing import Tuple, Union 5 | 6 | import hypothesis.strategies as st 7 | import torch 8 | import torch.nn as nn 9 | from hypothesis import given, settings 10 | 11 | from .common import GradSampleHooks_test, expander, shrinker 12 | 13 | 14 | class Conv3d_test(GradSampleHooks_test): 15 | @given( 16 | N=st.integers(1, 4), 17 | C=st.sampled_from([1, 3, 32]), 18 | D=st.integers(3, 6), 19 | H=st.integers(6, 10), 20 | W=st.integers(6, 10), 21 | out_channels_mapper=st.sampled_from([expander, shrinker]), 22 | kernel_size=st.sampled_from([2, 3, (1, 2, 3)]), 23 | stride=st.sampled_from([1, 2, (1, 2, 3)]), 24 | padding=st.sampled_from([0, 2, (1, 2, 3)]), 25 | dilation=st.just(1), 26 | groups=st.integers(1, 16), 27 | ) 28 | @settings(deadline=10000) 29 | def test_conv3d( 30 | self, 31 | N: int, 32 | C: int, 33 | D: int, 34 | H: int, 35 | W: int, 36 | out_channels_mapper: int, 37 | kernel_size: Union[int, Tuple[int]], 38 | stride: Union[int, Tuple[int]], 39 | padding: Union[int, Tuple[int]], 40 | dilation: int, 41 | groups: int, 42 | ): 43 | 44 | out_channels = out_channels_mapper(C) 45 | if ( 46 | C % groups != 0 or out_channels % groups != 0 47 | ): # since in_channels and out_channels must be divisible by groups 48 | return 49 | x = torch.randn([N, C, D, H, W]) 50 | conv = nn.Conv3d( 51 | in_channels=C, 52 | out_channels=out_channels, 53 | kernel_size=kernel_size, 54 | stride=stride, 55 | padding=padding, 56 | dilation=dilation, 57 | groups=groups, 58 | ) 59 | self.run_test(x, conv, batch_first=True, atol=10e-5, rtol=10e-3) 60 | -------------------------------------------------------------------------------- /scripts/install_via_pip.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | set -e 5 | 6 | PYTORCH_NIGHTLY=false 7 | DEPLOY=false 8 | CHOSEN_TORCH_VERSION=-1 9 | 10 | while getopts 'ncdv:' flag; do 11 | case "${flag}" in 12 | n) PYTORCH_NIGHTLY=true ;; 13 | c) CUDA=true ;; 14 | d) DEPLOY=true ;; 15 | v) CHOSEN_TORCH_VERSION=${OPTARG} ;; 16 | *) echo "usage: $0 [-n] [-d] [-v version]" >&2 17 | exit 1 ;; 18 | esac 19 | done 20 | 21 | # NOTE: Only Debian variants are supported, since this script is only 22 | # used by our tests on CircleCI. 23 | 24 | 25 | sudo apt-get update 26 | sudo apt-get upgrade ca-certificates -y 27 | curl -sL https://deb.nodesource.com/setup_14.x | sudo -E bash - 28 | sudo apt-get install -y nodejs 29 | curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - 30 | echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list 31 | sudo apt-get update && sudo apt-get install yarn 32 | 33 | # yarn needs terminal info 34 | export TERM=xterm 35 | 36 | # NOTE: We don't use sudo for the below installs, because we expect these to come from pyenv which 37 | # installs Python in a folder for which we have user access. 38 | 39 | # upgrade pip 40 | pip install --upgrade pip 41 | 42 | # install with dev dependencies 43 | pip install -e .[dev] --user 44 | 45 | # install pytorch nightly if asked for 46 | if [[ $PYTORCH_NIGHTLY == true ]]; then 47 | if [[ $CUDA == true ]]; then 48 | pip install --upgrade --pre torch torchvision torchcsprng -f https://download.pytorch.org/whl/nightly/cu101/torch_nightly.html 49 | else 50 | pip install --upgrade --pre torch torchvision torchcsprng -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html 51 | fi 52 | else 53 | # If no version specified, upgrade to latest release. 54 | if [[ $CHOSEN_TORCH_VERSION == -1 ]]; then 55 | pip install --upgrade torch 56 | else 57 | pip install torch=="$CHOSEN_TORCH_VERSION" 58 | fi 59 | fi 60 | 61 | # install deployment bits if asked for 62 | if [[ $DEPLOY == true ]]; then 63 | pip install beautifulsoup4 ipython nbconvert 64 | fi 65 | -------------------------------------------------------------------------------- /.github/workflows/test-installation.yml: -------------------------------------------------------------------------------- 1 | # This workflow makes sure edge case versions work 2 | name: Test Installation 3 | 4 | on: 5 | pull_request: 6 | branches: 7 | - main 8 | paths: 9 | - "**.py" 10 | - ".github/workflows/**.yml" 11 | 12 | jobs: 13 | test-installation: 14 | strategy: 15 | max-parallel: 48 16 | matrix: 17 | os: [windows-latest, ubuntu-latest, macos-latest] 18 | python-version: [3.6.8, 3.7, 3.8] 19 | torch-version: [1.5.0, 1.5.1, 1.6.0, 1.7.0, 1.7.1] 20 | # include: 21 | # - os: windows-latest 22 | # python-version: 3.9 23 | # torch-version: 1.7.1 24 | # - os: ubuntu-latest 25 | # python-version: 3.9 26 | # torch-version: 1.7.1 27 | # - os: macos-latest 28 | # python-version: 3.9 29 | # torch-version: 1.7.1 30 | 31 | runs-on: ${{ matrix.os }} 32 | steps: 33 | - uses: actions/checkout@v2 34 | 35 | - name: Set up Python ${{ matrix.python-version }} 36 | uses: actions/setup-python@v2 37 | with: 38 | python-version: ${{ matrix.python-version }} 39 | 40 | - name: Get pip cache dir 41 | id: pip-cache 42 | run: | 43 | echo "::set-output name=dir::$(pip cache dir)" 44 | 45 | - name: pip cache 46 | uses: actions/cache@v2 47 | with: 48 | path: ${{ steps.pip-cache.outputs.dir }} 49 | key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} 50 | restore-keys: | 51 | ${{ runner.os }}-pip- 52 | 53 | - name: Install PyTorch Linux and MacOS 54 | if: startsWith(runner.os, 'Windows') != true 55 | run: | 56 | ./scripts/pytorch_install.sh ${{ matrix.torch-version }} 57 | 58 | - name: Install PyTorch Windows 59 | if: startsWith(runner.os, 'Windows') 60 | run: | 61 | ./scripts/pytorch_install.ps1 ${{ matrix.torch-version }} 62 | 63 | - name: Install Opacus in Editing Mode 64 | run: | 65 | pip install -e . --default-timeout=60 66 | python -c "import opacus; print(opacus.__version__)" 67 | -------------------------------------------------------------------------------- /examples/scripts/make_small_imagenet_N_classes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # =================== 4 | # This script makes a smaller version of ImageNet, where only N classes are copied. 5 | # =================== 6 | 7 | 8 | usage() 9 | { 10 | echo "usage Example: bash make_small_imagenet_N_classes.sh -o Documents/imagenet_full_size -d Documents/imagenet_small_50_classes -n 50" 11 | echo "usage: bash make_small_imagenet_N_classes.sh -o -d -n " 12 | } 13 | 14 | if [ "$1" == "" ]; then # If arguments are not specified 15 | usage 16 | exit 1 17 | fi 18 | while [ "$1" != "" ]; do # Read the input arguments 19 | case $1 in 20 | -d | --destination ) shift 21 | destination=$1 22 | ;; 23 | -o | --origin ) shift 24 | origin=$1 25 | ;; 26 | -n | --N ) shift 27 | N=$1 28 | ;; 29 | -h | --help ) usage 30 | exit 31 | ;; 32 | * ) echo "ERROR: unknown parameter \"$1\"" 33 | usage 34 | exit 1 35 | esac 36 | shift 37 | done 38 | 39 | # If all necessary arguments are not supplied 40 | if [[ -z $destination || -z $origin || -z $N ]] 41 | then 42 | echo "You must specify all necessary parameters." 43 | usage 44 | exit 1 45 | fi 46 | 47 | # Get absolute path 48 | destination="$(readlink -f $destination)" 49 | origin="$(readlink -f $origin)" 50 | 51 | mkdir "$destination" 52 | mkdir "$destination/train" 53 | mkdir "$destination/val" 54 | 55 | echo 'Copying' 56 | for val_train_folder in val train; do # Do copying for both 'train' and 'val' folders 57 | cd "$origin/$val_train_folder" || { echo "Failure"; exit 1; } # change directory to origin's train or val folders 58 | find . -maxdepth 1 -mindepth 1 | head -n "$N" | xargs cp -ir -t "$destination/$val_train_folder" # select and copy N classes 59 | echo "Copying folder $val_train_folder is done." 60 | done 61 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/dp_rnn_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | import hypothesis.strategies as st 5 | import torch 6 | import torch.nn as nn 7 | from hypothesis import given, settings 8 | from opacus.layers import DPGRU, DPLSTM, DPRNN 9 | from opacus.utils.packed_sequences import _gen_packed_data 10 | 11 | from .common import GradSampleHooks_test 12 | 13 | 14 | MODELS = [ 15 | DPRNN, 16 | DPGRU, 17 | DPLSTM, 18 | ] 19 | 20 | 21 | class DPRNNAdapter(nn.Module): 22 | """ 23 | Adapter for DPRNN family. 24 | RNN returns a tuple, but our testing tools need the model to return a single tensor in output. 25 | We do this adaption here. 26 | """ 27 | 28 | def __init__(self, dp_rnn): 29 | super().__init__() 30 | self.dp_rnn = dp_rnn 31 | 32 | def forward(self, x): 33 | out, _rest = self.dp_rnn(x) 34 | return out 35 | 36 | 37 | class RNN_test(GradSampleHooks_test): 38 | @given( 39 | model=st.one_of([st.just(model) for model in MODELS]), 40 | N=st.integers(1, 3), 41 | T=st.integers(1, 3), 42 | D=st.integers(4, 5), 43 | H=st.integers(8, 10), 44 | num_layers=st.sampled_from([1, 2]), 45 | bias=st.booleans(), 46 | batch_first=st.booleans(), 47 | bidirectional=st.booleans(), 48 | using_packed_sequences=st.booleans(), 49 | packed_sequences_sorted=st.booleans(), 50 | ) 51 | @settings(deadline=30000) 52 | def test_rnn( 53 | self, 54 | model, 55 | N: int, 56 | T: int, 57 | D: int, 58 | H: int, 59 | num_layers: int, 60 | bias: bool, 61 | batch_first: bool, 62 | bidirectional: bool, 63 | using_packed_sequences: bool, 64 | packed_sequences_sorted: bool, 65 | ): 66 | rnn = model( 67 | D, 68 | H, 69 | num_layers=num_layers, 70 | batch_first=batch_first, 71 | bias=bias, 72 | bidirectional=bidirectional, 73 | ) 74 | rnn = DPRNNAdapter(rnn) 75 | 76 | if using_packed_sequences: 77 | x = _gen_packed_data(N, T, D, batch_first, packed_sequences_sorted) 78 | else: 79 | if batch_first: 80 | x = torch.randn([N, T, D]) 81 | else: 82 | x = torch.randn([T, N, D]) 83 | self.run_test(x, rnn, batch_first=batch_first) 84 | -------------------------------------------------------------------------------- /website/core/TutorialSidebar.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2019-present, Facebook, Inc. 3 | * 4 | * This source code is licensed under the BSD license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | * 7 | * @format 8 | */ 9 | 10 | const React = require('react'); 11 | const fs = require('fs-extra'); 12 | const path = require('path'); 13 | const join = path.join; 14 | const CWD = process.cwd(); 15 | 16 | const CompLibrary = require(join( 17 | CWD, 18 | '/node_modules/docusaurus/lib/core/CompLibrary.js', 19 | )); 20 | const SideNav = require(join( 21 | CWD, 22 | '/node_modules/docusaurus/lib/core/nav/SideNav.js', 23 | )); 24 | 25 | const Container = CompLibrary.Container; 26 | 27 | const OVERVIEW_ID = 'tutorial_overview'; 28 | 29 | class TutorialSidebar extends React.Component { 30 | render() { 31 | const {currentTutorialID} = this.props; 32 | const current = { 33 | id: currentTutorialID || OVERVIEW_ID, 34 | }; 35 | 36 | const toc = [ 37 | { 38 | type: 'CATEGORY', 39 | title: 'Tutorials', 40 | children: [ 41 | { 42 | type: 'LINK', 43 | item: { 44 | permalink: 'tutorials/', 45 | id: OVERVIEW_ID, 46 | title: 'Overview', 47 | }, 48 | }, 49 | ], 50 | }, 51 | ]; 52 | 53 | const jsonFile = join(CWD, 'tutorials.json'); 54 | const normJsonFile = path.normalize(jsonFile); 55 | const json = JSON.parse(fs.readFileSync(normJsonFile, {encoding: 'utf8'})); 56 | 57 | Object.keys(json).forEach(category => { 58 | const categoryItems = json[category]; 59 | const items = []; 60 | categoryItems.map(item => { 61 | items.push({ 62 | type: 'LINK', 63 | item: { 64 | permalink: `tutorials/${item.id}`, 65 | id: item.id, 66 | title: item.title, 67 | }, 68 | }); 69 | }); 70 | 71 | toc.push({ 72 | type: 'CATEGORY', 73 | title: category, 74 | children: items, 75 | }); 76 | }); 77 | 78 | return ( 79 | 80 | 87 | 88 | ); 89 | } 90 | } 91 | 92 | module.exports = TutorialSidebar; 93 | -------------------------------------------------------------------------------- /examples/mnist_README.md: -------------------------------------------------------------------------------- 1 | # First run 2 | To run a basic training script without differential privacy: 3 | ```shell 4 | python mnist.py --device=cpu --disable-dp --n=20 --lr=.1 -sr=0.004 5 | ``` 6 | The first time the script runs, it attempts to download the MNIST dataset from http://yann.lecun.com and place it in `../mnist/MNIST/raw`. If you prefer a different location or your execution environment does not have access to the outside world, download and unpack the dataset yourself and pass the location as `--data-root=custom_dir_name`. The script will expect to find under `custom_dir_name/MNIST/processed` two files: `test.pt` (7.9 MB) and `training.pt` (47.5 MB). 7 | 8 | If the run is successful, you will something similar to this (your exact accuracy and performance numbers will vary): 9 | ``` 10 | 100%|██████████| 240/240 [00:14<00:00, 16.58it/s] 11 | Train Epoch: 1 Loss: 0.536689 12 | 100%|██████████| 240/240 [00:14<00:00, 16.48it/s] 13 | Train Epoch: 2 Loss: 0.102070 14 | ... 15 | 100%|██████████| 240/240 [00:14<00:00, 17.02it/s] 16 | Train Epoch: 10 Loss: 0.025479 17 | 100%|██████████| 10/10 [00:01<00:00, 6.64it/s] 18 | 19 | Test set: Average loss: 0.0000, Accuracy: 9893/10000 (98.93%) 20 | ``` 21 | 22 | To train a differentially private model, run the following command: 23 | ```shell 24 | python mnist.py --device=cpu -n=15 --lr=.25 --sigma=1.3 -c=1.5 -sr=0.004 25 | ``` 26 | If the run is successful, expect to see 27 | ``` 28 | 100%|██████████| 240/240 [00:22<00:00, 10.48it/s] 29 | Train Epoch: 1 Loss: 0.912457 (ε = 0.71, δ = 1e-05) for α = 18.0 30 | ... 31 | 100%|██████████| 240/240 [00:22<00:00, 10.79it/s] 32 | Train Epoch: 15 Loss: 0.404850 (ε = 1.16, δ = 1e-05) for α = 17.0 33 | 100%|██████████| 10/10 [00:01<00:00, 6.76it/s] 34 | 35 | Test set: Average loss: 0.0004, Accuracy: 9486/10000 (94.86%) 36 | ``` 37 | 38 | # Sample parameter sets 39 | 40 | **Baseline: no differential privacy** 41 | 42 | Command: `--disable-dp --n=20 --lr=.1 -sr=0.004` 43 | 44 | Result: accuracy averaged over 10 runs 98.94% ± 0.32% 45 | 46 | **(6.86, 10-5)-DP** 47 | 48 | Command: `-n=45 --lr=.25 --sigma=.7 -c=1.5 -sr=0.004` 49 | 50 | Result: accuracy averaged over 10 runs 97.09% ± 0.17% 51 | 52 | **(2.91, 10-5)-DP** 53 | 54 | Command: `-n 60 --lr=.15 --sigma=1.1 -c=1.0 -sr=0.004` 55 | 56 | Result: accuracy averaged over 10 runs 96.78% ± 0.21% 57 | 58 | **(1.16, 10-5)-DP** 59 | 60 | Command: `-n=15 --lr=.25 --sigma=1.3 -c=1.5 -sr=0.004` 61 | 62 | Result: accuracy averaged over 10 runs 94.63% ± 0.34% 63 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/dp_multihead_attention_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | import hypothesis.strategies as st 5 | import torch 6 | import torch.nn as nn 7 | from hypothesis import given, settings 8 | from opacus.layers import DPMultiheadAttention 9 | 10 | from .common import GradSampleHooks_test 11 | 12 | 13 | class DPMultiheadAttentionAdapter(nn.Module): 14 | """ 15 | Adapter for DPMultiHeadAttention. 16 | This module takes three inputs, but our testing tools need that the model is given a single 17 | tensor, and returns a single tensor in output. 18 | 19 | To adapt for this, we stack the three input tensors required (q, k, v) over the LAST dimension, 20 | because our testing tools need to handle the `batch_first` argument which will manipulate x 21 | over the first (and potentially second) dimension. 22 | """ 23 | 24 | def __init__(self, *args, **kwargs): 25 | super().__init__() 26 | self.attn = DPMultiheadAttention(*args, **kwargs) 27 | 28 | def forward(self, x): 29 | q, k, v = x.unbind(-1) 30 | out, _attn_weights = self.attn(q, k, v) 31 | return out 32 | 33 | 34 | class MultiHeadAttention_test(GradSampleHooks_test): 35 | @given( 36 | N=st.integers(1, 4), 37 | T=st.integers(16, 20), 38 | D=st.sampled_from([4]), 39 | P=st.sampled_from([1, 2]), 40 | bias=st.booleans(), 41 | add_bias_kv=st.booleans(), 42 | add_zero_attn=st.booleans(), 43 | kv_dim=st.booleans(), 44 | ) 45 | @settings(deadline=10000) 46 | def test_multihead_attention( 47 | self, 48 | N: int, 49 | T: int, 50 | D: int, 51 | P: int, 52 | bias: bool, 53 | add_bias_kv: bool, 54 | add_zero_attn: bool, 55 | kv_dim: bool, 56 | ): 57 | 58 | if kv_dim: 59 | kdim, vdim = D, D 60 | else: 61 | kdim, vdim = None, None 62 | attn = DPMultiheadAttentionAdapter( 63 | D, 64 | P, 65 | bias=bias, 66 | add_bias_kv=add_bias_kv, 67 | add_zero_attn=add_zero_attn, 68 | dropout=0.0, 69 | kdim=kdim, 70 | vdim=vdim, 71 | ) 72 | q = torch.randn([T, N, D]) 73 | k = torch.randn([T, N, D]) 74 | v = torch.randn([T, N, D]) 75 | x = torch.stack((q, k, v), dim=-1) 76 | 77 | self.run_test(x, attn, batch_first=False) 78 | -------------------------------------------------------------------------------- /opacus/README.md: -------------------------------------------------------------------------------- 1 | # Developer Notes 2 | These developer notes can help you ramp up on this codebase. For any question, hit us up on the [forums](https://discuss.pytorch.org/c/opacus/29)! 3 | 4 | 5 | ## Supported modules 6 | Opacus only works with supported ``nn.Module``s. The following modules are supported: 7 | 8 | 1. Modules with no trainable parameters (eg ``nn.ReLU``, `nn.Tanh`) 9 | 2. Modules which are frozen. A nn.Module can be frozen in PyTorch by unsetting ``requires_grad`` 10 | in each of its parameters, ie `for p in module.parameters(): p.requires_grad = False`. 11 | 3. Explicitly supported modules (we keep a dictionary in opacus.SUPPORTED_LAYERS), eg ``nn.Conv2d``. 12 | 4. Any complex nn.Module that contains only supported nn.Modules. This means that most models 13 | will be compatible, given that we support most of the common building blocks. This however also 14 | means that Opacus support depends on how a specific ``nn.Module`` is implemented. For example, 15 | ``nn.LSTM`` *could* be written by using ``nn.Linear`` (which we support), but its actual 16 | implementation does not use it (so that it can fuse operators and be faster). Any layer that 17 | needs a rewrite to be supported is in the `/layers` folder. 18 | 19 | As an example, the following ``nn.Module`` is supported, because it's made entirely of supported 20 | nn.Modules: 21 | 22 | ```python 23 | class SampleConvNet(nn.Module): 24 | def __init__(self): 25 | super().__init__() 26 | self.conv1 = nn.Conv2d(1, 16, 8, 2, padding=3) 27 | self.conv2 = nn.Conv2d(16, 32, 4, 2) 28 | self.fc1 = nn.Linear(32 * 4 * 4, 32) 29 | self.fc2 = nn.Linear(32, 10) 30 | 31 | def forward(self, x): 32 | # x of shape [B, 1, 28, 28] 33 | x = F.relu(self.conv1(x)) # -> [B, 16, 14, 14] 34 | x = F.max_pool2d(x, 2, 1) # -> [B, 16, 13, 13] 35 | x = F.relu(self.conv2(x)) # -> [B, 32, 5, 5] 36 | x = F.max_pool2d(x, 2, 1) # -> [B, 32, 4, 4] 37 | x = x.view(-1, 32 * 4 * 4) # -> [B, 512] 38 | x = F.relu(self.fc1(x)) # -> [B, 32] 39 | x = self.fc2(x) # -> [B, 10] 40 | return x 41 | ``` 42 | 43 | ## Limitations of backward hooks 44 | The implementation of gradient clipping in autograd_grad_sample.py uses backward hooks to capture per-sample gradients. 45 | The `register_backward hook` function has a known issue being tracked at https://github.com/pytorch/pytorch/issues/598. However, this is the only known way of implementing this as of now (your suggestions and contributions are very welcome). The behavior has been verified to be correct for the layers currently supported by opacus. 46 | -------------------------------------------------------------------------------- /website/core/Tutorial.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2019-present, Facebook, Inc. 3 | * 4 | * This source code is licensed under the BSD license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | * 7 | * @format 8 | */ 9 | 10 | const React = require('react'); 11 | 12 | const fs = require('fs-extra'); 13 | const path = require('path'); 14 | const CWD = process.cwd(); 15 | 16 | const CompLibrary = require(`${CWD}/node_modules/docusaurus/lib/core/CompLibrary.js`); 17 | const Container = CompLibrary.Container; 18 | 19 | const TutorialSidebar = require(`${CWD}/core/TutorialSidebar.js`); 20 | 21 | function renderDownloadIcon() { 22 | return ( 23 | 37 | ); 38 | } 39 | 40 | class Tutorial extends React.Component { 41 | render() { 42 | const {baseUrl, tutorialID} = this.props; 43 | 44 | const htmlFile = `${CWD}/tutorials/${tutorialID}.html`; 45 | const normalizedHtmlFile = path.normalize(htmlFile); 46 | 47 | return ( 48 |
49 | 50 | 51 | 69 | ); 70 | } 71 | } 72 | 73 | module.exports = Tutorial; 74 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | import os 5 | import sys 6 | 7 | from setuptools import find_packages, setup 8 | 9 | 10 | # 3.6.8 is the final Windows binary release for 3.6.x 11 | REQUIRED_MAJOR = 3 12 | REQUIRED_MINOR = 6 13 | REQUIRED_MICRO = 8 14 | 15 | 16 | version = {} 17 | with open("opacus/version.py") as fp: 18 | exec(fp.read(), version) 19 | 20 | __version__ = version["__version__"] 21 | 22 | # Check for python version 23 | if sys.version_info < (REQUIRED_MAJOR, REQUIRED_MINOR, REQUIRED_MICRO): 24 | error = ( 25 | "Your version of python ({major}.{minor}.{micro}) is too old. You need " 26 | "python >= {required_major}.{required_minor}.{required_micro}" 27 | ).format( 28 | major=sys.version_info.major, 29 | minor=sys.version_info.minor, 30 | micro=sys.version_info.micro, 31 | required_major=REQUIRED_MAJOR, 32 | required_minor=REQUIRED_MINOR, 33 | required_micro=REQUIRED_MICRO, 34 | ) 35 | sys.exit(error) 36 | 37 | 38 | src_dir = os.path.abspath(os.path.dirname(__file__)) 39 | 40 | with open("README.md", "r", encoding="utf8") as fh: 41 | long_description = fh.read() 42 | 43 | requirements_txt = os.path.join(src_dir, "requirements.txt") 44 | with open("requirements.txt", encoding="utf8") as f: 45 | required = f.read().splitlines() 46 | 47 | with open("dev_requirements.txt", encoding="utf8") as f: 48 | dev_required = f.read().splitlines() 49 | 50 | setup( 51 | name="opacus", 52 | version=__version__, 53 | author="The Opacus Team", 54 | description="Train PyTorch models with Differential Privacy", 55 | long_description=long_description, 56 | long_description_content_type="text/markdown", 57 | url="https://opacus.ai", 58 | project_urls={ 59 | "Documentation": "https://opacus.ai/api", 60 | "Source": "https://github.com/pytorch/opacus", 61 | }, 62 | license="Apache-2.0", 63 | install_requires=required, 64 | extras_require={"dev": dev_required}, 65 | packages=find_packages(), 66 | keywords=[ 67 | "PyTorch", 68 | "Differential Privacy", 69 | "DP-SGD", 70 | "DP SGD", 71 | "Privacy Preserving Machine Learning", 72 | "PPML", 73 | "PPAI", 74 | ], 75 | classifiers=[ 76 | "Development Status :: 4 - Beta", 77 | "Intended Audience :: Developers", 78 | "Intended Audience :: Education", 79 | "Intended Audience :: Science/Research", 80 | "License :: OSI Approved :: Apache Software License", 81 | "Programming Language :: Python :: 3 :: Only", 82 | "Topic :: Scientific/Engineering", 83 | ], 84 | python_requires=f">={REQUIRED_MAJOR}.{REQUIRED_MINOR}.{REQUIRED_MICRO}", 85 | ) 86 | -------------------------------------------------------------------------------- /opacus/grad_sample/conv.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | from typing import Union 5 | 6 | import numpy as np 7 | import torch 8 | import torch.nn as nn 9 | from opacus.utils.tensor_utils import unfold3d 10 | 11 | from .utils import create_or_extend_grad_sample, register_grad_sampler 12 | 13 | 14 | @register_grad_sampler([nn.Conv1d, nn.Conv2d, nn.Conv3d]) 15 | def compute_conv_grad_sample( 16 | layer: Union[nn.Conv2d, nn.Conv1d], 17 | A: torch.Tensor, 18 | B: torch.Tensor, 19 | batch_dim: int = 0, 20 | ) -> None: 21 | """ 22 | Computes per sample gradients for convolutional layers 23 | 24 | Args: 25 | layer: Layer 26 | A: Activations 27 | B: Backpropagations 28 | batch_dim: Batch dimension position 29 | """ 30 | n = A.shape[0] 31 | # get A and B in shape depending on the Conv layer 32 | if type(layer) == nn.Conv2d: 33 | A = torch.nn.functional.unfold( 34 | A, 35 | layer.kernel_size, 36 | padding=layer.padding, 37 | stride=layer.stride, 38 | dilation=layer.dilation, 39 | ) 40 | B = B.reshape(n, -1, A.shape[-1]) 41 | elif type(layer) == nn.Conv1d: 42 | # unfold doesn't work for 3D tensors; so force it to be 4D 43 | A = A.unsqueeze(-2) # add the H dimension 44 | # set arguments to tuples with appropriate second element 45 | A = torch.nn.functional.unfold( 46 | A, 47 | (1, layer.kernel_size[0]), 48 | padding=(0, layer.padding[0]), 49 | stride=(1, layer.stride[0]), 50 | dilation=(1, layer.dilation[0]), 51 | ) 52 | B = B.reshape(n, -1, A.shape[-1]) 53 | elif type(layer) == nn.Conv3d: 54 | A = unfold3d( 55 | A, 56 | kernel_size=layer.kernel_size, 57 | padding=layer.padding, 58 | stride=layer.stride, 59 | dilation=layer.dilation, 60 | ) 61 | B = B.reshape(n, -1, A.shape[-1]) 62 | 63 | # n=batch_sz; o=num_out_channels; p=(num_in_channels/groups)*kernel_sz 64 | grad_sample = torch.einsum("noq,npq->nop", B, A) 65 | # rearrange the above tensor and extract diagonals. 66 | grad_sample = grad_sample.view( 67 | n, 68 | layer.groups, 69 | -1, 70 | layer.groups, 71 | int(layer.in_channels / layer.groups), 72 | np.prod(layer.kernel_size), 73 | ) 74 | grad_sample = torch.einsum("ngrg...->ngr...", grad_sample).contiguous() 75 | shape = [n] + list(layer.weight.shape) 76 | 77 | create_or_extend_grad_sample(layer.weight, grad_sample.view(shape), batch_dim) 78 | 79 | if layer.bias is not None: 80 | create_or_extend_grad_sample(layer.bias, torch.sum(B, dim=2), batch_dim) 81 | -------------------------------------------------------------------------------- /website/scripts/parse_sphinx.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | import argparse 5 | import os 6 | 7 | from bs4 import BeautifulSoup 8 | 9 | 10 | js_scripts = """ 11 | 13 | 14 | 15 | 16 | 17 | 18 | """ # noqa: E501 19 | 20 | search_js_scripts = """ 21 | 24 | 25 | 26 | """ 27 | 28 | 29 | def parse_sphinx(input_dir, output_dir): 30 | for cur, _, files in os.walk(input_dir): 31 | for fname in files: 32 | if fname.endswith(".html"): 33 | with open(os.path.join(cur, fname), "r") as f: 34 | soup = BeautifulSoup(f.read(), "html.parser") 35 | doc = soup.find("div", {"class": "document"}) 36 | wrapped_doc = doc.wrap(soup.new_tag("div", **{"class": "sphinx"})) 37 | # add js 38 | if fname == "search.html": 39 | out = js_scripts + search_js_scripts + str(wrapped_doc) 40 | else: 41 | out = js_scripts + str(wrapped_doc) 42 | output_path = os.path.join(output_dir, os.path.relpath(cur, input_dir)) 43 | os.makedirs(output_path, exist_ok=True) 44 | with open(os.path.join(output_path, fname), "w") as fout: 45 | fout.write(out) 46 | 47 | # update reference in JS file 48 | with open(os.path.join(input_dir, "_static/searchtools.js"), "r") as js_file: 49 | js = js_file.read() 50 | js = js.replace( 51 | "DOCUMENTATION_OPTIONS.URL_ROOT + '_sources/'", "'_sphinx-sources/'" 52 | ) 53 | with open(os.path.join(input_dir, "_static/searchtools.js"), "w") as js_file: 54 | js_file.write(js) 55 | 56 | 57 | if __name__ == "__main__": 58 | parser = argparse.ArgumentParser(description="Strip HTML body from Sphinx docs.") 59 | parser.add_argument( 60 | "-i", 61 | "--input_dir", 62 | metavar="path", 63 | required=True, 64 | help="Input directory for Sphinx HTML.", 65 | ) 66 | parser.add_argument( 67 | "-o", 68 | "--output_dir", 69 | metavar="path", 70 | required=True, 71 | help="Output directory in Docusaurus.", 72 | ) 73 | args = parser.parse_args() 74 | parse_sphinx(args.input_dir, args.output_dir) 75 | -------------------------------------------------------------------------------- /website/scripts/build_website.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | # run this script from the project root using `./scripts/build_website.sh` 5 | 6 | usage() { 7 | echo "Usage: $0 [-b]" 8 | echo "" 9 | echo "Build Opacus documentation." 10 | echo "" 11 | echo " -b Build static version of documentation (otherwise start server)" 12 | echo "" 13 | exit 1 14 | } 15 | 16 | BUILD_STATIC=false 17 | 18 | while getopts 'hb' flag; do 19 | case "${flag}" in 20 | h) 21 | usage 22 | ;; 23 | b) 24 | BUILD_STATIC=true 25 | ;; 26 | *) 27 | usage 28 | ;; 29 | esac 30 | done 31 | 32 | echo "-----------------------------------" 33 | echo "Generating API reference via Sphinx" 34 | echo "-----------------------------------" 35 | cd sphinx || exit 36 | make html 37 | cd .. || exit 38 | 39 | echo "-----------------------------------" 40 | echo "Building Opacus Docusaurus site" 41 | echo "-----------------------------------" 42 | yarn || exit 43 | 44 | # run script to parse html generated by sphinx 45 | echo "--------------------------------------------" 46 | echo "Parsing Sphinx docs and moving to Docusaurus" 47 | echo "--------------------------------------------" 48 | cd .. 49 | mkdir -p "website/pages/api/" 50 | 51 | cwd=$(pwd) 52 | python website/scripts/parse_sphinx.py -i "${cwd}/website/sphinx/build/html/" -o "${cwd}/website/pages/api/" || exit 1 53 | 54 | SPHINX_JS_DIR='website/sphinx/build/html/_static/' 55 | DOCUSAURUS_JS_DIR='website/static/js/' 56 | 57 | mkdir -p $DOCUSAURUS_JS_DIR 58 | 59 | # move JS files from /sphinx/build/html/_static/*: 60 | cp "${SPHINX_JS_DIR}documentation_options.js" "${DOCUSAURUS_JS_DIR}documentation_options.js" 61 | cp "${SPHINX_JS_DIR}jquery.js" "${DOCUSAURUS_JS_DIR}jquery.js" 62 | cp "${SPHINX_JS_DIR}underscore.js" "${DOCUSAURUS_JS_DIR}underscore.js" 63 | cp "${SPHINX_JS_DIR}doctools.js" "${DOCUSAURUS_JS_DIR}doctools.js" 64 | cp "${SPHINX_JS_DIR}language_data.js" "${DOCUSAURUS_JS_DIR}language_data.js" 65 | cp "${SPHINX_JS_DIR}searchtools.js" "${DOCUSAURUS_JS_DIR}searchtools.js" 66 | 67 | # searchindex.js is not static util 68 | cp "website/sphinx/build/html/searchindex.js" "${DOCUSAURUS_JS_DIR}searchindex.js" 69 | 70 | # copy module sources 71 | cp -r "website/sphinx/build/html/_sources/" "website/static/_sphinx-sources/" 72 | 73 | echo "-----------------------------------" 74 | echo "Generating tutorials" 75 | echo "-----------------------------------" 76 | mkdir -p "website/static/files" 77 | mkdir "website/tutorials" 78 | python website/scripts/parse_tutorials.py -w "${cwd}" || exit 1 79 | 80 | cd website || exit 81 | 82 | if [[ $BUILD_STATIC == true ]]; then 83 | echo "-----------------------------------" 84 | echo "Building static site" 85 | echo "-----------------------------------" 86 | yarn build 87 | else 88 | echo "-----------------------------------" 89 | echo "Starting local server" 90 | echo "-----------------------------------" 91 | yarn start 92 | fi 93 | -------------------------------------------------------------------------------- /examples/scripts/make_small_imagenet_sampled.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # =================== 4 | # This script makes a sampled version of ImageNet, with all 1000 classes. 5 | # =================== 6 | 7 | usage() 8 | { 9 | echo "usage Example: bash make_small_imagenet_N_classes.sh -o Documents/imagenet_full_size -d Documents/imagenet_small_100 -nt 100 -nv 10" 10 | echo "usage: bash make_small_imagenet_N_classes.sh -o -d -nt -nv " 11 | } 12 | 13 | if [ "$1" == "" ]; then # If arguments are not specified 14 | usage 15 | exit 1 16 | fi 17 | while [ "$1" != "" ]; do # Read the input arguments 18 | case $1 in 19 | -d | --destination ) shift 20 | destination=$1 21 | ;; 22 | -o | --origin ) shift 23 | origin=$1 24 | ;; 25 | -nt ) shift 26 | N_train=$1 27 | ;; 28 | -nv ) shift 29 | N_val=$1 30 | ;; 31 | -h | --help ) usage 32 | exit 33 | ;; 34 | * ) echo "ERROR: unknown parameter \"$1\"" 35 | usage 36 | exit 1 37 | esac 38 | shift 39 | done 40 | 41 | # If all necessary arguments are not supplied 42 | if [[ -z $destination || -z $origin || -z $N_train || -z $N_val ]] 43 | then 44 | echo "You must specify all necessary parameters." 45 | usage 46 | exit 1 47 | fi 48 | 49 | # Get absolute paths 50 | destination="$(readlink -f "$destination")" 51 | origin="$(readlink -f "$origin")" 52 | 53 | mkdir "$destination" 54 | mkdir "$destination/train" 55 | mkdir "$destination/val" 56 | 57 | echo 'Copying' 58 | for val_train_folder in val train; do # Do copying for both 'train' and 'val' folders 59 | if [[ $val_train_folder = 'train' ]]; then 60 | N=$N_train 61 | else 62 | N=$N_val 63 | fi 64 | cd "$origin/$val_train_folder" || { echo "Failure"; exit 1; } # change directory to origin's train or val folders 65 | for d in */ ; do # loop through the 1000 folders 66 | mkdir "$destination/$val_train_folder/$d" 67 | cd "$origin/$val_train_folder/$d" || { echo "Failure"; exit 1; } 68 | find . -maxdepth 1 -mindepth 1 -type f |sort -R |tail -"$N" |while read -r file; do # select N files from each 1000 folders 69 | cp "$file" "$destination/$val_train_folder/$d/$file" 70 | printf "." 71 | done 72 | done 73 | echo "Copying folder $val_train_folder is done." 74 | done 75 | -------------------------------------------------------------------------------- /.circleci/flake8_config.ini: -------------------------------------------------------------------------------- 1 | [flake8] 2 | select = B,C,E,F,P,W,B9 3 | max-line-length = 80 4 | # Main Explanation Docs: https://github.com/grantmcconnaughey/Flake8Rules 5 | ignore = 6 | # Black conflicts and overlaps. 7 | # Found in https://github.com/psf/black/issues/429 8 | B950, # Line too long. (Use `arc lint`'s LINEWRAP instead) 9 | E111, # Indentation is not a multiple of four. 10 | E115, # Expected an indented block (comment). 11 | E117, # Over-indented. 12 | E121, # Continuation line under-indented for hanging indent. 13 | E122, # Continuation line missing indentation or outdented. 14 | E123, # Closing bracket does not match indentation of opening bracket's line. 15 | E124, # Closing bracket does not match visual indentation. 16 | E125, # Continuation line with same indent as next logical line. 17 | E126, # Continuation line over-indented for hanging indent. 18 | E127, # Continuation line over-indented for visual indent. 19 | E128, # Continuation line under-indented for visual indent. 20 | E129, # Visually indented line with same indent as next logical line. 21 | E201, # Whitespace after '('. 22 | E202, # Whitespace before ')'. 23 | E203, # Whitespace before ':'. 24 | E221, # Multiple spaces before operator. 25 | E222, # Multiple spaces after operator. 26 | E225, # Missing whitespace around operator. 27 | E226, # Missing whitespace around arithmetic operator. 28 | E227, # Missing whitespace around bitwise or shift operator. 29 | E231, # Missing whitespace after ',', ';', or ':'. 30 | E241, # Multiple spaces after ','. 31 | E251, # Unexpected spaces around keyword / parameter equals. 32 | E261, # At least two spaces before inline comment. 33 | E262, # Inline comment should start with '# '. 34 | E265, # Block comment should start with '# '. 35 | E271, # Multiple spaces after keyword. 36 | E272, # Multiple spaces before keyword. 37 | E301, # Expected 1 blank line, found 0. 38 | E302, # Expected 2 blank lines, found 0. 39 | E303, # Too many blank lines (3). 40 | E305, # Expected 2 blank lines after end of function or class. 41 | E306, # Expected 1 blank line before a nested definition. 42 | E501, # Line too long (82 > 79 characters). 43 | E502, # The backslash is redundant between brackets. 44 | E701, # Multiple statements on one line (colon). 45 | E702, # Multiple statements on one line (semicolon). 46 | E703, # Statement ends with a semicolon. 47 | E704, # Multiple statements on one line (def). 48 | W291, # Trailing whitespace. 49 | W292, # No newline at end of file. 50 | W293, # Blank line contains whitespace. 51 | W391, # Blank line at end of file. 52 | 53 | # Too opinionated. 54 | E265, # Block comment should start with '# '. 55 | E266, # Too many leading '#' for block comment. 56 | E402, # Module level import not at top of file. 57 | E722, # Do not use bare except, specify exception instead. (Duplicate of B001) 58 | F811, # Redefinition of unused name from line n. 59 | P207, # (Duplicate of B003) 60 | P208, # (Duplicate of C403) 61 | W503 # Line break occurred before a binary operator. 62 | exclude = 63 | .hg, 64 | __pycache__, 65 | 66 | max-complexity = 12 67 | -------------------------------------------------------------------------------- /opacus/grad_sample/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | from typing import Sequence, Type, Union 5 | 6 | import torch 7 | import torch.nn as nn 8 | 9 | from .grad_sample_module import GradSampleModule 10 | 11 | 12 | def register_grad_sampler( 13 | target_class_or_classes: Union[Type[nn.Module], Sequence[Type[nn.Module]]] 14 | ): 15 | """ 16 | Registers the decorated function as the ``grad_sampler`` of ``target_class_or_classes``, which is 17 | the function that will be invoked every time you want to compute a per-sample gradient 18 | of ``target_class_or_classes``. The signature of every grad_sampler is always the same: 19 | 20 | >>> @register_grad_sampler(nn.MyCustomClass) 21 | >>> def compute_grad_sample(module, activations, backprops): 22 | >>> pass 23 | 24 | It may help you to take a look at the existing grad_samplers inside Opacus, under ``opacus.grad_sample.`` 25 | """ 26 | 27 | def decorator(f): 28 | target_classes = ( 29 | target_class_or_classes 30 | if isinstance(target_class_or_classes, Sequence) 31 | else [target_class_or_classes] 32 | ) 33 | for target_class in target_classes: 34 | GradSampleModule.GRAD_SAMPLERS[target_class] = f 35 | return f 36 | 37 | return decorator 38 | 39 | 40 | def create_or_extend_grad_sample( 41 | param: torch.Tensor, grad_sample: torch.Tensor, batch_dim: int 42 | ) -> None: 43 | """ 44 | Creates a ``grad_sample`` attribute in the given parameter, or appends to it 45 | if the ``grad_sample`` attribute already exists. 46 | 47 | Args: 48 | param: Parameter to which ``grad_sample`` will be added 49 | grad_sample: Per-sample gradients tensor. Must be of the same 50 | shape as ``param`` with extra batch dimension 51 | batch_dim: Position of the batch dimension in the shape of 52 | ``grad_sample`` 53 | """ 54 | 55 | if hasattr(param, "grad_sample"): 56 | param.grad_sample = torch.cat((param.grad_sample, grad_sample), batch_dim) 57 | else: 58 | param.grad_sample = grad_sample 59 | 60 | 61 | def create_or_accumulate_grad_sample( 62 | param: torch.Tensor, grad_sample: torch.Tensor, layer: nn.Module 63 | ) -> None: 64 | """ 65 | Creates a ``grad_sample`` attribute in the given parameter, or adds to it 66 | if the ``grad_sample`` attribute already exists. 67 | 68 | Args: 69 | param: Parameter to which ``grad_sample`` will be added 70 | grad_sample: Per-sample gradients tensor. Must be of the same 71 | shape as ``param`` with extra batch dimension 72 | """ 73 | 74 | if hasattr(param, "grad_sample"): 75 | param.grad_sample[: grad_sample.shape[0]] += grad_sample 76 | else: 77 | max_batch_len = layer.max_batch_len 78 | param.grad_sample = torch.zeros( 79 | torch.Size([max_batch_len]) + grad_sample.shape[1:], 80 | device=grad_sample.device, 81 | dtype=grad_sample.dtype, 82 | ) 83 | param.grad_sample[: grad_sample.shape[0]] = grad_sample 84 | -------------------------------------------------------------------------------- /website/siteConfig.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2017-present, Facebook, Inc. 3 | * 4 | * This source code is licensed under the BSD license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | * 7 | * @format 8 | */ 9 | 10 | // See https://docusaurus.io/docs/site-config for all the possible 11 | // site configuration options. 12 | 13 | // Define this so it can be easily modified in scripts (to host elsewhere) 14 | const baseUrl = '/'; 15 | 16 | // List of projects/orgs using your project for the users page. 17 | const users = []; 18 | 19 | const siteConfig = { 20 | title: 'Opacus', 21 | tagline: 'Train PyTorch models with Differential Privacy', 22 | url: 'https://opacus.ai', 23 | baseUrl: baseUrl, 24 | cleanUrl: true, // No .html extensions for paths 25 | cname: 'opacus.ai', 26 | 27 | // used for publishing and more 28 | organizationName: 'pytorch', 29 | projectName: 'opacus', 30 | 31 | // Google analytics 32 | gaTrackingId: 'UA-117752657-3', 33 | 34 | // links that will be used in the header navigation bar 35 | headerLinks: [ 36 | {doc: 'introduction', label: 'Introduction'}, 37 | {doc: 'faq', label: 'FAQ'}, 38 | {href: `${baseUrl}tutorials/`, label: 'Tutorials'}, 39 | {href: `${baseUrl}api/`, label: 'API Reference'}, 40 | {href: 'https://github.com/pytorch/opacus', label: 'GitHub'}, 41 | {search: true}, // position search box to the very right 42 | ], 43 | 44 | // add users to the website 45 | users, 46 | 47 | // search integration w/ algolia 48 | algolia: { 49 | apiKey: '207c27d819f967749142d8611de7cb19', 50 | indexName: 'opacus', 51 | }, 52 | 53 | // images for header/footer and favicon 54 | headerIcon: 'img/opacus_logo.svg', 55 | footerIcon: 'img/opacus_favicon.svg', 56 | favicon: 'img/opacus_favicon.svg', 57 | logo: 'img/opacus_logo_vertical.svg', 58 | 59 | // colors for website 60 | colors: { 61 | primaryColor: '#4283f4', 62 | secondaryColor: '#2af2bf', // green 63 | }, 64 | 65 | highlight: { 66 | theme: 'default', 67 | }, 68 | 69 | // custom scripts that are placed in of each page 70 | scripts: [ 71 | // Github buttons 72 | 'https://buttons.github.io/buttons.js', 73 | // Copy-to-clipboard button for code blocks 74 | `${baseUrl}js/code_block_buttons.js`, 75 | 'https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.0/clipboard.min.js', 76 | // Mathjax for rendering math content 77 | `${baseUrl}js/mathjax.js`, 78 | 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS_HTML', 79 | ], 80 | 81 | // CSS sources to load 82 | stylesheets: [`${baseUrl}css/code_block_buttons.css`], 83 | 84 | // enable on-page navigation for the current documentation page 85 | onPageNav: 'separate', 86 | 87 | // enable scroll to top button a the bottom of the site 88 | scrollToTop: true, 89 | 90 | // if true, expand/collapse links & subcategories in sidebar 91 | docsSideNavCollapsible: true, 92 | 93 | // URL for editing docs 94 | editUrl: 'https://github.com/pytorch/opacus/tree/main/docs/', 95 | 96 | // Disable logo text so we can just show the logo 97 | disableHeaderTitle: true, 98 | 99 | // Open Graph and Twitter card images 100 | ogImage: 'img/opacus_logo.svg', 101 | twitterImage: 'img/opacus_logo.svg', 102 | 103 | // show html docs generated by sphinx 104 | wrapPagesHTML: true, 105 | }; 106 | 107 | module.exports = siteConfig; 108 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to make participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies within all project spaces, and it also applies when 49 | an individual is representing the project or its community in public spaces. 50 | Examples of representing a project or community include using an official 51 | project e-mail address, posting via an official social media account, or acting 52 | as an appointed representative at an online or offline event. Representation of 53 | a project may be further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at . All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v0.15.0 4 | ### New Features 5 | * DDP support for faster distributed training (#196) 6 | * Support of GRU and RNN. Refactored LSTM implementation. (#222) 7 | * PyTorch Lightning Demo (#244) 8 | ### Bug fixes 9 | * Improve nn.Linear grad sampler memory consumption (#192) 10 | * Update Opacus to stop using deprecated torch.set_deterministic (#197) 11 | * Fix optimizer.step after engine.detach() 12 | * Test fixes. 13 | ### Miscellaneous 14 | * Better validation error reporting (#199) 15 | * grad sampler type checking (#241) 16 | 17 | ## v0.14.0 18 | ### New features 19 | * Major refactoring - per-sample gradient computation is separated into its own module - GradSampleModule (#175) 20 | * Improved RDP to (eps, delta)-DP conversion (#162) 21 | * Multi-GPU support (#166) 22 | ### Bug fixes 23 | * Handle empty batches in Poisson sampling (#164) 24 | * Fixed memory leak from no_grad execution (#180) 25 | 26 | ## v0.13.0 27 | ### New features 28 | * PackedSequence support for DPLSTM (#150) (thanks @touqir14 !) 29 | ### Miscellaneous 30 | * Pytest moved to dev installation (#144) 31 | 32 | ## v0.12.0 33 | This version introduces a **mildly-breaking change**: the privacy engine will now support sampling with variable batch size, just like in the Abadi et al. paper. To accommodate this feature, we have made `batch_size` a kwarg (no longer positional). We are also enforcing that all kwargs must not be specified positionally. If you had code that passed kwargs positionally, you will find an error (which will be very simple to fix). 34 | ### New features 35 | * Enforce kwargs to Privacy Engine (#136). 36 | * Fix batch construction and privacy engine (#128). (thanks @ConstanceBeguier!) 37 | * Compute required sigma to reach (epsilon, delta) budget (#126) 38 | * Friendly user message for unused parameters (#118). 39 | * Print helpful message when models are not in train mode (#113) 40 | ### Bug fixes 41 | * Now the Opacus package has a `__version__` attribute. 42 | * Fix immer security issue, fix website errors 43 | * Updated setup.py version requirements to support 3.6.8 for Windows (#108) (thanks @madhavajay!) 44 | ### Miscellaneous 45 | * Rewrote the grad_sample tests to use Hypothesis (#125). (thanks @touqir14!) 46 | 47 | ## v0.11.0 48 | ### New features 49 | * Extend DPLSTM to support multilayer, dropout (#101) 50 | * Modifications to Char LSTM name classification example 51 | * Introduce issue templates for GitHub (#102) 52 | * Added support for Conv3D layers 53 | ### Bug fixes 54 | * Linter fixes for Conv3D (#105) 55 | ### Miscellaneous 56 | * Make TorchCSPRNG an optional dependency (#106) 57 | * Removed unnecessary calls to zero_grad from examples and tutorials (#96) 58 | 59 | ## v0.10.1 60 | ### Bug fixes 61 | * Fix PyPI deployment (#91). 62 | ### Miscellaneous 63 | * Refactor grad sample tests (#90). 64 | * Avoid storing activations in certain scenarios (#87) 65 | 66 | ## v0.10.0 67 | ### New features 68 | * Reimplemented the Embedding layer, making it 9x faster with lower memory footprint (#73). 69 | * Reimplemented the DPLSTM layer, making it 2x faster with lower memory footprint. 70 | * Extended our Conv support to grouped convolutions (#78). 71 | ### Bug fixes 72 | * Small fixes to clipping logic (#45). 73 | ### Miscellaneous 74 | * Changed docstring style from numpy -> Google. 75 | * Throw an error if sample rate > 1 in privacy engine. 76 | * Migrated our IMDB example from TorchText -> HuggingFace (#85). 77 | * Added PRNG shuffling to our examples. 78 | 79 | ## v0.9.1 80 | ### Bug fixes 81 | * Compatibility with Python 3.6 (Minimum required version changed from 3.7 to 3.6.9). 82 | * Allow DP-LSTM to have null init. 83 | 84 | ## v0.9.0 85 | ### New Features 86 | * Initial commit. 87 | -------------------------------------------------------------------------------- /opacus/tests/dp_layers/dp_multihead_attention_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | from typing import Optional 5 | 6 | import hypothesis.strategies as st 7 | import pytest 8 | import torch 9 | import torch.nn as nn 10 | from hypothesis import given, settings 11 | from opacus.layers import DPMultiheadAttention 12 | 13 | from .common import DPModules_test 14 | 15 | 16 | def attn_train_fn( 17 | model: nn.Module, 18 | *args, 19 | **kwargs, 20 | ): 21 | model.train() 22 | criterion = nn.MSELoss() 23 | logits, attn_weights = model(*args, **kwargs) 24 | y = torch.zeros_like(logits) 25 | loss = criterion(logits, y) 26 | loss.backward() 27 | 28 | 29 | class DPMultiheadAttention_test(DPModules_test): 30 | @given( 31 | batch_size=st.integers(1, 5), 32 | src_seq_len=st.integers(1, 6), 33 | tgt_seq_len=st.integers(1, 6), 34 | num_heads=st.integers(1, 3), 35 | bias=st.booleans(), 36 | add_bias_kv=st.booleans(), 37 | add_zero_attn=st.booleans(), 38 | kdim=st.integers(2, 8) | st.none(), 39 | vdim=st.integers(2, 8) | st.none(), 40 | ) 41 | @settings(deadline=10000) 42 | @pytest.mark.skip( 43 | "Failing due to a known problem. Should be enabled after issue #123 is fixed" 44 | ) 45 | def test_attn( 46 | self, 47 | batch_size: int, 48 | src_seq_len: int, 49 | tgt_seq_len: int, 50 | num_heads: int, 51 | bias: bool, 52 | add_bias_kv: bool, 53 | add_zero_attn: bool, 54 | kdim: Optional[int], 55 | vdim: Optional[int], 56 | ): 57 | embed_dim = 4 * num_heads # embed_dim must be divisible by num_heads 58 | 59 | attn = nn.MultiheadAttention( 60 | embed_dim, 61 | num_heads, 62 | dropout=0.0, # Untestable between two different implementations 63 | bias=bias, 64 | add_bias_kv=add_bias_kv, 65 | add_zero_attn=add_zero_attn, 66 | kdim=kdim, 67 | vdim=vdim, 68 | ) 69 | dp_attn = DPMultiheadAttention( 70 | embed_dim, 71 | num_heads, 72 | dropout=0.0, # Untestable between two different implementations 73 | bias=bias, 74 | add_bias_kv=add_bias_kv, 75 | add_zero_attn=add_zero_attn, 76 | kdim=kdim, 77 | vdim=vdim, 78 | ) 79 | 80 | dp_attn.load_state_dict(attn.state_dict()) 81 | 82 | q = torch.randn(tgt_seq_len, batch_size, embed_dim) 83 | k = torch.randn( 84 | src_seq_len, batch_size, kdim if kdim is not None else embed_dim 85 | ) 86 | v = torch.randn( 87 | src_seq_len, batch_size, vdim if vdim is not None else embed_dim 88 | ) 89 | 90 | self.compare_forward_outputs( 91 | attn, 92 | dp_attn, 93 | q, 94 | k, 95 | v, 96 | output_names=("attn_out", "attn_out_weights"), 97 | atol=1e-5, 98 | rtol=1e-3, 99 | key_padding_mask=None, 100 | need_weights=True, 101 | attn_mask=None, 102 | ) 103 | 104 | self.compare_gradients( 105 | attn, 106 | dp_attn, 107 | attn_train_fn, 108 | q, 109 | k, 110 | v, 111 | atol=1e-5, 112 | rtol=1e-3, 113 | key_padding_mask=None, 114 | need_weights=True, 115 | attn_mask=None, 116 | ) 117 | -------------------------------------------------------------------------------- /website/static/img/opacus_favicon.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /website/core/Footer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2017-present, Facebook, Inc. 3 | * 4 | * This source code is licensed under the BSD license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | * 7 | * @format 8 | */ 9 | 10 | const PropTypes = require('prop-types'); 11 | const React = require('react'); 12 | 13 | function SocialFooter(props) { 14 | const repoUrl = `https://github.com/${props.config.organizationName}/${props.config.projectName}`; 15 | 16 | return ( 17 |
18 |
Social
19 | 30 |
31 | ); 32 | } 33 | 34 | SocialFooter.propTypes = { 35 | config: PropTypes.object, 36 | }; 37 | 38 | class Footer extends React.Component { 39 | docUrl(doc, language) { 40 | const baseUrl = this.props.config.baseUrl; 41 | const docsUrl = this.props.config.docsUrl; 42 | const docsPart = `${docsUrl ? `${docsUrl}/` : ''}`; 43 | const langPart = `${language ? `${language}/` : ''}`; 44 | return `${baseUrl}${docsPart}${langPart}${doc}`; 45 | } 46 | 47 | pageUrl(doc, language) { 48 | const baseUrl = this.props.config.baseUrl; 49 | return baseUrl + (language ? `${language}/` : '') + doc; 50 | } 51 | 52 | render() { 53 | const currentYear = new Date().getFullYear(); 54 | return ( 55 | 110 | ); 111 | } 112 | } 113 | 114 | module.exports = Footer; 115 | -------------------------------------------------------------------------------- /website/scripts/parse_tutorials.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | import argparse 5 | import json 6 | import os 7 | 8 | import nbformat 9 | from bs4 import BeautifulSoup 10 | from nbconvert import HTMLExporter 11 | 12 | 13 | TEMPLATE = """const CWD = process.cwd(); 14 | 15 | const React = require('react'); 16 | const Tutorial = require(`${{CWD}}/core/Tutorial.js`); 17 | 18 | class TutorialPage extends React.Component {{ 19 | render() {{ 20 | const {{config: siteConfig}} = this.props; 21 | const {{baseUrl}} = siteConfig; 22 | return ; 23 | }} 24 | }} 25 | 26 | module.exports = TutorialPage; 27 | 28 | """ 29 | 30 | JS_SCRIPTS = """ 31 | 34 | 37 | """ # noqa: E501 38 | 39 | 40 | def gen_tutorials(repo_dir: str) -> None: 41 | """Generate HTML tutorials for Docusaurus site from Jupyter notebooks. 42 | 43 | Also create ipynb and py versions of tutorial in Docusaurus site for 44 | download. 45 | """ 46 | with open(os.path.join(repo_dir, "website", "tutorials.json"), "r") as infile: 47 | tutorial_config = json.loads(infile.read()) 48 | 49 | tutorial_ids = {x["id"] for v in tutorial_config.values() for x in v} 50 | 51 | for tid in tutorial_ids: 52 | print("Generating {} tutorial".format(tid)) 53 | 54 | # convert notebook to HTML 55 | ipynb_in_path = os.path.join(repo_dir, "tutorials", "{}.ipynb".format(tid)) 56 | with open(ipynb_in_path, "r") as infile: 57 | nb_str = infile.read() 58 | nb = nbformat.reads(nb_str, nbformat.NO_CONVERT) 59 | 60 | # displayname is absent from notebook metadata 61 | nb["metadata"]["kernelspec"]["display_name"] = "python3" 62 | 63 | exporter = HTMLExporter(template_name="classic") 64 | html, meta = exporter.from_notebook_node(nb) 65 | 66 | # pull out html div for notebook 67 | soup = BeautifulSoup(html, "html.parser") 68 | nb_meat = soup.find("div", {"id": "notebook-container"}) 69 | del nb_meat.attrs["id"] 70 | nb_meat.attrs["class"] = ["notebook"] 71 | html_out = JS_SCRIPTS + str(nb_meat) 72 | 73 | # generate html file 74 | html_out_path = os.path.join( 75 | repo_dir, "website", "tutorials", "{}.html".format(tid) 76 | ) 77 | with open(html_out_path, "w") as html_outfile: 78 | html_outfile.write(html_out) 79 | 80 | # generate JS file 81 | script = TEMPLATE.format(tid) 82 | js_out_path = os.path.join( 83 | repo_dir, "website", "pages", "tutorials", "{}.js".format(tid) 84 | ) 85 | with open(js_out_path, "w") as js_outfile: 86 | js_outfile.write(script) 87 | 88 | # # output tutorial in both ipynb & py form 89 | # ipynb_out_path = os.path.join( 90 | # repo_dir, "website", "static", "files", "{}.ipynb".format(tid) 91 | # ) 92 | # with open(ipynb_out_path, "w") as ipynb_outfile: 93 | # ipynb_outfile.write(nb_str) 94 | # exporter = ScriptExporter() 95 | # script, meta = exporter.from_notebook_node(nb) 96 | # py_out_path = os.path.join( 97 | # repo_dir, "website", "static", "files", "{}.py".format(tid) 98 | # ) 99 | # with open(py_out_path, "w") as py_outfile: 100 | # py_outfile.write(script) 101 | 102 | 103 | if __name__ == "__main__": 104 | parser = argparse.ArgumentParser( 105 | description="Generate JS, HTML, ipynb, and py files for tutorials." 106 | ) 107 | parser.add_argument( 108 | "-w", "--repo_dir", metavar="path", required=True, help="repo directory." 109 | ) 110 | args = parser.parse_args() 111 | gen_tutorials(args.repo_dir) 112 | -------------------------------------------------------------------------------- /opacus/tests/dp_model_inspector_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | 5 | import unittest 6 | 7 | import torch 8 | import torch.nn as nn 9 | from opacus import dp_model_inspector as dp_inspector 10 | from opacus.utils.module_modification import convert_batchnorm_modules 11 | from torchvision import models 12 | 13 | 14 | class dp_model_inspector_test(unittest.TestCase): 15 | def test_raises_exception(self): 16 | inspector = dp_inspector.DPModelInspector() 17 | model = models.resnet50() 18 | with self.assertRaises(dp_inspector.IncompatibleModuleException): 19 | inspector.validate(model) 20 | 21 | def test_returns_False(self): 22 | inspector = dp_inspector.DPModelInspector(should_throw=False) 23 | model = models.resnet50() 24 | self.assertFalse(inspector.validate(model)) 25 | 26 | def test_raises_for_eval_mode(self): 27 | inspector = dp_inspector.DPModelInspector() 28 | model = models.resnet50() 29 | model = model.eval() 30 | with self.assertRaises(dp_inspector.IncompatibleModuleException): 31 | inspector.validate(model) 32 | 33 | def test_convert_batchnorm(self): 34 | inspector = dp_inspector.DPModelInspector() 35 | model = convert_batchnorm_modules(models.resnet50()) 36 | self.assertTrue(inspector.validate(model)) 37 | 38 | def test_running_stats(self): 39 | inspector = dp_inspector.DPModelInspector(should_throw=False) 40 | 41 | self.assertTrue(inspector.validate(nn.InstanceNorm1d(16))) 42 | self.assertTrue(inspector.validate(nn.InstanceNorm1d(16, affine=True))) 43 | self.assertTrue( 44 | inspector.validate(nn.InstanceNorm1d(16, track_running_stats=True)) 45 | ) 46 | self.assertFalse( 47 | inspector.validate( 48 | nn.InstanceNorm1d(16, affine=True, track_running_stats=True) 49 | ) 50 | ) 51 | 52 | def test_extra_param(self): 53 | inspector = dp_inspector.DPModelInspector(should_throw=False) 54 | 55 | class SampleNetWithExtraParam(nn.Module): 56 | def __init__(self): 57 | super().__init__() 58 | 59 | self.fc = nn.Linear(8, 16) 60 | self.extra_param = nn.Parameter(torch.Tensor(16, 2)) 61 | 62 | def forward(self, x): 63 | x = self.fc(x) 64 | x = x.matmul(self.extra_param) 65 | return x 66 | 67 | model = SampleNetWithExtraParam() 68 | self.assertFalse(inspector.validate(model)) 69 | 70 | model.extra_param.requires_grad = False 71 | self.assertTrue(inspector.validate(model)) 72 | 73 | def test_unsupported_layer(self): 74 | class SampleNetWithTransformer(nn.Module): 75 | def __init__(self): 76 | super().__init__() 77 | 78 | self.fc = nn.Linear(8, 16) 79 | self.encoder = nn.Transformer() 80 | 81 | def forward(self, x): 82 | x = self.fc(x) 83 | x = self.encoder(x) 84 | return x 85 | 86 | model = SampleNetWithTransformer() 87 | inspector = dp_inspector.DPModelInspector(should_throw=False) 88 | self.assertFalse(inspector.validate(model)) 89 | 90 | def test_conv2d(self): 91 | inspector = dp_inspector.DPModelInspector(should_throw=False) 92 | 93 | self.assertTrue( 94 | inspector.validate( 95 | nn.Conv2d(in_channels=3, out_channels=6, kernel_size=1, groups=1) 96 | ) 97 | ) 98 | self.assertTrue( 99 | inspector.validate( 100 | nn.Conv2d(in_channels=3, out_channels=6, kernel_size=1, groups=3) 101 | ) 102 | ) 103 | self.assertFalse( 104 | inspector.validate( 105 | nn.Conv2d(in_channels=6, out_channels=6, kernel_size=1, groups=2) 106 | ) 107 | ) 108 | -------------------------------------------------------------------------------- /website/pages/tutorials/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2019-present, Facebook, Inc. 3 | * 4 | * This source code is licensed under the BSD license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | * 7 | * @format 8 | */ 9 | 10 | const React = require('react'); 11 | 12 | const CWD = process.cwd(); 13 | 14 | const CompLibrary = require(`${CWD}/node_modules/docusaurus/lib/core/CompLibrary.js`); 15 | const Container = CompLibrary.Container; 16 | const MarkdownBlock = CompLibrary.MarkdownBlock; 17 | 18 | const TutorialSidebar = require(`${CWD}/core/TutorialSidebar.js`); 19 | 20 | class TutorialHome extends React.Component { 21 | render() { 22 | return ( 23 |
24 | 25 | 26 |
27 |
28 |

Tutorials

29 |
30 |

31 | This is the tutorials page. Navigate the sidebar to find various 32 | tutorials. 33 |

34 |

External Blog Posts

35 |

36 | 39 | Introducing Opacus 40 | 41 | , by Facebook AI 42 |

43 |

Differential Privacy Blog Post Series

44 |
    45 |
  1. 46 | 49 | DP-SGD Algorithm Explained 50 | 51 |
    2. 52 |
  2. 53 | 56 | Efficient Per-Sample Gradient Computation in Opacus 57 | 58 |
    3. 59 |
60 |

Videos*

61 |

* Note that Opacus API has changed over time and some of the code samples and demos in the videos may not work. The concepts presented in the videos though are concrete and still valid.

62 |
    63 |
  1. 64 | 67 | OpenMined PriCon 2020 Tutorial: DP Model Training with Opacus 68 | 69 |
    2. 70 |
  2. 71 | 74 | PyTorch Developer Day 2020: Differential Privacy on PyTorch 75 | 76 |
    3. 77 |
78 |

Blog Posts by OpenMined

79 |
    80 |
  1. 81 | 84 | Differentially Private Deep Learning In 20 Lines Of Code 85 | 86 |
    2. 87 |
  2. 88 | 91 | PySyft + Opacus: Federated Learning With Differential Privacy 92 | 93 |
    3. 94 |
95 |
96 | 97 |
98 | ); 99 | } 100 | } 101 | 102 | module.exports = TutorialHome; 103 | -------------------------------------------------------------------------------- /website/static/pygments.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 3 | */ 4 | 5 | .highlight .hll { background-color: #ffffcc } 6 | .highlight .c { color: #60a0b0; font-style: italic } /* Comment */ 7 | .highlight .err { border: 1px solid #FF0000 } /* Error */ 8 | .highlight .k { color: #007020; font-weight: bold } /* Keyword */ 9 | .highlight .o { color: #666666 } /* Operator */ 10 | .highlight .cm { color: #60a0b0; font-style: italic } /* Comment.Multiline */ 11 | .highlight .cp { color: #007020 } /* Comment.Preproc */ 12 | .highlight .c1 { color: #60a0b0; font-style: italic } /* Comment.Single */ 13 | .highlight .cs { color: #60a0b0; background-color: #fff0f0 } /* Comment.Special */ 14 | .highlight .gd { color: #A00000 } /* Generic.Deleted */ 15 | .highlight .ge { font-style: italic } /* Generic.Emph */ 16 | .highlight .gr { color: #FF0000 } /* Generic.Error */ 17 | .highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ 18 | .highlight .gi { color: #00A000 } /* Generic.Inserted */ 19 | .highlight .go { color: #808080 } /* Generic.Output */ 20 | .highlight .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ 21 | .highlight .gs { font-weight: bold } /* Generic.Strong */ 22 | .highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ 23 | .highlight .gt { color: #0040D0 } /* Generic.Traceback */ 24 | .highlight .kc { color: #007020; font-weight: bold } /* Keyword.Constant */ 25 | .highlight .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ 26 | .highlight .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ 27 | .highlight .kp { color: #007020 } /* Keyword.Pseudo */ 28 | .highlight .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ 29 | .highlight .kt { color: #902000 } /* Keyword.Type */ 30 | .highlight .m { color: #40a070 } /* Literal.Number */ 31 | .highlight .s { color: #4070a0 } /* Literal.String */ 32 | .highlight .na { color: #4070a0 } /* Name.Attribute */ 33 | .highlight .nb { color: #007020 } /* Name.Builtin */ 34 | .highlight .nc { color: #0e84b5; font-weight: bold } /* Name.Class */ 35 | .highlight .no { color: #60add5 } /* Name.Constant */ 36 | .highlight .nd { color: #555555; font-weight: bold } /* Name.Decorator */ 37 | .highlight .ni { color: #d55537; font-weight: bold } /* Name.Entity */ 38 | .highlight .ne { color: #007020 } /* Name.Exception */ 39 | .highlight .nf { color: #06287e } /* Name.Function */ 40 | .highlight .nl { color: #002070; font-weight: bold } /* Name.Label */ 41 | .highlight .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ 42 | .highlight .nt { color: #062873; font-weight: bold } /* Name.Tag */ 43 | .highlight .nv { color: #bb60d5 } /* Name.Variable */ 44 | .highlight .ow { color: #007020; font-weight: bold } /* Operator.Word */ 45 | .highlight .w { color: #bbbbbb } /* Text.Whitespace */ 46 | .highlight .mf { color: #40a070 } /* Literal.Number.Float */ 47 | .highlight .mh { color: #40a070 } /* Literal.Number.Hex */ 48 | .highlight .mi { color: #40a070 } /* Literal.Number.Integer */ 49 | .highlight .mo { color: #40a070 } /* Literal.Number.Oct */ 50 | .highlight .sb { color: #4070a0 } /* Literal.String.Backtick */ 51 | .highlight .sc { color: #4070a0 } /* Literal.String.Char */ 52 | .highlight .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ 53 | .highlight .s2 { color: #4070a0 } /* Literal.String.Double */ 54 | .highlight .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ 55 | .highlight .sh { color: #4070a0 } /* Literal.String.Heredoc */ 56 | .highlight .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ 57 | .highlight .sx { color: #c65d09 } /* Literal.String.Other */ 58 | .highlight .sr { color: #235388 } /* Literal.String.Regex */ 59 | .highlight .s1 { color: #4070a0 } /* Literal.String.Single */ 60 | .highlight .ss { color: #517918 } /* Literal.String.Symbol */ 61 | .highlight .bp { color: #007020 } /* Name.Builtin.Pseudo */ 62 | .highlight .vc { color: #bb60d5 } /* Name.Variable.Class */ 63 | .highlight .vg { color: #bb60d5 } /* Name.Variable.Global */ 64 | .highlight .vi { color: #bb60d5 } /* Name.Variable.Instance */ 65 | .highlight .il { color: #40a070 } /* Literal.Number.Integer.Long */ 66 | -------------------------------------------------------------------------------- /opacus/tests/multigpu_gradcheck.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | import os 4 | import sys 5 | import unittest 6 | 7 | import torch 8 | import torch.distributed as dist 9 | import torch.multiprocessing as mp 10 | import torch.nn as nn 11 | import torch.optim as optim 12 | from opacus import PrivacyEngine 13 | from opacus.layers import DifferentiallyPrivateDistributedDataParallel as DPDDP 14 | from torch.nn.parallel import DistributedDataParallel as DDP 15 | 16 | 17 | PRIVACY_ALPHAS = [1 + x / 10.0 for x in range(1, 100)] + list(range(12, 64)) 18 | 19 | 20 | def setup(rank, world_size): 21 | if sys.platform == "win32": 22 | # Distributed package only covers collective communications with Gloo 23 | # backend and FileStore on Windows platform. Set init_method parameter 24 | # in init_process_group to a local file. 25 | # Example init_method="file:///f:/libtmp/some_file" 26 | init_method = "file:///{your local file path}" 27 | 28 | # initialize the process group 29 | dist.init_process_group( 30 | "gloo", init_method=init_method, rank=rank, world_size=world_size 31 | ) 32 | else: 33 | os.environ["MASTER_ADDR"] = "localhost" 34 | os.environ["MASTER_PORT"] = "12355" 35 | 36 | # initialize the process group 37 | # dist.init_process_group("gloo", rank=rank, world_size=world_size) 38 | 39 | os.environ["RANK"] = str(rank) 40 | os.environ["WORLD_SIZE"] = str(world_size) 41 | torch.distributed.init_process_group( 42 | init_method="env://", 43 | backend="nccl", 44 | ) 45 | 46 | 47 | def cleanup(): 48 | dist.destroy_process_group() 49 | 50 | 51 | class ToyModel(nn.Module): 52 | def __init__(self): 53 | super(ToyModel, self).__init__() 54 | self.net1 = nn.Linear(10, 10) 55 | self.relu = nn.ReLU() 56 | self.net2 = nn.Linear(10, 5) 57 | 58 | def forward(self, x): 59 | return self.net2(self.relu(self.net1(x))) 60 | 61 | 62 | def demo_basic(rank, weight, world_size, dp): 63 | torch.manual_seed(world_size) 64 | batch_size = 32 65 | withdp = "with" + ("out " if not dp else "") 66 | print(f"Running basic DDP {withdp} differential privacy example on rank {rank}.") 67 | setup(rank, world_size) 68 | 69 | # create model and move it to GPU with id rank 70 | model = ToyModel().to(rank) 71 | if dp: 72 | ddp_model = DPDDP(model) 73 | engine = PrivacyEngine( 74 | ddp_model, 75 | batch_size=batch_size, 76 | sample_size=10 * batch_size, 77 | alphas=PRIVACY_ALPHAS, 78 | noise_multiplier=0, 79 | max_grad_norm=1e8, 80 | ) 81 | else: 82 | ddp_model = DDP(model, device_ids=[rank]) 83 | 84 | loss_fn = nn.MSELoss() 85 | optimizer = optim.SGD(ddp_model.parameters(), lr=1) 86 | if dp: 87 | engine.attach(optimizer) 88 | 89 | # if rank == 0: 90 | # print(model.net1.weight) 91 | optimizer.zero_grad() 92 | labels = torch.randn(batch_size, 5).to(rank) 93 | outputs = ddp_model(torch.randn(batch_size, 10).to(rank)) 94 | loss_fn(outputs, labels).backward() 95 | optimizer.step() 96 | # if rank == 0: 97 | # print(model.net1.weight) 98 | 99 | weight.copy_(model.net1.weight.data.cpu()) 100 | 101 | cleanup() 102 | 103 | 104 | def run_demo(demo_fn, weight, world_size, dp): 105 | mp.spawn(demo_fn, args=(weight, world_size, dp), nprocs=world_size, join=True) 106 | 107 | 108 | class GradientComputationTest(unittest.TestCase): 109 | def test_gradient_correct(self): 110 | # Tests that gradient is the same with DP or with DDP 111 | n_gpus = torch.cuda.device_count() 112 | self.assertTrue( 113 | n_gpus >= 2, f"Need at least 2 gpus but was provided only {n_gpus}." 114 | ) 115 | weight_dp, weight_nodp = torch.zeros(10, 10), torch.zeros(10, 10) 116 | run_demo(demo_basic, weight_dp, 2, dp=True) 117 | run_demo(demo_basic, weight_nodp, 2, dp=False) 118 | 119 | self.assertTrue(torch.norm(weight_dp - weight_nodp) < 1e-7) 120 | -------------------------------------------------------------------------------- /opacus/utils/packed_sequences.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | from typing import List, Optional 5 | 6 | import torch 7 | from torch.nn.utils.rnn import PackedSequence, pack_padded_sequence, pad_sequence 8 | 9 | 10 | def _gen_packed_data( 11 | minibatch_size: int, 12 | max_seq_length: int, 13 | input_dim: int, 14 | batch_first: bool, 15 | sorted_: Optional[bool] = False, 16 | ) -> PackedSequence: 17 | """ 18 | This is used to generate random PackedSequence data, sampled from a normal distribution, for testing DPLSTM. 19 | 20 | Args: 21 | minibatch_size : Total number of sequences to generate 22 | max_seq_length : The maximum number of timesteps of a sequence 23 | input_dim : The embedding dimension of a sequence at any timestep 24 | batch_first : If this is true, data is first generated using a padded sequence of dimension (minibatch_size x max_seq_len x input_dim) , else: (max_seq_length x minibatch_size x input_dim) 25 | sorted_ : If this is true then the original generated data used to produce the PackedSequence will already be ordered based on sequence lengths, else a random order and the 'sorted_indices' 26 | and 'unsorted_indices' fields will be None. 27 | 28 | Return Value: 29 | packed_data : A PackedSequence object with its data sampled from a normal distribution. 30 | """ 31 | 32 | if batch_first: 33 | data = [] 34 | seq_lengths = [] 35 | for _ in range(minibatch_size): 36 | seq_length = torch.randint(1, max_seq_length + 1, (1,)).item() 37 | seq_lengths.append(seq_length) 38 | data.append(torch.randn(seq_length, input_dim)) 39 | 40 | if sorted_: 41 | data = sorted(data, key=lambda x: x.shape[0], reverse=True) 42 | seq_lengths = sorted(seq_lengths, reverse=True) 43 | packed_data = pack_padded_sequence( 44 | pad_sequence(data, batch_first=True), 45 | seq_lengths, 46 | batch_first=True, 47 | enforce_sorted=True, 48 | ) 49 | else: 50 | packed_data = pack_padded_sequence( 51 | pad_sequence(data, batch_first=True), 52 | seq_lengths, 53 | batch_first=True, 54 | enforce_sorted=False, 55 | ) 56 | else: 57 | seq_lengths = [ 58 | torch.randint(1, max_seq_length + 1, (1,)).item() 59 | for _ in range(minibatch_size) 60 | ] 61 | if sorted_: 62 | seq_lengths = sorted(seq_lengths, reverse=True) 63 | padded_data = torch.zeros((max_seq_length, minibatch_size, input_dim)) 64 | for i in range(minibatch_size): 65 | padded_data[: seq_lengths[i], i, :] = torch.randn(seq_lengths[i], input_dim) 66 | 67 | if sorted_: 68 | packed_data = pack_padded_sequence( 69 | padded_data, seq_lengths, batch_first=False, enforce_sorted=True 70 | ) 71 | else: 72 | packed_data = pack_padded_sequence( 73 | padded_data, seq_lengths, batch_first=False, enforce_sorted=False 74 | ) 75 | 76 | return packed_data 77 | 78 | 79 | def compute_seq_lengths(batch_sizes: torch.Tensor) -> List[int]: 80 | """ 81 | Computes the sequence lengths of a PackedSequence represented with batch_sizes. 82 | 83 | Args: 84 | batch_sizes: Contains the batch sizes as stored in a PackedSequence 85 | 86 | Returns: 87 | running_seq_lengths: the length parameter used in the torch.nn.utils.rnn.packed_padded_sequence function 88 | to create a PackedSequence. It's a list of the same length as batch_sizes. 89 | """ 90 | 91 | max_batch_size = batch_sizes[0] 92 | if len(batch_sizes) == 1: 93 | return [1] * max_batch_size 94 | 95 | running_seq = 0 96 | running_seq_lengths = [] 97 | for i in range(1, len(batch_sizes)): 98 | delta = batch_sizes[i - 1].item() - batch_sizes[i].item() 99 | running_seq += 1 100 | running_seq_lengths += delta * [running_seq] 101 | 102 | running_seq += 1 103 | running_seq_lengths += batch_sizes[-1].item() * [running_seq] 104 | running_seq_lengths.reverse() 105 | return running_seq_lengths 106 | -------------------------------------------------------------------------------- /opacus/utils/module_inspection.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | r""" 4 | This module includes utils for inspecting model layers using specified 5 | predicates to check for conditions, getting layer type etc. 6 | """ 7 | from typing import Callable, Optional 8 | 9 | from torch import nn 10 | 11 | 12 | class ModelInspector: 13 | """ 14 | An inspector of models given a specific predicate. If a module 15 | has children the predicate is checked on all children recursively. 16 | 17 | Example: 18 | >>> inspector = ModelInspector('simple', lambda x: isinstance(x, Conv2d)) 19 | >>> print(inspector.validate(nn.Conv2d(1, 1, 1))) 20 | True 21 | """ 22 | 23 | def __init__( 24 | self, 25 | name: str, 26 | predicate: Callable[[nn.Module], bool], 27 | check_leaf_nodes_only: bool = True, 28 | message: Optional[str] = None, 29 | ): 30 | """ 31 | Args: 32 | name: String to represent the predicate. 33 | predicate: Callable boolean function which tests a hypothesis on a module. 34 | check_leaf_nodes_only: Flag to check only leaf nodes of a module. Here 35 | leaf nodes are the ones that have parameters of their own. 36 | message: Optional value to hold a message about violating this predicate. 37 | 38 | Notes: 39 | The predicates will not be applied on non-leaf modules unless 40 | ``check_leaf_nodes_only`` is set to False. E.g. A predicate like: 41 | 42 | ``lambda model: isinstance(model, nn.Sequential)`` 43 | 44 | will always return True unless ``check_leaf_nodes_only`` is set. 45 | """ 46 | self.name = name 47 | if check_leaf_nodes_only: 48 | self.predicate = ( 49 | lambda x: has_no_param(x) or not requires_grad(x) or predicate(x) 50 | ) 51 | else: 52 | self.predicate = predicate 53 | self.message = message 54 | self.violators = [] 55 | # List that contains the module names that have violated the 56 | # predicate. The list does not get automatically emptied if 57 | # the predicate is applied on multiple modules. 58 | 59 | def validate(self, model: nn.Module) -> bool: 60 | """ 61 | Checks if the provided module satisfies the predicate specified 62 | upon creation of the :class:`~opacus.utils.ModelInspector`. 63 | 64 | Args: 65 | model: PyTorch module on which the predicate must be evaluated 66 | and satisfied. 67 | 68 | Returns: 69 | Flag indicate if predicate is satisfied. 70 | """ 71 | valid = True 72 | for name, module in model.named_modules(): 73 | if not self.predicate(module): 74 | valid = False 75 | self.violators.append(f"{name} ({get_layer_type(module)})") 76 | return valid 77 | 78 | 79 | def has_no_param(module: nn.Module) -> bool: 80 | """ 81 | Checks if a module does not have any parameters. 82 | 83 | Args: 84 | module: The module on which this function is being evaluated. 85 | 86 | Returns: 87 | Flag indicating if the provided module does not have any 88 | parameters. 89 | """ 90 | has_params = any(p is not None for p in module.parameters(recurse=False)) 91 | return not has_params 92 | 93 | 94 | def requires_grad(module: nn.Module, recurse: bool = False) -> bool: 95 | """ 96 | Checks if any parameters in a specified module require gradients. 97 | 98 | Args: 99 | module: PyTorch module whose parameters are examined 100 | recurse: Flag specifying if the gradient requirement check should 101 | be applied recursively to sub-modules of the specified module 102 | 103 | Returns: 104 | Flag indicate if any parameters require gradients 105 | """ 106 | requires_grad = any(p.requires_grad for p in module.parameters(recurse)) 107 | return requires_grad 108 | 109 | 110 | def get_layer_type(layer: nn.Module) -> str: 111 | """ 112 | Returns the name of the type of the given layer. 113 | 114 | Args: 115 | layer: The module corresponding to the layer whose type 116 | is being queried. 117 | 118 | Returns: 119 | Name of the class of the layer 120 | """ 121 | return layer.__class__.__name__ 122 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Opacus 2 | 3 | We want to make contributing to Opacus is as easy and transparent as possible. 4 | 5 | 6 | ## Development installation 7 | 8 | To get the development installation with all the necessary dependencies for 9 | linting, testing, and building the documentation, run the following: 10 | ```bash 11 | git clone https://github.com/pytorch/opacus.git 12 | cd opacus 13 | pip install -e .[dev] 14 | ``` 15 | 16 | 17 | ## Our Development Process 18 | 19 | #### Code Style 20 | 21 | Opacus uses the [black](https://github.com/ambv/black) and [flake8](https://github.com/PyCQA/flake8) code formatter to 22 | enforce a common code style across the code base. black is installed easily via 23 | pip using `pip install black`, and run locally by calling 24 | ```bash 25 | black . 26 | flake8 . 27 | ``` 28 | from the repository root. No additional configuration should be needed (see the 29 | [black documentation](https://black.readthedocs.io/en/stable/installation_and_usage.html#usage) 30 | for advanced usage). 31 | 32 | Opacus also uses [isort](https://github.com/timothycrosley/isort) to sort imports 33 | alphabetically and separate into sections. isort is installed easily via 34 | pip using `pip install isort`, and run locally by calling 35 | ```bash 36 | isort -v -l 88 -o opacus --lines-after-imports 2 -m 3 --trailing-comma . 37 | ``` 38 | from the repository root. Configuration for isort is located in .isort.cfg. 39 | 40 | We feel strongly that having a consistent code style is extremely important, so 41 | CircleCI will fail on your PR if it does not adhere to the black or flake8 formatting style or isort import ordering. 42 | 43 | 44 | #### Type Hints 45 | 46 | Opacus is fully typed using python 3.6+ 47 | [type hints](https://www.python.org/dev/peps/pep-0484/). 48 | We expect any contributions to also use proper type annotations. 49 | While we currently do not enforce full consistency of these in our continuous integration 50 | test, you should strive to type check your code locally. For this we recommend 51 | using [mypy](http://mypy-lang.org/). 52 | 53 | 54 | #### Unit Tests 55 | 56 | To run the unit tests, you can either use `pytest` (if installed): 57 | ```bash 58 | pytest -ra 59 | ``` 60 | or python's `unittest`: 61 | ```bash 62 | python -m unittest 63 | ``` 64 | 65 | To get coverage reports we recommend using the `pytest-cov` plugin: 66 | ```bash 67 | pytest -ra --cov=. --cov-report term-missing 68 | ``` 69 | 70 | 71 | #### Documentation 72 | Opacus's website is also open source, and is part of this very repository (the 73 | code can be found in the [website](/website/) folder). 74 | It is built using [Docusaurus](https://docusaurus.io/), and consists of three 75 | main elements: 76 | 77 | 1. The documentation in Docusaurus itself (if you know Markdown, you can 78 | already contribute!). This lives in the [docs](/docs/). 79 | 2. The API reference, auto-generated from the docstrings using 80 | [Sphinx](http://www.sphinx-doc.org), and embedded into the Docusaurus website. 81 | The sphinx .rst source files for this live in [sphinx/source](/sphinx/source/). 82 | 3. The Jupyter notebook tutorials, parsed by `nbconvert`, and embedded into the 83 | Docusaurus website. These live in [tutorials](/tutorials/). 84 | 85 | To build the documentation you will need [Node](https://nodejs.org/en/) >= 8.x 86 | and [Yarn](https://yarnpkg.com/en/) >= 1.5. 87 | 88 | The following command will both build the docs and serve the site locally: 89 | ```bash 90 | ./scripts/build_docs.sh 91 | ``` 92 | 93 | ## Pull Requests 94 | We actively welcome your pull requests. 95 | 96 | 1. Fork the repo and create your branch from `main`. 97 | 2. If you have added code that should be tested, add unit tests. 98 | In other words, add unit tests. 99 | 3. If you have changed APIs, document the API change in the PR. 100 | Also update the documentation and make sure the documentation builds. 101 | 4. Ensure the test suite passes. 102 | 5. Make sure your code passes both `black` and `flake8` formatting checks. 103 | 104 | 105 | ## Issues 106 | 107 | We use GitHub issues to track public bugs. Please ensure your description is 108 | clear and has sufficient instructions to be able to reproduce the issue. 109 | 110 | Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe 111 | disclosure of security bugs. In those cases, please go through the process 112 | outlined on that page and do not file a public issue. 113 | 114 | 115 | ## License 116 | 117 | By contributing to Opacus, you agree that your contributions will be licensed 118 | under the LICENSE file in the root directory of this source tree. 119 | -------------------------------------------------------------------------------- /opacus/utils/tests/module_inspection_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | 5 | import unittest 6 | 7 | import torch.nn as nn 8 | from opacus.utils import module_inspection as mi 9 | from torchvision import models 10 | 11 | 12 | class utils_ModelInspector_test(unittest.TestCase): 13 | def setUp(self): 14 | def pred_supported(module): 15 | return isinstance(module, (nn.Conv2d, nn.Linear)) 16 | 17 | def pred_not_unsupported(module): 18 | return not isinstance(module, (nn.BatchNorm2d, nn.BatchNorm3d)) 19 | 20 | def pred_requires_grad(module): 21 | return all(p.requires_grad for p in module.parameters(recurse=False)) 22 | 23 | self.pred_supported = pred_supported 24 | self.pred_not_unsupported = pred_not_unsupported 25 | self.pred_mix = lambda m: (not pred_requires_grad(m)) or pred_not_unsupported(m) 26 | 27 | def test_validate_basic(self): 28 | inspector = mi.ModelInspector( 29 | "pred", lambda model: isinstance(model, nn.Linear) 30 | ) 31 | model = nn.Conv1d(1, 1, 1) 32 | valid = inspector.validate(model) 33 | self.assertFalse(valid, inspector.violators) 34 | 35 | def test_validate_positive_predicate_valid(self): 36 | # test when a positive predicate (e.g. supported) returns true 37 | inspector = mi.ModelInspector("pred", self.pred_supported) 38 | model = nn.Conv2d(1, 1, 1) 39 | valid = inspector.validate(model) 40 | self.assertTrue(valid) 41 | list_len = len(inspector.violators) 42 | self.assertEqual(list_len, 0, f"violators = {inspector.violators}") 43 | 44 | def test_validate_positive_predicate_invalid(self): 45 | # test when a positive predicate (e.g. supported) returns false 46 | inspector = mi.ModelInspector("pred", self.pred_supported) 47 | model = nn.Conv1d(1, 1, 1) 48 | valid = inspector.validate(model) 49 | self.assertFalse(valid) 50 | list_len = len(inspector.violators) 51 | self.assertEqual(list_len, 1, f"violators = {inspector.violators}") 52 | 53 | def test_validate_negative_predicate_ture(self): 54 | # test when a negative predicate (e.g. not unsupported) returns true 55 | inspector = mi.ModelInspector("pred1", self.pred_not_unsupported) 56 | model = nn.Sequential(nn.Conv2d(1, 1, 1), nn.Linear(1, 1)) 57 | valid = inspector.validate(model) 58 | self.assertTrue(valid) 59 | list_len = len(inspector.violators) 60 | self.assertEqual(list_len, 0) 61 | 62 | def test_validate_negative_predicate_False(self): 63 | # test when a negative predicate (e.g. not unsupported) returns false 64 | inspector = mi.ModelInspector("pred", self.pred_not_unsupported) 65 | model = nn.Sequential(nn.Conv2d(1, 1, 1), nn.BatchNorm2d(1)) 66 | valid = inspector.validate(model) 67 | self.assertFalse(valid) 68 | list_len = len(inspector.violators) 69 | self.assertEqual(list_len, 1, f"violators = {inspector.violators}") 70 | 71 | def test_validate_mix_predicate(self): 72 | # check with a mix predicate not requires grad or is not unsupported 73 | inspector = mi.ModelInspector("pred1", self.pred_mix) 74 | model = nn.Sequential(nn.Conv2d(1, 1, 1), nn.BatchNorm2d(1)) 75 | for p in model[1].parameters(): 76 | p.requires_grad = False 77 | valid = inspector.validate(model) 78 | self.assertTrue(valid) 79 | 80 | def test_check_everything_flag(self): 81 | # check to see if a model does not containt nn.sequential 82 | inspector = mi.ModelInspector( 83 | "pred", 84 | lambda model: not isinstance(model, nn.Sequential), 85 | check_leaf_nodes_only=False, 86 | ) 87 | model = nn.Sequential(nn.Conv1d(1, 1, 1)) 88 | valid = inspector.validate(model) 89 | self.assertFalse(valid, f"violators = {inspector.violators}") 90 | 91 | def test_complicated_case(self): 92 | def good(x): 93 | return isinstance(x, (nn.Conv2d, nn.Linear)) 94 | 95 | def bad(x): 96 | return isinstance(x, nn.modules.batchnorm._BatchNorm) 97 | 98 | inspector1 = mi.ModelInspector("good_or_bad", lambda x: good(x) or bad(x)) 99 | inspector2 = mi.ModelInspector("not_bad", lambda x: not bad(x)) 100 | model = models.resnet50() 101 | valid = inspector1.validate(model) 102 | self.assertTrue(valid, f"violators = {inspector1.violators}") 103 | self.assertEqual( 104 | len(inspector1.violators), 0, f"violators = {inspector1.violators}" 105 | ) 106 | valid = inspector2.validate(model) 107 | self.assertFalse(valid, f"violators = {inspector2.violators}") 108 | self.assertEqual( 109 | len(inspector2.violators), 53, f"violators = {inspector2.violators}" 110 | ) 111 | -------------------------------------------------------------------------------- /opacus/tests/dp_layers/dp_rnn_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | from typing import Optional, Tuple, Union 5 | 6 | import hypothesis.strategies as st 7 | import torch 8 | import torch.nn as nn 9 | from hypothesis import given, settings 10 | from opacus.layers import DPGRU, DPLSTM, DPRNN 11 | from opacus.utils.packed_sequences import _gen_packed_data 12 | from torch.nn.utils.rnn import PackedSequence 13 | 14 | from .common import DPModules_test 15 | 16 | 17 | def rnn_train_fn( 18 | model: nn.Module, 19 | x: Union[torch.Tensor, PackedSequence], 20 | state_init: Optional[Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]] = None, 21 | ): 22 | model.train() 23 | criterion = nn.MSELoss() 24 | logits, _ = model(x, state_init) 25 | if isinstance(logits, PackedSequence): 26 | y = torch.zeros_like(logits[0]) 27 | loss = criterion(logits[0], y) 28 | else: 29 | y = torch.zeros_like(logits) 30 | loss = criterion(logits, y) 31 | loss.backward() 32 | 33 | 34 | class DPLSTM_test(DPModules_test): 35 | @given( 36 | mode=st.one_of(st.just("rnn"), st.just("gru"), st.just("lstm")), 37 | batch_size=st.integers(1, 5), 38 | seq_len=st.integers(1, 6), 39 | emb_size=st.integers(5, 10), 40 | hidden_size=st.integers(3, 7), 41 | num_layers=st.integers(1, 3), 42 | bidirectional=st.booleans(), 43 | bias=st.booleans(), 44 | batch_first=st.booleans(), 45 | zero_init=st.booleans(), 46 | packed_input_flag=st.integers( 47 | 0, # no packed sequence input 48 | # 1, # packed sequence input in sorted order 49 | 2, # packed sequence input in unsorted order 50 | ), 51 | ) 52 | @settings(deadline=20000) 53 | def test_rnn( 54 | self, 55 | mode: str, 56 | batch_size: int, 57 | seq_len: int, 58 | emb_size: int, 59 | hidden_size: int, 60 | num_layers: int, 61 | bidirectional: bool, 62 | bias: bool, 63 | batch_first: bool, 64 | zero_init: bool, 65 | packed_input_flag: int, 66 | ): 67 | use_cn = False 68 | if mode == "rnn": 69 | original_rnn_class = nn.RNN 70 | dp_rnn_class = DPRNN 71 | elif mode == "gru": 72 | original_rnn_class = nn.GRU 73 | dp_rnn_class = DPGRU 74 | elif mode == "lstm": 75 | original_rnn_class = nn.LSTM 76 | dp_rnn_class = DPLSTM 77 | use_cn = True 78 | else: 79 | raise ValueError("Invalid RNN mode") 80 | 81 | rnn = original_rnn_class( 82 | emb_size, 83 | hidden_size, 84 | num_layers=num_layers, 85 | batch_first=batch_first, 86 | bidirectional=bidirectional, 87 | bias=bias, 88 | ) 89 | dp_rnn = dp_rnn_class( 90 | emb_size, 91 | hidden_size, 92 | num_layers=num_layers, 93 | batch_first=batch_first, 94 | bidirectional=bidirectional, 95 | bias=bias, 96 | ) 97 | 98 | dp_rnn.load_state_dict(rnn.state_dict()) 99 | 100 | if packed_input_flag == 0: 101 | x = ( 102 | torch.randn([batch_size, seq_len, emb_size]) 103 | if batch_first 104 | else torch.randn([seq_len, batch_size, emb_size]) 105 | ) 106 | elif packed_input_flag == 1: 107 | x = _gen_packed_data( 108 | batch_size, seq_len, emb_size, batch_first, sorted_=True 109 | ) 110 | elif packed_input_flag == 2: 111 | x = _gen_packed_data( 112 | batch_size, seq_len, emb_size, batch_first, sorted_=False 113 | ) 114 | else: 115 | raise ValueError("Invalid packed input flag") 116 | 117 | if zero_init: 118 | self.compare_forward_outputs( 119 | rnn, 120 | dp_rnn, 121 | x, 122 | output_names=("out", "hn", "cn") if use_cn else ("out", "hn"), 123 | atol=1e-5, 124 | rtol=1e-3, 125 | ) 126 | 127 | self.compare_gradients( 128 | rnn, 129 | dp_rnn, 130 | rnn_train_fn, 131 | x, 132 | atol=1e-5, 133 | rtol=1e-3, 134 | ) 135 | 136 | else: 137 | num_directions = 2 if bidirectional else 1 138 | h0 = torch.randn([num_layers * num_directions, batch_size, hidden_size]) 139 | c0 = torch.randn([num_layers * num_directions, batch_size, hidden_size]) 140 | self.compare_forward_outputs( 141 | rnn, 142 | dp_rnn, 143 | x, 144 | (h0, c0) if use_cn else h0, 145 | output_names=("out", "hn", "cn") if use_cn else ("out", "hn"), 146 | atol=1e-5, 147 | rtol=1e-3, 148 | ) 149 | self.compare_gradients( 150 | rnn, 151 | dp_rnn, 152 | rnn_train_fn, 153 | x, 154 | (h0, c0) if use_cn else h0, 155 | atol=1e-5, 156 | rtol=1e-3, 157 | ) 158 | -------------------------------------------------------------------------------- /opacus/layers/param_rename.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | 5 | from typing import Dict, Union 6 | 7 | import torch.nn as nn 8 | from torch import Tensor 9 | from torch.nn.modules.module import _IncompatibleKeys 10 | 11 | 12 | def filter_out_old_keys(self, state_dict, prefix, local_metadata): 13 | new_state_dict = { 14 | param_name: param_value 15 | for param_name, param_value in state_dict.items() 16 | if param_name not in self.old_to_new 17 | } 18 | return new_state_dict 19 | 20 | 21 | class RenameParamsMixin: 22 | """ 23 | This class defines a nn.Module whose parameters are renamed. This is useful when you want to 24 | reimplement a layer but make sure its state_dict and list of parameters are exactly the same 25 | as another reference layer so that you can have a drop-in replacement that does not depend on 26 | how your layer is actually implemented. In Opacus, this is used for DPLSTM, where our 27 | implementation leverages submodules and requires alignment to the state_dict of nn.LSTM. 28 | 29 | Example: 30 | 31 | class DPModel(RenameParamsMixin, nn.Module): 32 | def __init__(self, hidden_size): 33 | super().__init__() 34 | self.w = nn.Parameter(torch.zeros(hidden_size, requires_grad=True)) 35 | self.set_rename_map({"w": "weights"}) 36 | 37 | >>> model = DPModel(5) 38 | >>> model.state_dict() 39 | {'weights': tensor([0., 0., 0., 0., 0.])} 40 | """ 41 | 42 | def set_rename_map(self, rename_map: Dict[str, str]): 43 | """ 44 | Initializes internal state. Subclass this instead of ``torch.nn.Module`` whenever you need 45 | to rename your model's state. 46 | 47 | Args: 48 | rename_map: mapping from old name -> new name for each parameter you want renamed. 49 | Note that this must be a 1:1 mapping! 50 | """ 51 | self.old_to_new = rename_map 52 | self.new_to_old = {v: k for k, v in rename_map.items()} 53 | 54 | self._register_state_dict_hook(filter_out_old_keys) 55 | 56 | def _register_renamed_parameters(self): 57 | """ 58 | Internal function. This function simply registers parameters under their new name. They will 59 | automatically mask their duplicates coming from submodules. This trick works because 60 | self.parameters() proceeds recursively from the top, going into submodules after processing 61 | items at the current level, and will not return duplicates. 62 | """ 63 | for old_name, param in super().named_parameters(): 64 | if old_name in self.old_to_new: 65 | new_name = self.old_to_new[old_name] 66 | self.register_parameter(new_name, param) 67 | 68 | def __setattr__(self, name: str, value: Union[Tensor, nn.Module]) -> None: 69 | """ 70 | Whenever you set an attribute, eg `self.linear`, this is called to actually register it in 71 | any nn.Module. We rely on the masking trick explained in the docs for 72 | ``_register_renamed_parameters`` to make sure we replace things only once. If a new parameter 73 | in the rename list is detected, we rename and mask it so next time this is called we will 74 | no longer find it. 75 | """ 76 | super().__setattr__(name, value) 77 | try: 78 | self._register_renamed_parameters() 79 | except AttributeError: 80 | # At the very beginning of instantiation, this will fail because we do not yet have 81 | # self._parameters. Safe to ignore. 82 | pass 83 | 84 | def load_state_dict( 85 | self, 86 | state_dict: Dict[str, Tensor], 87 | strict: bool = True, 88 | ): 89 | """ 90 | Identical to ``torch.nn.Module.load_state_dict()`` but handles the renamed keys. 91 | """ 92 | 93 | # nn.Module recomputes its state_dict(), without calling the same logic as in self.state_dict() 94 | # This means that it will find both the old and the renamed parameters. Both point to the 95 | # same parameter object, so either of them will set it correctly. It will however complain 96 | # that some keys are missing (the "old" keys). We can safely ignore those and process them 97 | # accordingly 98 | 99 | missing_keys, unexpected_keys = super().load_state_dict( 100 | state_dict, strict=False 101 | ) 102 | missing_keys = [k for k in missing_keys if k not in self.old_to_new] 103 | if strict: 104 | error_msgs = [] 105 | if len(unexpected_keys) > 0: 106 | error_msgs.insert( 107 | 0, 108 | "Unexpected key(s) in state_dict: {}. ".format( 109 | ", ".join('"{}"'.format(k) for k in unexpected_keys) 110 | ), 111 | ) 112 | if len(missing_keys) > 0: 113 | error_msgs.insert( 114 | 0, 115 | "Missing key(s) in state_dict: {}. ".format( 116 | ", ".join('"{}"'.format(k) for k in missing_keys) 117 | ), 118 | ) 119 | 120 | if len(error_msgs) > 0: 121 | raise RuntimeError( 122 | "Error(s) in loading state_dict for {}:\n\t{}".format( 123 | self.__class__.__name__, "\n\t".join(error_msgs) 124 | ) 125 | ) 126 | return _IncompatibleKeys(missing_keys, unexpected_keys) 127 | -------------------------------------------------------------------------------- /opacus/scripts/compute_dp_sgd_privacy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | 4 | """ 5 | Command-line script for computing privacy of a model trained with DP-SGD. 6 | The script applies the RDP accountant to estimate privacy budget of an iterated 7 | Sampled Gaussian Mechanism. 8 | 9 | The code is mainly based on Google's TF Privacy: 10 | https://github.com/tensorflow/privacy/blob/master/tensorflow_privacy/privacy/analysis/compute_dp_sgd_privacy.py 11 | 12 | 13 | Example: 14 | 15 | To call this script from command line, you can enter: 16 | 17 | >>> python compute_dp_sgd_privacy.py --dataset-size=60000 --batch-size=256 --noise_multiplier=1.12 --epochs=60 --delta=1e-5 --a 10 20 100 18 | 19 | The training process with these parameters satisfies (epsilon,delta)-DP of (2.95, 1e-5). 20 | """ 21 | import argparse 22 | import math 23 | from typing import List, Tuple 24 | 25 | from opacus import privacy_analysis 26 | 27 | 28 | def _apply_dp_sgd_analysis( 29 | sample_rate: float, 30 | noise_multiplier: float, 31 | steps: int, 32 | alphas: List[float], 33 | delta: float, 34 | verbose: bool = True, 35 | ) -> Tuple[float, float]: 36 | """ 37 | Computes the privacy Epsilon at a given delta via RDP accounting and 38 | converting to an (epsilon, delta) guarantee for a target Delta. 39 | 40 | Args: 41 | sample_rate : The sample rate in SGD 42 | noise_multiplier : The ratio of the standard deviation of the Gaussian 43 | noise to the L2-sensitivity of the function to which the noise is added 44 | steps : The number of steps 45 | alphas : A list of RDP orders 46 | delta : Target delta 47 | verbose : If enabled, will print the results of DP-SGD analysis 48 | 49 | Returns: 50 | Pair of privacy loss epsilon and optimal order alpha 51 | """ 52 | rdp = privacy_analysis.compute_rdp(sample_rate, noise_multiplier, steps, alphas) 53 | eps, opt_alpha = privacy_analysis.get_privacy_spent(alphas, rdp, delta=delta) 54 | 55 | if verbose: 56 | print( 57 | f"DP-SGD with\n\tsampling rate = {100 * sample_rate:.3g}%," 58 | f"\n\tnoise_multiplier = {noise_multiplier}," 59 | f"\n\titerated over {steps} steps,\nsatisfies " 60 | f"differential privacy with\n\tepsilon = {eps:.3g}," 61 | f"\n\tdelta = {delta}." 62 | f"\nThe optimal alpha is {opt_alpha}." 63 | ) 64 | 65 | if opt_alpha == max(alphas) or opt_alpha == min(alphas): 66 | print( 67 | "The privacy estimate is likely to be improved by expanding " 68 | "the set of alpha orders." 69 | ) 70 | return eps, opt_alpha 71 | 72 | 73 | def compute_dp_sgd_privacy( 74 | sample_rate: float, 75 | noise_multiplier: float, 76 | epochs: int, 77 | delta: float, 78 | alphas: List[float], 79 | verbose: bool = True, 80 | ) -> Tuple[float, float]: 81 | """ 82 | Performs the DP-SGD privacy analysis. 83 | 84 | Finds sample rate and number of steps based on the input parameters, and calls 85 | DP-SGD privacy analysis to find the privacy loss epsilon and optimal order alpha. 86 | 87 | Args: 88 | sample_rate : probability of each sample from the dataset to be selected for a next batch 89 | noise_multiplier : The ratio of the standard deviation of the Gaussian noise 90 | to the L2-sensitivity of the function to which the noise is added 91 | epochs : Number of epochs 92 | delta : Target delta 93 | alphas : A list of RDP orders 94 | verbose : If enabled, will print the results of DP-SGD analysis 95 | 96 | Returns: 97 | Pair of privacy loss epsilon and optimal order alpha 98 | 99 | Raises: 100 | ValueError 101 | When batch size is greater than sample size 102 | """ 103 | if sample_rate > 1: 104 | raise ValueError("sample_rate must be no greater than 1") 105 | steps = epochs * math.ceil(1 / sample_rate) 106 | 107 | return _apply_dp_sgd_analysis( 108 | sample_rate, noise_multiplier, steps, alphas, delta, verbose 109 | ) 110 | 111 | 112 | def main(): 113 | parser = argparse.ArgumentParser(description="RDP computation") 114 | parser.add_argument( 115 | "-r", 116 | "--sample-rate", 117 | type=float, 118 | required=True, 119 | help="Input sample rate (probability of each sample from the dataset to be selected for a next batch)", 120 | ) 121 | parser.add_argument( 122 | "-n", 123 | "--noise-multiplier", 124 | type=float, 125 | required=True, 126 | help="Noise multiplier", 127 | ) 128 | parser.add_argument( 129 | "-e", 130 | "--epochs", 131 | type=int, 132 | required=True, 133 | help="Number of epochs to train", 134 | ) 135 | parser.add_argument( 136 | "-d", "--delta", type=float, default=1e-5, help="Targeted delta (default: 1e-5)" 137 | ) 138 | parser.add_argument( 139 | "-a", 140 | "--alphas", 141 | action="store", 142 | dest="alphas", 143 | type=float, 144 | nargs="+", 145 | default=[1 + x / 10.0 for x in range(1, 100)] + list(range(12, 64)), 146 | help="List of alpha values (alpha orders of Renyi-DP evaluation). " 147 | "A default list is provided. Else, space separated numbers. E.g.," 148 | "-a 10 100", 149 | ) 150 | 151 | args = parser.parse_args() 152 | 153 | compute_dp_sgd_privacy( 154 | args.sample_rate, 155 | args.noise_multiplier, 156 | args.epochs, 157 | args.delta, 158 | args.alphas, 159 | ) 160 | 161 | 162 | if __name__ == "__main__": 163 | main() 164 | -------------------------------------------------------------------------------- /website/static/img/opacus_logo.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /opacus/utils/uniform_sampler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | 4 | from typing import Optional 5 | 6 | import torch 7 | from torch.utils.data import Sampler 8 | 9 | 10 | class UniformWithReplacementSampler(Sampler): 11 | r""" 12 | This sampler samples elements according to the Sampled Gaussian Mechanism. 13 | Each sample is selected with a probability equal to ``sample_rate``. 14 | """ 15 | 16 | def __init__(self, num_samples: int, sample_rate: float, generator=None): 17 | r""" 18 | Args: 19 | num_samples (int): number of samples to draw. 20 | sample_rate (float): probability used in sampling. 21 | generator (Generator): Generator used in sampling. 22 | """ 23 | self.num_samples = num_samples 24 | self.sample_rate = sample_rate 25 | self.generator = generator 26 | if self.generator is None: 27 | generator = torch.Generator() 28 | generator.manual_seed( 29 | int(torch.empty((), dtype=torch.int64).random_().item()) 30 | ) 31 | 32 | if self.num_samples <= 0: 33 | raise ValueError( 34 | "num_samples should be a positive integer " 35 | "value, but got num_samples={}".format(self.num_samples) 36 | ) 37 | 38 | def __len__(self): 39 | return int(1 / self.sample_rate) 40 | 41 | def __iter__(self): 42 | num_batches = int(1 / self.sample_rate) 43 | while num_batches > 0: 44 | mask = ( 45 | torch.rand(self.num_samples, generator=self.generator) 46 | < self.sample_rate 47 | ) 48 | indices = mask.nonzero(as_tuple=False).reshape(-1).tolist() 49 | if len(indices) != 0: 50 | # We only output non-empty list of indices, otherwise the dataloader is unhappy 51 | # This is compensated by the privacy engine 52 | yield indices 53 | num_batches -= 1 54 | 55 | 56 | class DistributedPoissonBatchSampler(Sampler): 57 | """ 58 | Distributed batch sampler. 59 | 60 | Each batch is sampled as follows: 61 | 1. Shuffle the dataset (enabled by default) 62 | 2. Split the dataset among the replicas into chunks of equal size 63 | (plus or minus one sample) 64 | 3. Each replica selects each sample of its chunk independently 65 | with probability `sample_rate` 66 | 4. Each replica ouputs the selected samples, which form a local batch 67 | 68 | The sum of the lengths of the local batches follows a Poisson distribution. 69 | In particular, the expected length of each local batch is: 70 | `sample_rate * total_size / num_replicas` 71 | """ 72 | 73 | def __init__( 74 | self, 75 | total_size: int, 76 | sample_rate: float, 77 | num_replicas: Optional[int] = None, 78 | rank: Optional[int] = None, 79 | shuffle: bool = True, 80 | seed: int = 0, 81 | generator=None, 82 | ): 83 | self.total_size = total_size 84 | self.sample_rate = sample_rate 85 | self.generator = generator 86 | self.num_replicas = num_replicas 87 | self.rank = rank 88 | self.epoch = 0 89 | self.shuffle = shuffle 90 | self.seed = seed 91 | if self.generator is None: 92 | generator = torch.Generator() 93 | generator.manual_seed( 94 | int(torch.empty((), dtype=torch.int64).random_().item()) 95 | ) 96 | 97 | if self.total_size <= 0: 98 | raise ValueError( 99 | "total_size should be a positive integer " 100 | "value, but got total_size={}".format(self.total_size) 101 | ) 102 | 103 | # Size of the local dataset specific to the current replica 104 | self.num_samples = self.total_size // self.num_replicas 105 | if self.rank < self.total_size % self.num_replicas: 106 | # The first replicas get an extra datapoint if necessary (balanced) 107 | self.num_samples += 1 108 | 109 | # Number of batches: same as non-distributed Poisson sampling, but each batch is smaller 110 | self.num_batches = int(1 / self.sample_rate) 111 | 112 | def __len__(self) -> int: 113 | return self.num_batches 114 | 115 | def __iter__(self): 116 | if self.shuffle: 117 | # deterministically shuffle based on epoch and seed 118 | g = torch.Generator() 119 | g.manual_seed(self.seed + self.epoch) 120 | indices = torch.randperm(self.total_size, generator=g) # type: ignore 121 | else: 122 | indices = torch.arange(self.total_size) # type: ignore 123 | 124 | # Subset of the dataset assigned to this replica 125 | # NOTE: the first replicas might have 1 more sample. 126 | # (Different from the regular distributed loader that pads with more samples) 127 | indices = indices[self.rank : self.total_size : self.num_replicas] 128 | assert len(indices) == self.num_samples 129 | 130 | # Now, select a batch with Poisson subsampling 131 | for _ in range(self.num_batches): 132 | mask = ( 133 | torch.rand(self.num_samples, generator=self.generator) 134 | < self.sample_rate 135 | ) 136 | selected_examples = mask.nonzero(as_tuple=False).reshape(-1) 137 | if len(selected_examples) > 0: 138 | yield indices[selected_examples] 139 | 140 | def __len__(self) -> int: 141 | """ 142 | Expected number of batches. 143 | """ 144 | return self.num_batches 145 | 146 | def set_epoch(self, epoch: int) -> None: 147 | r""" 148 | Sets the epoch for this sampler. When :attr:`shuffle=True`, this ensures all replicas 149 | use a different random ordering for each epoch. Otherwise, the next iteration of this 150 | sampler will yield the same ordering. 151 | 152 | Args: 153 | epoch (int): Epoch number. 154 | """ 155 | self.epoch = epoch 156 | -------------------------------------------------------------------------------- /website/static/css/custom.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 3 | */ 4 | 5 | @import url('https://fonts.googleapis.com/css?family=Montserrat'); 6 | @import url('https://fonts.googleapis.com/css?family=IBM+Plex+Mono'); 7 | 8 | html body { 9 | font-family: 'Montserrat', sans-serif; 10 | overflow-x: hidden; 11 | } 12 | 13 | .navSearchWrapper { 14 | display: none; 15 | } 16 | 17 | .fixedHeaderContainer { 18 | background-color: #ffffff; 19 | border-bottom: 1px solid #e2e2e2; 20 | } 21 | 22 | .fixedHeaderContainer header .headerTitleWithLogo { 23 | display: block; 24 | color: #222222; 25 | } 26 | 27 | .navigationSlider .slidingNav ul { 28 | background: #ffffff; 29 | } 30 | 31 | .navigationSlider .slidingNav ul li a { 32 | color: #222222; 33 | background-color: #ffffff; 34 | } 35 | 36 | .navigationSlider .slidingNav ul li a:hover, 37 | .navigationSlider .slidingNav ul li a:focus { 38 | color: #f15a24; 39 | background-color: inherit; 40 | } 41 | 42 | .navigationSlider .slidingNav ul li.siteNavItemActive > a, 43 | .navigationSlider .slidingNav ul li.siteNavGroupActive > a { 44 | color: #222222; 45 | background-color: inherit; 46 | } 47 | 48 | .homeContainer { 49 | background-image: linear-gradient(to top left, #E6E6E6, white); 50 | } 51 | 52 | .productTitle { 53 | color: #ffffff; 54 | font-family: FreightSans, Helvetica Neue, Helvetica, Arial, sans-serif; 55 | font-size: 48px; 56 | font-weight: 400; 57 | } 58 | 59 | .splashLogo { 60 | display: block; 61 | margin: 0 auto; 62 | height: 380px; 63 | } 64 | 65 | .primaryLogoImage { 66 | height: 100%; 67 | width: 100%; 68 | } 69 | 70 | .projectTitle { 71 | color: black; 72 | /*font-variant: small-caps;*/ 73 | font-weight: 300; 74 | } 75 | 76 | .promoSection .button { 77 | border: 1px solid black; 78 | color: black; 79 | } 80 | 81 | .promoSection .button:hover { 82 | background-color: inherit; 83 | border: 1px solid #f15a24; 84 | color: #f15a24; 85 | } 86 | 87 | .landingPage { 88 | padding: 0px; 89 | } 90 | 91 | div.productShowcaseSection { 92 | padding-top: 40px; 93 | color: #6c6c6c; 94 | } 95 | 96 | .productShowcaseSection > h2 { 97 | font-variant: small-caps; 98 | font-weight: 360; 99 | margin: 0px; 100 | padding: 0px; 101 | color: #222222; 102 | } 103 | 104 | .productShowcaseSection p { 105 | font-weight: 360; 106 | } 107 | 108 | .productShowcaseSection div.container { 109 | padding: 40px 0px; 110 | } 111 | 112 | .productShowcaseSection div.blockImage { 113 | height: 80px; 114 | } 115 | 116 | .productShowcaseSection li { 117 | padding: 10px 0; 118 | } 119 | 120 | .productShowcaseSection pre { 121 | margin: 10px 0; 122 | } 123 | 124 | .productShowcaseSection code { 125 | background: #fff; 126 | } 127 | 128 | .container .wrapper .alignCenter h2 { 129 | color: #222222; 130 | } 131 | 132 | div#quickstart { 133 | background: #efefef; 134 | } 135 | 136 | div#quickstart ol { 137 | margin-bottom: 0px; 138 | } 139 | 140 | .nav-footer { 141 | background-color: #222222; 142 | } 143 | 144 | .nav-footer .sitemap a { 145 | color: #efefef; 146 | } 147 | 148 | .nav-footer .sitemap a:hover { 149 | color: #f15a24; 150 | } 151 | 152 | a, 153 | p a { 154 | color: #1c60f7; 155 | } 156 | 157 | a:hover, 158 | p a:hover { 159 | color: #1c60f7; 160 | } 161 | 162 | /* Style docs */ 163 | .toc .toggleNav .navGroup .navGroupCategoryTitle { 164 | color: #222222; 165 | } 166 | 167 | .toc .toggleNav ul li a { 168 | color: #6c6c6c; 169 | } 170 | 171 | .toc .toggleNav ul li a:hover { 172 | color: #f15a24; 173 | } 174 | 175 | .toc .toggleNav .navGroup .navListItemActive a { 176 | color: #1c60f7; 177 | } 178 | 179 | .mainContainer .wrapper .post .postHeaderTitle { 180 | color: #222222; 181 | } 182 | 183 | .mainContainer .wrapper .post h1, 184 | .mainContainer .wrapper .post h2, 185 | .mainContainer .wrapper .post h3 { 186 | color: #222222; 187 | } 188 | 189 | .mainContainer .wrapper .post { 190 | color: #6c6c6c; 191 | } 192 | 193 | .mainContainer .wrapper .post strong { 194 | color: #222222; 195 | } 196 | 197 | a.edit-page-link { 198 | color: #1c60f7; 199 | border: 1px solid #1c60f7; 200 | } 201 | 202 | a.edit-page-link:hover { 203 | color: #f15a24; 204 | border: 1px solid #f15a24; 205 | background-color: inherit; 206 | } 207 | 208 | a.docs-next, 209 | a.docs-prev { 210 | color: #1c60f7; 211 | border: 1px solid #1c60f7; 212 | } 213 | 214 | a.docs-next:hover, 215 | a.docs-prev:hover { 216 | color: #f15a24; 217 | border: 1px solid #f15a24; 218 | background-color: inherit; 219 | } 220 | 221 | /* Style tutorials */ 222 | .tutorialBody { 223 | margin-top: -20px; 224 | color: #6c6c6c; 225 | } 226 | 227 | .tutorialBody h1 { 228 | margin: 0px; 229 | } 230 | 231 | .tutorialBody h1, 232 | .tutorialBody h2, 233 | .tutorialBody h3 { 234 | color: #222222; 235 | } 236 | 237 | .tutorialBody pre { 238 | font-family: 'IBM Plex Mono', monospace; 239 | font-size: 14px; 240 | margin: 0px; 241 | } 242 | 243 | .tutorialBody .input_prompt, 244 | .tutorialBody .output_prompt { 245 | color: darkred; 246 | font-size: 12px; 247 | } 248 | 249 | .tutorialBody .highlight { 250 | background: #f3f4f7; 251 | padding: 10px 20px; 252 | border: lightgray 1px solid; 253 | border-radius: 3px; 254 | } 255 | 256 | .tutorialBody .cell { 257 | margin: 20px; 258 | } 259 | 260 | .tutorialBody .output_stderr { 261 | background-color: #fdede9; 262 | } 263 | 264 | .tutorialBody .anchor-link { 265 | color: lightgray; 266 | } 267 | 268 | .tutorialButtonWrapper { 269 | margin: 20px; 270 | } 271 | 272 | .tutorialButton { 273 | color: #1c60f7; 274 | border: 1px solid #1c60f7; 275 | } 276 | 277 | .tutorialButton svg { 278 | height: 15px; 279 | margin-right: 5px; 280 | } 281 | 282 | .tutorialButton:hover { 283 | color: #f15a24; 284 | border: 1px solid #f15a24; 285 | background-color: inherit; 286 | } 287 | 288 | .wrapper { 289 | max-width: 1400px; 290 | } 291 | 292 | @media only screen and (min-device-width: 360px) and (max-device-width: 736px) { 293 | } 294 | 295 | @media only screen and (min-width: 1024px) { 296 | } 297 | 298 | @media only screen and (max-width: 1023px) { 299 | } 300 | 301 | @media only screen and (min-width: 1400px) { 302 | } 303 | 304 | @media only screen and (min-width: 1500px) { 305 | } 306 | -------------------------------------------------------------------------------- /opacus/utils/module_modification.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | r""" 4 | This module includes utils for modifying model layers, replacing layers etc. 5 | """ 6 | from typing import Callable, Type 7 | 8 | from torch import nn 9 | 10 | 11 | def _replace_child( 12 | root: nn.Module, child_name: str, converter: Callable[[nn.Module], nn.Module] 13 | ) -> None: 14 | """ 15 | Converts a sub-module to a new module given a helper 16 | function, the root module and a string representing 17 | the name of the submodule to be replaced. 18 | 19 | Args: 20 | root: Root module whose sub module must be replaced. 21 | child_name: Name of submodule that must be replaced. 22 | converter: Function or a lambda that takes a module 23 | (the submodule to be replaced) and returns its 24 | replacement. 25 | """ 26 | # find the immediate parent 27 | parent = root 28 | nameList = child_name.split(".") 29 | for name in nameList[:-1]: 30 | parent = parent._modules[name] 31 | # set to identity 32 | parent._modules[nameList[-1]] = converter(parent._modules[nameList[-1]]) 33 | 34 | 35 | def replace_all_modules( 36 | root: nn.Module, 37 | target_class: Type[nn.Module], 38 | converter: Callable[[nn.Module], nn.Module], 39 | ) -> nn.Module: 40 | """ 41 | Converts all the submodules (of root) that have the same 42 | type as target_class, given a converter, a module root, 43 | and a target class type. 44 | 45 | This method is useful for replacing modules that are not 46 | supported by the Privacy Engine. 47 | 48 | Args: 49 | root: Model instance, potentially with sub-modules 50 | target_class: Target class that needs to be replaced. 51 | converter: Function or a lambda that converts an instance 52 | of a given target_class to another nn.Module. 53 | 54 | Returns: 55 | Module with all the target_class types replaced using the 56 | converter. root is modified and is equal to the return value. 57 | 58 | Example: 59 | >>> from torchvision.models import resnet18 60 | >>> from torch import nn 61 | >>> model = resnet18() 62 | >>> print(model.layer1[0].bn1) 63 | BatchNorm2d(64, eps=1e-05, ... 64 | >>> model = replace_all_modules(model, nn.BatchNorm2d, lambda _: nn.Identity()) 65 | >>> print(model.layer1[0].bn1) 66 | Identity() 67 | """ 68 | # base case 69 | if isinstance(root, target_class): 70 | return converter(root) 71 | 72 | for name, obj in root.named_modules(): 73 | if isinstance(obj, target_class): 74 | _replace_child(root, name, converter) 75 | return root 76 | 77 | 78 | def _batchnorm_to_instancenorm(module: nn.modules.batchnorm._BatchNorm) -> nn.Module: 79 | """ 80 | Converts a BatchNorm module to the corresponding InstanceNorm module 81 | 82 | Args: 83 | module: BatchNorm module to be replaced 84 | 85 | Returns: 86 | InstanceNorm module that can replace the BatchNorm module provided 87 | """ 88 | 89 | def matchDim(): 90 | if isinstance(module, nn.BatchNorm1d): 91 | return nn.InstanceNorm1d 92 | elif isinstance(module, nn.BatchNorm2d): 93 | return nn.InstanceNorm2d 94 | elif isinstance(module, nn.BatchNorm3d): 95 | return nn.InstanceNorm3d 96 | 97 | return matchDim()(module.num_features) 98 | 99 | 100 | def _batchnorm_to_groupnorm(module: nn.modules.batchnorm._BatchNorm) -> nn.Module: 101 | """ 102 | Converts a BatchNorm ``module`` to GroupNorm module. 103 | This is a helper function. 104 | 105 | Args: 106 | module: BatchNorm module to be replaced 107 | 108 | Returns: 109 | GroupNorm module that can replace the BatchNorm module provided 110 | 111 | Notes: 112 | A default value of 32 is chosen for the number of groups based on the 113 | paper *Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour* 114 | https://arxiv.org/pdf/1706.02677.pdf 115 | """ 116 | return nn.GroupNorm(min(32, module.num_features), module.num_features, affine=True) 117 | 118 | 119 | def nullify_batchnorm_modules(root: nn.Module) -> nn.Module: 120 | """ 121 | Replaces all the BatchNorm submodules (e.g. :class:`torch.nn.BatchNorm1d`, 122 | :class:`torch.nn.BatchNorm2d` etc.) in ``root`` with :class:`torch.nn.Identity`. 123 | 124 | Args: 125 | root: Module for which to replace BatchNorm submodules. 126 | 127 | Returns: 128 | Module with all the BatchNorm sub modules replaced with 129 | Identity. ``root`` is modified and is equal to the return value. 130 | 131 | Notes: 132 | Most of the times replacing a BatchNorm module with Identity 133 | will heavily affect convergence of the model. 134 | """ 135 | return replace_all_modules( 136 | root, nn.modules.batchnorm._BatchNorm, lambda _: nn.Identity() 137 | ) 138 | 139 | 140 | def convert_batchnorm_modules( 141 | model: nn.Module, 142 | converter: Callable[ 143 | [nn.modules.batchnorm._BatchNorm], nn.Module 144 | ] = _batchnorm_to_groupnorm, 145 | ) -> nn.Module: 146 | """ 147 | Converts all BatchNorm modules to another module 148 | (defaults to GroupNorm) that is privacy compliant. 149 | 150 | Args: 151 | model: Module instance, potentially with sub-modules 152 | converter: Function or a lambda that converts an instance of a 153 | Batchnorm to another nn.Module. 154 | 155 | Returns: 156 | Model with all the BatchNorm types replaced by another operation 157 | by using the provided converter, defaulting to GroupNorm if one 158 | isn't provided. 159 | 160 | Example: 161 | >>> from torchvision.models import resnet50 162 | >>> from torch import nn 163 | >>> model = resnet50() 164 | >>> print(model.layer1[0].bn1) 165 | BatchNorm2d module details 166 | >>> model = convert_batchnorm_modules(model) 167 | >>> print(model.layer1[0].bn1) 168 | GroupNorm module details 169 | """ 170 | return replace_all_modules(model, nn.modules.batchnorm._BatchNorm, converter) 171 | -------------------------------------------------------------------------------- /website/static/img/opacus_logo_vertical.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /opacus/utils/tensor_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved 3 | """ 4 | Utils for generating stats from torch tensors. 5 | """ 6 | from typing import Iterator, List, Tuple, Union 7 | 8 | import numpy as np 9 | import torch 10 | from torch.functional import F 11 | 12 | 13 | def calc_sample_norms( 14 | named_params: Iterator[Tuple[str, torch.Tensor]], flat: bool = True 15 | ) -> List[torch.Tensor]: 16 | r""" 17 | Calculates the norm of the given tensors for each sample. 18 | 19 | This function calculates the overall norm of the given tensors for each sample, 20 | assuming the each batch's dim is zero. 21 | 22 | Args: 23 | named_params: An iterator of tuples with name being a 24 | string and param being a tensor of shape ``[B, ...]`` where ``B`` 25 | is the size of the batch and is the 0th dimension. 26 | flat: A flag, when set to `True` returns a flat norm over all 27 | layers norms 28 | 29 | Example: 30 | >>> t1 = torch.rand((2, 5)) 31 | >>> t2 = torch.rand((2, 5)) 32 | >>> calc_sample_norms([("1", t1), ("2", t2)]) 33 | [tensor([1.5117, 1.0618])] 34 | 35 | Returns: 36 | A list of tensor norms where length of the list is the number of layers 37 | """ 38 | norms = [param.view(len(param), -1).norm(2, dim=-1) for name, param in named_params] 39 | # calc norm over all layer norms if flat = True 40 | if flat: 41 | norms = [torch.stack(norms, dim=0).norm(2, dim=0)] 42 | return norms 43 | 44 | 45 | def calc_sample_norms_one_layer(param: torch.Tensor) -> torch.Tensor: 46 | r""" 47 | Calculates the norm of the given tensor (a single parameter) for each sample. 48 | 49 | This function calculates the overall norm of the given tensor for each sample, 50 | assuming the each batch's dim is zero. 51 | 52 | It is equivalent to: 53 | ``` 54 | calc_sample_norms(named_params=((None, param),))[0] 55 | ``` 56 | 57 | Args: 58 | param: A tensor of shape ``[B, ...]`` where ``B`` 59 | is the size of the batch and is the 0th dimension. 60 | 61 | Example: 62 | >>> t1 = torch.rand((2, 5)) 63 | >>> calc_sample_norms_one_layer(t1) 64 | tensor([1.4757, 0.8128]) 65 | 66 | Returns: 67 | A tensor of norms 68 | """ 69 | norms = param.view(len(param), -1).norm(2, dim=-1) 70 | return norms 71 | 72 | 73 | def sum_over_all_but_batch_and_last_n( 74 | tensor: torch.Tensor, n_dims: int 75 | ) -> torch.Tensor: 76 | r""" 77 | Calculates the sum over all dimensions, except the first 78 | (batch dimension), and excluding the last n_dims. 79 | 80 | This function will ignore the first dimension and it will 81 | not aggregate over the last n_dims dimensions. 82 | 83 | Args: 84 | tensor: An input tensor of shape ``(B, ..., X[n_dims-1])``. 85 | n_dims: Number of dimensions to keep. 86 | 87 | Example: 88 | >>> tensor = torch.ones(1, 2, 3, 4, 5) 89 | >>> sum_over_all_but_batch_and_last_n(tensor, n_dims=2).shape 90 | torch.Size([1, 4, 5]) 91 | 92 | Returns: 93 | A tensor of shape ``(B, ..., X[n_dims-1])`` 94 | """ 95 | if tensor.dim() == n_dims + 1: 96 | return tensor 97 | else: 98 | dims = list(range(1, tensor.dim() - n_dims)) 99 | return tensor.sum(dim=dims) 100 | 101 | 102 | def unfold3d( 103 | tensor: torch.Tensor, 104 | kernel_size: Union[int, Tuple[int, int, int]], 105 | padding: Union[int, Tuple[int, int, int]] = 0, 106 | stride: Union[int, Tuple[int, int, int]] = 1, 107 | dilation: Union[int, Tuple[int, int, int]] = 1, 108 | ): 109 | r""" 110 | Extracts sliding local blocks from an batched input tensor. 111 | 112 | :class:`torch.nn.Unfold` only supports 4D inputs (batched image-like tensors). 113 | This method implements the same action for 5D inputs 114 | 115 | Args: 116 | tensor: An input tensor of shape ``(B, C, D, H, W)``. 117 | kernel_size: the size of the sliding blocks 118 | padding: implicit zero padding to be added on both sides of input 119 | stride: the stride of the sliding blocks in the input spatial dimensions 120 | dilation: the spacing between the kernel points. 121 | 122 | Example: 123 | >>> B, C, D, H, W = 3, 4, 5, 6, 7 124 | >>> tensor = torch.arange(1,B*C*D*H*W+1.).view(B,C,D,H,W) 125 | >>> unfold3d(tensor, kernel_size=2, padding=0, stride=1).shape 126 | torch.Size([3, 32, 120]) 127 | 128 | Returns: 129 | A tensor of shape ``(B, C * np.product(kernel_size), L)``, where L - output spatial dimensions. 130 | See :class:`torch.nn.Unfold` for more details 131 | """ 132 | 133 | if len(tensor.shape) != 5: 134 | raise ValueError( 135 | f"Input tensor must be of the shape [B, C, D, H, W]. Got{tensor.shape}" 136 | ) 137 | 138 | if isinstance(kernel_size, int): 139 | kernel_size = (kernel_size, kernel_size, kernel_size) 140 | 141 | if isinstance(padding, int): 142 | padding = (padding, padding, padding) 143 | 144 | if isinstance(stride, int): 145 | stride = (stride, stride, stride) 146 | 147 | if isinstance(dilation, int): 148 | dilation = (dilation, dilation, dilation) 149 | 150 | if dilation != (1, 1, 1): 151 | raise NotImplementedError(f"dilation={dilation} not supported. We'd love a PR!") 152 | 153 | batch_size, channels, _, _, _ = tensor.shape 154 | 155 | # Input shape: (B, C, D, H, W) 156 | tensor = F.pad( 157 | tensor, (padding[2], padding[2], padding[1], padding[1], padding[0], padding[0]) 158 | ) 159 | # Output shape: (B, C, D+2*padding[2], H+2*padding[1], W+2*padding[0]) 160 | 161 | tensor = tensor.unfold(dimension=2, size=kernel_size[0], step=stride[0]) 162 | tensor = tensor.unfold(dimension=3, size=kernel_size[1], step=stride[1]) 163 | tensor = tensor.unfold(dimension=4, size=kernel_size[2], step=stride[2]) 164 | # Output shape: (B, C, D_out, H_out, W_out, kernel_size[0], kernel_size[1], kernel_size[2]) 165 | # For D_out, H_out, W_out definitions see :class:`torch.nn.Unfold` 166 | 167 | tensor = tensor.permute(0, 2, 3, 4, 1, 5, 6, 7) 168 | # Output shape: (B, D_out, H_out, W_out, C, kernel_size[0], kernel_size[1], kernel_size[2]) 169 | 170 | tensor = tensor.reshape(batch_size, -1, channels * np.prod(kernel_size)).transpose( 171 | 1, 2 172 | ) 173 | # Output shape: (B, D_out * H_out * W_out, C * kernel_size[0] * kernel_size[1] * kernel_size[2] 174 | 175 | return tensor 176 | --------------------------------------------------------------------------------