├── benchmarks ├── __init__.py ├── config.json ├── generate_report.py ├── check_threshold.py └── tests │ └── helpers.py ├── opacus ├── utils │ ├── __init__.py │ ├── adaptive_clipping │ │ ├── __init__.py │ │ └── README.md │ └── fsdp_utils.py ├── accountants │ ├── analysis │ │ ├── __init__.py │ │ └── prv │ │ │ ├── __init__.py │ │ │ └── compose.py │ ├── __init__.py │ ├── registry.py │ ├── gdp.py │ ├── utils.py │ └── rdp.py ├── tests │ ├── __init__.py │ ├── dp_layers │ │ └── __init__.py │ ├── schedulers │ │ ├── __init__.py │ │ ├── noise_scheduler_test.py │ │ └── grad_clip_scheduler_test.py │ ├── validators │ │ ├── __init__.py │ │ ├── gru_test.py │ │ ├── lstm_test.py │ │ ├── multihead_attention_test.py │ │ ├── batch_norm_test.py │ │ └── instance_norm_test.py │ ├── grad_samples │ │ ├── __init__.py │ │ ├── instance_norm1d_test.py │ │ ├── instance_norm2d_test.py │ │ ├── instance_norm3d_test.py │ │ ├── sequence_bias_test.py │ │ ├── embedding_test.py │ │ ├── group_norm_test.py │ │ ├── linear_test.py │ │ ├── embedding_bag_test.py │ │ ├── conv1d_test.py │ │ ├── layer_norm_test.py │ │ ├── rms_norm_test.py │ │ ├── conv3d_test.py │ │ ├── dp_rnn_test.py │ │ └── dp_multihead_attention_test.py │ ├── dpdataloader_test.py │ ├── utils.py │ └── poisson_test.py ├── version.py ├── layers │ └── __init__.py ├── __init__.py ├── schedulers │ └── __init__.py ├── optimizers │ ├── utils.py │ ├── perlayeroptimizer.py │ ├── ddpoptimizer.py │ ├── ddpoptimizer_fast_gradient_clipping.py │ └── __init__.py ├── validators │ ├── __init__.py │ ├── lstm.py │ ├── gru.py │ ├── instance_norm.py │ ├── multihead_attention.py │ ├── errors.py │ └── utils.py ├── grad_sample │ ├── dp_multihead_attention.py │ ├── gsm_no_op.py │ ├── rms_norm.py │ ├── group_norm.py │ ├── dp_rnn.py │ ├── layer_norm.py │ ├── gsm_exp_weights.py │ ├── instance_norm.py │ └── __init__.py ├── distributed.py ├── lightning.py └── README.md ├── website ├── static │ ├── .nojekyll │ ├── img │ │ ├── oss_logo.png │ │ ├── expanding_arrows.svg │ │ ├── pytorch_logo.svg │ │ └── modular.svg │ └── css │ │ └── code_block_buttons.css ├── sidebars.json ├── requirements.txt ├── sphinx │ ├── source │ │ ├── dp_rnn.rst │ │ ├── scripts.rst │ │ ├── layers.rst │ │ ├── distributed.rst │ │ ├── data_loader.rst │ │ ├── noise_scheduler.rst │ │ ├── accounting │ │ │ ├── rdp.rst │ │ │ ├── utils.rst │ │ │ ├── gdp.rst │ │ │ ├── iaccountant.rst │ │ │ └── accounting.rst │ │ ├── utils │ │ │ ├── module_utils.rst │ │ │ ├── tensor_utils.rst │ │ │ ├── packed_sequences.rst │ │ │ ├── uniform_sampler.rst │ │ │ ├── fast_gradient_clipping_utils.rst │ │ │ └── utils.rst │ │ ├── optim │ │ │ ├── dp_optimizer.rst │ │ │ ├── dp_ddp_optimizer.rst │ │ │ ├── dp_per_layer_optimizer.rst │ │ │ ├── dp_ddp_per_layer_optimizer.rst │ │ │ ├── dp_optimizer_fast_gradient_clipping.rst │ │ │ ├── dp_ddp_optimizer_fast_gradient_clipping.rst │ │ │ └── optimizers.rst │ │ ├── batch_memory_manager.rst │ │ ├── grad_sample_module.rst │ │ ├── compute_dp_sgd_privacy.rst │ │ ├── privacy_engine.rst │ │ ├── validator.rst │ │ ├── grad_sample_module_fast_gradient_clipping.rst │ │ ├── dp_multihead_attention.rst │ │ └── index.rst │ ├── Makefile │ └── make.bat ├── package.json ├── tutorials.json ├── core │ ├── TutorialSidebar.js │ └── Tutorial.js └── scripts │ └── parse_sphinx.py ├── MANIFEST.in ├── requirements.txt ├── tutorials ├── img │ ├── BERT.png │ ├── optimizer.png │ └── make_private.png └── README.md ├── docs ├── img │ └── epsilon-delta-dp.png └── introduction.md ├── .github ├── ISSUE_TEMPLATE │ ├── config.yml │ ├── documentation.md │ ├── feature-request.md │ └── bug-report.md ├── workflows │ ├── test-deploy.yml │ ├── python-publish.yml │ ├── test-installation.yml │ └── deploy.yml └── PULL_REQUEST_TEMPLATE.md ├── examples ├── README.md ├── __init__.py ├── char-lstm_README.md ├── imdb_README.md ├── mnist_README.md └── scripts │ ├── make_small_imagenet_N_classes.sh │ └── make_small_imagenet_sampled.sh ├── dev_requirements.txt ├── scripts ├── pytorch_install.ps1 ├── pytorch_install.sh └── install_via_pip.sh ├── research ├── README.md └── disk_optimizer │ ├── KFprivacy_engine.py │ ├── ReadMe.md │ └── optimizers │ └── KFperlayeroptimizer.py ├── .gitignore ├── conftest.py └── setup.py /benchmarks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /opacus/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /website/static/.nojekyll: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /opacus/accountants/analysis/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | include dev_requirements.txt 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy>=1.15 2 | torch>=2.6.0 3 | scipy>=1.2 4 | opt-einsum>=3.3.0 5 | -------------------------------------------------------------------------------- /website/sidebars.json: -------------------------------------------------------------------------------- 1 | { 2 | "docs": { 3 | "About": ["introduction", "faq"] 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /tutorials/img/BERT.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meta-pytorch/opacus/HEAD/tutorials/img/BERT.png -------------------------------------------------------------------------------- /website/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx 2 | sphinx-autodoc-typehints 3 | bs4 4 | nbformat 5 | nbconvert 6 | -------------------------------------------------------------------------------- /docs/img/epsilon-delta-dp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meta-pytorch/opacus/HEAD/docs/img/epsilon-delta-dp.png -------------------------------------------------------------------------------- /tutorials/img/optimizer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meta-pytorch/opacus/HEAD/tutorials/img/optimizer.png -------------------------------------------------------------------------------- /website/sphinx/source/dp_rnn.rst: -------------------------------------------------------------------------------- 1 | DPRNN 2 | ======= 3 | .. automodule:: opacus.layers.dp_rnn 4 | :members: 5 | -------------------------------------------------------------------------------- /website/sphinx/source/scripts.rst: -------------------------------------------------------------------------------- 1 | Scripts 2 | ======= 3 | .. toctree:: 4 | 5 | compute_dp_sgd_privacy 6 | -------------------------------------------------------------------------------- /tutorials/img/make_private.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meta-pytorch/opacus/HEAD/tutorials/img/make_private.png -------------------------------------------------------------------------------- /website/static/img/oss_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meta-pytorch/opacus/HEAD/website/static/img/oss_logo.png -------------------------------------------------------------------------------- /website/sphinx/source/layers.rst: -------------------------------------------------------------------------------- 1 | DP Layers 2 | ========= 3 | .. toctree:: 4 | 5 | dp_multihead_attention 6 | dp_rnn 7 | -------------------------------------------------------------------------------- /website/sphinx/source/distributed.rst: -------------------------------------------------------------------------------- 1 | Distributed 2 | =============== 3 | 4 | .. automodule:: opacus.distributed 5 | :members: -------------------------------------------------------------------------------- /website/sphinx/source/data_loader.rst: -------------------------------------------------------------------------------- 1 | DP Data Loader 2 | ============== 3 | 4 | .. automodule:: opacus.data_loader 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/noise_scheduler.rst: -------------------------------------------------------------------------------- 1 | Noise Scheduler 2 | =============== 3 | 4 | .. automodule:: opacus.scheduler 5 | :members: -------------------------------------------------------------------------------- /website/sphinx/source/accounting/rdp.rst: -------------------------------------------------------------------------------- 1 | RDPAccountant 2 | ================= 3 | 4 | .. automodule:: opacus.accountants.rdp 5 | :members: -------------------------------------------------------------------------------- /website/sphinx/source/accounting/utils.rst: -------------------------------------------------------------------------------- 1 | Utils 2 | ================= 3 | 4 | .. automodule:: opacus.accountants.utils 5 | :members: 6 | 7 | -------------------------------------------------------------------------------- /website/sphinx/source/utils/module_utils.rst: -------------------------------------------------------------------------------- 1 | Module Utils 2 | ============= 3 | 4 | .. automodule:: opacus.utils.module_utils 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/utils/tensor_utils.rst: -------------------------------------------------------------------------------- 1 | Tensor Utils 2 | ============= 3 | 4 | .. automodule:: opacus.utils.tensor_utils 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/accounting/gdp.rst: -------------------------------------------------------------------------------- 1 | GaussianAccountant 2 | ================== 3 | 4 | .. automodule:: opacus.accountants.gdp 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/optim/dp_optimizer.rst: -------------------------------------------------------------------------------- 1 | DPOptimizer 2 | ============== 3 | 4 | .. automodule:: opacus.optimizers.optimizer 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/batch_memory_manager.rst: -------------------------------------------------------------------------------- 1 | Batch Memory Manager 2 | =============== 3 | 4 | .. automodule:: opacus.utils.batch_memory_manager 5 | :members: -------------------------------------------------------------------------------- /website/sphinx/source/utils/packed_sequences.rst: -------------------------------------------------------------------------------- 1 | Packed Sequences 2 | ================ 3 | 4 | .. automodule:: opacus.utils.packed_sequences 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/utils/uniform_sampler.rst: -------------------------------------------------------------------------------- 1 | Uniform Sampler 2 | ================ 3 | 4 | .. automodule:: opacus.utils.uniform_sampler 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/accounting/iaccountant.rst: -------------------------------------------------------------------------------- 1 | IAccountant 2 | ================= 3 | 4 | .. automodule:: opacus.accountants.accountant 5 | :members: 6 | 7 | -------------------------------------------------------------------------------- /website/sphinx/source/grad_sample_module.rst: -------------------------------------------------------------------------------- 1 | GradSampleModule 2 | ================ 3 | 4 | .. automodule:: opacus.grad_sample.grad_sample_module 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/accounting/accounting.rst: -------------------------------------------------------------------------------- 1 | Privacy Accounting 2 | ================== 3 | 4 | .. toctree:: 5 | 6 | iaccountant 7 | rdp 8 | gdp 9 | utils -------------------------------------------------------------------------------- /website/sphinx/source/optim/dp_ddp_optimizer.rst: -------------------------------------------------------------------------------- 1 | DistributedDPOptimizer 2 | ======================= 3 | 4 | .. automodule:: opacus.optimizers.ddpoptimizer 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/compute_dp_sgd_privacy.rst: -------------------------------------------------------------------------------- 1 | Compute DP-SGD Privacy 2 | ====================== 3 | 4 | .. automodule:: opacus.scripts.compute_dp_sgd_privacy 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/optim/dp_per_layer_optimizer.rst: -------------------------------------------------------------------------------- 1 | DPPerLayerOptimizer 2 | ==================== 3 | 4 | .. automodule:: opacus.optimizers.perlayeroptimizer 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/privacy_engine.rst: -------------------------------------------------------------------------------- 1 | Privacy Engine 2 | ============== 3 | 4 | .. automodule:: opacus.privacy_engine 5 | 6 | .. autoclass:: PrivacyEngine 7 | :members: 8 | -------------------------------------------------------------------------------- /website/sphinx/source/utils/fast_gradient_clipping_utils.rst: -------------------------------------------------------------------------------- 1 | Fast Gradient Clipping Utils 2 | ============= 3 | 4 | .. automodule:: opacus.utils.fast_gradient_clipping_utils 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/optim/dp_ddp_per_layer_optimizer.rst: -------------------------------------------------------------------------------- 1 | DistributedPerLayerOptimizer 2 | ============================= 3 | 4 | .. automodule:: opacus.optimizers.ddp_perlayeroptimizer 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/optim/dp_optimizer_fast_gradient_clipping.rst: -------------------------------------------------------------------------------- 1 | DPOptimizerFastGradientClipping 2 | ============== 3 | 4 | .. automodule:: opacus.optimizers.optimizer_fast_gradient_clipping 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/utils/utils.rst: -------------------------------------------------------------------------------- 1 | Utils 2 | ===== 3 | .. toctree:: 4 | 5 | module_utils 6 | tensor_utils 7 | packed_sequences 8 | uniform_sampler 9 | fast_gradient_clipping_utils 10 | -------------------------------------------------------------------------------- /website/sphinx/source/validator.rst: -------------------------------------------------------------------------------- 1 | ModuleValidator 2 | =============== 3 | 4 | .. automodule:: opacus.validators.module_validator 5 | :members: 6 | 7 | .. automodule:: opacus.validators.utils 8 | :members: -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | contact_links: 3 | - name: Questions 4 | url: https://discuss.pytorch.org/c/opacus/29 5 | about: Ask questions and discuss with other community members 6 | -------------------------------------------------------------------------------- /website/sphinx/source/grad_sample_module_fast_gradient_clipping.rst: -------------------------------------------------------------------------------- 1 | GradSampleModuleFastGradientClipping 2 | ================ 3 | 4 | .. automodule:: opacus.grad_sample.grad_sample_module_fast_gradient_clipping 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/optim/dp_ddp_optimizer_fast_gradient_clipping.rst: -------------------------------------------------------------------------------- 1 | DistributedDPOptimizerFastGradientClipping 2 | ============== 3 | 4 | .. automodule:: opacus.optimizers.ddpoptimizer_fast_gradient_clipping 5 | :members: 6 | -------------------------------------------------------------------------------- /website/sphinx/source/dp_multihead_attention.rst: -------------------------------------------------------------------------------- 1 | DPMultiheadAttention 2 | ======================== 3 | 4 | .. automodule:: opacus.layers.dp_multihead_attention 5 | 6 | .. autoclass:: SequenceBias 7 | :members: 8 | 9 | .. autoclass:: DPMultiheadAttention 10 | :members: 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/documentation.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F4DA Documentation" 3 | about: Report an issue related to https://pytorch.org/docs 4 | 5 | --- 6 | 7 | ## 📚 Documentation 8 | 9 | 10 | -------------------------------------------------------------------------------- /website/sphinx/source/optim/optimizers.rst: -------------------------------------------------------------------------------- 1 | Optimizers 2 | ========== 3 | .. toctree:: 4 | 5 | dp_optimizer 6 | dp_optimizer_fast_gradient_clipping 7 | dp_per_layer_optimizer 8 | dp_ddp_optimizer 9 | dp_ddp_optimizer_fast_gradient_clipping 10 | dp_ddp_per_layer_optimizer 11 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | This folder contains multiple examples to get you started on training differentially private models! 3 | 4 | Note that you may not have all the required packages. You can install opacus's dev version, which will 5 | bring in all the required packages in these examples: 6 | 7 | ``` 8 | pip install opacus[dev] 9 | ``` 10 | -------------------------------------------------------------------------------- /opacus/utils/adaptive_clipping/__init__.py: -------------------------------------------------------------------------------- 1 | from .adaptive_clipping_utils import ( 2 | DPLossFastGradientAdaptiveClipping, 3 | DPTensorFastGradientAdaptiveClipping, 4 | PrivacyEngineAdaptiveClipping, 5 | ) 6 | 7 | 8 | __all__ = [ 9 | "DPTensorFastGradientAdaptiveClipping", 10 | "DPLossFastGradientAdaptiveClipping", 11 | "PrivacyEngineAdaptiveClipping", 12 | ] 13 | -------------------------------------------------------------------------------- /dev_requirements.txt: -------------------------------------------------------------------------------- 1 | torch 2 | torchvision>=0.9.1 3 | tqdm>=4.40 4 | requests>=2.25.1 5 | black 6 | pytest 7 | flake8 8 | sphinx 9 | sphinx-autodoc-typehints 10 | mypy>=0.760 11 | isort>=5.0.0 12 | hypothesis 13 | tensorboard 14 | datasets 15 | transformers 16 | scikit-learn 17 | pytorch-lightning 18 | lightning-bolts 19 | jsonargparse[signatures]>=3.19.3 # required for Lightning CLI 20 | coverage 21 | -------------------------------------------------------------------------------- /tutorials/README.md: -------------------------------------------------------------------------------- 1 | # Tutorials 2 | This folder contains multiple tutorials to get you started on training differentially private models! We recommend "building_text_classifier.ipynb" to experiment with latest Opacus features such as Fast Gradient Clipping, LoRA, and fine-tuning Hugging Face Transformers. 3 | 4 | Note that you may not have all the required packages. You can install opacus's dev version, which will 5 | bring in all the required packages in these tutorials: 6 | 7 | ``` 8 | pip install opacus[dev] 9 | ``` 10 | -------------------------------------------------------------------------------- /opacus/accountants/analysis/prv/__init__.py: -------------------------------------------------------------------------------- 1 | from .compose import compose_heterogeneous 2 | from .domain import Domain, compute_safe_domain_size 3 | from .prvs import ( 4 | DiscretePRV, 5 | PoissonSubsampledGaussianPRV, 6 | TruncatedPrivacyRandomVariable, 7 | discretize, 8 | ) 9 | 10 | 11 | __all__ = [ 12 | "DiscretePRV", 13 | "Domain", 14 | "PoissonSubsampledGaussianPRV", 15 | "TruncatedPrivacyRandomVariable", 16 | "compose_heterogeneous", 17 | "compute_safe_domain_size", 18 | "discretize", 19 | ] 20 | -------------------------------------------------------------------------------- /website/sphinx/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = source 8 | BUILDDIR = build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 20 | -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /opacus/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /opacus/tests/dp_layers/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /opacus/tests/schedulers/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /opacus/tests/validators/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /website/static/img/expanding_arrows.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /opacus/version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | __version__ = "1.5.4" 17 | -------------------------------------------------------------------------------- /website/sphinx/source/index.rst: -------------------------------------------------------------------------------- 1 | :github_url: https://github.com/pytorch/opacus 2 | 3 | Opacus API Reference 4 | ==================== 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | :caption: Contents: 9 | 10 | .. toctree:: 11 | :maxdepth: 2 12 | :caption: API Reference 13 | 14 | privacy_engine 15 | grad_sample_module 16 | grad_sample_module_fast_gradient_clipping 17 | optim/optimizers 18 | data_loader 19 | accounting/accounting 20 | validator 21 | distributed 22 | noise_scheduler 23 | batch_memory_manager 24 | layers 25 | utils/utils 26 | scripts 27 | 28 | Indices and Tables 29 | ================== 30 | 31 | * :ref:`genindex` 32 | * :ref:`modindex` 33 | * :ref:`search` 34 | -------------------------------------------------------------------------------- /.github/workflows/test-deploy.yml: -------------------------------------------------------------------------------- 1 | name: Test Github Pages deployment 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | # Review gh actions docs if you want to further define triggers, paths, etc 8 | # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#on 9 | 10 | jobs: 11 | test-deploy: 12 | name: Test deployment 13 | runs-on: ubuntu-latest 14 | defaults: 15 | run: 16 | working-directory: website 17 | steps: 18 | - uses: actions/checkout@v3 19 | - name: Install dependencies 20 | run: | 21 | python -m pip install --upgrade pip 22 | python -m pip install opacus sphinx sphinx-autodoc-typehints nbsphinx bs4 23 | yarn install 24 | - name: Test build website 25 | run: ./scripts/build_website.sh -b 26 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F680 Feature Request" 3 | about: Submit a proposal/request for a new feature in Opacus 4 | 5 | --- 6 | 7 | ## 🚀 Feature 8 | 9 | 10 | ## Motivation 11 | 12 | 13 | 14 | ## Pitch 15 | 16 | 17 | 18 | ## Alternatives 19 | 20 | 21 | 22 | ## Additional context 23 | 24 | 25 | -------------------------------------------------------------------------------- /benchmarks/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "linear": { 3 | "input_shape": [], 4 | "in_features": 512, 5 | "out_features": 512 6 | }, 7 | "conv": { 8 | "in_channels": 64, 9 | "input_shape": [50, 100], 10 | "out_channels": 64, 11 | "kernel_size": 8 12 | }, 13 | "layernorm": { 14 | "input_shape": [64], 15 | "D": 1 16 | }, 17 | "instancenorm": { 18 | "num_features": 256, 19 | "input_shape": [64], 20 | "affine": true 21 | }, 22 | "groupnorm": { 23 | "input_shape": [], 24 | "num_groups": 16, 25 | "num_channels": 256 26 | }, 27 | "embedding": { 28 | "input_shape": [], 29 | "num_embeddings": 20000, 30 | "embedding_dim": 100 31 | }, 32 | "mha": { 33 | "source_seq_len": 128, 34 | "targ_seq_len": 64, 35 | "embed_dim": 100, 36 | "num_heads": 4 37 | }, 38 | "rnn_base": { 39 | "seq_len": 128, 40 | "input_size": 100, 41 | "hidden_size": 100 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /docs/introduction.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: introduction 3 | title: Introduction 4 | --- 5 | 6 | Opacus is a library that enables training PyTorch models with differential privacy. It supports training with minimal code changes required on the client, has little impact on training performance, and allows the client to online track the privacy budget expended at any given moment. 7 | 8 | Please refer to [this post](https://ai.facebook.com/blog/introducing-opacus-a-high-speed-library-for-training-pytorch-models-with-differential-privacy/) to read more about Opacus. 9 | 10 | ## Target audience 11 | Opacus is aimed at two target audiences: 12 | 13 | 1. ML practitioners will find this to be a gentle introduction to training a model with differential privacy as it requires minimal code changes. 14 | 2. Differential Privacy scientists will find this easy to experiment and tinker with, allowing them to focus on what matters. 15 | -------------------------------------------------------------------------------- /scripts/pytorch_install.ps1: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | [string]$TORCH_VERSION=$args[0] 17 | If ($TORCH_VERSION -eq "2.6.0") { 18 | $TORCHVISION_VERSION="0.21.0" 19 | } 20 | pip install torch==$TORCH_VERSION torchvision==$TORCHVISION_VERSION -f https://download.pytorch.org/whl/torch_stable.html 21 | -------------------------------------------------------------------------------- /website/static/img/pytorch_logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 8 | pytorch_logo 9 | 11 | 13 | 14 | -------------------------------------------------------------------------------- /website/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "scripts": { 3 | "examples": "docusaurus-examples", 4 | "start": "docusaurus-start", 5 | "build": "docusaurus-build", 6 | "publish-gh-pages": "docusaurus-publish", 7 | "write-translations": "docusaurus-write-translations", 8 | "version": "docusaurus-version", 9 | "rename-version": "docusaurus-rename-version" 10 | }, 11 | "devDependencies": { 12 | "docusaurus": "^1.14.7" 13 | }, 14 | "dependencies": { 15 | "@babel/helper-compilation-targets": "^8.0.0-alpha.14", 16 | "bl": "^5.0.0", 17 | "browserslist": "^4.21.4", 18 | "prismjs": "^1.29.0" 19 | }, 20 | "resolutions": { 21 | "trim-newlines": "^4.0.2", 22 | "normalize-url": "^6.1.0", 23 | "highlight.js": "^11.8.0", 24 | "react-dev-utils": "^12.0.0", 25 | "immer": "^10.0.0", 26 | "prismjs": "^1.29.0", 27 | "bl": "^5.0.0", 28 | "glob-parent": "^6.0.2", 29 | "browserslist": "^4.21.4" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /website/sphinx/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /website/tutorials.json: -------------------------------------------------------------------------------- 1 | { 2 | "Using Opacus": [ 3 | { 4 | "id": "building_text_classifier", 5 | "title": "Building text classifier with Fast Gradient Clipping DP-SGD" 6 | }, 7 | { 8 | "id": "building_image_classifier", 9 | "title": "Building image classifier with Differential Privacy" 10 | }, 11 | { 12 | "id": "building_lstm_name_classifier", 13 | "title": "Training a differentially private LSTM model for name classification" 14 | }, 15 | { 16 | "id": "intro_to_advanced_features", 17 | "title": "Deep dive into advanced features of Opacus" 18 | }, 19 | { 20 | "id": "guide_to_module_validator", 21 | "title": "Guide to Module Validator and Fixer" 22 | }, 23 | { 24 | "id": "guide_to_grad_sampler", 25 | "title": "Guide to grad samplers" 26 | }, 27 | { 28 | "id": "ddp_tutorial", 29 | "title": "Training on multiple GPUs with DistributedDataParallel" 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /scripts/pytorch_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -e 17 | TORCH_VERSION=$1 18 | 19 | if [ "$TORCH_VERSION" = "2.6.0" ] 20 | then 21 | TORCHVISION_VERSION="0.21.0" 22 | fi 23 | 24 | pip install torch=="${TORCH_VERSION}" --extra-index-url https://download.pytorch.org/whl/cpu 25 | pip install torchvision==${TORCHVISION_VERSION} --extra-index-url https://download.pytorch.org/whl/cpu 26 | -------------------------------------------------------------------------------- /opacus/layers/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .dp_multihead_attention import DPMultiheadAttention, SequenceBias 17 | from .dp_rnn import DPGRU, DPLSTM, DPRNN 18 | from .param_rename import RenameParamsMixin 19 | 20 | 21 | __all__ = [ 22 | "DPRNN", 23 | "DPGRU", 24 | "DPLSTM", 25 | "DPMultiheadAttention", 26 | "RenameParamsMixin", 27 | "SequenceBias", 28 | ] 29 | -------------------------------------------------------------------------------- /opacus/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from . import utils 17 | from .grad_sample import GradSampleModule, GradSampleModuleFastGradientClipping 18 | from .privacy_engine import PrivacyEngine 19 | from .version import __version__ 20 | 21 | 22 | __all__ = [ 23 | "PrivacyEngine", 24 | "GradSampleModule", 25 | "GradSampleModuleFastGradientClipping", 26 | "utils", 27 | "__version__", 28 | ] 29 | -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflows will upload a Python Package using Twine when a release is created 2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 3 | 4 | name: Upload Opacus Python Package to PyPI 5 | 6 | on: 7 | release: 8 | types: [created] 9 | 10 | jobs: 11 | deploy: 12 | 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Set up Python 18 | uses: actions/setup-python@v2 19 | with: 20 | python-version: '3.9' 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install --upgrade setuptools wheel twine 25 | - name: Build and publish 26 | env: 27 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 28 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 29 | run: | 30 | rm -rf dist/ 31 | python setup.py sdist bdist_wheel 32 | twine upload dist/* 33 | -------------------------------------------------------------------------------- /opacus/accountants/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .accountant import IAccountant 16 | from .gdp import GaussianAccountant 17 | from .prv import PRVAccountant 18 | from .rdp import RDPAccountant 19 | from .registry import create_accountant, register_accountant 20 | 21 | 22 | __all__ = [ 23 | "IAccountant", 24 | "GaussianAccountant", 25 | "RDPAccountant", 26 | "PRVAccountant", 27 | "register_accountant", 28 | "create_accountant", 29 | ] 30 | -------------------------------------------------------------------------------- /opacus/schedulers/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .grad_clip_scheduler import ( 17 | ExponentialGradClip, 18 | LambdaGradClip, 19 | StepGradClip, 20 | _GradClipScheduler, 21 | ) 22 | from .noise_scheduler import ExponentialNoise, LambdaNoise, StepNoise, _NoiseScheduler 23 | 24 | 25 | __all__ = [ 26 | "_GradClipScheduler", 27 | "ExponentialGradClip", 28 | "LambdaGradClip", 29 | "StepGradClip", 30 | "_NoiseScheduler", 31 | "ExponentialNoise", 32 | "LambdaNoise", 33 | "StepNoise", 34 | ] 35 | -------------------------------------------------------------------------------- /opacus/optimizers/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import List 16 | 17 | import torch.nn as nn 18 | from torch.optim import Optimizer 19 | 20 | 21 | def params(optimizer: Optimizer) -> List[nn.Parameter]: 22 | """ 23 | Return all parameters controlled by the optimizer 24 | Args: 25 | optimizer: optimizer 26 | 27 | Returns: 28 | Flat list of parameters from all ``param_groups`` 29 | """ 30 | ret = [] 31 | for param_group in optimizer.param_groups: 32 | ret += [p for p in param_group["params"] if p.requires_grad] 33 | return ret 34 | -------------------------------------------------------------------------------- /examples/char-lstm_README.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | This example demonstrates how to run a privacy-preserving LSTM (with DP) on a name classification task. This is the same task as in this old PyTorch tutorial https://pytorch.org/tutorials/intermediate/char_rnn_classification_tutorial.html, with a rewrite that aligns its style to modern PyTorch. 4 | 5 | # Sample run 6 | 7 | Download the training zip from https://download.pytorch.org/tutorial/data.zip and then extract it to a local directory. In this example, we assume that the data path is /home/data_folder/names/*.txt 8 | 9 | Run with dp: 10 | 11 | ``` 12 | python char-lstm-classification.py --epochs=50 --learning-rate=2.0 --hidden-size=128 --delta=8e-5 --batch-size 64 --n-layers=1 --sigma=1.0 --max-per-sample-grad-norm=1.5 --device=cuda:0 --data-root="/my/folder/data/names/" --test-every 5 13 | ``` 14 | 15 | You should get something like this: Test Accuracy: 0.739542 (ε = 11.83, δ = 8e-05) for α = 2.7 16 | 17 | Run without dp: 18 | 19 | ``` 20 | python char-lstm-classification.py --epochs=50 --learning-rate=0.5 --hidden-size=128 --batch-size 64 --n-layers=1 --disable-dp --device=cuda:1 --data-root="/my/folder/data/names/" --test-every 5 21 | ``` 22 | You should get something like this: Test Accuracy: 0.760716 23 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Types of changes 2 | 3 | 4 | 5 | - [ ] Bug fix (non-breaking change which fixes an issue) 6 | - [ ] New feature (non-breaking change which adds functionality) 7 | - [ ] Breaking change (fix or feature that would cause existing functionality to change) 8 | - [ ] Docs change / refactoring / dependency upgrade 9 | 10 | ## Motivation and Context / Related issue 11 | 12 | 13 | 14 | 15 | 16 | ## How Has This Been Tested (if it applies) 17 | 18 | 19 | 20 | ## Checklist 21 | 22 | 23 | 24 | 25 | - [ ] The documentation is up-to-date with the changes I made. 26 | - [ ] I have read the **CONTRIBUTING** document and completed the CLA (see **CONTRIBUTING**). 27 | - [ ] All tests passed, and additional code has been covered with new tests. 28 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/instance_norm1d_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import hypothesis.strategies as st 17 | import torch 18 | import torch.nn as nn 19 | from hypothesis import given, settings 20 | 21 | from .common import GradSampleHooks_test 22 | 23 | 24 | class InstanceNorm1d_test(GradSampleHooks_test): 25 | @given(N=st.integers(1, 4), C=st.integers(1, 3), W=st.integers(5, 10)) 26 | @settings(deadline=60000) 27 | def test_3d_input(self, N: int, C: int, W: int): 28 | x = torch.randn([N, C, W]) 29 | norm = nn.InstanceNorm1d(num_features=C, affine=True, track_running_stats=False) 30 | 31 | self.run_test(x, norm, batch_first=True) 32 | -------------------------------------------------------------------------------- /website/static/img/modular.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/workflows/test-installation.yml: -------------------------------------------------------------------------------- 1 | # This workflow makes sure edge case versions work 2 | name: Test Installation 3 | 4 | on: 5 | pull_request: 6 | branches: 7 | - main 8 | paths: 9 | - "**.py" 10 | - ".github/workflows/**.yml" 11 | 12 | jobs: 13 | test-installation: 14 | strategy: 15 | max-parallel: 48 16 | matrix: 17 | os: [windows-latest, ubuntu-latest, macos-latest] 18 | python-version: ["3.9", "3.10"] 19 | torch-version: [2.6.0] 20 | 21 | runs-on: ${{ matrix.os }} 22 | steps: 23 | - uses: actions/checkout@v2 24 | 25 | - name: Set up Python ${{ matrix.python-version }} 26 | uses: actions/setup-python@v2 27 | with: 28 | python-version: ${{ matrix.python-version }} 29 | 30 | - name: Install PyTorch Linux and MacOS 31 | if: startsWith(runner.os, 'Windows') != true 32 | run: | 33 | ./scripts/pytorch_install.sh ${{ matrix.torch-version }} 34 | 35 | - name: Install PyTorch Windows 36 | if: startsWith(runner.os, 'Windows') 37 | run: | 38 | ./scripts/pytorch_install.ps1 ${{ matrix.torch-version }} 39 | 40 | - name: Install Opacus in Editing Mode 41 | run: | 42 | pip install -e . --default-timeout=60 43 | python -c "import opacus; print(opacus.__version__)" 44 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/instance_norm2d_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import hypothesis.strategies as st 17 | import torch 18 | import torch.nn as nn 19 | from hypothesis import given, settings 20 | 21 | from .common import GradSampleHooks_test 22 | 23 | 24 | class InstanceNorm2d_test(GradSampleHooks_test): 25 | @given( 26 | N=st.integers(1, 4), 27 | C=st.integers(1, 3), 28 | W=st.integers(5, 10), 29 | H=st.integers(4, 8), 30 | ) 31 | @settings(deadline=60000) 32 | def test_4d_input(self, N: int, C: int, W: int, H: int): 33 | x = torch.randn([N, C, H, W]) 34 | norm = nn.InstanceNorm2d(num_features=C, affine=True, track_running_stats=False) 35 | self.run_test(x, norm, batch_first=True) 36 | -------------------------------------------------------------------------------- /opacus/validators/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # we import fix and validate methods from all submodules here 17 | # to ensure that when `opacus.validators` is imported, 18 | # we call register_module_validator and register_module_fixer 19 | # on respective methods 20 | 21 | from .batch_norm import fix, validate # noqa 22 | from .gru import fix, validate # noqa 23 | from .instance_norm import fix, validate # noqa 24 | from .lstm import fix, validate # noqa 25 | from .module_validator import ModuleValidator 26 | from .multihead_attention import fix, validate # noqa 27 | from .utils import register_module_fixer, register_module_validator 28 | 29 | 30 | __all__ = [ 31 | "ModuleValidator", 32 | "register_module_validator", 33 | "register_module_fixer", 34 | ] 35 | -------------------------------------------------------------------------------- /website/static/css/code_block_buttons.css: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* "Copy" code block button */ 18 | pre { 19 | position: relative; 20 | } 21 | 22 | pre .btnIcon { 23 | position: absolute; 24 | top: 4px; 25 | z-index: 2; 26 | cursor: pointer; 27 | border: 1px solid transparent; 28 | padding: 0; 29 | color: #000; 30 | background-color: transparent; 31 | height: 30px; 32 | transition: all .25s ease-out; 33 | } 34 | 35 | pre .btnIcon:hover { 36 | text-decoration: none; 37 | } 38 | 39 | .btnIcon__body { 40 | align-items: center; 41 | display: flex; 42 | } 43 | 44 | .btnIcon svg { 45 | fill: currentColor; 46 | margin-right: .4em; 47 | } 48 | 49 | .btnIcon__label { 50 | font-size: 11px; 51 | } 52 | 53 | .btnClipboard { 54 | right: 10px; 55 | } 56 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/instance_norm3d_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import hypothesis.strategies as st 17 | import torch 18 | import torch.nn as nn 19 | from hypothesis import given, settings 20 | 21 | from .common import GradSampleHooks_test 22 | 23 | 24 | class InstanceNorm3d_test(GradSampleHooks_test): 25 | @given( 26 | N=st.integers(1, 4), 27 | C=st.integers(1, 3), 28 | W=st.integers(5, 10), 29 | H=st.integers(4, 8), 30 | Z=st.integers(1, 4), 31 | ) 32 | @settings(deadline=60000) 33 | def test_5d_input(self, N: int, C: int, W: int, H: int, Z: int): 34 | x = torch.randn([N, C, Z, H, W]) 35 | norm = nn.InstanceNorm3d(num_features=C, affine=True, track_running_stats=False) 36 | self.run_test(x, norm, batch_first=True) 37 | -------------------------------------------------------------------------------- /opacus/grad_sample/dp_multihead_attention.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from typing import Dict, List 18 | 19 | import torch 20 | import torch.nn as nn 21 | from opacus.layers.dp_multihead_attention import SequenceBias 22 | 23 | from .utils import register_grad_sampler 24 | 25 | 26 | @register_grad_sampler(SequenceBias) 27 | def compute_sequence_bias_grad_sample( 28 | layer: SequenceBias, activations: List[torch.Tensor], backprops: torch.Tensor 29 | ) -> Dict[nn.Parameter, torch.Tensor]: 30 | """ 31 | Computes per sample gradients for ``SequenceBias`` layer 32 | 33 | Args: 34 | layer: Layer 35 | activations: Activations 36 | backprops: Backpropagations 37 | """ 38 | ret = {} 39 | if layer.bias.requires_grad: 40 | ret[layer.bias] = backprops[:, -1] 41 | return ret 42 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/sequence_bias_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import hypothesis.strategies as st 17 | import torch 18 | from hypothesis import given, settings 19 | from opacus.layers import SequenceBias 20 | 21 | from .common import GradSampleHooks_test 22 | 23 | 24 | class SequenceBias_test(GradSampleHooks_test): 25 | @given( 26 | N=st.integers(0, 4), 27 | T=st.integers(10, 20), 28 | D=st.integers(4, 8), 29 | batch_first=st.booleans(), 30 | ) 31 | @settings(deadline=60000) 32 | def test_batch_second(self, N: int, T: int, D: int, batch_first: bool): 33 | seqbias = SequenceBias(D, batch_first) 34 | if batch_first: 35 | x = torch.randn([N, T, D]) 36 | else: 37 | x = torch.randn([T, N, D]) 38 | self.run_test(x, seqbias, batch_first, ew_compatible=False) 39 | -------------------------------------------------------------------------------- /benchmarks/generate_report.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | 17 | from benchmarks.utils import generate_report 18 | 19 | 20 | if __name__ == "__main__": 21 | parser = argparse.ArgumentParser() 22 | parser.add_argument( 23 | "--path-to-results", 24 | default="./results/raw", 25 | type=str, 26 | help="the path that `run_benchmarks.py` has saved results to.", 27 | ) 28 | parser.add_argument( 29 | "--save-path", 30 | default="./results/report.csv", 31 | type=str, 32 | help="path to save the output.", 33 | ) 34 | 35 | parser.add_argument( 36 | "--format", 37 | default="csv", 38 | type=str, 39 | help="output format", 40 | choices=["csv", "pkl"], 41 | ) 42 | args = parser.parse_args() 43 | 44 | generate_report(args.path_to_results, args.save_path, args.format) 45 | -------------------------------------------------------------------------------- /benchmarks/check_threshold.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | 17 | import pandas as pd 18 | 19 | 20 | if __name__ == "__main__": 21 | parser = argparse.ArgumentParser() 22 | parser.add_argument( 23 | "--report-path", 24 | type=str, 25 | help="path to the report produced by generate_report.py", 26 | ) 27 | parser.add_argument( 28 | "--metric", 29 | type=str, 30 | help="Metric to be checked", 31 | choices=["runtime", "memory"], 32 | ) 33 | parser.add_argument( 34 | "--column", 35 | type=str, 36 | help="Report column to be checked", 37 | ) 38 | parser.add_argument( 39 | "--threshold", 40 | type=float, 41 | ) 42 | args = parser.parse_args() 43 | 44 | r = pd.read_pickle(args.report_path).fillna(0) 45 | if (r.loc[:, (args.metric, args.column)] < args.threshold).all(): 46 | exit(0) 47 | else: 48 | exit(1) 49 | -------------------------------------------------------------------------------- /opacus/grad_sample/gsm_no_op.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import torch 17 | import torch.nn as nn 18 | from opacus.grad_sample.gsm_base import AbstractGradSampleModule 19 | 20 | 21 | class GradSampleModuleNoOp(AbstractGradSampleModule): 22 | """ 23 | NoOp GradSampleModule. 24 | Only wraps the module. The main goal of this class is to provide the same API for all methods. 25 | See README.md for more details 26 | """ 27 | 28 | def __init__( 29 | self, 30 | m: nn.Module, 31 | *, 32 | batch_first=True, 33 | loss_reduction="mean", 34 | ): 35 | if not batch_first: 36 | raise NotImplementedError 37 | 38 | super().__init__( 39 | m, 40 | batch_first=batch_first, 41 | loss_reduction=loss_reduction, 42 | ) 43 | 44 | def forward(self, x: torch.Tensor, *args, **kwargs): 45 | return self._module.forward(x, *args, **kwargs) 46 | -------------------------------------------------------------------------------- /examples/imdb_README.md: -------------------------------------------------------------------------------- 1 | # Simple run 2 | run with dp: 3 | ``` 4 | python imdb.py --device cuda:0 5 | 6 | ``` 7 | for with no dp: 8 | ``` 9 | python imdb.py --device cuda:0 --disable-dp 10 | ``` 11 | 12 | Default parameters are tweaked for optimal performance with DP 13 | 14 | # Expeted results 15 | We use the same architecture and dataset as described in https://arxiv.org/pdf/1911.11607.pdf. 16 | 17 | Reference implementation for the paper (tensorflow privacy): https://github.com/tensorflow/privacy/blob/master/research/GDP_2019/imdb_tutorial.py 18 | 19 | Unless specified otherwise, we set `batch_size=64` for a faster iteration (opacus has higher requirements in memory compared to tensorflow-privacy). We didn't use virtual batches for this experiment, and therefowe were limited to the given batch size. 20 | All metrics are calculated on test data after training for 20 epochs 21 | 22 | | Approach | Accuracy | 23 | | ----------------------------------------|:--------:| 24 | | Claimed in paper (no DP) | 84% | 25 | | tf_privacy (no DP) | 84% | 26 | | opacus (no DP) | 84% | 27 | | Claimed in paper (with DP) | 84% | 28 | | tf_privacy (with DP) | 80% | 29 | | tf_privacy (with DP, bsz=512) | 84% | 30 | | opacus (Adam, lr = 0.02) | 74% | 31 | | opacus (Adam, lr = 0.02, 100 epochs) | 78% | 32 | | opacus (Adagrad, lr = 0.02) | 59% | 33 | | opacus (Adagrad, lr = 0.1) | 65% | 34 | | opacus (SGD, lr = 0.1) | 63% | 35 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Deploy to GitHub Pages 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | deploy: 10 | name: Deploy to GitHub Pages 11 | runs-on: ubuntu-latest 12 | permissions: 13 | # Grant write permission here so it can be pushed to gh-pages branch 14 | contents: write 15 | defaults: 16 | run: 17 | working-directory: website 18 | steps: 19 | - uses: actions/checkout@v3 20 | - name: Install dependencies 21 | run: | 22 | python -m pip install --upgrade pip 23 | python -m pip install opacus sphinx sphinx-autodoc-typehints nbsphinx bs4 24 | yarn install 25 | - name: Build website 26 | run: ./scripts/build_website.sh -b 27 | 28 | # Popular action to deploy to GitHub Pages: 29 | # Docs: https://github.com/peaceiris/actions-gh-pages#%EF%B8%8F-docusaurus 30 | - name: Deploy to GitHub Pages 31 | uses: peaceiris/actions-gh-pages@v3 32 | with: 33 | github_token: ${{ secrets.GITHUB_TOKEN }} 34 | # Build output to publish to the `gh-pages` branch: 35 | publish_dir: ./website/build/opacus/ 36 | # The following lines assign commit authorship to the official 37 | # GH-Actions bot for deploys to `gh-pages` branch: 38 | # https://github.com/actions/checkout/issues/13#issuecomment-724415212 39 | # The GH actions bot is used by default if you didn't specify the two fields. 40 | # You can swap them out with your own user credentials. 41 | user_name: github-actions[bot] 42 | user_email: 41898282+github-actions[bot]@users.noreply.github.com 43 | -------------------------------------------------------------------------------- /opacus/grad_sample/rms_norm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from typing import Dict, List 18 | 19 | import torch 20 | import torch.nn as nn 21 | import torch.nn.functional as F 22 | from opacus.utils.tensor_utils import sum_over_all_but_batch_and_last_n 23 | 24 | from .utils import register_grad_sampler 25 | 26 | 27 | @register_grad_sampler(nn.RMSNorm) 28 | def compute_rms_norm_grad_sample( 29 | layer: nn.RMSNorm, 30 | activations: List[torch.Tensor], 31 | backprops: torch.Tensor, 32 | ) -> Dict[nn.Parameter, torch.Tensor]: 33 | """ 34 | Computes per sample gradients for RMSNorm 35 | 36 | Args: 37 | layer: Layer 38 | activations: Activations 39 | backprops: Backpropagations 40 | """ 41 | activations = activations[0] 42 | ret = {} 43 | if layer.weight.requires_grad: 44 | ret[layer.weight] = sum_over_all_but_batch_and_last_n( 45 | F.rms_norm(activations, layer.normalized_shape, eps=layer.eps) * backprops, 46 | layer.weight.dim(), 47 | ) 48 | return ret 49 | -------------------------------------------------------------------------------- /opacus/grad_sample/group_norm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from typing import Dict, List 18 | 19 | import torch 20 | import torch.nn as nn 21 | import torch.nn.functional as F 22 | 23 | from .utils import register_grad_sampler 24 | 25 | 26 | @register_grad_sampler(nn.GroupNorm) 27 | def compute_group_norm_grad_sample( 28 | layer: nn.GroupNorm, 29 | activations: List[torch.Tensor], 30 | backprops: torch.Tensor, 31 | ) -> Dict[nn.Parameter, torch.Tensor]: 32 | """ 33 | Computes per sample gradients for GroupNorm 34 | 35 | Args: 36 | layer: Layer 37 | activations: Activations 38 | backprops: Backpropagations 39 | """ 40 | activations = activations[0] 41 | ret = {} 42 | if layer.weight.requires_grad: 43 | gs = F.group_norm(activations, layer.num_groups, eps=layer.eps) * backprops 44 | ret[layer.weight] = torch.einsum("ni...->ni", gs) 45 | if layer.bias is not None and layer.bias.requires_grad: 46 | ret[layer.bias] = torch.einsum("ni...->ni", backprops) 47 | return ret 48 | -------------------------------------------------------------------------------- /opacus/tests/validators/gru_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import unittest 17 | 18 | import torch.nn as nn 19 | from opacus.layers import DPGRU 20 | from opacus.utils.module_utils import are_state_dict_equal 21 | from opacus.validators.errors import ShouldReplaceModuleError 22 | from opacus.validators.module_validator import ModuleValidator 23 | 24 | 25 | class GRUValidator_test(unittest.TestCase): 26 | def setUp(self) -> None: 27 | self.gru = nn.GRU(8, 4) 28 | self.mv = ModuleValidator.VALIDATORS 29 | self.mf = ModuleValidator.FIXERS 30 | 31 | def test_validate(self) -> None: 32 | val_gru = self.mv[type(self.gru)](self.gru) 33 | self.assertEqual(len(val_gru), 1) 34 | self.assertTrue(isinstance(val_gru[0], ShouldReplaceModuleError)) 35 | 36 | def test_fix(self) -> None: 37 | fix_gru = self.mf[type(self.gru)](self.gru) 38 | self.assertTrue(isinstance(fix_gru, DPGRU)) 39 | self.assertTrue( 40 | are_state_dict_equal(self.gru.state_dict(), fix_gru.state_dict()) 41 | ) 42 | -------------------------------------------------------------------------------- /opacus/tests/validators/lstm_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import unittest 17 | 18 | import torch.nn as nn 19 | from opacus.layers import DPLSTM 20 | from opacus.utils.module_utils import are_state_dict_equal 21 | from opacus.validators.errors import ShouldReplaceModuleError 22 | from opacus.validators.module_validator import ModuleValidator 23 | 24 | 25 | class LSTMValidator_test(unittest.TestCase): 26 | def setUp(self) -> None: 27 | self.lstm = nn.LSTM(8, 4) 28 | self.mv = ModuleValidator.VALIDATORS 29 | self.mf = ModuleValidator.FIXERS 30 | 31 | def test_validate(self) -> None: 32 | val_lstm = self.mv[type(self.lstm)](self.lstm) 33 | self.assertEqual(len(val_lstm), 1) 34 | self.assertTrue(isinstance(val_lstm[0], ShouldReplaceModuleError)) 35 | 36 | def test_fix(self) -> None: 37 | fix_lstm = self.mf[type(self.lstm)](self.lstm) 38 | self.assertTrue(isinstance(fix_lstm, DPLSTM)) 39 | self.assertTrue( 40 | are_state_dict_equal(self.lstm.state_dict(), fix_lstm.state_dict()) 41 | ) 42 | -------------------------------------------------------------------------------- /research/README.md: -------------------------------------------------------------------------------- 1 | # New Methods and Extensions of Opacus 2 | 3 | This directory contains novel methods built on top of Opacus that enhance DP-SGD. These contributions, made by the community, stem from research demonstrating potential improvements in differentially private model training. By consolidating these methods within the Opacus repository, we facilitate new research and provide a broader array of tools for DP-ML practitioners. 4 | 5 | 6 | ## Contributions 7 | We warmly welcome and encourage contributions of new methods! To contribute, please follow these steps: 8 | 9 | 1. Fork the repo and create your branch from `main`. 10 | 2. Place the new method in a separate subfolder within the `research` directory. 11 | 3. The new folder should include a `README.md` that explains the method at a high level, demonstrates usage (e.g., introducing new parameters to the `PrivacyEngine`), and cites relevant sources. The subfolder name should aptly represent the method. 12 | 4. Format code using `black`, `flake8`, and `isort` following the instructions under `Code Style` [here](https://github.com/pytorch/opacus/blob/main/CONTRIBUTING.md). 13 | 5. Add copyright headers to each `.py` file contributed in the format `# Copyright (c) [copy-right holder]`. 14 | 15 | More detailed PR instructions can be found [here](https://github.com/pytorch/opacus/blob/main/CONTRIBUTING.md). 16 | 17 | Feel free to reach out with any questions about the process or to discuss whether your method is a good fit for the repository. 18 | 19 | ## Notes 20 | Please note that the code provided in this directory will not be maintained by the Opacus team, which may lead to compatibility issues with future changes. If you have any questions, please reach out to the PR contributor directly. 21 | -------------------------------------------------------------------------------- /opacus/tests/validators/multihead_attention_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import unittest 17 | 18 | import torch.nn as nn 19 | from opacus.layers import DPMultiheadAttention 20 | from opacus.utils.module_utils import are_state_dict_equal 21 | from opacus.validators.errors import ShouldReplaceModuleError 22 | from opacus.validators.module_validator import ModuleValidator 23 | 24 | 25 | class MultiheadAttentionValidator_test(unittest.TestCase): 26 | def setUp(self) -> None: 27 | self.mha = nn.MultiheadAttention(8, 4) 28 | self.mv = ModuleValidator.VALIDATORS 29 | self.mf = ModuleValidator.FIXERS 30 | 31 | def test_validate(self) -> None: 32 | val_mha = self.mv[type(self.mha)](self.mha) 33 | self.assertEqual(len(val_mha), 1) 34 | self.assertTrue(isinstance(val_mha[0], ShouldReplaceModuleError)) 35 | 36 | def test_fix(self) -> None: 37 | fix_mha = self.mf[type(self.mha)](self.mha) 38 | self.assertTrue(isinstance(fix_mha, DPMultiheadAttention)) 39 | self.assertTrue( 40 | are_state_dict_equal(self.mha.state_dict(), fix_mha.state_dict()) 41 | ) 42 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F41B Bug Report" 3 | about: Submit a bug report to help us improve Opacus 4 | 5 | --- 6 | 7 | ## 🐛 Bug 8 | 9 | 10 | 11 | 12 | ## Please reproduce using our [template Colab](https://colab.research.google.com/drive/1R-dnKipK9LOVV4_oKbuHoq4VvGKGbDnd?usp=sharing) and post here the link 13 | 14 | 15 | ## To Reproduce 16 | > :warning: **We cannot help you without you sharing reproducible code**. Do not ignore this part :) 17 | Steps to reproduce the behavior: 18 | 19 | 1. 20 | 2. 21 | 3. 22 | 23 | 24 | 25 | ## Expected behavior 26 | 27 | 28 | 29 | ## Environment 30 | 31 | Please copy and paste the output from our 32 | [environment collection script](https://raw.githubusercontent.com/pytorch/pytorch/master/torch/utils/collect_env.py) 33 | (or fill out the checklist below manually). 34 | 35 | You can get the script and run it with: 36 | 37 | ``` 38 | wget https://raw.githubusercontent.com/pytorch/pytorch/master/torch/utils/collect_env.py 39 | # For security purposes, please check the contents of collect_env.py before running it. 40 | python collect_env.py 41 | ``` 42 | 43 | - PyTorch Version (e.g., 1.0): 44 | - OS (e.g., Linux): 45 | - How you installed PyTorch (`conda`, `pip`, source): 46 | - Build command you used (if compiling from source): 47 | - Python version: 48 | - CUDA/cuDNN version: 49 | - GPU models and configuration: 50 | - Any other relevant information: 51 | 52 | ## Additional context 53 | 54 | 55 | -------------------------------------------------------------------------------- /opacus/grad_sample/dp_rnn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from typing import Dict, List 18 | 19 | import torch 20 | import torch.nn as nn 21 | from opacus.layers.dp_rnn import RNNLinear 22 | 23 | from .utils import register_grad_sampler 24 | 25 | 26 | @register_grad_sampler(RNNLinear) 27 | def compute_rnn_linear_grad_sample( 28 | layer: RNNLinear, activations: List[torch.Tensor], backprops: torch.Tensor 29 | ) -> Dict[nn.Parameter, torch.Tensor]: 30 | """ 31 | Computes per sample gradients for ``RNNLinear`` layer. The RNN-like (DPLSTM, DPGRU) models 32 | are written using this layer as its building block. 33 | 34 | class 35 | 36 | Args: 37 | layer: Layer 38 | activations: Activations 39 | backprops: Backpropagations 40 | """ 41 | activations = activations[0] 42 | 43 | activations = activations.to(backprops.dtype) 44 | 45 | ret = {} 46 | if layer.weight.requires_grad: 47 | gs = torch.einsum("n...i,n...j->nij", backprops, activations) 48 | ret[layer.weight] = gs 49 | if layer.bias is not None and layer.bias.requires_grad: 50 | ret[layer.bias] = torch.einsum("n...k->nk", backprops) 51 | return ret 52 | -------------------------------------------------------------------------------- /opacus/grad_sample/layer_norm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from typing import Dict, List 18 | 19 | import torch 20 | import torch.nn as nn 21 | import torch.nn.functional as F 22 | from opacus.utils.tensor_utils import sum_over_all_but_batch_and_last_n 23 | 24 | from .utils import register_grad_sampler 25 | 26 | 27 | @register_grad_sampler(nn.LayerNorm) 28 | def compute_layer_norm_grad_sample( 29 | layer: nn.LayerNorm, 30 | activations: List[torch.Tensor], 31 | backprops: torch.Tensor, 32 | ) -> Dict[nn.Parameter, torch.Tensor]: 33 | """ 34 | Computes per sample gradients for LayerNorm 35 | 36 | Args: 37 | layer: Layer 38 | activations: Activations 39 | backprops: Backpropagations 40 | """ 41 | activations = activations[0] 42 | ret = {} 43 | if layer.weight.requires_grad: 44 | ret[layer.weight] = sum_over_all_but_batch_and_last_n( 45 | F.layer_norm(activations, layer.normalized_shape, eps=layer.eps) 46 | * backprops, 47 | layer.weight.dim(), 48 | ) 49 | if layer.bias.requires_grad: 50 | ret[layer.bias] = sum_over_all_but_batch_and_last_n(backprops, layer.bias.dim()) 51 | return ret 52 | -------------------------------------------------------------------------------- /opacus/grad_sample/gsm_exp_weights.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import torch 17 | import torch.nn as nn 18 | from opacus.grad_sample.gsm_base import AbstractGradSampleModule 19 | 20 | 21 | class GradSampleModuleExpandedWeights(AbstractGradSampleModule): 22 | """ 23 | ExpandedWeights-based implementation of AbstractGradSampleModule 24 | 25 | Computes per-sample gradients using PyTorch built-in mechanism of ExpandedWeights. 26 | See README.md for more details 27 | """ 28 | 29 | def __init__( 30 | self, 31 | m: nn.Module, 32 | *, 33 | batch_first=True, 34 | loss_reduction="mean", 35 | ): 36 | if not batch_first: 37 | raise NotImplementedError 38 | 39 | super().__init__( 40 | m, 41 | batch_first=batch_first, 42 | loss_reduction=loss_reduction, 43 | ) 44 | 45 | def forward(self, x: torch.Tensor, *args, **kwargs): 46 | from torch.nn.utils._per_sample_grad import call_for_per_sample_grads 47 | 48 | return call_for_per_sample_grads( 49 | module=self._module, 50 | batch_size=x.shape[0], 51 | loss_reduction=self.loss_reduction, 52 | )(x, *args, **kwargs) 53 | -------------------------------------------------------------------------------- /opacus/grad_sample/instance_norm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from typing import Dict, List, Union 17 | 18 | import torch 19 | import torch.nn as nn 20 | import torch.nn.functional as F 21 | 22 | from .utils import register_grad_sampler 23 | 24 | 25 | @register_grad_sampler( 26 | [ 27 | nn.InstanceNorm1d, 28 | nn.InstanceNorm2d, 29 | nn.InstanceNorm3d, 30 | ] 31 | ) 32 | def compute_instance_norm_grad_sample( 33 | layer: Union[ 34 | nn.InstanceNorm1d, 35 | nn.InstanceNorm2d, 36 | nn.InstanceNorm3d, 37 | ], 38 | activations: List[torch.Tensor], 39 | backprops: torch.Tensor, 40 | ) -> Dict[nn.Parameter, torch.Tensor]: 41 | """ 42 | Computes per sample gradients for InstanceNorm layers 43 | 44 | Args: 45 | layer: Layer 46 | activations: Activations 47 | backprops: Backpropagations 48 | """ 49 | activations = activations[0] 50 | ret = {} 51 | if layer.weight.requires_grad: 52 | gs = F.instance_norm(activations, eps=layer.eps) * backprops 53 | ret[layer.weight] = torch.einsum("ni...->ni", gs) 54 | if layer.bias is not None and layer.bias.requires_grad: 55 | ret[layer.bias] = torch.einsum("ni...->ni", backprops) 56 | return ret 57 | -------------------------------------------------------------------------------- /opacus/validators/lstm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from typing import List 17 | 18 | import torch.nn as nn 19 | from opacus.layers import DPLSTM 20 | 21 | from .errors import ShouldReplaceModuleError, UnsupportedModuleError 22 | from .utils import register_module_fixer, register_module_validator 23 | 24 | 25 | @register_module_validator(nn.LSTM) 26 | def validate(module: nn.LSTM) -> List[UnsupportedModuleError]: 27 | return [ 28 | ShouldReplaceModuleError( 29 | "We do not support nn.LSTM because its implementation uses special " 30 | "modules. We have written a DPLSTM class that is a drop-in replacement " 31 | "which is compatible with our Grad Sample hooks. Please run the recommended " 32 | "replacement!" 33 | ) 34 | ] 35 | 36 | 37 | @register_module_fixer(nn.LSTM) 38 | def fix(module: nn.LSTM) -> DPLSTM: 39 | dplstm = DPLSTM( 40 | input_size=module.input_size, 41 | hidden_size=module.hidden_size, 42 | num_layers=module.num_layers, 43 | bias=module.bias, 44 | batch_first=module.batch_first, 45 | dropout=module.dropout, 46 | bidirectional=module.bidirectional, 47 | ) 48 | dplstm.load_state_dict(module.state_dict()) 49 | return dplstm 50 | -------------------------------------------------------------------------------- /opacus/distributed.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import torch 17 | import torch.nn as nn 18 | 19 | 20 | def average_gradients(model: nn.Module) -> None: 21 | """ 22 | For all parameters of a given ``model`` averages gradients over all workers 23 | 24 | Args: 25 | model: model 26 | 27 | Returns: 28 | None 29 | """ 30 | world_size = torch.distributed.get_world_size() 31 | for param in model.parameters(): 32 | if not param.requires_grad: 33 | continue 34 | torch.distributed.all_reduce(param.grad, op=torch.distributed.ReduceOp.SUM) 35 | param.grad /= world_size 36 | 37 | 38 | class DifferentiallyPrivateDistributedDataParallel(nn.Module): 39 | """ 40 | Implements distributed data parallelism that is based on 41 | ``torch.distributed`` package at the module level. 42 | 43 | """ 44 | 45 | def __init__(self, model: nn.Module): 46 | super().__init__() 47 | 48 | # Synchronize the model 49 | params = list(model.parameters()) 50 | with torch.no_grad(): 51 | for p in params: 52 | torch.distributed.broadcast(p.data, 0) 53 | 54 | self.module = model 55 | 56 | def forward(self, *args, **kwargs): 57 | return self.module(*args, **kwargs) 58 | -------------------------------------------------------------------------------- /opacus/validators/gru.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from typing import List 17 | 18 | import torch.nn as nn 19 | from opacus.layers import DPGRU 20 | 21 | from .errors import ShouldReplaceModuleError, UnsupportedModuleError 22 | from .utils import register_module_fixer, register_module_validator 23 | 24 | 25 | @register_module_validator(nn.GRU) 26 | def validate(module: nn.GRU) -> List[UnsupportedModuleError]: 27 | return [ 28 | ShouldReplaceModuleError( 29 | "We do not support nn.GRU because its implementation uses special " 30 | "modules. We have written a GRU class that is a drop-in replacement " 31 | "which is compatible with our Grad Sample hooks. Please run the recommended " 32 | "replacement!" 33 | ) 34 | ] 35 | 36 | 37 | @register_module_fixer(nn.GRU) 38 | def fix(module: nn.GRU) -> DPGRU: 39 | dpgru = DPGRU( 40 | input_size=module.input_size, 41 | hidden_size=module.hidden_size, 42 | num_layers=module.num_layers, 43 | bias=module.bias, 44 | batch_first=module.batch_first, 45 | dropout=module.dropout, 46 | bidirectional=module.bidirectional, 47 | proj_size=module.proj_size, 48 | ) 49 | dpgru.load_state_dict(module.state_dict()) 50 | return dpgru 51 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/embedding_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import hypothesis.strategies as st 17 | import torch 18 | import torch.nn as nn 19 | from hypothesis import given, settings 20 | 21 | from .common import GradSampleHooks_test 22 | 23 | 24 | class Embedding_test(GradSampleHooks_test): 25 | @given( 26 | N=st.integers(0, 4), 27 | T=st.integers(1, 5), 28 | Q=st.integers(1, 4), 29 | R=st.integers(1, 2), 30 | V=st.integers(2, 32), 31 | D=st.integers(10, 17), 32 | dim=st.integers(2, 4), 33 | batch_first=st.booleans(), 34 | ) 35 | @settings(deadline=60000) 36 | def test_input_across_dims( 37 | self, 38 | N: int, 39 | T: int, 40 | Q: int, 41 | R: int, 42 | V: int, 43 | D: int, 44 | dim: int, 45 | batch_first: bool, 46 | ): 47 | if dim == 1: # TODO: fix when dim is 1 48 | size = [T] 49 | elif dim == 2: 50 | size = [N, T] 51 | elif dim == 3: 52 | size = [N, T, Q] 53 | elif dim == 4: 54 | size = [N, T, Q, R] 55 | 56 | emb = nn.Embedding(V, D) 57 | x = torch.randint(low=0, high=V, size=size) 58 | self.run_test(x, emb, batch_first=batch_first, ew_compatible=N > 0) 59 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/group_norm_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from typing import Union 17 | 18 | import hypothesis.strategies as st 19 | import torch 20 | import torch.nn as nn 21 | from hypothesis import given, settings 22 | 23 | from .common import GradSampleHooks_test 24 | 25 | 26 | class GroupNorm_test(GradSampleHooks_test): 27 | """ 28 | We only test the case with ``affine=True`` here, because it is the only case that will actually 29 | compute a gradient. There is no grad_sample from this module otherwise. 30 | """ 31 | 32 | @given( 33 | N=st.integers(0, 4), 34 | C=st.integers(1, 8), 35 | H=st.integers(5, 10), 36 | W=st.integers(4, 8), 37 | num_groups=st.sampled_from([1, 4, "C"]), 38 | ) 39 | @settings(deadline=60000) 40 | def test_3d_input_groups( 41 | self, 42 | N: int, 43 | C: int, 44 | H: int, 45 | W: int, 46 | num_groups: Union[int, str], 47 | ): 48 | if num_groups == "C": 49 | num_groups = C 50 | 51 | if C % num_groups != 0: 52 | return 53 | 54 | x = torch.randn([N, C, H, W]) 55 | norm = nn.GroupNorm(num_groups=num_groups, num_channels=C, affine=True) 56 | self.run_test(x, norm, batch_first=True, ew_compatible=N > 0) 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # Downloaded data 7 | data/ 8 | 9 | # Buck Targets 10 | TARGETS 11 | 12 | # Distribution / packaging 13 | .Python 14 | env/ 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | 32 | # watchman 33 | # Watchman helper 34 | .watchmanconfig 35 | 36 | # OSX 37 | *.DS_Store 38 | 39 | # Atom plugin files and ctags 40 | .ftpconfig 41 | .ftpconfig.cson 42 | .ftpignore 43 | *.tags 44 | *.tags1 45 | 46 | # PyInstaller 47 | # Usually these files are written by a python script from a template 48 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 49 | *.manifest 50 | *.spec 51 | 52 | # Installer logs 53 | pip-log.txt 54 | pip-delete-this-directory.txt 55 | 56 | # Unit test / coverage reports 57 | htmlcov/ 58 | .tox/ 59 | .coverage 60 | .coverage.* 61 | .cache 62 | nosetests.xml 63 | coverage.xml 64 | *.cover 65 | .hypothesis/ 66 | .pytest_cache/ 67 | 68 | # PyBuilder 69 | target/ 70 | 71 | # Jupyter Notebook 72 | .ipynb_checkpoints 73 | 74 | # pyenv 75 | .python-version 76 | 77 | # dotenv 78 | .env 79 | 80 | # virtualenv 81 | .venv 82 | venv/ 83 | ENV/ 84 | 85 | # mypy 86 | .mypy_cache/ 87 | 88 | # vim 89 | *.swp 90 | 91 | # Sphinx documentation 92 | sphinx/build/ 93 | 94 | # Docusaurus 95 | website/build/ 96 | website/i18n/ 97 | website/node_modules/ 98 | 99 | # PyCharm 100 | .idea/ 101 | 102 | ## Generated for tutorials 103 | website/_tutorials/ 104 | website/static/files/ 105 | website/pages/tutorials/* 106 | !website/pages/tutorials/index.js 107 | 108 | ## Generated for Sphinx 109 | website/pages/api/ 110 | website/static/js/* 111 | !website/static/js/mathjax.js 112 | !website/static/js/code_block_buttons.js 113 | website/static/_sphinx-sources/ 114 | spelling_wordlist.txt 115 | -------------------------------------------------------------------------------- /opacus/tests/dpdataloader_test.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import unittest 16 | 17 | import torch 18 | from opacus.data_loader import DPDataLoader 19 | from torch.utils.data import DataLoader, TensorDataset 20 | 21 | 22 | class DPDataLoaderTest(unittest.TestCase): 23 | def setUp(self) -> None: 24 | self.data_size = 10 25 | self.dimension = 7 26 | self.num_classes = 11 27 | 28 | def test_collate_classes(self) -> None: 29 | x = torch.randn(self.data_size, self.dimension) 30 | y = torch.randint(low=0, high=self.num_classes, size=(self.data_size,)) 31 | 32 | dataset = TensorDataset(x, y) 33 | data_loader = DPDataLoader(dataset, sample_rate=1e-5) 34 | 35 | x_b, y_b = next(iter(data_loader)) 36 | self.assertEqual(x_b.size(0), 0) 37 | self.assertEqual(y_b.size(0), 0) 38 | 39 | def test_collate_tensor(self) -> None: 40 | x = torch.randn(self.data_size, self.dimension) 41 | 42 | dataset = TensorDataset(x) 43 | data_loader = DPDataLoader(dataset, sample_rate=1e-5) 44 | 45 | (s,) = next(iter(data_loader)) 46 | 47 | self.assertEqual(s.size(0), 0) 48 | 49 | def test_drop_last_true(self) -> None: 50 | x = torch.randn(self.data_size, self.dimension) 51 | 52 | dataset = TensorDataset(x) 53 | data_loader = DataLoader(dataset, drop_last=True) 54 | _ = DPDataLoader.from_data_loader(data_loader) 55 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/linear_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import hypothesis.strategies as st 17 | import torch 18 | import torch.nn as nn 19 | from hypothesis import given, settings 20 | 21 | from .common import GradSampleHooks_test 22 | 23 | 24 | class Linear_test(GradSampleHooks_test): 25 | @given( 26 | N=st.integers(0, 4), 27 | Z=st.integers(1, 4), 28 | H=st.integers(1, 3), 29 | W=st.integers(10, 17), 30 | input_dim=st.integers(2, 4), 31 | bias=st.booleans(), 32 | batch_first=st.booleans(), 33 | ) 34 | @settings(deadline=60000) 35 | def test_input_bias( 36 | self, 37 | N: int, 38 | Z: int, 39 | W: int, 40 | H: int, 41 | input_dim: int, 42 | bias: bool, 43 | batch_first: bool, 44 | ): 45 | if input_dim == 2: 46 | if not batch_first: 47 | return # see https://github.com/pytorch/opacus/pull/265 48 | else: 49 | x_shape = [N, W] 50 | if input_dim == 3: 51 | x_shape = [N, Z, W] 52 | if input_dim == 4: 53 | x_shape = [N, Z, H, W] 54 | 55 | linear = nn.Linear(W, W + 2, bias=bias) 56 | x = torch.randn(x_shape) 57 | if not batch_first: 58 | x = x.transpose(0, 1) 59 | self.run_test(x, linear, batch_first=batch_first, ew_compatible=N > 0) 60 | -------------------------------------------------------------------------------- /opacus/validators/instance_norm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from typing import List, Union 17 | 18 | import torch.nn as nn 19 | from opacus.utils.module_utils import clone_module 20 | 21 | from .errors import IllegalModuleConfigurationError, UnsupportedModuleError 22 | from .utils import register_module_fixer, register_module_validator 23 | 24 | 25 | INSTANCENORM = Union[nn.InstanceNorm1d, nn.InstanceNorm2d, nn.InstanceNorm3d] 26 | 27 | 28 | @register_module_validator([nn.InstanceNorm1d, nn.InstanceNorm2d, nn.InstanceNorm3d]) 29 | def validate(module: INSTANCENORM) -> List[UnsupportedModuleError]: 30 | return ( 31 | [ 32 | IllegalModuleConfigurationError( 33 | "We do not support tracking running stats with differential privacy. " 34 | "To support it, we would have to add a DP mechanism for these statistics too, " 35 | "which would incur a privacy cost for little value in model accuracy. " 36 | "Just say no to running stats :)" 37 | ) 38 | ] 39 | if module.track_running_stats 40 | else [] 41 | ) 42 | 43 | 44 | @register_module_fixer([nn.InstanceNorm1d, nn.InstanceNorm2d, nn.InstanceNorm3d]) 45 | def fix(module: INSTANCENORM) -> INSTANCENORM: 46 | if len(validate(module)) == 0: 47 | return module 48 | # else 49 | new_module = clone_module(module) 50 | new_module.track_running_stats = False 51 | return new_module 52 | -------------------------------------------------------------------------------- /opacus/validators/multihead_attention.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from typing import List 17 | 18 | import torch.nn as nn 19 | from opacus.layers import DPMultiheadAttention 20 | 21 | from .errors import ShouldReplaceModuleError, UnsupportedModuleError 22 | from .utils import register_module_fixer, register_module_validator 23 | 24 | 25 | @register_module_validator(nn.MultiheadAttention) 26 | def validate(module: nn.MultiheadAttention) -> List[UnsupportedModuleError]: 27 | return [ 28 | ShouldReplaceModuleError( 29 | "We do not support nn.MultiheadAttention because its implementation uses special " 30 | "modules. We have written a DPMultiheadAttention class that is a drop-in replacement " 31 | "which is compatible with our Grad Sample hooks. Please run the recommended " 32 | "replacement!" 33 | ) 34 | ] 35 | 36 | 37 | @register_module_fixer(nn.MultiheadAttention) 38 | def fix(module: nn.MultiheadAttention) -> DPMultiheadAttention: 39 | dp_attn = DPMultiheadAttention( 40 | embed_dim=module.embed_dim, 41 | num_heads=module.num_heads, 42 | dropout=module.dropout, 43 | bias=module.in_proj_bias is not None, 44 | add_bias_kv=module.bias_k is not None, 45 | add_zero_attn=module.add_zero_attn, 46 | kdim=module.kdim, 47 | vdim=module.vdim, 48 | batch_first=module.batch_first, 49 | ) 50 | dp_attn.load_state_dict(module.state_dict()) 51 | return dp_attn 52 | -------------------------------------------------------------------------------- /opacus/validators/errors.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | class UnsupportedError(ValueError): 18 | """ 19 | Raised for unsupported components in Opacus. 20 | """ 21 | 22 | pass 23 | 24 | 25 | class UnsupportedModuleError(UnsupportedError): 26 | """ 27 | Raised for unsupported modules in Opacus. 28 | """ 29 | 30 | 31 | class UnsupportableModuleError(UnsupportedModuleError): 32 | """ 33 | Raised whenever there is a module we can't support ever. 34 | BatchNorm is the largest offender. 35 | """ 36 | 37 | pass 38 | 39 | 40 | class NotYetSupportedModuleError(UnsupportedModuleError): 41 | """ 42 | Raised whenever there is a module that we don't yet support. 43 | This is the "catch-all": the number of modules we won't ever support 44 | is very short, so a priori if we don't support it now it doesn't mean 45 | we can't extend support later (and PRs are welcome!!). 46 | """ 47 | 48 | pass 49 | 50 | 51 | class ShouldReplaceModuleError(UnsupportedModuleError): 52 | """ 53 | Raised whenever there is a module that we don't support as-is but we do support via 54 | replacement (and we have made a replacement ourselves). 55 | """ 56 | 57 | pass 58 | 59 | 60 | class IllegalModuleConfigurationError(UnsupportedModuleError): 61 | """ 62 | Raised whenever there is an illegal configuration of the training setup. 63 | Examples include, not setting the module to training mode, enabling running stats 64 | in InstanceNorm, etc. 65 | """ 66 | 67 | pass 68 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/embedding_bag_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import hypothesis.strategies as st 17 | import torch 18 | import torch.nn as nn 19 | from hypothesis import given, settings 20 | 21 | from .common import GradSampleHooks_test 22 | 23 | 24 | class Embedding_bag_test(GradSampleHooks_test): 25 | @given( 26 | N=st.integers(4, 8), 27 | sz=st.integers(3, 7), 28 | V=st.integers(10, 32), 29 | D=st.integers(10, 17), 30 | mode=st.sampled_from(["sum", "mean"]), 31 | ) 32 | @settings(deadline=60000) 33 | def test_input_across_dims( 34 | self, 35 | N: int, 36 | sz: int, 37 | V: int, 38 | D: int, 39 | mode: str, 40 | ): 41 | emb = nn.EmbeddingBag(num_embeddings=V, embedding_dim=D, mode=mode) 42 | 43 | sizes = torch.randint(low=1, high=sz + 1, size=(N,)) 44 | offsets = torch.LongTensor([0] + torch.cumsum(sizes, dim=0).tolist()[:-1]) 45 | input = [] 46 | for size in sizes: 47 | input += [torch.randperm(V)[:size]] 48 | 49 | input = torch.cat(input, dim=0) 50 | 51 | def chunk_method(x): 52 | input, offsets = x 53 | for i_offset, offset in enumerate(offsets): 54 | if i_offset < len(offsets) - 1: 55 | next_offset = offsets[i_offset + 1] 56 | else: 57 | next_offset = len(input) 58 | yield (input[offset:next_offset], torch.LongTensor([0])) 59 | 60 | self.run_test( 61 | (input, offsets), emb, chunk_method=chunk_method, ew_compatible=False 62 | ) 63 | -------------------------------------------------------------------------------- /benchmarks/tests/helpers.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import Optional 16 | 17 | import pytest 18 | import torch 19 | 20 | 21 | skipifnocuda = pytest.mark.skipif( 22 | not torch.cuda.is_available(), reason="Requires CUDA." 23 | ) 24 | 25 | 26 | def get_n_byte_tensor(n: int, device: Optional[torch.device] = None) -> torch.Tensor: 27 | """Returns a torch.int8 tensor of size n. 28 | 29 | Args: 30 | n: size of the tensor to allocate 31 | device: torch.device to allocate tensor on 32 | 33 | Returns: 34 | torch.int8 tensor of size n on device 35 | 36 | Notes: Though 1 int8 = 1 byte, memory is allocated in blocks, such that the size 37 | of the tensor in bytes >= n. 38 | """ 39 | return torch.zeros(n, dtype=torch.int8, device=device) 40 | 41 | 42 | def get_actual_memory_allocated(n: int, device: torch.device) -> int: 43 | """ 44 | Returns the CUDA memory allocated for a torch.int8 tensor of size n. 45 | 46 | Args: 47 | n: size of the tensor to get allocated memory for 48 | device: torch.device to allocate tensor on 49 | 50 | Returns: 51 | Number of bytes of CUDA memory allocated for a torch.int8 tensor of size n. 52 | 53 | Notes: Should only be called on CUDA devices. Resets CUDA memory statistics. 54 | """ 55 | assert device.type == "cuda" 56 | prev_memory_allocated = torch.cuda.memory_allocated(device) 57 | tensor = get_n_byte_tensor(n, device=device) 58 | memory_allocated = torch.cuda.memory_allocated(device) 59 | del tensor 60 | torch.cuda.reset_peak_memory_stats(device) 61 | assert prev_memory_allocated == torch.cuda.memory_allocated(device) 62 | return memory_allocated - prev_memory_allocated 63 | -------------------------------------------------------------------------------- /opacus/lightning.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from typing import Any, Optional 17 | 18 | import pytorch_lightning as pl 19 | import torch 20 | from opacus.data_loader import DPDataLoader 21 | 22 | 23 | class DPLightningDataModule(pl.LightningDataModule): 24 | def __init__( 25 | self, 26 | datamodule: pl.LightningDataModule, 27 | generator: Optional[torch.Generator] = None, 28 | ): 29 | super().__init__() 30 | self.datamodule = datamodule 31 | self.generator = generator 32 | 33 | def prepare_data(self) -> None: 34 | self.datamodule.prepare_data() 35 | 36 | def setup(self, stage: Optional[str] = None) -> None: 37 | self.datamodule.setup(stage) 38 | 39 | def train_dataloader(self): 40 | dataloader = self.datamodule.train_dataloader() 41 | return DPDataLoader.from_data_loader(dataloader, distributed=False) 42 | 43 | def val_dataloader(self): 44 | return self.datamodule.val_dataloader() 45 | 46 | def test_dataloader(self): 47 | return self.datamodule.test_dataloader() 48 | 49 | def predict_dataloader(self): 50 | return self.datamodule.predict_dataloader() 51 | 52 | def transfer_batch_to_device( 53 | self, batch: Any, device: torch.device, dataloader_idx: int 54 | ) -> Any: 55 | return self.datamodule.transfer_batch_to_device(batch, device, dataloader_idx) 56 | 57 | def on_before_batch_transfer(self, batch: Any, dataloader_idx: int) -> Any: 58 | return self.datamodule.on_before_batch_transfer(batch, dataloader_idx) 59 | 60 | def on_after_batch_transfer(self, batch: Any, dataloader_idx: int) -> Any: 61 | return self.datamodule.on_after_batch_transfer(batch, dataloader_idx) 62 | -------------------------------------------------------------------------------- /opacus/utils/adaptive_clipping/README.md: -------------------------------------------------------------------------------- 1 | # Adaptive Clipping (with Ghost Clipping) 2 | 3 | Adaptive clipping [1] adapts the clipping norm (and amount of noise) during training to a quantile of per-sample gradient norms. It can reduce hyper-parameter tuning efforts and improve model accuracy by injecting less noise. 4 | 5 | It is supported with: 6 | - Ghost clipping 7 | - Distributed data parallel training 8 | 9 | It is **not** currently supported with: 10 | - Vanilla DP-SGD 11 | - Virtual batch sizes via Batch Memory Manager 12 | 13 | ## Overview 14 | 15 | `PrivacyEngineAdaptiveClipping` is the entry-point for adaptive clipping training. It extends `PrivacyEngine` with additional arguments for adaptive clipping: 16 | 17 | * `target_unclipped_quantile`: the quantile of per-sample gradient norms at which to clip (between 0 and 1) 18 | * `min_clipbound`: the minimum allowed clipping norm 19 | * `max_clipbound`: the maximum allowed clipping norm 20 | * `clipbound_learning_rate`: the learning rate for tracking the true quantile 21 | * `max_grad_norm`: the initial clipping norm (used at step 0) 22 | 23 | The main hyper-parameter to tune is `target_unclipped_quantile`, which replaces tuning the clipping norm (`max_grad_norm`) in constant clipping DP-SGD. This parameter can be easier to tune, since the search is over a smaller range of values. 24 | 25 | 26 | ## Example usage 27 | 28 | ```python 29 | from opacus.utils.adaptive_clipping.adaptive_clipping_utils import PrivacyEngineAdaptiveClipping 30 | 31 | # ... 32 | privacy_engine = PrivacyEngineAdaptiveClipping() 33 | model, optimizer, criterion, train_loader = privacy_engine.make_private( 34 | module=model, 35 | optimizer=optimizer, 36 | data_loader=train_loader, 37 | criterion=criterion, 38 | noise_multiplier=args.sigma, 39 | max_grad_norm=10, # initial clipping norm 40 | grad_sample_mode="ghost", 41 | target_unclipped_quantile=0.5, # key parameter, may need tuning 42 | min_clipbound=1, # default value 43 | max_clipbound=1e8, # default value 44 | clipbound_learning_rate=0.2 # default value, tuning not recommended 45 | ) 46 | # ... 47 | ``` 48 | 49 | Note that `grad_sample_mode` must be set to `"ghost"` for adaptive clipping to work. 50 | 51 | ## References 52 | 53 | [1] Galen Andrew, Om Thakkar, H. Brendan McMahan, Swaroop Ramaswamy, "Differentially Private Learning with Adaptive Clipping", NeurIPS, 2021. 54 | -------------------------------------------------------------------------------- /opacus/tests/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import torch 16 | import torch.nn as nn 17 | import torch.nn.functional as F 18 | 19 | 20 | class BasicSupportedModule(nn.Module): 21 | def __init__(self): 22 | super().__init__() 23 | self.conv = nn.Conv1d(in_channels=16, out_channels=8, kernel_size=2) 24 | self.gn = nn.GroupNorm(num_groups=2, num_channels=8) 25 | self.fc = nn.Linear(in_features=4, out_features=8) 26 | self.ln = nn.LayerNorm([8, 8]) 27 | 28 | def forward(self, x): 29 | x = self.conv(x) 30 | x = self.gn(x) 31 | x = self.fc(x) 32 | x = self.ln(x) 33 | return x 34 | 35 | 36 | class CustomLinearModule(nn.Module): 37 | def __init__(self, in_features: int, out_features: int): 38 | super().__init__() 39 | self._weight = nn.Parameter(torch.randn(out_features, in_features)) 40 | self._bias = nn.Parameter(torch.randn(out_features)) 41 | 42 | def forward(self, x): 43 | return F.linear(x, self._weight, self._bias) 44 | 45 | 46 | class MatmulModule(nn.Module): 47 | def __init__(self, input_features: int, output_features: int): 48 | super().__init__() 49 | self.weight = nn.Parameter(torch.randn(input_features, output_features)) 50 | 51 | def forward(self, x): 52 | return torch.matmul(x, self.weight) 53 | 54 | 55 | class LinearWithExtraParam(nn.Module): 56 | def __init__(self, in_features: int, out_features: int, hidden_dim: int = 8): 57 | super().__init__() 58 | self.fc = nn.Linear(in_features, hidden_dim) 59 | self.extra_param = nn.Parameter(torch.randn(hidden_dim, out_features)) 60 | 61 | def forward(self, x): 62 | x = self.fc(x) 63 | x = x.matmul(self.extra_param) 64 | return x 65 | -------------------------------------------------------------------------------- /opacus/accountants/registry.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import Dict, Type 16 | 17 | from .accountant import IAccountant 18 | from .gdp import GaussianAccountant 19 | from .prv import PRVAccountant 20 | from .rdp import RDPAccountant 21 | 22 | 23 | _ACCOUNTANTS: Dict[str, Type[IAccountant]] = { 24 | "rdp": RDPAccountant, 25 | "gdp": GaussianAccountant, 26 | "prv": PRVAccountant, 27 | } 28 | 29 | 30 | def register_accountant( 31 | mechanism: str, accountant: Type[IAccountant], force: bool = False 32 | ): 33 | r""" 34 | Register a new accountant class to be used with a specified mechanism name. 35 | 36 | Args: 37 | mechanism: Name of the mechanism to register the accountant for 38 | accountant: Accountant class (subclass of IAccountant) to register 39 | force: If True, overwrites existing accountant for the specified mechanism. 40 | 41 | Raises: 42 | ValueError: If the mechanism is already registered. 43 | """ 44 | if mechanism in _ACCOUNTANTS and not force: 45 | raise ValueError(f"Accountant for mechanism {mechanism} is already registered") 46 | 47 | _ACCOUNTANTS[mechanism] = accountant 48 | 49 | 50 | def create_accountant(mechanism: str) -> IAccountant: 51 | r""" 52 | Creates and returns an accountant instance for the specified privacy mechanism. 53 | 54 | Args: 55 | mechanism: Name of the privacy accounting mechanism to use. 56 | 57 | Returns: 58 | An instance of the appropriate accountant class (subclass of IAccountant) 59 | for the specified mechanism. 60 | 61 | Raises: 62 | ValueError: If the specified mechanism is not registered. 63 | """ 64 | if mechanism in _ACCOUNTANTS: 65 | return _ACCOUNTANTS[mechanism]() 66 | 67 | raise ValueError(f"Unexpected accounting mechanism: {mechanism}") 68 | -------------------------------------------------------------------------------- /opacus/tests/validators/batch_norm_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import unittest 17 | 18 | import torch.nn as nn 19 | from opacus.validators.errors import ShouldReplaceModuleError 20 | from opacus.validators.module_validator import ModuleValidator 21 | 22 | 23 | class BatchNormValidator_test(unittest.TestCase): 24 | def setUp(self) -> None: 25 | self.bn1 = nn.BatchNorm1d(4) 26 | self.bn2 = nn.BatchNorm2d(4) 27 | self.bn3 = nn.BatchNorm3d(4) 28 | self.bns = nn.SyncBatchNorm(4) 29 | self.mv = ModuleValidator.VALIDATORS 30 | self.mf = ModuleValidator.FIXERS 31 | 32 | def test_validate(self) -> None: 33 | val1 = self.mv[type(self.bn1)](self.bn1) 34 | val2 = self.mv[type(self.bn2)](self.bn2) 35 | val3 = self.mv[type(self.bn3)](self.bn3) 36 | vals = self.mv[type(self.bns)](self.bns) 37 | 38 | self.assertEqual(len(val1), 1) 39 | self.assertEqual(len(val2), 1) 40 | self.assertEqual(len(val3), 1) 41 | self.assertEqual(len(vals), 1) 42 | 43 | self.assertTrue(isinstance(val1[0], ShouldReplaceModuleError)) 44 | self.assertTrue(isinstance(val2[0], ShouldReplaceModuleError)) 45 | self.assertTrue(isinstance(val3[0], ShouldReplaceModuleError)) 46 | self.assertTrue(isinstance(vals[0], ShouldReplaceModuleError)) 47 | 48 | def test_fix(self) -> None: 49 | fix1 = self.mf[type(self.bn1)](self.bn1) 50 | fix2 = self.mf[type(self.bn2)](self.bn2) 51 | fix3 = self.mf[type(self.bn3)](self.bn3) 52 | fixs = self.mf[type(self.bns)](self.bns) 53 | 54 | self.assertTrue(isinstance(fix1, nn.GroupNorm)) 55 | self.assertTrue(isinstance(fix2, nn.GroupNorm)) 56 | self.assertTrue(isinstance(fix3, nn.GroupNorm)) 57 | self.assertTrue(isinstance(fixs, nn.GroupNorm)) 58 | -------------------------------------------------------------------------------- /examples/mnist_README.md: -------------------------------------------------------------------------------- 1 | # First run 2 | To run a basic training script without differential privacy: 3 | ```shell 4 | python mnist.py --device=cpu --disable-dp --n=20 --lr=.1 -b=240 5 | ``` 6 | The first time the script runs, it attempts to download the MNIST dataset from http://yann.lecun.com and place it in `../mnist/MNIST/raw`. If you prefer a different location or your execution environment does not have access to the outside world, download and unpack the dataset yourself and pass the location as `--data-root=custom_dir_name`. The script will expect to find under `custom_dir_name/MNIST/processed` two files: `test.pt` (7.9 MB) and `training.pt` (47.5 MB). 7 | 8 | If the run is successful, you will something similar to this (your exact accuracy and performance numbers will vary): 9 | ``` 10 | 100%|██████████| 240/240 [00:14<00:00, 16.58it/s] 11 | Train Epoch: 1 Loss: 0.536689 12 | 100%|██████████| 240/240 [00:14<00:00, 16.48it/s] 13 | Train Epoch: 2 Loss: 0.102070 14 | ... 15 | 100%|██████████| 240/240 [00:14<00:00, 17.02it/s] 16 | Train Epoch: 10 Loss: 0.025479 17 | 100%|██████████| 10/10 [00:01<00:00, 6.64it/s] 18 | 19 | Test set: Average loss: 0.0000, Accuracy: 9893/10000 (98.93%) 20 | ``` 21 | 22 | To train a differentially private model, run the following command: 23 | ```shell 24 | python mnist.py --device=cpu -n=15 --lr=.25 --sigma=1.3 -c=1.5 -b=240 25 | ``` 26 | If the run is successful, expect to see 27 | ``` 28 | 100%|██████████| 240/240 [00:22<00:00, 10.48it/s] 29 | Train Epoch: 1 Loss: 0.912457 (ε = 0.71, δ = 1e-05) for α = 18.0 30 | ... 31 | 100%|██████████| 240/240 [00:22<00:00, 10.79it/s] 32 | Train Epoch: 15 Loss: 0.404850 (ε = 1.16, δ = 1e-05) for α = 17.0 33 | 100%|██████████| 10/10 [00:01<00:00, 6.76it/s] 34 | 35 | Test set: Average loss: 0.0004, Accuracy: 9486/10000 (94.86%) 36 | ``` 37 | 38 | # Sample parameter sets 39 | 40 | **Baseline: no differential privacy** 41 | 42 | Command: `--disable-dp --n=20 --lr=.1 -b=240` 43 | 44 | Result: accuracy averaged over 10 runs 98.94% ± 0.32% 45 | 46 | **(6.86, 10-5)-DP** 47 | 48 | Command: `-n=45 --lr=.25 --sigma=.7 -c=1.5 -b=240` 49 | 50 | Result: accuracy averaged over 10 runs 97.09% ± 0.17% 51 | 52 | **(2.91, 10-5)-DP** 53 | 54 | Command: `-n 60 --lr=.15 --sigma=1.1 -c=1.0 -b=240` 55 | 56 | Result: accuracy averaged over 10 runs 96.78% ± 0.21% 57 | 58 | **(1.16, 10-5)-DP** 59 | 60 | Command: `-n=15 --lr=.25 --sigma=1.3 -c=1.5 -b=240` 61 | 62 | Result: accuracy averaged over 10 runs 94.63% ± 0.34% 63 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/conv1d_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from typing import Callable 17 | 18 | import hypothesis.strategies as st 19 | import torch 20 | import torch.nn as nn 21 | from hypothesis import given, settings 22 | 23 | from .common import GradSampleHooks_test, expander, shrinker 24 | 25 | 26 | class Conv1d_test(GradSampleHooks_test): 27 | @given( 28 | N=st.integers(0, 4), 29 | C=st.sampled_from([1, 3, 32]), 30 | W=st.integers(6, 10), 31 | out_channels_mapper=st.sampled_from([expander, shrinker]), 32 | kernel_size=st.integers(2, 3), 33 | stride=st.integers(1, 2), 34 | padding=st.sampled_from([0, 1, 2, "same", "valid"]), 35 | dilation=st.integers(1, 2), 36 | groups=st.integers(1, 12), 37 | ) 38 | @settings(deadline=60000) 39 | def test_conv1d( 40 | self, 41 | N: int, 42 | C: int, 43 | W: int, 44 | out_channels_mapper: Callable[[int], int], 45 | kernel_size: int, 46 | stride: int, 47 | padding: int, 48 | dilation: int, 49 | groups: int, 50 | ): 51 | if padding == "same" and stride != 1: 52 | return 53 | out_channels = out_channels_mapper(C) 54 | if ( 55 | C % groups != 0 or out_channels % groups != 0 56 | ): # since in_channels and out_channels must be divisible by groups 57 | return 58 | 59 | x = torch.randn([N, C, W]) 60 | conv = nn.Conv1d( 61 | in_channels=C, 62 | out_channels=out_channels, 63 | kernel_size=kernel_size, 64 | stride=stride, 65 | padding=padding, 66 | dilation=dilation, 67 | groups=groups, 68 | ) 69 | self.run_test( 70 | x, conv, batch_first=True, atol=10e-5, rtol=10e-4, ew_compatible=N > 0 71 | ) 72 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/layer_norm_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import hypothesis.strategies as st 17 | import torch 18 | import torch.nn as nn 19 | from hypothesis import given, settings 20 | 21 | from .common import GradSampleHooks_test 22 | 23 | 24 | class LayerNorm_test(GradSampleHooks_test): 25 | @given( 26 | N=st.integers(1, 4), 27 | Z=st.integers(1, 4), 28 | H=st.integers(1, 3), 29 | W=st.integers(5, 10), 30 | input_dim=st.integers(2, 4), 31 | norm_dim=st.integers(1, 3), 32 | ) 33 | @settings(deadline=60000) 34 | def test_input_norm( 35 | self, N: int, Z: int, W: int, H: int, input_dim: int, norm_dim: int 36 | ): 37 | if norm_dim >= input_dim: 38 | return 39 | normalized_shape, x_shape = self.get_x_shape_and_norm_shape( 40 | H, N, W, Z, input_dim, norm_dim 41 | ) 42 | 43 | norm = nn.LayerNorm(normalized_shape, elementwise_affine=True) 44 | x = torch.randn(x_shape) 45 | self.run_test(x, norm, batch_first=True) 46 | 47 | @staticmethod 48 | def get_x_shape_and_norm_shape(H, N, W, Z, input_dim, norm_dim): 49 | if norm_dim == 1: 50 | normalized_shape = W 51 | if input_dim == 2: 52 | x_shape = [N, W] 53 | if input_dim == 3: 54 | x_shape = [N, Z, W] 55 | if input_dim == 4: 56 | x_shape = [N, Z, H, W] 57 | elif norm_dim == 2: 58 | if input_dim == 3: 59 | normalized_shape = [Z, W] 60 | x_shape = [N, Z, W] 61 | if input_dim == 4: 62 | normalized_shape = [H, W] 63 | x_shape = [N, Z, H, W] 64 | elif norm_dim == 3: 65 | normalized_shape = [Z, H, W] 66 | x_shape = [N, Z, H, W] 67 | return normalized_shape, x_shape 68 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/rms_norm_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import hypothesis.strategies as st 17 | import torch 18 | import torch.nn as nn 19 | from hypothesis import given, settings 20 | 21 | from .common import GradSampleHooks_test 22 | 23 | 24 | class RMSNorm_test(GradSampleHooks_test): 25 | @given( 26 | N=st.integers(1, 4), 27 | Z=st.integers(1, 4), 28 | H=st.integers(1, 3), 29 | W=st.integers(5, 10), 30 | input_dim=st.integers(2, 4), 31 | norm_dim=st.integers(1, 3), 32 | ) 33 | @settings(deadline=60000) 34 | def test_input_norm( 35 | self, N: int, Z: int, W: int, H: int, input_dim: int, norm_dim: int 36 | ): 37 | if norm_dim >= input_dim: 38 | return 39 | normalized_shape, x_shape = self.get_x_shape_and_norm_shape( 40 | H, N, W, Z, input_dim, norm_dim 41 | ) 42 | 43 | norm = nn.RMSNorm(normalized_shape, elementwise_affine=True) 44 | x = torch.randn(x_shape) 45 | self.run_test(x, norm, batch_first=True, ew_compatible=False) 46 | 47 | @staticmethod 48 | def get_x_shape_and_norm_shape(H, N, W, Z, input_dim, norm_dim): 49 | if norm_dim == 1: 50 | normalized_shape = W 51 | if input_dim == 2: 52 | x_shape = [N, W] 53 | if input_dim == 3: 54 | x_shape = [N, Z, W] 55 | if input_dim == 4: 56 | x_shape = [N, Z, H, W] 57 | elif norm_dim == 2: 58 | if input_dim == 3: 59 | normalized_shape = [Z, W] 60 | x_shape = [N, Z, W] 61 | if input_dim == 4: 62 | normalized_shape = [H, W] 63 | x_shape = [N, Z, H, W] 64 | elif norm_dim == 3: 65 | normalized_shape = [Z, H, W] 66 | x_shape = [N, Z, H, W] 67 | return normalized_shape, x_shape 68 | -------------------------------------------------------------------------------- /scripts/install_via_pip.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -e 17 | 18 | PYTORCH_NIGHTLY=false 19 | DEPLOY=false 20 | CHOSEN_TORCH_VERSION=-1 21 | 22 | while getopts 'ncdv:' flag; do 23 | case "${flag}" in 24 | n) PYTORCH_NIGHTLY=true ;; 25 | c) CUDA=true ;; 26 | d) DEPLOY=true ;; 27 | v) CHOSEN_TORCH_VERSION=${OPTARG} ;; 28 | *) echo "usage: $0 [-n] [-d] [-v version]" >&2 29 | exit 1 ;; 30 | esac 31 | done 32 | 33 | # NOTE: Only Debian variants are supported, since this script is only 34 | # used by our tests on CircleCI. 35 | 36 | curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - 37 | echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list 38 | sudo apt-get -y update && sudo apt-get -y install yarn 39 | 40 | # yarn needs terminal info 41 | export TERM=xterm 42 | 43 | # NOTE: We don't use sudo for the below installs, because we expect these to come from pyenv which 44 | # installs Python in a folder for which we have user access. 45 | 46 | # upgrade pip 47 | pip install --upgrade pip 48 | 49 | # install with dev dependencies 50 | pip install -e .[dev] --user 51 | 52 | # install pytorch nightly if asked for 53 | if [[ $PYTORCH_NIGHTLY == true ]]; then 54 | if [[ $CUDA == true ]]; then 55 | pip install --upgrade --pre torch torchvision -f https://download.pytorch.org/whl/nightly/cu101/torch_nightly.html 56 | else 57 | pip install --upgrade --pre torch torchvision -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html 58 | fi 59 | else 60 | # If no version specified, upgrade to latest release. 61 | if [[ $CHOSEN_TORCH_VERSION == -1 ]]; then 62 | pip install --upgrade torch 63 | else 64 | pip install torch=="$CHOSEN_TORCH_VERSION" 65 | fi 66 | fi 67 | 68 | # install deployment bits if asked for 69 | if [[ $DEPLOY == true ]]; then 70 | pip install beautifulsoup4 ipython nbconvert 71 | fi 72 | -------------------------------------------------------------------------------- /opacus/grad_sample/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .conv import compute_conv_grad_sample # noqa 17 | from .dp_multihead_attention import compute_sequence_bias_grad_sample # noqa 18 | from .dp_rnn import compute_rnn_linear_grad_sample # noqa 19 | from .embedding import compute_embedding_grad_sample # noqa 20 | from .embedding_norm_sample import compute_embedding_norm_sample # noqa 21 | from .grad_sample_module import GradSampleModule, create_or_accumulate_grad_sample 22 | from .grad_sample_module_fast_gradient_clipping import ( # noqa 23 | GradSampleModuleFastGradientClipping, 24 | ) 25 | from .grad_sample_module_fast_gradient_clipping_fsdp import ( # noqa 26 | GradSampleModuleFastGradientClippingFSDP, 27 | ) 28 | from .grad_sample_module_fast_gradient_clipping_tp import ( # noqa 29 | GradSampleModuleFastGradientClippingTP, 30 | ) 31 | from .group_norm import compute_group_norm_grad_sample # noqa 32 | from .gsm_base import AbstractGradSampleModule 33 | from .gsm_exp_weights import GradSampleModuleExpandedWeights 34 | from .gsm_no_op import GradSampleModuleNoOp 35 | from .instance_norm import compute_instance_norm_grad_sample # noqa 36 | from .layer_norm import compute_layer_norm_grad_sample # noqa 37 | from .linear import compute_linear_grad_sample # noqa 38 | from .rms_norm import compute_rms_norm_grad_sample # noqa 39 | from .utils import ( 40 | get_gsm_class, 41 | register_grad_sampler, 42 | register_norm_sampler, 43 | wrap_model, 44 | ) 45 | 46 | 47 | __all__ = [ 48 | "GradSampleModule", 49 | "GradSampleModuleFastGradientClipping", 50 | "GradSampleModuleFastGradientClippingFSDP", 51 | "GradSampleModuleFastGradientClippingTP", 52 | "GradSampleModuleExpandedWeights", 53 | "GradSampleModuleNoOp", 54 | "AbstractGradSampleModule", 55 | "register_grad_sampler", 56 | "register_norm_sampler", 57 | "create_or_accumulate_grad_sample", 58 | "wrap_model", 59 | "get_gsm_class", 60 | ] 61 | -------------------------------------------------------------------------------- /opacus/utils/fsdp_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from typing import Iterable 17 | 18 | import torch 19 | import torch.nn as nn 20 | from opacus.grad_sample.grad_sample_module_fast_gradient_clipping_fsdp import ( 21 | GradSampleModuleFastGradientClippingFSDP, 22 | ) 23 | from opacus.utils.module_utils import has_trainable_params 24 | from torch.distributed.fsdp import MixedPrecisionPolicy, fully_shard 25 | 26 | 27 | def has_params(module: nn.Module) -> bool: 28 | return len(list(module.parameters(recurse=False))) > 0 29 | 30 | 31 | def iterate_submodules(module: nn.Module) -> Iterable[nn.Module]: 32 | if has_params(module): 33 | yield module 34 | 35 | for m in module.children(): 36 | yield from iterate_submodules(m) 37 | 38 | 39 | def FSDP2Wrapper(model: nn.Module, **kwargs) -> nn.Module: 40 | sampler_classes = set( 41 | list(GradSampleModuleFastGradientClippingFSDP.GRAD_SAMPLERS.keys()) 42 | + list(GradSampleModuleFastGradientClippingFSDP.NORM_SAMPLERS.keys()) 43 | ) 44 | mp_policy = kwargs.get("mp_policy", MixedPrecisionPolicy()) 45 | opacus_high_precision_layers = kwargs.get("opacus_high_precision_layers", []) 46 | for module in iterate_submodules(model): 47 | if (type(module) in sampler_classes) or (not has_trainable_params(module)): 48 | if len(opacus_high_precision_layers) > 0 and isinstance( 49 | module, opacus_high_precision_layers 50 | ): 51 | # For certain layers, higher precision is needed to stablize the training of DP-SGD. 52 | fully_shard( 53 | module, 54 | mp_policy=MixedPrecisionPolicy( 55 | param_dtype=torch.get_default_dtype() 56 | ), 57 | ) 58 | else: 59 | fully_shard(module, mp_policy=mp_policy) 60 | model = fully_shard(model, mp_policy=mp_policy) 61 | return model 62 | -------------------------------------------------------------------------------- /opacus/README.md: -------------------------------------------------------------------------------- 1 | # Developer Notes 2 | These developer notes can help you ramp up on this codebase. For any question, hit us up on the [forums](https://discuss.pytorch.org/c/opacus/29)! 3 | 4 | 5 | ## Supported modules 6 | Opacus only works with supported ``nn.Module``s. The following modules are supported: 7 | 8 | 1. Modules with no trainable parameters (eg ``nn.ReLU``, `nn.Tanh`) 9 | 2. Modules which are frozen. A nn.Module can be frozen in PyTorch by unsetting ``requires_grad`` 10 | in each of its parameters, ie `for p in module.parameters(): p.requires_grad = False`. 11 | 3. Explicitly supported modules and settings (see `grad_sample/` folder), eg ``nn.Conv2d``. 12 | 4. Any complex nn.Module that contains only supported nn.Modules. This means that most models 13 | will be compatible, given that we support most of the common building blocks. This however also 14 | means that Opacus support depends on how a specific ``nn.Module`` is implemented. For example, 15 | ``nn.LSTM`` *could* be written by using ``nn.Linear`` (which we support), but its actual 16 | implementation does not use it (so that it can fuse operators and be faster). Any layer that 17 | needs a rewrite to be supported is in the `grad_sample/` folder. 18 | 19 | As an example, the following ``nn.Module`` is supported, because it's made entirely of supported 20 | nn.Modules: 21 | 22 | ```python 23 | class SampleConvNet(nn.Module): 24 | def __init__(self): 25 | super().__init__() 26 | self.conv1 = nn.Conv2d(1, 16, 8, 2, padding=3) 27 | self.conv2 = nn.Conv2d(16, 32, 4, 2) 28 | self.fc1 = nn.Linear(32 * 4 * 4, 32) 29 | self.fc2 = nn.Linear(32, 10) 30 | 31 | def forward(self, x): 32 | # x of shape [B, 1, 28, 28] 33 | x = F.relu(self.conv1(x)) # -> [B, 16, 14, 14] 34 | x = F.max_pool2d(x, 2, 1) # -> [B, 16, 13, 13] 35 | x = F.relu(self.conv2(x)) # -> [B, 32, 5, 5] 36 | x = F.max_pool2d(x, 2, 1) # -> [B, 32, 4, 4] 37 | x = x.view(-1, 32 * 4 * 4) # -> [B, 512] 38 | x = F.relu(self.fc1(x)) # -> [B, 32] 39 | x = self.fc2(x) # -> [B, 10] 40 | return x 41 | ``` 42 | 43 | ## Limitations of backward hooks 44 | The implementation of gradient clipping in autograd_grad_sample.py uses backward hooks to capture per-sample gradients. 45 | The `register_backward hook` function has a known issue being tracked at https://github.com/pytorch/pytorch/issues/598. However, this is the only known way of implementing this as of now (your suggestions and contributions are very welcome). The behavior has been verified to be correct for the layers currently supported by Opacus. 46 | -------------------------------------------------------------------------------- /conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | import torch 17 | from opacus import PrivacyEngine 18 | from torch import nn 19 | from torch.utils.data import DataLoader, TensorDataset 20 | 21 | 22 | class MyCustomModel(nn.Module): 23 | """Demo module to use in doctests""" 24 | 25 | def __init__(self): 26 | super().__init__() 27 | self.f = nn.Linear(5, 2) 28 | 29 | def forward(self, x): 30 | x = self.f(x) 31 | return x 32 | 33 | 34 | def create_demo_dataloader(): 35 | dataset = TensorDataset(torch.randn(64, 5), torch.randint(0, 2, (64,))) 36 | dataloader = DataLoader(dataset, batch_size=8) 37 | return dataloader 38 | 39 | 40 | def _init_private_training(): 41 | model = MyCustomModel() 42 | optimizer = torch.optim.SGD(model.parameters(), lr=0.05) 43 | data_loader = create_demo_dataloader() 44 | privacy_engine = PrivacyEngine() 45 | 46 | model, optimizer, data_loader = privacy_engine.make_private( 47 | module=model, 48 | optimizer=optimizer, 49 | data_loader=data_loader, 50 | noise_multiplier=1.0, 51 | max_grad_norm=1.0, 52 | ) 53 | 54 | return model, optimizer, data_loader 55 | 56 | 57 | @pytest.fixture(autouse=True) 58 | def create_namespace(doctest_namespace): 59 | """ 60 | Initialize namespace for doctest. 61 | Everything added to `doctest_namespace` will be available in the doctest. 62 | """ 63 | from typing import Any, Dict, List, Set, Tuple, Union # noqa 64 | 65 | import numpy as np # noqa 66 | import opacus # noqa 67 | import torch # noqa 68 | from torch import nn # noqa 69 | 70 | # Adding all imports in the doctest namespace 71 | doctest_namespace.update(**locals()) 72 | 73 | doctest_namespace["MyCustomModel"] = MyCustomModel 74 | doctest_namespace["TensorDataset"] = TensorDataset 75 | doctest_namespace["demo_dataloader"] = create_demo_dataloader() 76 | doctest_namespace["_init_private_training"] = _init_private_training 77 | -------------------------------------------------------------------------------- /research/disk_optimizer/KFprivacy_engine.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Xinwei Zhang 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import List, Union 16 | 17 | from opacus.optimizers import DPOptimizer 18 | from opacus.privacy_engine import PrivacyEngine 19 | from torch import optim 20 | 21 | from .optimizers import KF_DPOptimizer, get_optimizer_class 22 | 23 | 24 | class KF_PrivacyEngine(PrivacyEngine): 25 | def __init__(self, *, accountant: str = "prv", secure_mode: bool = False): 26 | super().__init__(accountant=accountant, secure_mode=secure_mode) 27 | 28 | def _prepare_optimizer( 29 | self, 30 | *, 31 | optimizer: optim.Optimizer, 32 | noise_multiplier: float, 33 | max_grad_norm: Union[float, List[float]], 34 | expected_batch_size: int, 35 | loss_reduction: str = "mean", 36 | distributed: bool = False, 37 | clipping: str = "flat", 38 | noise_generator=None, 39 | grad_sample_mode="hooks", 40 | kalman: bool = False, 41 | **kwargs, 42 | ) -> DPOptimizer: 43 | if kalman and isinstance(optimizer, KF_DPOptimizer): 44 | optimizer = optimizer.original_optimizer 45 | elif not kalman and isinstance(optimizer, DPOptimizer): 46 | optimizer = optimizer.original_optimizer 47 | 48 | generator = None 49 | if self.secure_mode: 50 | generator = self.secure_rng 51 | elif noise_generator is not None: 52 | generator = noise_generator 53 | 54 | optim_class = get_optimizer_class( 55 | clipping=clipping, 56 | distributed=distributed, 57 | grad_sample_mode=grad_sample_mode, 58 | kalman=kalman, 59 | ) 60 | 61 | return optim_class( 62 | optimizer=optimizer, 63 | noise_multiplier=noise_multiplier, 64 | max_grad_norm=max_grad_norm, 65 | expected_batch_size=expected_batch_size, 66 | loss_reduction=loss_reduction, 67 | generator=generator, 68 | secure_mode=self.secure_mode, 69 | **kwargs, 70 | ) 71 | -------------------------------------------------------------------------------- /opacus/accountants/gdp.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import warnings 16 | 17 | from .accountant import IAccountant 18 | from .analysis import gdp as privacy_analysis 19 | 20 | 21 | class GaussianAccountant(IAccountant): 22 | def __init__(self): 23 | warnings.warn( 24 | "GDP accounting is experimental and can underestimate privacy expenditure." 25 | "Proceed with caution. More details: https://arxiv.org/pdf/2106.02848.pdf" 26 | ) 27 | super().__init__() 28 | 29 | def step(self, *, noise_multiplier: float, sample_rate: float): 30 | if len(self.history) >= 1: 31 | last_noise_multiplier, last_sample_rate, num_steps = self.history.pop() 32 | if ( 33 | last_noise_multiplier != noise_multiplier 34 | or last_sample_rate != sample_rate 35 | ): 36 | raise ValueError( 37 | "Noise multiplier and sample rate have to stay constant in GaussianAccountant." 38 | ) 39 | else: 40 | self.history = [ 41 | (last_noise_multiplier, last_sample_rate, num_steps + 1) 42 | ] 43 | 44 | else: 45 | self.history = [(noise_multiplier, sample_rate, 1)] 46 | 47 | def get_epsilon(self, delta: float, poisson: bool = True, **kwargs) -> float: 48 | """ 49 | Return privacy budget (epsilon) expended so far. 50 | 51 | Args: 52 | delta: target delta 53 | poisson: ``True`` is input batches was sampled via Poisson sampling, 54 | ``False`` otherwise 55 | """ 56 | 57 | compute_eps = ( 58 | privacy_analysis.compute_eps_poisson 59 | if poisson 60 | else privacy_analysis.compute_eps_uniform 61 | ) 62 | noise_multiplier, sample_rate, num_steps = self.history[-1] 63 | return compute_eps( 64 | steps=num_steps, 65 | noise_multiplier=noise_multiplier, 66 | sample_rate=sample_rate, 67 | delta=delta, 68 | ) 69 | 70 | def __len__(self): 71 | return len(self.history) 72 | 73 | @classmethod 74 | def mechanism(cls) -> str: 75 | return "gdp" 76 | -------------------------------------------------------------------------------- /opacus/optimizers/perlayeroptimizer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import annotations 16 | 17 | from typing import List, Optional 18 | 19 | import torch 20 | from opacus.optimizers.utils import params 21 | from torch.optim import Optimizer 22 | 23 | from .optimizer import DPOptimizer, _check_processed_flag, _mark_as_processed 24 | 25 | 26 | class DPPerLayerOptimizer(DPOptimizer): 27 | """ 28 | :class:`~opacus.optimizers.optimizer.DPOptimizer` that implements 29 | per layer clipping strategy 30 | """ 31 | 32 | def __init__( 33 | self, 34 | optimizer: Optimizer, 35 | *, 36 | noise_multiplier: float, 37 | max_grad_norm: List[float], 38 | expected_batch_size: Optional[int], 39 | loss_reduction: str = "mean", 40 | generator=None, 41 | secure_mode: bool = False, 42 | **kwargs, 43 | ): 44 | assert len(max_grad_norm) == len(params(optimizer)) 45 | self.max_grad_norms = max_grad_norm 46 | max_grad_norm = torch.norm(torch.Tensor(self.max_grad_norms), p=2).item() 47 | super().__init__( 48 | optimizer, 49 | noise_multiplier=noise_multiplier, 50 | max_grad_norm=max_grad_norm, 51 | expected_batch_size=expected_batch_size, 52 | loss_reduction=loss_reduction, 53 | generator=generator, 54 | secure_mode=secure_mode, 55 | **kwargs, 56 | ) 57 | 58 | def clip_and_accumulate(self): 59 | for p, max_grad_norm in zip(self.params, self.max_grad_norms): 60 | _check_processed_flag(p.grad_sample) 61 | 62 | grad_sample = self._get_flat_grad_sample(p) 63 | per_sample_norms = grad_sample.norm( 64 | 2, dim=tuple(range(1, grad_sample.ndim)) 65 | ) 66 | per_sample_clip_factor = (max_grad_norm / (per_sample_norms + 1e-6)).clamp( 67 | max=1.0 68 | ) 69 | grad = torch.einsum("i,i...", per_sample_clip_factor, grad_sample) 70 | 71 | if p.summed_grad is not None: 72 | p.summed_grad += grad 73 | else: 74 | p.summed_grad = grad 75 | 76 | _mark_as_processed(p.grad_sample) 77 | -------------------------------------------------------------------------------- /opacus/tests/validators/instance_norm_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import unittest 17 | 18 | import torch.nn as nn 19 | from opacus.validators.errors import IllegalModuleConfigurationError 20 | from opacus.validators.module_validator import ModuleValidator 21 | 22 | 23 | class InstanceNormValidator_test(unittest.TestCase): 24 | def setUp(self) -> None: 25 | self.in1 = nn.InstanceNorm1d(4, affine=True, track_running_stats=True) 26 | self.in2 = nn.InstanceNorm2d(4, affine=False, track_running_stats=True) 27 | self.in3 = nn.InstanceNorm3d(4, affine=False, track_running_stats=True) 28 | self.in_no_stats = nn.InstanceNorm3d(4, affine=True) 29 | 30 | self.mv = ModuleValidator.VALIDATORS 31 | self.mf = ModuleValidator.FIXERS 32 | 33 | def test_validate(self) -> None: 34 | val1 = self.mv[type(self.in1)](self.in1) 35 | val2 = self.mv[type(self.in2)](self.in2) 36 | val3 = self.mv[type(self.in3)](self.in3) 37 | vals = self.mv[type(self.in_no_stats)](self.in_no_stats) 38 | 39 | self.assertEqual(len(val1), 1) 40 | self.assertEqual(len(val2), 1) 41 | self.assertEqual(len(val3), 1) 42 | self.assertEqual(len(vals), 0) 43 | 44 | self.assertTrue(isinstance(val1[0], IllegalModuleConfigurationError)) 45 | self.assertTrue(isinstance(val2[0], IllegalModuleConfigurationError)) 46 | self.assertTrue(isinstance(val3[0], IllegalModuleConfigurationError)) 47 | 48 | def test_fix(self) -> None: 49 | fix1 = self.mf[type(self.in1)](self.in1) 50 | fix2 = self.mf[type(self.in2)](self.in2) 51 | fix3 = self.mf[type(self.in3)](self.in3) 52 | fixs = self.mf[type(self.in_no_stats)](self.in_no_stats) 53 | 54 | self.assertTrue(isinstance(fix1, nn.InstanceNorm1d)) 55 | self.assertFalse(fix1.track_running_stats) 56 | 57 | self.assertTrue(isinstance(fix2, nn.InstanceNorm2d)) 58 | self.assertFalse(fix2.track_running_stats) 59 | 60 | self.assertTrue(isinstance(fix3, nn.InstanceNorm3d)) 61 | self.assertFalse(fix3.track_running_stats) 62 | 63 | self.assertTrue(isinstance(fixs, nn.InstanceNorm3d)) 64 | self.assertTrue(fixs is self.in_no_stats) 65 | -------------------------------------------------------------------------------- /examples/scripts/make_small_imagenet_N_classes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | usage() 18 | { 19 | echo "usage Example: bash make_small_imagenet_N_classes.sh -o Documents/imagenet_full_size -d Documents/imagenet_small_50_classes -n 50" 20 | echo "usage: bash make_small_imagenet_N_classes.sh -o -d -n " 21 | } 22 | 23 | if [ "$1" == "" ]; then # If arguments are not specified 24 | usage 25 | exit 1 26 | fi 27 | while [ "$1" != "" ]; do # Read the input arguments 28 | case $1 in 29 | -d | --destination ) shift 30 | destination=$1 31 | ;; 32 | -o | --origin ) shift 33 | origin=$1 34 | ;; 35 | -n | --N ) shift 36 | N=$1 37 | ;; 38 | -h | --help ) usage 39 | exit 40 | ;; 41 | * ) echo "ERROR: unknown parameter \"$1\"" 42 | usage 43 | exit 1 44 | esac 45 | shift 46 | done 47 | 48 | # If all necessary arguments are not supplied 49 | if [[ -z $destination || -z $origin || -z $N ]] 50 | then 51 | echo "You must specify all necessary parameters." 52 | usage 53 | exit 1 54 | fi 55 | 56 | # Get absolute path 57 | destination="$(readlink -f $destination)" 58 | origin="$(readlink -f $origin)" 59 | 60 | mkdir "$destination" 61 | mkdir "$destination/train" 62 | mkdir "$destination/val" 63 | 64 | echo 'Copying' 65 | for val_train_folder in val train; do # Do copying for both 'train' and 'val' folders 66 | cd "$origin/$val_train_folder" || { echo "Failure"; exit 1; } # change directory to origin's train or val folders 67 | find . -maxdepth 1 -mindepth 1 | head -n "$N" | xargs cp -ir -t "$destination/$val_train_folder" # select and copy N classes 68 | echo "Copying folder $val_train_folder is done." 69 | done 70 | -------------------------------------------------------------------------------- /opacus/optimizers/ddpoptimizer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import annotations 16 | 17 | from typing import Callable, Optional 18 | 19 | import torch 20 | from torch.optim import Optimizer 21 | 22 | from .optimizer import DPOptimizer 23 | 24 | 25 | class DistributedDPOptimizer(DPOptimizer): 26 | """ 27 | :class:`~opacus.optimizers.optimizer.DPOptimizer` compatible with 28 | distributed data processing 29 | """ 30 | 31 | def __init__( 32 | self, 33 | optimizer: Optimizer, 34 | *, 35 | noise_multiplier: float, 36 | max_grad_norm: float, 37 | expected_batch_size: Optional[int], 38 | loss_reduction: str = "mean", 39 | generator=None, 40 | secure_mode: bool = False, 41 | **kwargs, 42 | ): 43 | super().__init__( 44 | optimizer, 45 | noise_multiplier=noise_multiplier, 46 | max_grad_norm=max_grad_norm, 47 | expected_batch_size=expected_batch_size, 48 | loss_reduction=loss_reduction, 49 | generator=generator, 50 | secure_mode=secure_mode, 51 | **kwargs, 52 | ) 53 | self.rank = torch.distributed.get_rank() 54 | self.world_size = torch.distributed.get_world_size() 55 | 56 | def add_noise(self): 57 | # Noise only gets added to the first worker 58 | if self.rank == 0: 59 | super().add_noise() 60 | else: 61 | for p in self.params: 62 | p.grad = p.summed_grad.view_as(p) 63 | 64 | def reduce_gradients(self): 65 | for p in self.params: 66 | if not p.requires_grad: 67 | continue 68 | torch.distributed.all_reduce(p.grad, op=torch.distributed.ReduceOp.SUM) 69 | if self.loss_reduction == "mean": 70 | p.grad /= self.world_size 71 | 72 | def step( 73 | self, closure: Optional[Callable[[], float]] = None 74 | ) -> Optional[torch.Tensor]: 75 | if closure is not None: 76 | with torch.enable_grad(): 77 | closure() 78 | 79 | if self.pre_step(): 80 | self.reduce_gradients() 81 | return self.original_optimizer.step() 82 | else: 83 | return None 84 | -------------------------------------------------------------------------------- /opacus/tests/poisson_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import unittest 17 | 18 | import numpy as np 19 | import torch 20 | from opacus.utils.uniform_sampler import UniformWithReplacementSampler 21 | 22 | 23 | class PoissonSamplingTest(unittest.TestCase): 24 | def _init_data(self, seed=0): 25 | generator = torch.Generator() 26 | generator.manual_seed(seed) 27 | sampler = UniformWithReplacementSampler( 28 | num_samples=len(self.dataset), 29 | sample_rate=self.batch_size / len(self.dataset), 30 | generator=generator, 31 | ) 32 | dataloader = torch.utils.data.DataLoader(self.dataset, batch_sampler=sampler) 33 | 34 | return sampler, dataloader 35 | 36 | def setUp(self) -> None: 37 | self.data_size = 100 38 | self.batch_size = 10 39 | self.dataset = [ 40 | (torch.randn(10), torch.randn(10)) for _ in range(self.data_size) 41 | ] 42 | 43 | self.sampler, self.dataloader = self._init_data(seed=7) 44 | 45 | def test_length(self) -> None: 46 | self.assertEqual(len(self.sampler), 10) 47 | self.assertEqual(len(self.dataloader), 10) 48 | 49 | def test_batch_sizes(self) -> None: 50 | batch_sizes = [] 51 | for x, _y in self.dataloader: 52 | batch_sizes.append(x.shape[0]) 53 | 54 | self.assertGreater(len(set(batch_sizes)), 1) 55 | self.assertAlmostEqual(np.mean(batch_sizes), self.batch_size, delta=2) 56 | 57 | def test_same_seed(self) -> None: 58 | batch_sizes1 = [] 59 | for x, _y in self.dataloader: 60 | batch_sizes1.append(x.shape[0]) 61 | 62 | _, dataloader = self._init_data(seed=7) 63 | batch_sizes2 = [] 64 | for x, _y in dataloader: 65 | batch_sizes2.append(x.shape[0]) 66 | 67 | self.assertEqual(batch_sizes1, batch_sizes2) 68 | 69 | def test_different_seed(self) -> None: 70 | batch_sizes1 = [] 71 | for x, _y in self.dataloader: 72 | batch_sizes1.append(x.shape[0]) 73 | 74 | _, dataloader = self._init_data(seed=8) 75 | batch_sizes2 = [] 76 | for x, _y in dataloader: 77 | batch_sizes2.append(x.shape[0]) 78 | 79 | self.assertNotEqual(batch_sizes1, batch_sizes2) 80 | -------------------------------------------------------------------------------- /opacus/tests/schedulers/noise_scheduler_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import unittest 17 | 18 | import torch 19 | from opacus import PrivacyEngine 20 | from opacus.schedulers import ExponentialNoise, LambdaNoise, StepNoise 21 | from torch import nn, optim 22 | from torch.utils.data import DataLoader, TensorDataset 23 | 24 | 25 | class NoiseSchedulerTest(unittest.TestCase): 26 | def setUp(self): 27 | n_data, dim = 100, 10 28 | data = torch.randn(n_data, dim) 29 | model = nn.Linear(10, 10) 30 | optimizer = optim.SGD(model.parameters(), lr=0.1) 31 | data_loader = DataLoader(TensorDataset(data), batch_size=10) 32 | self.engine = PrivacyEngine() 33 | 34 | self.module, self.optimizer, self.data_loader = self.engine.make_private( 35 | module=model, 36 | optimizer=optimizer, 37 | data_loader=data_loader, 38 | noise_multiplier=1.0, 39 | max_grad_norm=1.0, 40 | ) 41 | 42 | def test_exponential_scheduler(self): 43 | gamma = 0.99 44 | scheduler = ExponentialNoise(self.optimizer, gamma=gamma) 45 | 46 | self.assertEqual(self.optimizer.noise_multiplier, 1.0) 47 | scheduler.step() 48 | self.assertEqual(self.optimizer.noise_multiplier, gamma) 49 | 50 | def test_step_scheduler(self): 51 | gamma = 0.1 52 | step_size = 2 53 | scheduler = StepNoise(self.optimizer, step_size=step_size, gamma=gamma) 54 | 55 | self.assertEqual(self.optimizer.noise_multiplier, 1.0) 56 | scheduler.step() 57 | self.assertEqual(self.optimizer.noise_multiplier, 1.0) 58 | scheduler.step() 59 | self.assertEqual(self.optimizer.noise_multiplier, gamma) 60 | scheduler.step() 61 | self.assertEqual(self.optimizer.noise_multiplier, gamma) 62 | scheduler.step() 63 | self.assertEqual(self.optimizer.noise_multiplier, gamma**2) 64 | 65 | def test_lambda_scheduler(self): 66 | def noise_lambda(epoch): 67 | return 1 - epoch / 10 68 | 69 | scheduler = LambdaNoise(self.optimizer, noise_lambda=noise_lambda) 70 | 71 | self.assertEqual(self.optimizer.noise_multiplier, 1.0) 72 | scheduler.step() 73 | self.assertEqual(self.optimizer.noise_multiplier, noise_lambda(1)) 74 | -------------------------------------------------------------------------------- /research/disk_optimizer/ReadMe.md: -------------------------------------------------------------------------------- 1 | # DiSK: Differentially Private Optimizer with Simplified Kalman Filter for Noise Reduction 2 | 3 | ## Introduction 4 | This part of the code introduces a new component to the optimizer named DiSK. The code uses a simplifed Kalman to improve the privatized gradient estimate. Speficially, the privatized minibatch gradient is replaced with: 5 | 6 | 7 | $$\mathbb{g}_{t+\frac{1}{2}}(\xi) = \frac{1-\kappa}{\kappa\gamma}\nabla f(x_t + \gamma(x_t-x_{t-1});\xi) + \Big(1- \frac{1-\kappa}{\kappa\gamma}\Big)\nabla f(x_t;\xi)$$ 8 | $$\mathbb{g_{t+\frac{1}{2}}} = \frac{1}{B}\sum_{\xi \in \mathcal{B}_t} \mathrm{clip}_C\left(\mathbb{g}_{t+\frac{1}{2}}(\xi)\right) + w_t$$ 9 | $$g_{t}= (1-\kappa)g_{t-1} + \kappa g_{t+\frac{1}{2}}$$ 10 | 11 | A detailed description of the algorithm can be found at [Here](https://arxiv.org/abs/2410.03883). 12 | 13 | ## Usage 14 | The code provides a modified privacy engine with three extra arguments: 15 | * kamlan: bool=False 16 | * kappa: float=0.7 17 | * gamma: float=0.5 18 | 19 | To use DiSK, follow the steps: 20 | 21 | **Step I:** Import KF_PrivacyEngine from KFprivacy_engine.py and set ```kalman=True``` 22 | 23 | **Step II:** Define a closure (see [here](https://pytorch.org/docs/stable/optim.html#optimizer-step-closure) for example) to compute loss and backward **without** ```zero_grad()``` and perform ```optimizer.step(closure)``` 24 | 25 | Example of using the DiSK optimizers: 26 | 27 | ```python 28 | from KFprivacy_engine import KF_PrivacyEngine 29 | # ... 30 | # follow the same steps as original opacus training scripts 31 | privacy_engine = KF_PrivacyEngine() 32 | model, optimizer, train_loader = privacy_engine.make_private( 33 | module=model, 34 | optimizer=optimizer, 35 | data_loader=train_loader, 36 | noise_multiplier=args.sigma, 37 | max_grad_norm=max_grad_norm, 38 | clipping=clipping, 39 | grad_sample_mode=args.grad_sample_mode, 40 | kalman=True, # need this argument 41 | kappa=0.7, # optional 42 | gamma=0.5 # optional 43 | ) 44 | 45 | # ... 46 | # during training: 47 | def closure(): # compute loss and backward, an example adapting the one used in examples/cifar10.py 48 | output = model(images) 49 | loss = criterion(output, target) 50 | loss.backward() 51 | return output, loss 52 | output, loss = optimizer.step(closure) 53 | optimizer.zero_grad() 54 | # compute other matrices 55 | # ... 56 | ``` 57 | 58 | ## Citation 59 | Consider citing the paper is you use DiSK in your papers, as follows: 60 | 61 | ``` 62 | @article{zhang2024disk, 63 | title={{DiSK}: Differentially private optimizer with simplified kalman filter for noise reduction}, 64 | author={Zhang, Xinwei and Bu, Zhiqi and Balle, Borja and Hong, Mingyi and Razaviyayn, Meisam and Mirrokni, Vahab}, 65 | journal={arXiv preprint arXiv:2410.03883}, 66 | year={2024} 67 | } 68 | ``` 69 | 70 | Contributer: Xinwei Zhang. Email: [zhan6234@umn.edu](mailto:zhan6234@umn.edu) 71 | 72 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/conv3d_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from typing import Tuple, Union 17 | 18 | import hypothesis.strategies as st 19 | import torch 20 | import torch.nn as nn 21 | from hypothesis import given, settings 22 | 23 | from .common import GradSampleHooks_test, expander, shrinker 24 | 25 | 26 | class Conv3d_test(GradSampleHooks_test): 27 | @given( 28 | N=st.integers(0, 4), 29 | C=st.sampled_from([1, 3, 32]), 30 | D=st.integers(3, 6), 31 | H=st.integers(6, 10), 32 | W=st.integers(6, 10), 33 | out_channels_mapper=st.sampled_from([expander, shrinker]), 34 | kernel_size=st.sampled_from([2, 3, (1, 2, 3)]), 35 | stride=st.sampled_from([1, 2, (1, 2, 3)]), 36 | padding=st.sampled_from([0, 2, (1, 2, 3), "same", "valid"]), 37 | dilation=st.sampled_from([1, (1, 2, 2)]), 38 | groups=st.integers(1, 16), 39 | ) 40 | @settings(deadline=60000) 41 | def test_conv3d( 42 | self, 43 | N: int, 44 | C: int, 45 | D: int, 46 | H: int, 47 | W: int, 48 | out_channels_mapper: int, 49 | kernel_size: Union[int, Tuple[int]], 50 | stride: Union[int, Tuple[int]], 51 | padding: Union[int, Tuple[int]], 52 | dilation: int, 53 | groups: int, 54 | ): 55 | if padding == "same" and stride != 1: 56 | return 57 | out_channels = out_channels_mapper(C) 58 | if ( 59 | C % groups != 0 or out_channels % groups != 0 60 | ): # since in_channels and out_channels must be divisible by groups 61 | return 62 | x = torch.randn([N, C, D, H, W]) 63 | conv = nn.Conv3d( 64 | in_channels=C, 65 | out_channels=out_channels, 66 | kernel_size=kernel_size, 67 | stride=stride, 68 | padding=padding, 69 | dilation=dilation, 70 | groups=groups, 71 | ) 72 | is_ew_compatible = ( 73 | dilation == 1 and padding != "same" and N > 0 74 | ) # TODO add support for padding = 'same' with EW 75 | self.run_test( 76 | x, 77 | conv, 78 | batch_first=True, 79 | atol=10e-5, 80 | rtol=10e-3, 81 | ew_compatible=is_ew_compatible, 82 | ) 83 | -------------------------------------------------------------------------------- /research/disk_optimizer/optimizers/KFperlayeroptimizer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Xinwei Zhang 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import annotations 16 | 17 | import logging 18 | from typing import List, Optional 19 | 20 | import torch 21 | from opacus.optimizers.optimizer import _check_processed_flag, _mark_as_processed 22 | from opacus.optimizers.utils import params 23 | from torch.optim import Optimizer 24 | 25 | from .KFoptimizer import KF_DPOptimizer 26 | 27 | 28 | logger = logging.getLogger(__name__) 29 | logger.disabled = True 30 | 31 | 32 | class KF_DPPerLayerOptimizer(KF_DPOptimizer): 33 | def __init__( 34 | self, 35 | optimizer: Optimizer, 36 | *, 37 | noise_multiplier: float, 38 | max_grad_norm: List[float], 39 | expected_batch_size: Optional[int], 40 | loss_reduction: str = "mean", 41 | generator=None, 42 | secure_mode: bool = False, 43 | kappa=0.7, 44 | gamma=0.5, 45 | **kwargs, 46 | ): 47 | assert len(max_grad_norm) == len(params(optimizer)) 48 | self.max_grad_norms = max_grad_norm 49 | max_grad_norm = torch.norm(torch.Tensor(self.max_grad_norms), p=2).item() 50 | super().__init__( 51 | optimizer, 52 | noise_multiplier=noise_multiplier, 53 | max_grad_norm=max_grad_norm, 54 | expected_batch_size=expected_batch_size, 55 | loss_reduction=loss_reduction, 56 | generator=generator, 57 | secure_mode=secure_mode, 58 | kappa=kappa, 59 | gamma=gamma, 60 | **kwargs, 61 | ) 62 | 63 | def clip_and_accumulate(self): 64 | for p, max_grad_norm in zip(self.params, self.max_grad_norms): 65 | _check_processed_flag(p.grad_sample) 66 | 67 | grad_sample = self._get_flat_grad_sample(p) 68 | per_sample_norms = grad_sample.norm( 69 | 2, dim=tuple(range(1, grad_sample.ndim)) 70 | ) 71 | per_sample_clip_factor = (max_grad_norm / (per_sample_norms + 1e-6)).clamp( 72 | max=1.0 73 | ) 74 | grad = torch.einsum("i,i...", per_sample_clip_factor, grad_sample) 75 | 76 | if p.summed_grad is not None: 77 | p.summed_grad += grad 78 | else: 79 | p.summed_grad = grad 80 | 81 | _mark_as_processed(p.grad_sample) 82 | -------------------------------------------------------------------------------- /opacus/accountants/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import Optional 16 | 17 | from opacus.accountants import create_accountant 18 | 19 | 20 | MAX_SIGMA = 1e6 21 | 22 | 23 | def get_noise_multiplier( 24 | *, 25 | target_epsilon: float, 26 | target_delta: float, 27 | sample_rate: float, 28 | epochs: Optional[int] = None, 29 | steps: Optional[int] = None, 30 | accountant: str = "rdp", 31 | epsilon_tolerance: float = 0.01, 32 | **kwargs, 33 | ) -> float: 34 | r""" 35 | Computes the noise level sigma to reach a total budget of (target_epsilon, target_delta) 36 | at the end of epochs, with a given sample_rate 37 | 38 | Args: 39 | target_epsilon: the privacy budget's epsilon 40 | target_delta: the privacy budget's delta 41 | sample_rate: the sampling rate (usually batch_size / n_data) 42 | epochs: the number of epochs to run 43 | steps: number of steps to run 44 | accountant: accounting mechanism used to estimate epsilon 45 | epsilon_tolerance: precision for the binary search 46 | Returns: 47 | The noise level sigma to ensure privacy budget of (target_epsilon, target_delta) 48 | """ 49 | if (steps is None) == (epochs is None): 50 | raise ValueError( 51 | "get_noise_multiplier takes as input EITHER a number of steps or a number of epochs" 52 | ) 53 | if steps is None: 54 | steps = int(epochs / sample_rate) 55 | 56 | eps_high = float("inf") 57 | accountant = create_accountant(mechanism=accountant) 58 | 59 | sigma_low, sigma_high = 0, 10 60 | while eps_high > target_epsilon: 61 | sigma_high = 2 * sigma_high 62 | accountant.history = [(sigma_high, sample_rate, steps)] 63 | eps_high = accountant.get_epsilon(delta=target_delta, **kwargs) 64 | if sigma_high > MAX_SIGMA: 65 | raise ValueError("The privacy budget is too low.") 66 | 67 | while target_epsilon - eps_high > epsilon_tolerance: 68 | sigma = (sigma_low + sigma_high) / 2 69 | accountant.history = [(sigma, sample_rate, steps)] 70 | eps = accountant.get_epsilon(delta=target_delta, **kwargs) 71 | 72 | if eps < target_epsilon: 73 | sigma_high = sigma 74 | eps_high = eps 75 | else: 76 | sigma_low = sigma 77 | 78 | return sigma_high 79 | -------------------------------------------------------------------------------- /opacus/optimizers/ddpoptimizer_fast_gradient_clipping.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import annotations 16 | 17 | from typing import Callable, Optional 18 | 19 | import torch 20 | from torch.optim import Optimizer 21 | 22 | from .optimizer_fast_gradient_clipping import DPOptimizerFastGradientClipping 23 | 24 | 25 | class DistributedDPOptimizerFastGradientClipping(DPOptimizerFastGradientClipping): 26 | """ 27 | :class:`opacus.optimizers.optimizer.DPOptimizer` compatible with 28 | distributed data processing 29 | """ 30 | 31 | def __init__( 32 | self, 33 | optimizer: Optimizer, 34 | *, 35 | noise_multiplier: float, 36 | max_grad_norm: float, 37 | expected_batch_size: Optional[int], 38 | loss_reduction: str = "mean", 39 | generator=None, 40 | secure_mode: bool = False, 41 | **kwargs, 42 | ): 43 | super().__init__( 44 | optimizer, 45 | noise_multiplier=noise_multiplier, 46 | max_grad_norm=max_grad_norm, 47 | expected_batch_size=expected_batch_size, 48 | loss_reduction=loss_reduction, 49 | generator=generator, 50 | secure_mode=secure_mode, 51 | **kwargs, 52 | ) 53 | self.rank = torch.distributed.get_rank() 54 | self.world_size = torch.distributed.get_world_size() 55 | 56 | def add_noise(self): 57 | # Noise only gets added to the first worker 58 | if self.rank == 0: 59 | super().add_noise() 60 | else: 61 | for p in self.params: 62 | p.grad = p.summed_grad.view_as(p) 63 | 64 | def reduce_gradients(self): 65 | for p in self.params: 66 | if not p.requires_grad: 67 | continue 68 | torch.distributed.all_reduce(p.grad, op=torch.distributed.ReduceOp.SUM) 69 | if self.loss_reduction == "mean": 70 | p.grad /= self.world_size 71 | 72 | def step( 73 | self, closure: Optional[Callable[[], float]] = None 74 | ) -> Optional[torch.Tensor]: 75 | if closure is not None: 76 | with torch.enable_grad(): 77 | closure() 78 | 79 | if self.pre_step(): 80 | self.reduce_gradients() 81 | return self.original_optimizer.step() 82 | else: 83 | return None 84 | -------------------------------------------------------------------------------- /opacus/tests/schedulers/grad_clip_scheduler_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import unittest 17 | 18 | import torch 19 | from opacus import PrivacyEngine 20 | from opacus.schedulers import ExponentialGradClip, LambdaGradClip, StepGradClip 21 | from torch import nn, optim 22 | from torch.utils.data import DataLoader, TensorDataset 23 | 24 | 25 | class GradClipSchedulerTest(unittest.TestCase): 26 | def setUp(self): 27 | n_data, dim = 100, 10 28 | data = torch.randn(n_data, dim) 29 | model = nn.Linear(10, 10) 30 | optimizer = optim.SGD(model.parameters(), lr=0.1) 31 | data_loader = DataLoader(TensorDataset(data), batch_size=10) 32 | self.engine = PrivacyEngine() 33 | 34 | self.module, self.optimizer, self.data_loader = self.engine.make_private( 35 | module=model, 36 | optimizer=optimizer, 37 | data_loader=data_loader, 38 | noise_multiplier=1.0, 39 | max_grad_norm=1.0, 40 | ) 41 | 42 | def test_exponential_scheduler(self): 43 | gamma = 0.99 44 | scheduler = ExponentialGradClip(self.optimizer, gamma=gamma) 45 | 46 | self.assertEqual(self.optimizer.max_grad_norm, 1.0) 47 | scheduler.step() 48 | self.assertEqual(self.optimizer.max_grad_norm, gamma) 49 | 50 | def test_step_scheduler(self): 51 | gamma = 0.1 52 | step_size = 2 53 | scheduler = StepGradClip(self.optimizer, step_size=step_size, gamma=gamma) 54 | 55 | self.assertEqual(self.optimizer.max_grad_norm, 1.0) 56 | scheduler.step() 57 | self.assertEqual(self.optimizer.max_grad_norm, 1.0) 58 | scheduler.step() 59 | self.assertEqual(self.optimizer.max_grad_norm, gamma) 60 | scheduler.step() 61 | self.assertEqual(self.optimizer.max_grad_norm, gamma) 62 | scheduler.step() 63 | self.assertEqual(self.optimizer.max_grad_norm, gamma**2) 64 | 65 | def test_lambda_scheduler(self): 66 | def scheduler_function(epoch): 67 | return 1 - epoch / 10 68 | 69 | scheduler = LambdaGradClip( 70 | self.optimizer, scheduler_function=scheduler_function 71 | ) 72 | 73 | self.assertEqual(self.optimizer.max_grad_norm, 1.0) 74 | scheduler.step() 75 | self.assertEqual(self.optimizer.max_grad_norm, scheduler_function(1)) 76 | -------------------------------------------------------------------------------- /opacus/accountants/analysis/prv/compose.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import List 16 | 17 | import numpy as np 18 | from scipy.fft import irfft, rfft 19 | from scipy.signal import convolve 20 | 21 | from .prvs import DiscretePRV 22 | 23 | 24 | def _compose_fourier(dprv: DiscretePRV, num_self_composition: int) -> DiscretePRV: 25 | if len(dprv) % 2 != 0: 26 | raise ValueError("Can only compose evenly sized discrete PRVs") 27 | 28 | composed_pmf = irfft(rfft(dprv.pmf) ** num_self_composition) 29 | 30 | m = num_self_composition - 1 31 | if num_self_composition % 2 == 0: 32 | m += len(composed_pmf) // 2 33 | composed_pmf = np.roll(composed_pmf, m) 34 | 35 | domain = dprv.domain.shift_right(dprv.domain.shifts * (num_self_composition - 1)) 36 | 37 | return DiscretePRV(pmf=composed_pmf, domain=domain) 38 | 39 | 40 | def _compose_two(dprv_left: DiscretePRV, dprv_right: DiscretePRV) -> DiscretePRV: 41 | pmf = convolve(dprv_left.pmf, dprv_right.pmf, mode="same") 42 | domain = dprv_left.domain.shift_right(dprv_right.domain.shifts) 43 | return DiscretePRV(pmf=pmf, domain=domain) 44 | 45 | 46 | def _compose_convolution_tree(dprvs: List[DiscretePRV]) -> DiscretePRV: 47 | # repeatedly convolve neighbouring PRVs until we only have one left 48 | while len(dprvs) > 1: 49 | dprvs_conv = [] 50 | if len(dprvs) % 2 == 1: 51 | dprvs_conv.append(dprvs.pop()) 52 | 53 | for dprv_left, dprv_right in zip(dprvs[:-1:2], dprvs[1::2]): 54 | dprvs_conv.append(_compose_two(dprv_left, dprv_right)) 55 | 56 | dprvs = dprvs_conv 57 | return dprvs[0] 58 | 59 | 60 | def compose_heterogeneous( 61 | dprvs: List[DiscretePRV], num_self_compositions: List[int] 62 | ) -> DiscretePRV: 63 | r""" 64 | Compose a heterogenous list of PRVs with multiplicity. We use FFT to compose 65 | identical PRVs with themselves first, then pairwise convolve the remaining PRVs. 66 | 67 | This is the approach taken in https://github.com/microsoft/prv_accountant 68 | """ 69 | if len(dprvs) != len(num_self_compositions): 70 | raise ValueError("dprvs and num_self_compositions must have the same length") 71 | 72 | dprvs = [ 73 | _compose_fourier(dprv, num_self_composition) 74 | for dprv, num_self_composition in zip(dprvs, num_self_compositions) 75 | ] 76 | return _compose_convolution_tree(dprvs) 77 | -------------------------------------------------------------------------------- /website/core/TutorialSidebar.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * @format 17 | */ 18 | 19 | const React = require('react'); 20 | const fs = require('fs-extra'); 21 | const path = require('path'); 22 | const join = path.join; 23 | const CWD = process.cwd(); 24 | 25 | const CompLibrary = require(join( 26 | CWD, 27 | '/node_modules/docusaurus/lib/core/CompLibrary.js', 28 | )); 29 | const SideNav = require(join( 30 | CWD, 31 | '/node_modules/docusaurus/lib/core/nav/SideNav.js', 32 | )); 33 | 34 | const Container = CompLibrary.Container; 35 | 36 | const OVERVIEW_ID = 'tutorial_overview'; 37 | 38 | class TutorialSidebar extends React.Component { 39 | render() { 40 | const {currentTutorialID} = this.props; 41 | const current = { 42 | id: currentTutorialID || OVERVIEW_ID, 43 | }; 44 | 45 | const toc = [ 46 | { 47 | type: 'CATEGORY', 48 | title: 'Tutorials', 49 | children: [ 50 | { 51 | type: 'LINK', 52 | item: { 53 | permalink: 'tutorials/', 54 | id: OVERVIEW_ID, 55 | title: 'Overview', 56 | }, 57 | }, 58 | ], 59 | }, 60 | ]; 61 | 62 | const jsonFile = join(CWD, 'tutorials.json'); 63 | const normJsonFile = path.normalize(jsonFile); 64 | const json = JSON.parse(fs.readFileSync(normJsonFile, {encoding: 'utf8'})); 65 | 66 | Object.keys(json).forEach(category => { 67 | const categoryItems = json[category]; 68 | const items = []; 69 | categoryItems.map(item => { 70 | items.push({ 71 | type: 'LINK', 72 | item: { 73 | permalink: `tutorials/${item.id}`, 74 | id: item.id, 75 | title: item.title, 76 | }, 77 | }); 78 | }); 79 | 80 | toc.push({ 81 | type: 'CATEGORY', 82 | title: category, 83 | children: items, 84 | }); 85 | }); 86 | 87 | return ( 88 | 89 | 96 | 97 | ); 98 | } 99 | } 100 | 101 | module.exports = TutorialSidebar; 102 | -------------------------------------------------------------------------------- /website/core/Tutorial.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * @format 17 | */ 18 | 19 | const React = require('react'); 20 | 21 | const fs = require('fs-extra'); 22 | const path = require('path'); 23 | const CWD = process.cwd(); 24 | 25 | const CompLibrary = require(`${CWD}/node_modules/docusaurus/lib/core/CompLibrary.js`); 26 | const Container = CompLibrary.Container; 27 | 28 | const TutorialSidebar = require(`${CWD}/core/TutorialSidebar.js`); 29 | 30 | function renderDownloadIcon() { 31 | return ( 32 | 46 | ); 47 | } 48 | 49 | class Tutorial extends React.Component { 50 | render() { 51 | const {baseUrl, tutorialID} = this.props; 52 | 53 | const htmlFile = `${CWD}/tutorials/${tutorialID}.html`; 54 | const normalizedHtmlFile = path.normalize(htmlFile); 55 | 56 | return ( 57 |
58 | 59 | 60 | 78 | ); 79 | } 80 | } 81 | 82 | module.exports = Tutorial; 83 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/dp_rnn_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import hypothesis.strategies as st 17 | import torch 18 | import torch.nn as nn 19 | from hypothesis import given, settings 20 | from opacus.layers import DPGRU, DPLSTM, DPRNN 21 | from opacus.utils.packed_sequences import _gen_packed_data 22 | 23 | from .common import GradSampleHooks_test 24 | 25 | 26 | MODELS = [ 27 | DPRNN, 28 | DPGRU, 29 | DPLSTM, 30 | ] 31 | 32 | 33 | class DPRNNAdapter(nn.Module): 34 | """ 35 | Adapter for DPRNN family. 36 | RNN returns a tuple, but our testing tools need the model to return a single tensor in output. 37 | We do this adaption here. 38 | """ 39 | 40 | def __init__(self, dp_rnn): 41 | super().__init__() 42 | self.dp_rnn = dp_rnn 43 | 44 | def forward(self, x): 45 | out, _rest = self.dp_rnn(x) 46 | return out 47 | 48 | 49 | class RNN_test(GradSampleHooks_test): 50 | @given( 51 | model=st.one_of([st.just(model) for model in MODELS]), 52 | N=st.integers(1, 3), 53 | T=st.integers(1, 3), 54 | D=st.integers(4, 5), 55 | H=st.integers(8, 10), 56 | num_layers=st.sampled_from([1, 2]), 57 | bias=st.booleans(), 58 | batch_first=st.booleans(), 59 | bidirectional=st.booleans(), 60 | using_packed_sequences=st.booleans(), 61 | packed_sequences_sorted=st.booleans(), 62 | ) 63 | @settings(deadline=60000) 64 | def test_rnn( 65 | self, 66 | model, 67 | N: int, 68 | T: int, 69 | D: int, 70 | H: int, 71 | num_layers: int, 72 | bias: bool, 73 | batch_first: bool, 74 | bidirectional: bool, 75 | using_packed_sequences: bool, 76 | packed_sequences_sorted: bool, 77 | ): 78 | torch.use_deterministic_algorithms(False) 79 | rnn = model( 80 | D, 81 | H, 82 | num_layers=num_layers, 83 | batch_first=batch_first, 84 | bias=bias, 85 | bidirectional=bidirectional, 86 | ) 87 | rnn = DPRNNAdapter(rnn) 88 | 89 | if using_packed_sequences: 90 | x = _gen_packed_data(N, T, D, batch_first, packed_sequences_sorted) 91 | else: 92 | if batch_first: 93 | x = torch.randn([N, T, D]) 94 | else: 95 | x = torch.randn([T, N, D]) 96 | 97 | self.run_test(x, rnn, batch_first=batch_first, ew_compatible=False) 98 | -------------------------------------------------------------------------------- /opacus/tests/grad_samples/dp_multihead_attention_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import hypothesis.strategies as st 17 | import torch 18 | import torch.nn as nn 19 | from hypothesis import given, settings 20 | from opacus.layers import DPMultiheadAttention 21 | 22 | from .common import GradSampleHooks_test 23 | 24 | 25 | class DPMultiheadAttentionAdapter(nn.Module): 26 | """ 27 | Adapter for DPMultiHeadAttention. 28 | This module takes three inputs, but our testing tools need that the model is given a single 29 | tensor, and returns a single tensor in output. 30 | 31 | To adapt for this, we stack the three input tensors required (q, k, v) over the LAST dimension, 32 | because our testing tools need to handle the `batch_first` argument which will manipulate x 33 | over the first (and potentially second) dimension. 34 | """ 35 | 36 | def __init__(self, *args, **kwargs): 37 | super().__init__() 38 | self.attn = DPMultiheadAttention(*args, **kwargs) 39 | 40 | def forward(self, x): 41 | q, k, v = x.unbind(-1) 42 | out, _attn_weights = self.attn(q, k, v) 43 | return out 44 | 45 | 46 | class MultiHeadAttention_test(GradSampleHooks_test): 47 | @given( 48 | N=st.integers(1, 4), 49 | T=st.integers(16, 20), 50 | D=st.sampled_from([4]), 51 | P=st.sampled_from([1, 2]), 52 | bias=st.booleans(), 53 | add_bias_kv=st.booleans(), 54 | add_zero_attn=st.booleans(), 55 | kv_dim=st.booleans(), 56 | test_or_check=st.integers(1, 2), 57 | ) 58 | @settings(deadline=60000) 59 | def test_multihead_attention( 60 | self, 61 | N: int, 62 | T: int, 63 | D: int, 64 | P: int, 65 | bias: bool, 66 | add_bias_kv: bool, 67 | add_zero_attn: bool, 68 | kv_dim: bool, 69 | test_or_check: int, 70 | ): 71 | if kv_dim: 72 | kdim, vdim = D, D 73 | else: 74 | kdim, vdim = None, None 75 | attn = DPMultiheadAttentionAdapter( 76 | D, 77 | P, 78 | bias=bias, 79 | add_bias_kv=add_bias_kv, 80 | add_zero_attn=add_zero_attn, 81 | dropout=0.0, 82 | kdim=kdim, 83 | vdim=vdim, 84 | ) 85 | q = torch.randn([T, N, D]) 86 | k = torch.randn([T, N, D]) 87 | v = torch.randn([T, N, D]) 88 | x = torch.stack((q, k, v), dim=-1) 89 | 90 | self.run_test(x, attn, batch_first=False) 91 | -------------------------------------------------------------------------------- /opacus/validators/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from typing import Sequence, Union 17 | 18 | from .module_validator import ModuleValidator 19 | 20 | 21 | DEFAULT_MODULE_VALIDATOR = ModuleValidator 22 | 23 | 24 | def register_module_validator( 25 | target_class_or_classes: Union[type, Sequence[type]], 26 | validator_class: type = DEFAULT_MODULE_VALIDATOR, 27 | ): 28 | """ 29 | Registers the decorated function as the ``validator`` of ``target_class_or_classes``, which is 30 | the function that will be invoked every time you want to validate that a module is compatible 31 | for training with Opacus. 32 | You may supply your own validator_class that holds the registry of VALIDATORS. 33 | The signature of every validator is always the same: 34 | 35 | >>> @register_module_validator(MyCustomModel) 36 | ... def validate(module: nn.Module, **kwargs) -> List[opacus.validators.errors.UnsupportedError]: 37 | ... pass 38 | 39 | It may help you to take a look at the existing validator inside Opacus, under ``opacus.validators.`` 40 | """ 41 | 42 | def decorator(f): 43 | target_classes = ( 44 | target_class_or_classes 45 | if isinstance(target_class_or_classes, Sequence) 46 | else [target_class_or_classes] 47 | ) 48 | for target_class in target_classes: 49 | validator_class.VALIDATORS[target_class] = f 50 | return f 51 | 52 | return decorator 53 | 54 | 55 | def register_module_fixer( 56 | target_class_or_classes: Union[type, Sequence[type]], 57 | validator_class: type = DEFAULT_MODULE_VALIDATOR, 58 | ): 59 | """ 60 | Registers the decorated function as the ``fixer`` of ``target_class_or_classes``, which is 61 | the function that will be invoked every time you want to fix an incompatoble module to make 62 | it work for training with Opacus. 63 | You may supply your own validator_class that holds the registry of FIXERS. 64 | The signature of every fixer is always the same: 65 | 66 | >>> @register_module_fixer(MyCustomModel) 67 | ... def fix(module: nn.Module, **kwargs) -> nn.Module: 68 | ... pass 69 | 70 | It may help you to take a look at the existing fixers inside Opacus, under ``opacus.validators.`` 71 | """ 72 | 73 | def decorator(f): 74 | target_classes = ( 75 | target_class_or_classes 76 | if isinstance(target_class_or_classes, Sequence) 77 | else [target_class_or_classes] 78 | ) 79 | for target_class in target_classes: 80 | validator_class.FIXERS[target_class] = f 81 | return f 82 | 83 | return decorator 84 | -------------------------------------------------------------------------------- /opacus/optimizers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .adaclipoptimizer import AdaClipDPOptimizer 16 | from .ddp_perlayeroptimizer import SimpleDistributedPerLayerOptimizer 17 | from .ddpoptimizer import DistributedDPOptimizer 18 | from .ddpoptimizer_fast_gradient_clipping import ( 19 | DistributedDPOptimizerFastGradientClipping, 20 | ) 21 | from .fsdpoptimizer_fast_gradient_clipping import FSDPOptimizerFastGradientClipping 22 | from .optimizer import DPOptimizer 23 | from .optimizer_fast_gradient_clipping import DPOptimizerFastGradientClipping 24 | from .perlayeroptimizer import DPPerLayerOptimizer 25 | 26 | 27 | __all__ = [ 28 | "AdaClipDPOptimizer", 29 | "DistributedDPOptimizer", 30 | "DPOptimizer", 31 | "DPOptimizerFastGradientClipping", 32 | "DistributedDPOptimizerFastGradientlipping", 33 | "FSDPOptimizerFastGradientClipping", 34 | "DPPerLayerOptimizer", 35 | "SimpleDistributedPerLayerOptimizer", 36 | ] 37 | 38 | 39 | def get_optimizer_class(clipping: str, distributed: bool, grad_sample_mode: str = None): 40 | if grad_sample_mode == "ghost": 41 | if clipping == "flat" and distributed is False: 42 | return DPOptimizerFastGradientClipping 43 | elif clipping == "flat" and distributed is True: 44 | return DistributedDPOptimizerFastGradientClipping 45 | else: 46 | raise ValueError( 47 | f"Unsupported combination of parameters. Clipping: {clipping} and grad_sample_mode: {grad_sample_mode}" 48 | ) 49 | elif grad_sample_mode == "ghost_fsdp": 50 | if clipping == "flat" and distributed is True: 51 | return FSDPOptimizerFastGradientClipping 52 | else: 53 | raise ValueError( 54 | f"Unsupported combination of parameters. Clipping: {clipping}, distributed: {distributed}, and grad_sample_mode: {grad_sample_mode}" 55 | ) 56 | elif clipping == "flat" and distributed is False: 57 | return DPOptimizer 58 | elif clipping == "flat" and distributed is True: 59 | return DistributedDPOptimizer 60 | elif clipping == "per_layer" and distributed is False: 61 | return DPPerLayerOptimizer 62 | elif clipping == "per_layer" and distributed is True: 63 | if grad_sample_mode == "hooks" or grad_sample_mode == "ew": 64 | return SimpleDistributedPerLayerOptimizer 65 | else: 66 | raise ValueError(f"Unexpected grad_sample_mode: {grad_sample_mode}") 67 | elif clipping == "adaptive" and distributed is False: 68 | return AdaClipDPOptimizer 69 | raise ValueError( 70 | f"Unexpected optimizer parameters. Clipping: {clipping}, distributed: {distributed}" 71 | ) 72 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import sys 18 | 19 | from setuptools import find_packages, setup 20 | 21 | 22 | REQUIRED_MAJOR = 3 23 | REQUIRED_MINOR = 7 24 | REQUIRED_MICRO = 5 25 | 26 | version = {} 27 | with open("opacus/version.py") as fp: 28 | exec(fp.read(), version) 29 | 30 | __version__ = version["__version__"] 31 | 32 | # Check for python version 33 | if sys.version_info < (REQUIRED_MAJOR, REQUIRED_MINOR, REQUIRED_MICRO): 34 | error = ( 35 | "Your version of python ({major}.{minor}.{micro}) is too old. You need " 36 | "python >= {required_major}.{required_minor}.{required_micro}" 37 | ).format( 38 | major=sys.version_info.major, 39 | minor=sys.version_info.minor, 40 | micro=sys.version_info.micro, 41 | required_major=REQUIRED_MAJOR, 42 | required_minor=REQUIRED_MINOR, 43 | required_micro=REQUIRED_MICRO, 44 | ) 45 | sys.exit(error) 46 | 47 | 48 | src_dir = os.path.abspath(os.path.dirname(__file__)) 49 | 50 | with open("README.md", "r", encoding="utf8") as fh: 51 | long_description = fh.read() 52 | 53 | requirements_txt = os.path.join(src_dir, "requirements.txt") 54 | with open("requirements.txt", encoding="utf8") as f: 55 | required = f.read().splitlines() 56 | 57 | with open("dev_requirements.txt", encoding="utf8") as f: 58 | dev_required = f.read().splitlines() 59 | 60 | setup( 61 | name="opacus", 62 | version=__version__, 63 | author="The Opacus Team", 64 | description="Train PyTorch models with Differential Privacy", 65 | long_description=long_description, 66 | long_description_content_type="text/markdown", 67 | url="https://opacus.ai", 68 | project_urls={ 69 | "Documentation": "https://opacus.ai/api", 70 | "Source": "https://github.com/pytorch/opacus", 71 | }, 72 | license="Apache-2.0", 73 | install_requires=required, 74 | extras_require={"dev": dev_required}, 75 | packages=find_packages(), 76 | keywords=[ 77 | "PyTorch", 78 | "Differential Privacy", 79 | "DP-SGD", 80 | "DP SGD", 81 | "Privacy Preserving Machine Learning", 82 | "PPML", 83 | "PPAI", 84 | ], 85 | classifiers=[ 86 | "Development Status :: 4 - Beta", 87 | "Intended Audience :: Developers", 88 | "Intended Audience :: Education", 89 | "Intended Audience :: Science/Research", 90 | "License :: OSI Approved :: Apache Software License", 91 | "Programming Language :: Python :: 3 :: Only", 92 | "Topic :: Scientific/Engineering", 93 | ], 94 | python_requires=f">={REQUIRED_MAJOR}.{REQUIRED_MINOR}.{REQUIRED_MICRO}", 95 | ) 96 | -------------------------------------------------------------------------------- /opacus/accountants/rdp.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import List, Optional, Tuple, Union 16 | 17 | from .accountant import IAccountant 18 | from .analysis import rdp as privacy_analysis 19 | 20 | 21 | class RDPAccountant(IAccountant): 22 | DEFAULT_ALPHAS = [1 + x / 10.0 for x in range(1, 100)] + list(range(12, 64)) 23 | 24 | def __init__(self): 25 | super().__init__() 26 | 27 | def step(self, *, noise_multiplier: float, sample_rate: float): 28 | if len(self.history) >= 1: 29 | last_noise_multiplier, last_sample_rate, num_steps = self.history.pop() 30 | if ( 31 | last_noise_multiplier == noise_multiplier 32 | and last_sample_rate == sample_rate 33 | ): 34 | self.history.append( 35 | (last_noise_multiplier, last_sample_rate, num_steps + 1) 36 | ) 37 | else: 38 | self.history.append( 39 | (last_noise_multiplier, last_sample_rate, num_steps) 40 | ) 41 | self.history.append((noise_multiplier, sample_rate, 1)) 42 | 43 | else: 44 | self.history.append((noise_multiplier, sample_rate, 1)) 45 | 46 | def get_privacy_spent( 47 | self, *, delta: float, alphas: Optional[List[Union[float, int]]] = None 48 | ) -> Tuple[float, float]: 49 | if not self.history: 50 | return 0, 0 51 | 52 | if alphas is None: 53 | alphas = self.DEFAULT_ALPHAS 54 | rdp = sum( 55 | [ 56 | privacy_analysis.compute_rdp( 57 | q=sample_rate, 58 | noise_multiplier=noise_multiplier, 59 | steps=num_steps, 60 | orders=alphas, 61 | ) 62 | for (noise_multiplier, sample_rate, num_steps) in self.history 63 | ] 64 | ) 65 | eps, best_alpha = privacy_analysis.get_privacy_spent( 66 | orders=alphas, rdp=rdp, delta=delta 67 | ) 68 | return float(eps), float(best_alpha) 69 | 70 | def get_epsilon( 71 | self, 72 | delta: float, 73 | alphas: Optional[List[Union[float, int]]] = None, 74 | **kwargs, 75 | ): 76 | """ 77 | Return privacy budget (epsilon) expended so far. 78 | 79 | Args: 80 | delta: target delta 81 | alphas: List of RDP orders (alphas) used to search for the optimal conversion 82 | between RDP and (epd, delta)-DP 83 | """ 84 | eps, _ = self.get_privacy_spent(delta=delta, alphas=alphas) 85 | return eps 86 | 87 | def __len__(self): 88 | return len(self.history) 89 | 90 | @classmethod 91 | def mechanism(cls) -> str: 92 | return "rdp" 93 | -------------------------------------------------------------------------------- /examples/scripts/make_small_imagenet_sampled.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | usage() 17 | { 18 | echo "usage Example: bash make_small_imagenet_N_classes.sh -o Documents/imagenet_full_size -d Documents/imagenet_small_100 -nt 100 -nv 10" 19 | echo "usage: bash make_small_imagenet_N_classes.sh -o -d -nt -nv " 20 | } 21 | 22 | if [ "$1" == "" ]; then # If arguments are not specified 23 | usage 24 | exit 1 25 | fi 26 | while [ "$1" != "" ]; do # Read the input arguments 27 | case $1 in 28 | -d | --destination ) shift 29 | destination=$1 30 | ;; 31 | -o | --origin ) shift 32 | origin=$1 33 | ;; 34 | -nt ) shift 35 | N_train=$1 36 | ;; 37 | -nv ) shift 38 | N_val=$1 39 | ;; 40 | -h | --help ) usage 41 | exit 42 | ;; 43 | * ) echo "ERROR: unknown parameter \"$1\"" 44 | usage 45 | exit 1 46 | esac 47 | shift 48 | done 49 | 50 | # If all necessary arguments are not supplied 51 | if [[ -z $destination || -z $origin || -z $N_train || -z $N_val ]] 52 | then 53 | echo "You must specify all necessary parameters." 54 | usage 55 | exit 1 56 | fi 57 | 58 | # Get absolute paths 59 | destination="$(readlink -f "$destination")" 60 | origin="$(readlink -f "$origin")" 61 | 62 | mkdir "$destination" 63 | mkdir "$destination/train" 64 | mkdir "$destination/val" 65 | 66 | echo 'Copying' 67 | for val_train_folder in val train; do # Do copying for both 'train' and 'val' folders 68 | if [[ $val_train_folder = 'train' ]]; then 69 | N=$N_train 70 | else 71 | N=$N_val 72 | fi 73 | cd "$origin/$val_train_folder" || { echo "Failure"; exit 1; } # change directory to origin's train or val folders 74 | for d in */ ; do # loop through the 1000 folders 75 | mkdir "$destination/$val_train_folder/$d" 76 | cd "$origin/$val_train_folder/$d" || { echo "Failure"; exit 1; } 77 | find . -maxdepth 1 -mindepth 1 -type f |sort -R |tail -"$N" |while read -r file; do # select N files from each 1000 folders 78 | cp "$file" "$destination/$val_train_folder/$d/$file" 79 | printf "." 80 | done 81 | done 82 | echo "Copying folder $val_train_folder is done." 83 | done 84 | -------------------------------------------------------------------------------- /website/scripts/parse_sphinx.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import argparse 17 | import os 18 | 19 | from bs4 import BeautifulSoup 20 | 21 | 22 | js_scripts = """ 23 | 25 | 26 | 27 | 28 | 29 | 30 | """ # noqa: E501 31 | 32 | search_js_scripts = """ 33 | 36 | 37 | 38 | """ 39 | 40 | 41 | def parse_sphinx(input_dir, output_dir): 42 | for cur, _, files in os.walk(input_dir): 43 | for fname in files: 44 | if fname.endswith(".html"): 45 | with open(os.path.join(cur, fname), "r") as f: 46 | soup = BeautifulSoup(f.read(), "html.parser") 47 | doc = soup.find("div", {"class": "document"}) 48 | wrapped_doc = doc.wrap(soup.new_tag("div", **{"class": "sphinx"})) 49 | # add js 50 | if fname == "search.html": 51 | out = js_scripts + search_js_scripts + str(wrapped_doc) 52 | else: 53 | out = js_scripts + str(wrapped_doc) 54 | output_path = os.path.join(output_dir, os.path.relpath(cur, input_dir)) 55 | os.makedirs(output_path, exist_ok=True) 56 | with open(os.path.join(output_path, fname), "w") as fout: 57 | fout.write(out) 58 | 59 | # update reference in JS file 60 | with open(os.path.join(input_dir, "_static/searchtools.js"), "r") as js_file: 61 | js = js_file.read() 62 | js = js.replace( 63 | "DOCUMENTATION_OPTIONS.URL_ROOT + '_sources/'", "'_sphinx-sources/'" 64 | ) 65 | with open(os.path.join(input_dir, "_static/searchtools.js"), "w") as js_file: 66 | js_file.write(js) 67 | 68 | 69 | if __name__ == "__main__": 70 | parser = argparse.ArgumentParser(description="Strip HTML body from Sphinx docs.") 71 | parser.add_argument( 72 | "-i", 73 | "--input_dir", 74 | metavar="path", 75 | required=True, 76 | help="Input directory for Sphinx HTML.", 77 | ) 78 | parser.add_argument( 79 | "-o", 80 | "--output_dir", 81 | metavar="path", 82 | required=True, 83 | help="Output directory in Docusaurus.", 84 | ) 85 | args = parser.parse_args() 86 | parse_sphinx(args.input_dir, args.output_dir) 87 | --------------------------------------------------------------------------------