├── .conda └── meta.yaml ├── .github └── ISSUE_TEMPLATE │ ├── ---bug-report.md │ ├── ---documentation-examples.md │ ├── ---feature-request.md │ └── ---refactor.md ├── .gitignore ├── .isort.cfg ├── .pre-commit-config.yaml ├── .pylintrc ├── .travis.yml ├── LICENSE ├── README.md ├── docs ├── Makefile ├── requirements.txt └── source │ ├── .gitignore │ ├── _static │ └── .gitkeep │ ├── beta_features.rst │ ├── conf.py │ ├── constraints.rst │ ├── distributions.rst │ ├── functions.rst │ ├── index.rst │ ├── kernels.rst │ ├── lazy.rst │ ├── likelihoods.rst │ ├── marginal_log_likelihoods.rst │ ├── means.rst │ ├── models.rst │ ├── module.rst │ ├── priors.rst │ ├── settings.rst │ ├── utils.rst │ └── variational.rst ├── environment.yml ├── examples ├── .gitignore ├── 00_Basic_Usage │ ├── Hyperparameters.ipynb │ ├── README.rst │ ├── Saving_and_Loading_Models.ipynb │ └── index.rst ├── 01_Exact_GPs │ ├── GP_Regression_Fully_Bayesian.ipynb │ ├── README.rst │ ├── Simple_GP_Regression.ipynb │ ├── Simple_GP_Regression_GPflow.ipynb │ ├── Simple_GP_Regression_GPflow_regression_1D.csv │ ├── Spectral_Mixture_GP_Regression.ipynb │ └── index.rst ├── 02_Scalable_Exact_GPs │ ├── Grid_GP_Regression.ipynb │ ├── KISSGP_Regression.ipynb │ ├── KeOps_GP_Regression.ipynb │ ├── README.rst │ ├── SGPR_Regression_CUDA.ipynb │ ├── Scalable_Kernel_Interpolation_for_Products_CUDA.ipynb │ ├── Simple_GP_Regression_CUDA.ipynb │ ├── Simple_GP_Regression_With_LOVE_Fast_Variances_CUDA.ipynb │ ├── Simple_MultiGPU_GP_Regression.ipynb │ └── index.rst ├── 03_Multitask_Exact_GPs │ ├── Batch_Independent_Multioutput_GP.ipynb │ ├── Hadamard_Multitask_GP_Regression.ipynb │ ├── ModelList_GP_Regression.ipynb │ ├── Multitask_GP_Regression.ipynb │ ├── README.rst │ └── index.rst ├── 04_Variational_and_Approximate_GPs │ ├── Approximate_GP_Objective_Functions.ipynb │ ├── GP_Regression_with_Uncertain_Inputs.ipynb │ ├── Modifying_the_variational_strategy_and_distribution.ipynb │ ├── Non_Gaussian_Likelihoods.ipynb │ ├── README.rst │ ├── SVGP_Multitask_GP_Regression.ipynb │ ├── SVGP_Regression_CUDA.ipynb │ └── index.rst ├── 05_Deep_Gaussian_Processes │ ├── Deep_Gaussian_Processes.ipynb │ ├── README.rst │ └── index.rst ├── 06_PyTorch_NN_Integration_DKL │ ├── .gitignore │ ├── Deep_Kernel_Learning_DenseNet_CIFAR_Tutorial.ipynb │ ├── KISSGP_Deep_Kernel_Regression_CUDA.ipynb │ ├── README.rst │ ├── densenet.py │ └── index.rst ├── 07_Pyro_Integration │ ├── Clustered_Multitask_GP_Regression.ipynb │ ├── Cox_Process_Example.ipynb │ ├── Pyro_GPyTorch_High_Level.ipynb │ ├── Pyro_GPyTorch_Low_Level.ipynb │ ├── README.rst │ └── index.rst ├── 08_Advanced_Usage │ ├── README.rst │ ├── Simple_Batch_Mode_GP_Regression.ipynb │ ├── Simple_GP_Regression_Derivative_Information_1d.ipynb │ ├── Simple_GP_Regression_Derivative_Information_2d.ipynb │ ├── TorchScript_Exact_Models.ipynb │ ├── TorchScript_Variational_Models.ipynb │ ├── index.rst │ └── svgp_elevators.pt ├── LBFGS.py ├── README.rst └── index.rst ├── gpytorch ├── __init__.py ├── beta_features.py ├── constraints │ ├── __init__.py │ └── constraints.py ├── distributions │ ├── __init__.py │ ├── delta.py │ ├── distribution.py │ ├── multitask_multivariate_normal.py │ └── multivariate_normal.py ├── functions │ ├── __init__.py │ ├── _dsmm.py │ ├── _inv_matmul.py │ ├── _inv_quad.py │ ├── _inv_quad_log_det.py │ ├── _log_normal_cdf.py │ ├── _matmul.py │ ├── _root_decomposition.py │ ├── matern_covariance.py │ └── rbf_covariance.py ├── kernels │ ├── __init__.py │ ├── additive_structure_kernel.py │ ├── cosine_kernel.py │ ├── cylindrical_kernel.py │ ├── grid_interpolation_kernel.py │ ├── grid_kernel.py │ ├── index_kernel.py │ ├── inducing_point_kernel.py │ ├── keops │ │ ├── __init__.py │ │ ├── keops_kernel.py │ │ ├── matern_kernel.py │ │ └── rbf_kernel.py │ ├── kernel.py │ ├── lcm_kernel.py │ ├── linear_kernel.py │ ├── matern_kernel.py │ ├── multi_device_kernel.py │ ├── multitask_kernel.py │ ├── newton_girard_additive_kernel.py │ ├── periodic_kernel.py │ ├── polynomial_kernel.py │ ├── polynomial_kernel_grad.py │ ├── product_structure_kernel.py │ ├── rbf_kernel.py │ ├── rbf_kernel_grad.py │ ├── rq_kernel.py │ ├── scale_kernel.py │ └── spectral_mixture_kernel.py ├── lazy │ ├── __init__.py │ ├── added_diag_lazy_tensor.py │ ├── batch_repeat_lazy_tensor.py │ ├── block_diag_lazy_tensor.py │ ├── block_interleaved_lazy_tensor.py │ ├── block_lazy_tensor.py │ ├── cached_cg_lazy_tensor.py │ ├── cat_lazy_tensor.py │ ├── chol_lazy_tensor.py │ ├── constant_mul_lazy_tensor.py │ ├── diag_lazy_tensor.py │ ├── interpolated_lazy_tensor.py │ ├── keops_lazy_tensor.py │ ├── kronecker_product_lazy_tensor.py │ ├── lazy_evaluated_kernel_tensor.py │ ├── lazy_tensor.py │ ├── lazy_tensor_representation_tree.py │ ├── matmul_lazy_tensor.py │ ├── mul_lazy_tensor.py │ ├── non_lazy_tensor.py │ ├── psd_sum_lazy_tensor.py │ ├── root_lazy_tensor.py │ ├── sum_batch_lazy_tensor.py │ ├── sum_lazy_tensor.py │ ├── toeplitz_lazy_tensor.py │ └── zero_lazy_tensor.py ├── likelihoods │ ├── __init__.py │ ├── bernoulli_likelihood.py │ ├── gaussian_likelihood.py │ ├── likelihood.py │ ├── likelihood_list.py │ ├── multitask_gaussian_likelihood.py │ ├── noise_models.py │ └── softmax_likelihood.py ├── means │ ├── __init__.py │ ├── constant_mean.py │ ├── constant_mean_grad.py │ ├── linear_mean.py │ ├── mean.py │ ├── multitask_mean.py │ └── zero_mean.py ├── mlls │ ├── __init__.py │ ├── _approximate_mll.py │ ├── added_loss_term.py │ ├── deep_approximate_mll.py │ ├── exact_marginal_log_likelihood.py │ ├── gamma_robust_variational_elbo.py │ ├── inducing_point_kernel_added_loss_term.py │ ├── marginal_log_likelihood.py │ ├── noise_model_added_loss_term.py │ ├── predictive_log_likelihood.py │ ├── sum_marginal_log_likelihood.py │ └── variational_elbo.py ├── models │ ├── __init__.py │ ├── approximate_gp.py │ ├── deep_gps │ │ ├── __init__.py │ │ └── deep_gp.py │ ├── exact_gp.py │ ├── exact_prediction_strategies.py │ ├── gp.py │ ├── model_list.py │ └── pyro │ │ ├── __init__.py │ │ ├── _pyro_mixin.py │ │ └── pyro_gp.py ├── module.py ├── priors │ ├── __init__.py │ ├── horseshoe_prior.py │ ├── lkj_prior.py │ ├── prior.py │ ├── smoothed_box_prior.py │ ├── torch_priors.py │ ├── utils.py │ └── wishart_prior.py ├── settings.py ├── test │ ├── __init__.py │ ├── base_kernel_test_case.py │ ├── base_likelihood_test_case.py │ ├── base_mean_test_case.py │ ├── base_test_case.py │ ├── lazy_tensor_test_case.py │ ├── model_test_case.py │ ├── utils.py │ └── variational_test_case.py ├── utils │ ├── __init__.py │ ├── broadcasting.py │ ├── cholesky.py │ ├── deprecation.py │ ├── errors.py │ ├── fft.py │ ├── getitem.py │ ├── grid.py │ ├── interpolation.py │ ├── lanczos.py │ ├── linear_cg.py │ ├── memoize.py │ ├── pivoted_cholesky.py │ ├── quadrature.py │ ├── sparse.py │ ├── stochastic_lq.py │ ├── toeplitz.py │ └── transforms.py └── variational │ ├── __init__.py │ ├── _variational_distribution.py │ ├── _variational_strategy.py │ ├── additive_grid_interpolation_variational_strategy.py │ ├── cholesky_variational_distribution.py │ ├── delta_variational_distribution.py │ ├── grid_interpolation_variational_strategy.py │ ├── mean_field_variational_distribution.py │ ├── multitask_variational_strategy.py │ ├── orthogonally_decoupled_variational_strategy.py │ ├── unwhitened_variational_strategy.py │ ├── variational_strategy.py │ └── whitened_variational_strategy.py ├── readthedocs.yml ├── requirements.txt ├── setup.cfg ├── setup.py └── test ├── __init__.py ├── constraints ├── __init__.py └── test_constraints.py ├── distributions ├── __init__.py ├── test_delta.py ├── test_multitask_multivariate_normal.py └── test_multivariate_normal.py ├── examples ├── __init__.py ├── old_variational_strategy_model.pth ├── test_batch_gp_regression.py ├── test_batch_multitask_gp_regression.py ├── test_batch_svgp_gp_regression.py ├── test_decoupled_svgp_regression.py ├── test_fixed_noise_fanatasy_updates.py ├── test_grid_gp_regression.py ├── test_hadamard_multitask_gp_regression.py ├── test_independent_multitask_gp_regression.py ├── test_kissgp_additive_classification.py ├── test_kissgp_additive_regression.py ├── test_kissgp_dkl_regression.py ├── test_kissgp_gp_classification.py ├── test_kissgp_gp_regression.py ├── test_kissgp_kronecker_product_classification.py ├── test_kissgp_kronecker_product_regression.py ├── test_kissgp_multiplicative_regression.py ├── test_kissgp_variational_regression.py ├── test_kissgp_white_noise_regression.py ├── test_kronecker_multitask_gp_regression.py ├── test_kronecker_multitask_ski_gp_regression.py ├── test_lcm_kernel_regression.py ├── test_model_list_gp_regression.py ├── test_pyro_integration.py ├── test_sgpr_regression.py ├── test_simple_gp_classification.py ├── test_simple_gp_regression.py ├── test_spectral_mixture_gp_regression.py ├── test_svgp_gp_classification.py ├── test_svgp_gp_regression.py ├── test_unwhitened_svgp_regression.py └── test_white_noise_regression.py ├── functions ├── __init__.py ├── test_dsmm.py ├── test_inv_matmul.py ├── test_inv_quad.py ├── test_inv_quad_log_det.py ├── test_log_normal_cdf.py ├── test_matern_covariance.py ├── test_matmul.py ├── test_rbf_covariance.py └── test_root_decomposition.py ├── kernels ├── __init__.py ├── keops │ ├── __init__.py │ ├── test_matern_kernel.py │ └── test_rbf_kernel.py ├── test_additive_kernel.py ├── test_cosine_kernel.py ├── test_cylindrical_kernel.py ├── test_grid_interpolation_kernel.py ├── test_grid_kernel.py ├── test_linear_kernel.py ├── test_matern_kernel.py ├── test_newton_girard_additive_kernel.py ├── test_periodic_kernel.py ├── test_polynomial_kernel.py ├── test_polynomial_kernel_grad.py ├── test_rbf_kernel.py ├── test_rbf_kernel_grad.py ├── test_rq_kernel.py ├── test_scale_kernel.py └── test_spectral_mixture_kernel.py ├── lazy ├── __init__.py ├── test_added_diag_lazy_tensor.py ├── test_batch_repeat_lazy_tensor.py ├── test_block_diag_lazy_tensor.py ├── test_block_interleaved_lazy_tensor.py ├── test_cached_cg_lazy_tensor.py ├── test_cat_lazy_tensor.py ├── test_chol_lazy_tensor.py ├── test_constant_mul_lazy_tensor.py ├── test_diag_lazy_tensor.py ├── test_interpolated_lazy_tensor.py ├── test_kronecker_product_lazy_tensor.py ├── test_lazy_evaluated_kernel_tensor.py ├── test_matmul_lazy_tensor.py ├── test_mul_lazy_tensor.py ├── test_non_lazy_tensor.py ├── test_psd_sum_lazy_tensor.py ├── test_root_lazy_tensor.py ├── test_sum_batch_lazy_tensor.py ├── test_sum_lazy_tensor.py ├── test_toeplitz_lazy_tensor.py └── test_zero_lazy_tensor.py ├── likelihoods ├── __init__.py ├── test_bernoulli_likelihood.py ├── test_gaussian_likelihood.py ├── test_general_multitask_gaussian_likelihood.py ├── test_multitask_gaussian_likelihood.py └── test_softmax_likelihood.py ├── means ├── __init__.py ├── test_constant_mean.py ├── test_constant_mean_grad.py ├── test_linear_mean.py ├── test_multitask_mean.py └── test_zero_mean.py ├── models ├── __init__.py ├── test_exact_gp.py ├── test_model_list.py └── test_variational_gp.py ├── priors ├── __init__.py ├── test_gamma_prior.py ├── test_horseshoe_prior.py ├── test_lkj_prior.py ├── test_multivariate_normal_prior.py ├── test_normal_prior.py └── test_smoothed_box_prior.py ├── utils ├── __init__.py ├── test_cholesky.py ├── test_fft.py ├── test_getitem.py ├── test_grid.py ├── test_interpolation.py ├── test_lanczos.py ├── test_linear_cg.py ├── test_pivoted_cholesky.py ├── test_quadrature.py ├── test_sparse.py └── test_toeplitz.py └── variational ├── __init__.py ├── test_grid_interpolation_variational_strategy.py ├── test_multitask_variational_strategy.py ├── test_orthogonally_decoupled_variational_strategy.py ├── test_unwhitened_variational_strategy.py ├── test_variational_strategy.py └── test_whitened_variational_strategy.py /.conda/meta.yaml: -------------------------------------------------------------------------------- 1 | {% set data = load_setup_py_data(setup_file="../setup.py", from_recipe_dir=True) %} 2 | 3 | package: 4 | name: {{ data.get("name")|lower }} 5 | version: {{ data.get("version") }} 6 | 7 | source: 8 | path: ../ 9 | 10 | build: 11 | noarch: python 12 | script: "$PYTHON ./setup.py install --single-version-externally-managed --record=record.txt" 13 | 14 | requirements: 15 | host: 16 | - python>=3.6 17 | 18 | run: 19 | - pytorch>=1.3 20 | 21 | test: 22 | imports: 23 | - gpytorch 24 | - gpytorch.distributions 25 | - gpytorch.functions 26 | - gpytorch.kernels 27 | - gpytorch.lazy 28 | - gpytorch.likelihoods 29 | - gpytorch.means 30 | - gpytorch.mlls 31 | - gpytorch.models 32 | - gpytorch.priors 33 | - gpytorch.utils 34 | - gpytorch.variational 35 | 36 | about: 37 | home: https://gpytorch.ai 38 | license: MIT 39 | license_file: LICENSE 40 | summary: An implementation of Gaussian Processes in Pytorch 41 | doc_url: https://gpytorch.readthedocs.io/en/latest/ 42 | dev_url: https://github.com/cornellius-gp/gpytorch 43 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/---bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F41B Bug report" 3 | about: Create a report to help us improve 4 | title: "[Bug]" 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | # 🐛 Bug 11 | 12 | 13 | 14 | ## To reproduce 15 | 16 | ** Code snippet to reproduce ** 17 | ```python 18 | # Your code goes here 19 | # Please make sure it does not require any external dependencies (other than PyTorch!) 20 | # (We much prefer small snippets rather than links to existing libraries!) 21 | ``` 22 | 23 | ** Stack trace/error message ** 24 | ``` 25 | // Paste the bad output here! 26 | ``` 27 | 28 | ## Expected Behavior 29 | 30 | 31 | 32 | ## System information 33 | 34 | **Please complete the following information:** 35 | - 36 | - 37 | - 38 | 39 | ## Additional context 40 | Add any other context about the problem here. 41 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/---documentation-examples.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F4DA Documentation/Examples" 3 | about: Describe this issue template's purpose here. 4 | title: "[Docs]" 5 | labels: documentation 6 | assignees: '' 7 | 8 | --- 9 | 10 | # 📚 Documentation/Examples 11 | 12 | ** Is there documentation missing? ** 13 | 14 | 15 | ** Is documentation wrong? ** 16 | 17 | 18 | ** Is there a feature that needs some example code? ** 19 | 20 | 21 | ** Think you know how to fix the docs? ** (If so, we'd love a pull request from you!) 22 | 23 | - Link to [GPyTorch documentation](https://gpytorch.readthedocs.io) 24 | - Link to [GPyTorch examples](https://github.com/cornellius-gp/gpytorch/tree/master/examples) 25 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/---feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F680 Feature request" 3 | about: Suggest an idea for this project 4 | title: "[Feature Request]" 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | # 🚀 Feature Request 11 | 12 | 13 | 14 | ## Motivation 15 | 16 | **Is your feature request related to a problem? Please describe.** 17 | 18 | 19 | 20 | ## Pitch 21 | 22 | **Describe the solution you'd like** 23 | 24 | 25 | **Describe alternatives you've considered** 26 | 27 | 28 | **Are you willing to open a pull request?** (We LOVE contributions!!!) 29 | 30 | ## Additional context 31 | 32 | 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/---refactor.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F527 Refactor" 3 | about: Propose a refactor/speedup/improvement to GPyTorch's internals 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | 12 | - 13 | - 14 | - 15 | - 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Atom plugin files and ctags 2 | .ftpconfig 3 | .ftpconfig.cson 4 | .ftpignore 5 | *.tags 6 | *.tags1 7 | 8 | # VS Code settings stuff 9 | .vscode 10 | 11 | # Project specific 12 | gpytorch/libfft 13 | .pytest_cache 14 | 15 | # Byte-compiled / optimized / DLL files 16 | __pycache__/ 17 | *.py[cod] 18 | *$py.class 19 | 20 | # C extensions 21 | *.so 22 | 23 | # Distribution / packaging 24 | .Python 25 | env/ 26 | build/ 27 | develop-eggs/ 28 | dist/ 29 | downloads/ 30 | eggs/ 31 | .eggs/ 32 | lib/ 33 | lib64/ 34 | parts/ 35 | sdist/ 36 | var/ 37 | wheels/ 38 | pip-wheel-metadata/ 39 | *.egg-info/ 40 | .installed.cfg 41 | *.egg 42 | 43 | # PyInstaller 44 | # Usually these files are written by a python script from a template 45 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 46 | *.manifest 47 | *.spec 48 | 49 | # Installer logs 50 | pip-log.txt 51 | pip-delete-this-directory.txt 52 | 53 | # Unit test / coverage reports 54 | htmlcov/ 55 | .tox/ 56 | .coverage 57 | .coverage.* 58 | .cache 59 | nosetests.xml 60 | coverage.xml 61 | *.cover 62 | .hypothesis/ 63 | 64 | # Translations 65 | *.mo 66 | *.pot 67 | 68 | # Django stuff: 69 | *.log 70 | local_settings.py 71 | 72 | # Flask stuff: 73 | instance/ 74 | .webassets-cache 75 | 76 | # Scrapy stuff: 77 | .scrapy 78 | 79 | # Sphinx documentation 80 | docs/_build/ 81 | 82 | # PyBuilder 83 | target/ 84 | 85 | # Jupyter Notebook 86 | .ipynb_checkpoints 87 | 88 | # pyenv 89 | .python-version 90 | 91 | # celery beat schedule file 92 | celerybeat-schedule 93 | 94 | # SageMath parsed files 95 | *.sage.py 96 | 97 | # dotenv 98 | .env 99 | 100 | # virtualenv 101 | .venv 102 | venv/ 103 | ENV/ 104 | 105 | # Spyder project settings 106 | .spyderproject 107 | .spyproject 108 | 109 | # Rope project settings 110 | .ropeproject 111 | 112 | # mkdocs documentation 113 | /site 114 | 115 | # mypy 116 | .mypy_cache/ 117 | 118 | # vim 119 | *.swp 120 | 121 | # watchman 122 | .watchmanconfig 123 | 124 | # Ignore macOS stuff 125 | # General 126 | .DS_Store 127 | .AppleDouble 128 | .LSOverride 129 | 130 | # Icon must end with two \r 131 | Icon 132 | 133 | 134 | # Thumbnails 135 | ._* 136 | 137 | # Files that might appear in the root of a volume 138 | .DocumentRevisions-V100 139 | .fseventsd 140 | .Spotlight-V100 141 | .TemporaryItems 142 | .Trashes 143 | .VolumeIcon.icns 144 | .com.apple.timemachine.donotpresent 145 | 146 | # Directories potentially created on remote AFP share 147 | .AppleDB 148 | .AppleDesktop 149 | Network Trash Folder 150 | Temporary Items 151 | .apdisk 152 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | known_third_party = matplotlib,numpy,setuptools,sphinx_rtd_theme,torch 3 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v2.4.0 4 | hooks: 5 | - id: flake8 6 | args: [--config=setup.cfg] 7 | exclude: ^(examples/*)|(docs/*) 8 | - id: check-byte-order-marker 9 | - id: check-case-conflict 10 | - id: check-merge-conflict 11 | - id: end-of-file-fixer 12 | - id: forbid-new-submodules 13 | - id: mixed-line-ending 14 | args: [--fix=lf] 15 | - id: trailing-whitespace 16 | - id: debug-statements 17 | - repo: https://github.com/ambv/black 18 | rev: 19.10b0 19 | hooks: 20 | - id: black 21 | exclude: ^(build/*)|(docs/*)|(examples/*) 22 | args: [-l 120, --target-version=py36] 23 | - repo: https://github.com/pre-commit/mirrors-isort 24 | rev: v4.3.21 25 | hooks: 26 | - id: isort 27 | language_version: python3 28 | exclude: ^(build/*)|(docs/*)|(examples/*) 29 | args: [-w 120, -m 3, -tc, --project=gpytorch] 30 | - repo: https://github.com/jumanjihouse/pre-commit-hooks 31 | rev: 1.11.0 32 | hooks: 33 | - id: require-ascii 34 | exclude: ^(examples/LBFGS.py)|(examples/.*\.ipynb) 35 | - id: script-must-have-extension 36 | - id: forbid-binary 37 | exclude: ^(examples/*) 38 | - repo: https://github.com/Lucas-C/pre-commit-hooks 39 | rev: v1.1.7 40 | hooks: 41 | - id: forbid-crlf 42 | - id: forbid-tabs 43 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [TYPECHECK] 2 | 3 | # List of members which are set dynamically and missed by Pylint inference 4 | # system, and so shouldn't trigger E1101 when accessed. 5 | generated-members=numpy.*, torch.* 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | branches: 4 | only: 5 | - master 6 | 7 | python: 8 | - "3.6" 9 | env: 10 | - PYTORCH_VERSION=master 11 | - PYTORCH_VERSION=stable 12 | - PYTORCH_VERSION=stable WITH_PYRO=true 13 | - PYTORCH_VERSION=stable WITH_PYRO=true EXAMPLES=true 14 | 15 | install: 16 | - if [[ $PYTORCH_VERSION = "master" ]]; then 17 | pip install numpy; 18 | pip install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html; 19 | python setup.py build develop; 20 | else 21 | pip install torch==1.3.1+cpu -f https://download.pytorch.org/whl/torch_stable.html; 22 | fi 23 | - if [[ $WITH_PYRO == true ]]; then 24 | pip install git+https://github.com/pyro-ppl/pyro@dev; 25 | fi 26 | - if [[ $EXAMPLES == true ]]; then 27 | pip install pytest nbval jupyter tqdm matplotlib torchvision scipy; 28 | fi 29 | 30 | script: 31 | - python -c "import torch; print('PyTorch Version:', torch.__version__)" 32 | - if [[ $EXAMPLES == true ]]; then 33 | python setup.py build develop; 34 | grep -l smoke_test examples/**/*.ipynb | xargs grep -L 'smoke_test = False' | CI=true xargs pytest --nbval-lax --current-env; 35 | else 36 | python -m unittest discover; 37 | fi 38 | 39 | matrix: 40 | include: 41 | - env: LINT_CHECK 42 | python: "3.6" 43 | install: pip install flake8 flake8-print 44 | script: flake8 45 | - env: PRECOMMIT_CHECK 46 | python: "3.6" 47 | install: pip install pre-commit; pre-commit install; pre-commit run seed-isort-config || true 48 | script: pre-commit run --files test/**/*.py gpytorch/**/*.py 49 | - env: DOCS_CHECK 50 | addons: 51 | apt_packages: 52 | - pandoc 53 | python: "3.6" 54 | install: pip install "nbformat<=4.4" IPython ipykernel "sphinx<3.0.0" sphinx_rtd_theme nbsphinx m2r attrs==19.1 55 | script: python setup.py build_sphinx 56 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Jake Gardner 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = GPyTorch 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | nbformat<=4.4 2 | ipython 3 | ipykernel 4 | sphinx 5 | sphinx_rtd_theme 6 | nbsphinx 7 | m2r 8 | -------------------------------------------------------------------------------- /docs/source/.gitignore: -------------------------------------------------------------------------------- 1 | examples 2 | -------------------------------------------------------------------------------- /docs/source/_static/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jwangjie/gpytorch/15979dacf1997af7daf0fdeddbdbfcef0730b007/docs/source/_static/.gitkeep -------------------------------------------------------------------------------- /docs/source/beta_features.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | 4 | gpytorch.beta_features 5 | =================================== 6 | 7 | .. currentmodule:: gpytorch.beta_features 8 | 9 | .. automodule:: gpytorch.beta_features 10 | :members: 11 | -------------------------------------------------------------------------------- /docs/source/constraints.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | 4 | gpytorch.constraints 5 | =================================== 6 | 7 | .. automodule:: gpytorch.constraints 8 | .. currentmodule:: gpytorch.constraints 9 | 10 | 11 | Parameter Constraints 12 | ----------------------------- 13 | 14 | :hidden:`Interval` 15 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 16 | 17 | .. autoclass:: Interval 18 | :members: 19 | 20 | :hidden:`GreaterThan` 21 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 22 | 23 | .. autoclass:: GreaterThan 24 | :members: 25 | 26 | :hidden:`Positive` 27 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 28 | 29 | .. autoclass:: Positive 30 | :members: 31 | 32 | :hidden:`LessThan` 33 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 34 | 35 | .. autoclass:: LessThan 36 | :members: 37 | -------------------------------------------------------------------------------- /docs/source/distributions.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | 4 | gpytorch.distributions 5 | =================================== 6 | 7 | GPyTorch distribution objects are essentially the same as torch distribution objects. 8 | For the most part, GpyTorch relies on torch's distribution library. 9 | However, we offer two custom distributions. 10 | 11 | We implement a custom :obj:`~gpytorch.distributions.MultivariateNormal` that accepts 12 | :obj:`~gpytorch.lazy.LazyTensor` objects for covariance matrices. This allows us to use custom 13 | linear algebra operations, which makes this more efficient than PyTorch's MVN implementation. 14 | 15 | In addition, we implement a :obj:`~gpytorch.distributions.MultitaskMultivariateNormal` which 16 | can be used with multi-output Gaussian process models. 17 | 18 | .. note:: 19 | 20 | If Pyro is available, all GPyTorch distribution objects inherit Pyro's distribution methods 21 | as well. 22 | 23 | .. automodule:: gpytorch.distributions 24 | .. currentmodule:: gpytorch.distributions 25 | 26 | 27 | Distribution 28 | ----------------------------- 29 | 30 | .. autoclass:: Distribution 31 | :members: 32 | 33 | 34 | MultivariateNormal 35 | ----------------------------- 36 | 37 | .. autoclass:: MultivariateNormal 38 | :members: 39 | 40 | 41 | MultitaskMultivariateNormal 42 | ---------------------------------- 43 | 44 | .. autoclass:: MultitaskMultivariateNormal 45 | :members: 46 | 47 | 48 | Delta 49 | ---------------------------------- 50 | 51 | .. class:: Delta(v, log_density=0.0, event_dim=0, validate_args=None) 52 | 53 | (Borrowed from Pyro.) Degenerate discrete distribution (a single point). 54 | 55 | Discrete distribution that assigns probability one to the single element in 56 | its support. Delta distribution parameterized by a random choice should not 57 | be used with MCMC based inference, as doing so produces incorrect results. 58 | 59 | :param v: The single support element. 60 | :param log_density: An optional density for this Delta. This is useful to 61 | keep the class of Delta distributions closed under differentiable 62 | transformation. 63 | :param event_dim: Optional event dimension, defaults to zero. 64 | :type v: torch.Tensor 65 | :type log_density: torch.Tensor 66 | :type event_dim: int 67 | -------------------------------------------------------------------------------- /docs/source/functions.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | 4 | gpytorch.functions 5 | =================================== 6 | 7 | .. currentmodule:: gpytorch.functions 8 | 9 | 10 | Functions 11 | ---------------- 12 | 13 | .. automodule:: gpytorch 14 | 15 | .. autofunction:: add_diag 16 | 17 | .. autofunction:: add_jitter 18 | 19 | .. autofunction:: dsmm 20 | 21 | .. autofunction:: inv_matmul 22 | 23 | .. autofunction:: inv_quad 24 | 25 | .. autofunction:: inv_quad_logdet 26 | 27 | .. autofunction:: matmul 28 | 29 | .. autofunction:: logdet 30 | 31 | .. autofunction:: log_normal_cdf 32 | 33 | .. autofunction:: root_decomposition 34 | 35 | .. autofunction:: root_inv_decomposition 36 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. GPyTorch documentation master file, created by 2 | sphinx-quickstart on Tue Aug 21 09:04:16 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | :github_url: https://github.com/cornellius-gp/gpytorch 7 | 8 | GPyTorch's documentation 9 | ==================================== 10 | 11 | .. toctree:: 12 | :glob: 13 | :maxdepth: 1 14 | :caption: Tutorials: 15 | 16 | examples/01_Exact_GPs/Simple_GP_Regression.ipynb 17 | 18 | .. toctree:: 19 | :glob: 20 | :maxdepth: 2 21 | :caption: Examples: 22 | 23 | examples/**/index 24 | 25 | .. toctree:: 26 | :maxdepth: 1 27 | :caption: Package Reference 28 | 29 | models 30 | likelihoods 31 | kernels 32 | means 33 | marginal_log_likelihoods 34 | constraints 35 | distributions 36 | priors 37 | variational 38 | 39 | .. toctree:: 40 | :maxdepth: 1 41 | :caption: Settings and Beta Features 42 | 43 | settings 44 | beta_features 45 | 46 | .. toctree:: 47 | :maxdepth: 1 48 | :caption: Advanced Package Reference 49 | 50 | module 51 | lazy 52 | functions 53 | utils 54 | 55 | 56 | 57 | Indices and tables 58 | ================== 59 | 60 | * :ref:`genindex` 61 | * :ref:`modindex` 62 | * :ref:`search` 63 | 64 | 65 | Research references 66 | ====================== 67 | 68 | * Gardner, Jacob R., Geoff Pleiss, David Bindel, Kilian Q. Weinberger, and Andrew Gordon Wilson. " GPyTorch: Blackbox Matrix-Matrix Gaussian Process Inference with GPU Acceleration." In NeurIPS (2018). 69 | * Pleiss, Geoff, Jacob R. Gardner, Kilian Q. Weinberger, and Andrew Gordon Wilson. "Constant-Time Predictive Distributions for Gaussian Processes." In ICML (2018). 70 | * Gardner, Jacob R., Geoff Pleiss, Ruihan Wu, Kilian Q. Weinberger, and Andrew Gordon Wilson. "Product Kernel Interpolation for Scalable Gaussian Processes." In AISTATS (2018). 71 | * Wilson, Andrew G., Zhiting Hu, Ruslan R. Salakhutdinov, and Eric P. Xing. "Stochastic variational deep kernel learning." In NeurIPS (2016). 72 | * Wilson, Andrew, and Hannes Nickisch. "Kernel interpolation for scalable structured Gaussian processes (KISS-GP)." In ICML (2015). 73 | * Hensman, James, Alexander G. de G. Matthews, and Zoubin Ghahramani. "Scalable variational Gaussian process classification." In AISTATS (2015). 74 | -------------------------------------------------------------------------------- /docs/source/likelihoods.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | 4 | gpytorch.likelihoods 5 | =================================== 6 | 7 | .. automodule:: gpytorch.likelihoods 8 | .. currentmodule:: gpytorch.likelihoods 9 | 10 | 11 | Likelihood 12 | -------------------- 13 | 14 | .. autoclass:: Likelihood 15 | :members: 16 | 17 | 18 | One-Dimensional Likelihoods 19 | ----------------------------- 20 | 21 | Likelihoods for GPs that are distributions of scalar functions. 22 | (I.e. for a specific :math:`\mathbf x` we expect that :math:`f(\mathbf x) \in \mathbb{R}`.) 23 | 24 | One-dimensional likelihoods should extend :obj:`gpytoch.likelihoods._OneDimensionalLikelihood` to 25 | reduce the variance when computing approximate GP objective functions. 26 | (Variance reduction is accomplished by using 1D Gauss-Hermite quadrature rather than MC-integration). 27 | 28 | 29 | :hidden:`GaussianLikelihood` 30 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 31 | 32 | .. autoclass:: GaussianLikelihood 33 | :members: 34 | 35 | 36 | :hidden:`BernoulliLikelihood` 37 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 38 | 39 | .. autoclass:: BernoulliLikelihood 40 | :members: 41 | 42 | 43 | Multi-Dimensional Likelihoods 44 | ----------------------------- 45 | 46 | Likelihoods for GPs that are distributions of vector-valued functions. 47 | (I.e. for a specific :math:`\mathbf x` we expect that :math:`f(\mathbf x) \in \mathbb{R}^t`, 48 | where :math:`t` is the number of output dimensions.) 49 | 50 | :hidden:`MultitaskGaussianLikelihood` 51 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 52 | 53 | .. autoclass:: MultitaskGaussianLikelihood 54 | :members: 55 | 56 | :hidden:`SoftmaxLikelihood` 57 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 58 | 59 | .. autoclass:: SoftmaxLikelihood 60 | :members: 61 | -------------------------------------------------------------------------------- /docs/source/marginal_log_likelihoods.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | 4 | gpytorch.mlls 5 | =================================== 6 | 7 | These are modules to compute (or approximate/bound) the marginal log likelihood 8 | (MLL) of the GP model when applied to data. I.e., given a GP :math:`f \sim 9 | \mathcal{GP}(\mu, K)`, and data :math:`\mathbf X, \mathbf y`, these modules 10 | compute/approximate 11 | 12 | .. math:: 13 | 14 | \begin{equation*} 15 | \mathcal{L} = p_f(\mathbf y \! \mid \! \mathbf X) 16 | = \int p \left( \mathbf y \! \mid \! f(\mathbf X) \right) \: p(f(\mathbf X) \! \mid \! \mathbf X) \: d f 17 | \end{equation*} 18 | 19 | This is computed exactly when the GP inference is computed exactly (e.g. regression w/ a Gaussian likelihood). 20 | It is approximated/bounded for GP models that use approximate inference. 21 | 22 | These models are typically used as the "loss" functions for GP models (though note that the output of 23 | these functions must be negated for optimization). 24 | 25 | .. automodule:: gpytorch.mlls 26 | .. currentmodule:: gpytorch.mlls 27 | 28 | 29 | Exact GP Inference 30 | ----------------------------- 31 | 32 | These are MLLs for use with :obj:`~gpytorch.models.ExactGP` modules. They compute the MLL exactly. 33 | 34 | :hidden:`ExactMarginalLogLikelihood` 35 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 36 | 37 | .. autoclass:: ExactMarginalLogLikelihood 38 | :members: 39 | 40 | 41 | Approximate GP Inference 42 | ----------------------------------- 43 | 44 | These are MLLs for use with :obj:`~gpytorch.models.ApproximateGP` modules. They are designed for 45 | when exact inference is intractable (either when the likelihood is non-Gaussian likelihood, or when 46 | there is too much data for an ExactGP model). 47 | 48 | :hidden:`VariationalELBO` 49 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 50 | 51 | .. autoclass:: VariationalELBO 52 | :members: 53 | 54 | :hidden:`PredictiveLogLikelihood` 55 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 56 | 57 | .. autoclass:: PredictiveLogLikelihood 58 | :members: 59 | 60 | :hidden:`GammaRobustVariationalELBO` 61 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 62 | 63 | .. autoclass:: GammaRobustVariationalELBO 64 | :members: 65 | 66 | :hidden:`DeepApproximateMLL` 67 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 68 | 69 | .. autoclass:: DeepApproximateMLL 70 | :members: 71 | -------------------------------------------------------------------------------- /docs/source/means.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | 4 | gpytorch.means 5 | =================================== 6 | 7 | .. automodule:: gpytorch.means 8 | .. currentmodule:: gpytorch.means 9 | 10 | 11 | Mean 12 | ---------------- 13 | 14 | .. autoclass:: Mean 15 | :members: 16 | 17 | 18 | Standard Means 19 | ----------------------------- 20 | 21 | :hidden:`ZeroMean` 22 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 23 | 24 | .. autoclass:: ZeroMean 25 | :members: 26 | 27 | :hidden:`ConstantMean` 28 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 29 | 30 | .. autoclass:: ConstantMean 31 | :members: 32 | 33 | 34 | Specialty Means 35 | ----------------------------------- 36 | 37 | :hidden:`MultitaskMean` 38 | ~~~~~~~~~~~~~~~~~~~~~~~ 39 | 40 | .. autoclass:: MultitaskMean 41 | :members: 42 | 43 | :hidden:`ConstantMeanGrad` 44 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 45 | 46 | .. autoclass:: ConstantMeanGrad 47 | :members: 48 | -------------------------------------------------------------------------------- /docs/source/models.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | 4 | gpytorch.models 5 | =================================== 6 | 7 | .. automodule:: gpytorch.models 8 | .. currentmodule:: gpytorch.models 9 | 10 | 11 | Models for Exact GP Inference 12 | ----------------------------- 13 | 14 | :hidden:`ExactGP` 15 | ~~~~~~~~~~~~~~~~~ 16 | 17 | .. autoclass:: ExactGP 18 | :members: 19 | 20 | 21 | Models for Approximate GP Inference 22 | ----------------------------------- 23 | 24 | :hidden:`ApproximateGP` 25 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 26 | 27 | .. autoclass:: ApproximateGP 28 | :members: 29 | 30 | 31 | Models for Deep GPs 32 | ----------------------------------- 33 | 34 | :hidden:`deep_gps.DeepGP` 35 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 36 | 37 | .. autoclass:: gpytorch.models.deep_gps.DeepGP 38 | :members: 39 | 40 | :hidden:`deep_gps.DeepGPLayer` 41 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 42 | 43 | .. autoclass:: gpytorch.models.deep_gps.DeepGPLayer 44 | :members: 45 | 46 | 47 | Models for integrating with Pyro 48 | ----------------------------------- 49 | 50 | :hidden:`PyroGP` 51 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 52 | 53 | .. autoclass:: PyroGP 54 | :members: 55 | -------------------------------------------------------------------------------- /docs/source/module.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | 4 | gpytorch.Module 5 | =================================== 6 | 7 | .. currentmodule:: gpytorch.Module 8 | 9 | 10 | .. autoclass:: gpytorch.Module 11 | :members: 12 | 13 | -------------------------------------------------------------------------------- /docs/source/priors.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | 4 | gpytorch.priors 5 | =================================== 6 | 7 | .. automodule:: gpytorch.priors 8 | .. currentmodule:: gpytorch.priors 9 | 10 | 11 | Prior 12 | ---------------- 13 | 14 | .. autoclass:: Prior 15 | :members: 16 | 17 | 18 | Standard Priors 19 | ----------------------------- 20 | 21 | :hidden:`GammaPrior` 22 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 23 | 24 | .. autoclass:: GammaPrior 25 | :members: 26 | 27 | :hidden:`LKJCovariancePrior` 28 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 29 | 30 | .. autoclass:: LKJCovariancePrior 31 | :members: 32 | 33 | :hidden:`MultivariateNormalPrior` 34 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 35 | 36 | .. autoclass:: MultivariateNormalPrior 37 | :members: 38 | 39 | :hidden:`NormalPrior` 40 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 41 | 42 | .. autoclass:: NormalPrior 43 | :members: 44 | 45 | :hidden:`SmoothedBoxPrior` 46 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 47 | 48 | .. autoclass:: SmoothedBoxPrior 49 | :members: 50 | -------------------------------------------------------------------------------- /docs/source/settings.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | 4 | gpytorch.settings 5 | =================================== 6 | 7 | .. currentmodule:: gpytorch.settings 8 | 9 | .. automodule:: gpytorch.settings 10 | :members: 11 | -------------------------------------------------------------------------------- /docs/source/utils.rst: -------------------------------------------------------------------------------- 1 | .. role:: hidden 2 | :class: hidden-section 3 | 4 | gpytorch.utils 5 | =================================== 6 | 7 | .. currentmodule:: gpytorch.utils 8 | 9 | 10 | Utilities 11 | ---------------- 12 | 13 | .. automodule:: gpytorch.utils 14 | :members: 15 | 16 | Lanczos Utilities 17 | ~~~~~~~~~~~~~~~~~ 18 | 19 | .. automodule:: gpytorch.utils.lanczos 20 | :members: 21 | 22 | Pivoted Cholesky Utilities 23 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 24 | 25 | .. automodule:: gpytorch.utils.pivoted_cholesky 26 | :members: 27 | 28 | Quadrature Utilities 29 | ~~~~~~~~~~~~~~~~~~~~ 30 | 31 | .. automodule:: gpytorch.utils.quadrature 32 | :members: 33 | 34 | Sparse Utilities 35 | ~~~~~~~~~~~~~~~~~ 36 | 37 | .. automodule:: gpytorch.utils.sparse 38 | :members: 39 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: gpytorch 2 | channels: 3 | - pytorch 4 | dependencies: 5 | - pytorch 6 | -------------------------------------------------------------------------------- /examples/.gitignore: -------------------------------------------------------------------------------- 1 | *.ipynb_checkpoints 2 | *.mat 3 | *.pth 4 | *.dat 5 | **/_data 6 | -------------------------------------------------------------------------------- /examples/00_Basic_Usage/README.rst: -------------------------------------------------------------------------------- 1 | index.rst -------------------------------------------------------------------------------- /examples/00_Basic_Usage/index.rst: -------------------------------------------------------------------------------- 1 | Basic Usage 2 | ============== 3 | 4 | This folder contains notebooks for basic usage of the package, e.g. things like dealing with hyperparameters, 5 | parameter constraints and priors, and saving and loading models. 6 | 7 | Before checking these out, you may want to check out our `simple GP regression tutorial`_ that details the anatomy of a GPyTorch model. 8 | 9 | - Check out our `Tutorial on Hyperparameters`_ for information on things like raw versus actual 10 | parameters, constraints, priors and more. 11 | - The `Saving and Loading Models`_ notebook details how to save and load GPyTorch models 12 | on disk. 13 | 14 | .. toctree:: 15 | :maxdepth: 1 16 | :hidden: 17 | 18 | Hyperparameters.ipynb 19 | Saving_and_Loading_Models.ipynb 20 | 21 | .. _simple GP regression tutorial: 22 | ../01_Exact_GPs/Simple_GP_Regression.ipynb 23 | 24 | .. _Tutorial on Hyperparameters: 25 | Hyperparameters.ipynb 26 | 27 | .. _Saving and Loading Models: 28 | Saving_and_Loading_Models.ipynb 29 | -------------------------------------------------------------------------------- /examples/01_Exact_GPs/README.rst: -------------------------------------------------------------------------------- 1 | index.rst -------------------------------------------------------------------------------- /examples/01_Exact_GPs/Simple_GP_Regression_GPflow_regression_1D.csv: -------------------------------------------------------------------------------- 1 | 8.658165855998894989e-01,1.525531433714437224e+00 2 | 6.661700880180961848e-01,3.643420296823000282e+00 3 | 8.049218148148531427e-01,3.010885733911660811e+00 4 | 7.714303440386238719e-01,3.774442382979624977e+00 5 | 1.479047835465483463e-01,3.368763948379832396e+00 6 | 2.633173728814863779e-02,3.868970757495299839e+00 7 | 3.718859661709991604e-01,3.493356575175871281e+00 8 | 8.897812990554012647e-01,1.428453882063584146e+00 9 | 2.432357456111999827e-01,3.871535091569236364e+00 10 | 9.666105548197428066e-01,5.550645204060850268e+00 11 | 4.866105548197428066e-01,-1.850645204060850268e+00 12 | -------------------------------------------------------------------------------- /examples/01_Exact_GPs/index.rst: -------------------------------------------------------------------------------- 1 | Exact GPs (Regression) 2 | ======================== 3 | 4 | Regression with a Gaussian noise model is the cannonical example of Gaussian processes. 5 | These examples will work for small to medium sized datasets (~2,000 data points). 6 | All examples here use exact GP inference. 7 | 8 | - `Simple GP Regression`_ is the basic tutorial for regression in GPyTorch. 9 | - `Spectral Mixture Regression`_ extends on the previous example with a more complex kernel. 10 | - `Fully Bayesian GP Regression`_ demonstrates how to perform fully Bayesian inference by sampling the GP hyperparameters 11 | using NUTS. (This example requires Pyro to be installed). 12 | 13 | .. toctree:: 14 | :maxdepth: 1 15 | :hidden: 16 | 17 | Simple_GP_Regression.ipynb 18 | Spectral_Mixture_GP_Regression.ipynb 19 | GP_Regression_Fully_Bayesian.ipynb 20 | 21 | .. _Simple GP Regression: 22 | ./Simple_GP_Regression.ipynb 23 | 24 | .. _Spectral Mixture Regression: 25 | ./Spectral_Mixture_GP_Regression.ipynb 26 | 27 | .. _Fully Bayesian GP Regression: 28 | ./GP_Regression_Fully_Bayesian.ipynb 29 | -------------------------------------------------------------------------------- /examples/02_Scalable_Exact_GPs/README.rst: -------------------------------------------------------------------------------- 1 | index.rst -------------------------------------------------------------------------------- /examples/03_Multitask_Exact_GPs/README.rst: -------------------------------------------------------------------------------- 1 | index.rst -------------------------------------------------------------------------------- /examples/03_Multitask_Exact_GPs/index.rst: -------------------------------------------------------------------------------- 1 | Multitask/Multioutput GPs with Exact Inference 2 | ================================================ 3 | 4 | Exact GPs can be used to model vector valued functions, or functions that represent multiple tasks. 5 | There are several different cases: 6 | 7 | Multi-output (vector valued functions) 8 | ---------------------------------------- 9 | 10 | - **Correlated output dimensions**: this is the most common use case. 11 | See the `Multitask GP Regression`_ example, which implements the inference strategy defined in `Bonilla et al., 2008`_. 12 | - **Independent output dimensions**: here we will use an independent GP for each output. 13 | 14 | - If the outputs share the same kernel and mean, you can train a `Batch Independent Multioutput GP`_. 15 | - Otherwise, you can train a `ModelList Multioutput GP`_. 16 | 17 | .. toctree:: 18 | :maxdepth: 1 19 | :hidden: 20 | 21 | Multitask_GP_Regression.ipynb 22 | Batch_Independent_Multioutput_GP.ipynb 23 | ModelList_GP_Regression.ipynb 24 | 25 | Scalar function with multiple tasks 26 | ---------------------------------------- 27 | 28 | See the `Hadamard Multitask GP Regression`_ example. 29 | This setting should be used only when each input corresponds to a single task. 30 | 31 | .. toctree:: 32 | :maxdepth: 1 33 | :hidden: 34 | 35 | Hadamard_Multitask_GP_Regression.ipynb 36 | 37 | 38 | .. _Multitask GP Regression: 39 | ./Multitask_GP_Regression.ipynb 40 | 41 | .. _Bonilla et al., 2008: 42 | https://papers.nips.cc/paper/3189-multi-task-gaussian-process-prediction 43 | 44 | .. _Batch Independent Multioutput GP: 45 | ./Batch_Independent_Multioutput_GP.ipynb 46 | 47 | .. _ModelList Multioutput GP: 48 | ./ModelList_GP_Regression.ipynb 49 | 50 | .. _Hadamard Multitask GP Regression: 51 | ./Hadamard_Multitask_GP_Regression.ipynb 52 | -------------------------------------------------------------------------------- /examples/04_Variational_and_Approximate_GPs/README.rst: -------------------------------------------------------------------------------- 1 | index.rst -------------------------------------------------------------------------------- /examples/05_Deep_Gaussian_Processes/README.rst: -------------------------------------------------------------------------------- 1 | index.rst -------------------------------------------------------------------------------- /examples/05_Deep_Gaussian_Processes/index.rst: -------------------------------------------------------------------------------- 1 | .. toctree:: 2 | :glob: 3 | :maxdepth: 1 4 | :hidden: 5 | 6 | Deep_Gaussian_Processes.ipynb 7 | -------------------------------------------------------------------------------- /examples/06_PyTorch_NN_Integration_DKL/.gitignore: -------------------------------------------------------------------------------- 1 | data/* 2 | *.dat 3 | -------------------------------------------------------------------------------- /examples/06_PyTorch_NN_Integration_DKL/README.rst: -------------------------------------------------------------------------------- 1 | index.rst -------------------------------------------------------------------------------- /examples/06_PyTorch_NN_Integration_DKL/index.rst: -------------------------------------------------------------------------------- 1 | PyTorch NN Integration (Deep Kernel Learning) 2 | =============================================== 3 | 4 | Because GPyTorch is built on top of PyTorch, you can seamlessly integrate existing PyTorch modules into GPyTorch models. 5 | This makes it possible to combine neural networks with GPs, either with exact or approximate inference. 6 | 7 | Here we provide some examples of **Deep Kernel Learning**, which are GP models that use kernels parameterized by neural networks. 8 | 9 | - **Exact inference GP + NN**: see the `Exact DKL`_ (*deep kernel learning*) example, based on `Wilson et al., 2015`_. 10 | - **Approximate inference GP + NN**: see the `CIFAR-10 Classification SVDKL`_ (*stochastic variational deep kernel learning*) example, based on `Wilson et al., 2016`_. 11 | 12 | .. toctree:: 13 | :glob: 14 | :maxdepth: 1 15 | :hidden: 16 | 17 | KISSGP_Deep_Kernel_Regression_CUDA.ipynb 18 | Deep_Kernel_Learning_DenseNet_CIFAR_Tutorial.ipynb 19 | 20 | .. _Exact DKL: 21 | KISSGP_Deep_Kernel_Regression_CUDA.ipynb 22 | 23 | .. _CIFAR-10 Classification SVDKL: 24 | Deep_Kernel_Learning_DenseNet_CIFAR_Tutorial.ipynb 25 | 26 | .. _Wilson et al., 2015: 27 | https://arxiv.org/abs/1511.02222 28 | 29 | .. _Wilson et al., 2016: 30 | https://arxiv.org/abs/1611.00336 31 | -------------------------------------------------------------------------------- /examples/07_Pyro_Integration/README.rst: -------------------------------------------------------------------------------- 1 | index.rst -------------------------------------------------------------------------------- /examples/07_Pyro_Integration/index.rst: -------------------------------------------------------------------------------- 1 | Pyro Integration 2 | =================== 3 | 4 | GPyTorch can optionally work with the Pyro probablistic programming language. 5 | This makes it possible to use Pyro's advanced inference algorithms, or to incorporate GPs as part of larger probablistic models. 6 | GPyTorch offers two ways of integrating with Pyro: 7 | 8 | High-level Pyro Interface (for predictive models) 9 | -------------------------------------------------- 10 | 11 | The high-level interface provides a simple wrapper around :obj:`~gpytorch.models.ApproximateGP` that makes it 12 | possible to use Pyro's inference tools with GPyTorch models. 13 | It is best designed for: 14 | 15 | - Developing models that will be used for predictive tasks 16 | - GPs with likelihoods that have additional latent variables 17 | 18 | The `Pyro + GPyTorch High-Level Introduction`_ gives an overview of the high-level interface. 19 | For a more in-depth example that shows off the power of the integration, see the `Clustered Multitask GP Example`_. 20 | 21 | .. toctree:: 22 | :glob: 23 | :maxdepth: 1 24 | :hidden: 25 | 26 | Pyro_GPyTorch_High_Level.ipynb 27 | Clustered_Multitask_GP_Regression.ipynb 28 | 29 | 30 | Low-level Pyro Interface (for latent function inference) 31 | ---------------------------------------------------------- 32 | 33 | The low-level interface simply provides tools to compute GP latent functions, and requires users to write their own :meth:`model` and :meth:`guide` functions. 34 | It is best designed for: 35 | 36 | - Performing inference on probabilistic models that involve GPs 37 | - Models with complicated likelihoods 38 | 39 | The `Pyro + GPyTorch Low-Level Introduction`_ gives an overview of the low-level interface. 40 | The `Cox Process Example`_ is a more in-depth example of a model that can be built using this interface. 41 | 42 | .. toctree:: 43 | :glob: 44 | :maxdepth: 1 45 | :hidden: 46 | 47 | Pyro_GPyTorch_Low_Level.ipynb 48 | Cox_Process_Example.ipynb 49 | 50 | .. _Pyro + GPyTorch High-Level Introduction: 51 | Pyro_GPyTorch_High_Level.ipynb 52 | 53 | .. _Clustered Multitask GP Example: 54 | Clustered_Multitask_GP_Regression.ipynb 55 | 56 | .. _Pyro + GPyTorch Low-Level Introduction: 57 | Pyro_GPyTorch_Low_Level.ipynb 58 | 59 | .. _Cox Process Example: 60 | Cox_Process_Example.ipynb 61 | -------------------------------------------------------------------------------- /examples/08_Advanced_Usage/README.rst: -------------------------------------------------------------------------------- 1 | index.rst -------------------------------------------------------------------------------- /examples/08_Advanced_Usage/index.rst: -------------------------------------------------------------------------------- 1 | Advanced Usage 2 | =============================================== 3 | 4 | Here are some examples highlighting GPyTorch's more advanced features. 5 | 6 | Batch GPs 7 | ----------- 8 | 9 | GPyTorch makes it possible to train/perform inference with a batch of Gaussian processes in parallel. 10 | This can be useful for a number of applications: 11 | 12 | - Modeling a function with multiple (independent) outputs 13 | - Performing efficient cross-validation 14 | - Parallel acquisition function sampling for Bayesian optimization 15 | - And more! 16 | 17 | Here we highlight a number of common batch GP scenarios and how to construct them in GPyTorch. 18 | 19 | - **Multi-output functions (with independent outputs).** Batch GPs are extremely efficient at modelling multi-output functions, when each of the output functions 20 | are **independent**. See the `Batch Independent Multioutput GP`_ example for more details. 21 | 22 | - **For cross validation**, or for some BayesOpt applications, it may make sense to evaluate the GP on different batches of test data. 23 | This can be accomplished by using a standard (non-batch) GP model. 24 | At test time, feeding a `b x n x d` tensor into the model will then return `b` batches of `n` test points. 25 | See the `Batch Mode Regression`_ example for more details. 26 | 27 | .. toctree:: 28 | :glob: 29 | :maxdepth: 1 30 | :hidden: 31 | 32 | Simple_Batch_Mode_GP_Regression.ipynb 33 | 34 | 35 | GPs with Derivatives 36 | ---------------------- 37 | 38 | Derivative information can be used by GPs to accelerate Bayesian optimization. 39 | See the `1D derivatives GP example`_ or the `2D derivatives GP example`_ for examples on using GPs with derivative information. 40 | 41 | .. toctree:: 42 | :glob: 43 | :maxdepth: 1 44 | :hidden: 45 | 46 | Simple_GP_Regression_Derivative_Information_1d.ipynb 47 | Simple_GP_Regression_Derivative_Information_2d.ipynb 48 | 49 | .. _Batch Independent Multioutput GP: 50 | Batch_Independent_Multioutput_GP.ipynb 51 | 52 | .. _Batch Mode Regression: 53 | Simple_Batch_Mode_GP_Regression.ipynb 54 | 55 | .. _1D derivatives GP example: 56 | Batch_Independent_Multioutput_GP.ipynb 57 | 58 | .. _2D derivatives GP example: 59 | Simple_Batch_Mode_GP_Regression.ipynb: 60 | 61 | 62 | Converting Models to TorchScript 63 | ---------------------------------- 64 | 65 | In order to deploy GPs in production code, it can be desirable to avoid using PyTorch directly for performance reasons. 66 | Fortunarely, PyTorch offers a mechanism caled TorchScript to aid in this. In these example notebooks, we'll demonstrate 67 | how to convert both an exact GP and a variational GP to a ScriptModule that can then be used for example in LibTorch. 68 | 69 | .. toctree:: 70 | :glob: 71 | :maxdepth: 1 72 | :hidden: 73 | 74 | TorchScript_Exact_Models.ipynb 75 | TorchScript_Variational_Models.ipynb 76 | -------------------------------------------------------------------------------- /examples/08_Advanced_Usage/svgp_elevators.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jwangjie/gpytorch/15979dacf1997af7daf0fdeddbdbfcef0730b007/examples/08_Advanced_Usage/svgp_elevators.pt -------------------------------------------------------------------------------- /examples/README.rst: -------------------------------------------------------------------------------- 1 | index.rst -------------------------------------------------------------------------------- /examples/index.rst: -------------------------------------------------------------------------------- 1 | Overview of Examples 2 | ===================== 3 | 4 | Here are numerous ipython notebooks that demonstrate the use of GPyTorch. For an overview of the notebooks in this folder, check out our newly updated `Documentation`_ guide to the notebooks. 5 | 6 | .. _Documentation: 7 | https://gpytorch.readthedocs.io/en/latest/ 8 | -------------------------------------------------------------------------------- /gpytorch/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from . import ( 3 | beta_features, 4 | distributions, 5 | kernels, 6 | lazy, 7 | likelihoods, 8 | means, 9 | mlls, 10 | models, 11 | priors, 12 | settings, 13 | utils, 14 | variational, 15 | ) 16 | from .functions import ( # Deprecated 17 | add_diag, 18 | add_jitter, 19 | dsmm, 20 | inv_matmul, 21 | inv_quad, 22 | inv_quad_logdet, 23 | log_normal_cdf, 24 | logdet, 25 | matmul, 26 | root_decomposition, 27 | root_inv_decomposition, 28 | ) 29 | from .lazy import cat, delazify, lazify 30 | from .mlls import ExactMarginalLogLikelihood 31 | from .module import Module 32 | 33 | __version__ = "1.0.1" 34 | 35 | __all__ = [ 36 | # Submodules 37 | "distributions", 38 | "kernels", 39 | "lazy", 40 | "likelihoods", 41 | "means", 42 | "mlls", 43 | "models", 44 | "priors", 45 | "utils", 46 | "variational", 47 | # Classes 48 | "Module", 49 | "ExactMarginalLogLikelihood", 50 | # Functions 51 | "add_diag", 52 | "add_jitter", 53 | "cat", 54 | "delazify", 55 | "dsmm", 56 | "inv_matmul", 57 | "inv_quad", 58 | "inv_quad_logdet", 59 | "lazify", 60 | "logdet", 61 | "log_normal_cdf", 62 | "matmul", 63 | "root_decomposition", 64 | "root_inv_decomposition", 65 | # Context managers 66 | "beta_features", 67 | "settings", 68 | # Other 69 | "__version__", 70 | ] 71 | -------------------------------------------------------------------------------- /gpytorch/beta_features.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import warnings 4 | 5 | from .settings import _feature_flag, _value_context 6 | 7 | 8 | class _moved_beta_feature(object): 9 | def __init__(self, new_cls, orig_name=None): 10 | self.new_cls = new_cls 11 | self.orig_name = orig_name if orig_name is not None else "gpytorch.settings.{}".format(new_cls.__name__) 12 | 13 | def __call__(self, *args, **kwargs): 14 | warnings.warn( 15 | "`{}` has moved to `gpytorch.settings.{}`.".format(self.orig_name, self.new_cls.__name__), 16 | DeprecationWarning, 17 | ) 18 | return self.new_cls(*args, **kwargs) 19 | 20 | def __getattr__(self, name): 21 | return getattr(self.new_cls, name) 22 | 23 | 24 | class checkpoint_kernel(_value_context): 25 | """ 26 | Should the kernel be computed in chunks with checkpointing or not? (Default, no) 27 | 28 | If `split_size = 0`: 29 | The kernel is computed explicitly. During training, the kernel matrix is 30 | kept in memory for the backward pass. This is the fastest option but the 31 | most memory intensive. 32 | If `split_size > 0`: 33 | The kernel is never fully computed or stored. Instead, the kernel is only 34 | accessed through matrix multiplication. The matrix multiplication is 35 | computed in `segments` chunks. This is slower, but requires significantly less memory. 36 | 37 | Default: 0 38 | """ 39 | 40 | _global_value = 0 41 | 42 | 43 | class default_preconditioner(_feature_flag): 44 | """ 45 | Add a diagonal correction to scalable inducing point methods 46 | """ 47 | 48 | pass 49 | 50 | 51 | __all__ = ["checkpoint_kernel", "default_preconditioner"] 52 | -------------------------------------------------------------------------------- /gpytorch/constraints/__init__.py: -------------------------------------------------------------------------------- 1 | from .constraints import GreaterThan, Interval, LessThan, Positive 2 | 3 | __all__ = ["GreaterThan", "Interval", "LessThan", "Positive"] 4 | -------------------------------------------------------------------------------- /gpytorch/distributions/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from .delta import Delta 4 | from .distribution import Distribution 5 | from .multitask_multivariate_normal import MultitaskMultivariateNormal 6 | from .multivariate_normal import MultivariateNormal 7 | 8 | # Get the set of distributions from either PyTorch or Pyro 9 | try: 10 | # If pyro is installed, use that set of base distributions 11 | import pyro.distributions as base_distributions 12 | except ImportError: 13 | # Otherwise, use PyTorch 14 | import torch.distributions as base_distributions 15 | 16 | 17 | __all__ = ["Delta", "Distribution", "MultivariateNormal", "MultitaskMultivariateNormal", "base_distributions"] 18 | -------------------------------------------------------------------------------- /gpytorch/distributions/distribution.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from torch.distributions import Distribution as TDistribution 4 | 5 | 6 | class _DistributionBase(TDistribution): 7 | """ 8 | The base class of Distributions. (Same as torch.distribution.Distribution 9 | or pyro.distribution.Distribution). 10 | """ 11 | 12 | @property 13 | def islazy(self): 14 | return self._islazy 15 | 16 | def __add__(self, other): 17 | raise NotImplementedError() 18 | 19 | def __div__(self, other): 20 | raise NotImplementedError() 21 | 22 | def __mul__(self, other): 23 | raise NotImplementedError() 24 | 25 | 26 | try: 27 | # If pyro is installed, add the TorchDistributionMixin 28 | from pyro.distributions.torch_distribution import TorchDistributionMixin 29 | 30 | class Distribution(_DistributionBase, TorchDistributionMixin): 31 | pass 32 | 33 | 34 | except ImportError: 35 | 36 | class Distribution(_DistributionBase): 37 | pass 38 | -------------------------------------------------------------------------------- /gpytorch/functions/_dsmm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from torch.autograd import Function 4 | 5 | from ..utils.sparse import bdsmm 6 | 7 | 8 | class DSMM(Function): 9 | @staticmethod 10 | def forward(ctx, sparse, dense): 11 | ctx.sparse = sparse 12 | return bdsmm(ctx.sparse, dense) 13 | 14 | @staticmethod 15 | def backward(ctx, grad_output): 16 | return None, bdsmm(ctx.sparse.transpose(-1, -2), grad_output) 17 | -------------------------------------------------------------------------------- /gpytorch/functions/_matmul.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from torch.autograd import Function 4 | 5 | from .. import settings 6 | 7 | 8 | class Matmul(Function): 9 | @staticmethod 10 | def forward(ctx, representation_tree, rhs, *matrix_args): 11 | ctx.representation_tree = representation_tree 12 | orig_rhs = rhs 13 | 14 | if rhs.ndimension() == 1: 15 | is_vector = True 16 | rhs = rhs.unsqueeze(-1) 17 | else: 18 | is_vector = False 19 | 20 | lazy_tsr = ctx.representation_tree(*matrix_args) 21 | res = lazy_tsr._matmul(rhs) 22 | 23 | to_save = [orig_rhs] + list(matrix_args) 24 | ctx.save_for_backward(*to_save) 25 | if settings.memory_efficient.off(): 26 | ctx._lazy_tsr = lazy_tsr 27 | 28 | # Squeeze if necessary 29 | if is_vector: 30 | res = res.squeeze(-1) 31 | return res 32 | 33 | @staticmethod 34 | def backward(ctx, grad_output): 35 | rhs = ctx.saved_tensors[0] 36 | matrix_args = ctx.saved_tensors[1:] 37 | rhs_shape = rhs.shape 38 | 39 | rhs_grad = None 40 | arg_grads = [None] * len(matrix_args) 41 | 42 | # input_1 gradient 43 | if any(ctx.needs_input_grad[2:]): 44 | rhs = rhs.unsqueeze(-1) if (rhs.ndimension() == 1) else rhs 45 | grad_output_matrix = grad_output.unsqueeze(-1) if grad_output.ndimension() == 1 else grad_output 46 | arg_grads = ctx.representation_tree(*matrix_args)._quad_form_derivative(grad_output_matrix, rhs) 47 | 48 | # input_2 gradient 49 | if ctx.needs_input_grad[1]: 50 | if hasattr(ctx, "_lazy_tsr"): 51 | lazy_tsr = ctx._lazy_tsr 52 | else: 53 | lazy_tsr = ctx.representation_tree(*matrix_args) 54 | 55 | if grad_output.dim() == 1: 56 | # Confusing Cublas_Sgemv bug when grad_output is single dimensional on GPU. 57 | rhs_grad = lazy_tsr._t_matmul(grad_output.unsqueeze(-1)).squeeze(-1) 58 | else: 59 | rhs_grad = lazy_tsr._t_matmul(grad_output) 60 | 61 | # For broadcasting 62 | if rhs_grad.dim() > len(rhs_shape): 63 | rhs_grad = rhs_grad.reshape(-1, *rhs_shape).sum(0) 64 | 65 | return tuple([None] + [rhs_grad] + list(arg_grads)) 66 | -------------------------------------------------------------------------------- /gpytorch/functions/matern_covariance.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | 5 | 6 | class MaternCovariance(torch.autograd.Function): 7 | @staticmethod 8 | def forward(ctx, x1, x2, lengthscale, nu, dist_func): 9 | if any(ctx.needs_input_grad[:2]): 10 | raise RuntimeError("MaternCovariance cannot compute gradients with " "respect to x1 and x2") 11 | if lengthscale.size(-1) > 1: 12 | raise ValueError("MaternCovariance cannot handle multiple lengthscales") 13 | # Subtract mean for numerical stability. Won't affect computations 14 | # because covariance matrix is stationary. 15 | needs_grad = any(ctx.needs_input_grad) 16 | mean = x1.reshape(-1, x1.size(-1)).mean(0)[(None,) * (x1.dim() - 1)] 17 | x1_ = (x1 - mean).div(lengthscale) 18 | x2_ = (x2 - mean).div(lengthscale) 19 | scaled_unitless_dist = dist_func(x1_, x2_).mul_(math.sqrt(2 * nu)) 20 | if nu == 0.5: 21 | # 1 kernel sized Tensor if no grad else 2 22 | scaled_unitless_dist_ = scaled_unitless_dist.clone() if needs_grad else scaled_unitless_dist 23 | exp_component = scaled_unitless_dist_.neg_().exp_() 24 | covar_mat = exp_component 25 | if needs_grad: 26 | d_output_d_input = scaled_unitless_dist.div_(lengthscale).mul_(exp_component) 27 | elif nu == 1.5: 28 | # 2 kernel sized Tensors if no grad else 3 29 | if needs_grad: 30 | scaled_unitless_dist_ = scaled_unitless_dist.clone() 31 | linear_term = scaled_unitless_dist.clone().add_(1) 32 | exp_component = scaled_unitless_dist.neg_().exp_() 33 | covar_mat = linear_term.mul_(exp_component) 34 | if needs_grad: 35 | d_output_d_input = scaled_unitless_dist_.pow_(2).div_(lengthscale).mul_(exp_component) 36 | elif nu == 2.5: 37 | # 3 kernel sized Tensors if no grad else 4 38 | linear_term = scaled_unitless_dist.clone().add_(1) 39 | quadratic_term = scaled_unitless_dist.clone().pow_(2).div_(3) 40 | exp_component = scaled_unitless_dist.neg_().exp_() 41 | if needs_grad: 42 | covar_mat = (linear_term + quadratic_term).mul_(exp_component) 43 | d_output_d_input = linear_term.mul_(quadratic_term).mul_(exp_component).div_(lengthscale) 44 | else: 45 | covar_mat = exp_component.mul_(linear_term.add_(quadratic_term)) 46 | if needs_grad: 47 | ctx.save_for_backward(d_output_d_input) 48 | return covar_mat 49 | 50 | @staticmethod 51 | def backward(ctx, grad_output): 52 | d_output_d_input = ctx.saved_tensors[0] 53 | lengthscale_grad = grad_output * d_output_d_input 54 | return None, None, lengthscale_grad, None, None 55 | -------------------------------------------------------------------------------- /gpytorch/functions/rbf_covariance.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class RBFCovariance(torch.autograd.Function): 5 | @staticmethod 6 | def forward(ctx, x1, x2, lengthscale, sq_dist_func): 7 | if any(ctx.needs_input_grad[:2]): 8 | raise RuntimeError("RBFCovariance cannot compute gradients with " "respect to x1 and x2") 9 | if lengthscale.size(-1) > 1: 10 | raise ValueError("RBFCovariance cannot handle multiple lengthscales") 11 | needs_grad = any(ctx.needs_input_grad) 12 | x1_ = x1.div(lengthscale) 13 | x2_ = x2.div(lengthscale) 14 | unitless_sq_dist = sq_dist_func(x1_, x2_) 15 | # clone because inplace operations will mess with what's saved for backward 16 | unitless_sq_dist_ = unitless_sq_dist.clone() if needs_grad else unitless_sq_dist 17 | covar_mat = unitless_sq_dist_.div_(-2.0).exp_() 18 | if needs_grad: 19 | d_output_d_input = unitless_sq_dist.mul_(covar_mat).div_(lengthscale) 20 | ctx.save_for_backward(d_output_d_input) 21 | return covar_mat 22 | 23 | @staticmethod 24 | def backward(ctx, grad_output): 25 | d_output_d_input = ctx.saved_tensors[0] 26 | lengthscale_grad = grad_output * d_output_d_input 27 | return None, None, lengthscale_grad, None 28 | -------------------------------------------------------------------------------- /gpytorch/kernels/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from . import keops 3 | from .additive_structure_kernel import AdditiveStructureKernel 4 | from .cosine_kernel import CosineKernel 5 | from .cylindrical_kernel import CylindricalKernel 6 | from .grid_interpolation_kernel import GridInterpolationKernel 7 | from .grid_kernel import GridKernel 8 | from .index_kernel import IndexKernel 9 | from .inducing_point_kernel import InducingPointKernel 10 | from .kernel import AdditiveKernel, Kernel, ProductKernel 11 | from .lcm_kernel import LCMKernel 12 | from .linear_kernel import LinearKernel 13 | from .matern_kernel import MaternKernel 14 | from .multi_device_kernel import MultiDeviceKernel 15 | from .multitask_kernel import MultitaskKernel 16 | from .newton_girard_additive_kernel import NewtonGirardAdditiveKernel 17 | from .periodic_kernel import PeriodicKernel 18 | from .polynomial_kernel import PolynomialKernel 19 | from .polynomial_kernel_grad import PolynomialKernelGrad 20 | from .product_structure_kernel import ProductStructureKernel 21 | from .rbf_kernel import RBFKernel 22 | from .rbf_kernel_grad import RBFKernelGrad 23 | from .rq_kernel import RQKernel 24 | from .scale_kernel import ScaleKernel 25 | from .spectral_mixture_kernel import SpectralMixtureKernel 26 | 27 | __all__ = [ 28 | "keops", 29 | "Kernel", 30 | "AdditiveKernel", 31 | "AdditiveStructureKernel", 32 | "CylindricalKernel", 33 | "MultiDeviceKernel", 34 | "CosineKernel", 35 | "GridKernel", 36 | "GridInterpolationKernel", 37 | "IndexKernel", 38 | "InducingPointKernel", 39 | "LCMKernel", 40 | "LinearKernel", 41 | "MaternKernel", 42 | "MultitaskKernel", 43 | "NewtonGirardAdditiveKernel", 44 | "PeriodicKernel", 45 | "PolynomialKernel", 46 | "PolynomialKernelGrad", 47 | "ProductKernel", 48 | "ProductStructureKernel", 49 | "RBFKernel", 50 | "RBFKernelGrad", 51 | "RQKernel", 52 | "ScaleKernel", 53 | "SpectralMixtureKernel", 54 | ] 55 | -------------------------------------------------------------------------------- /gpytorch/kernels/additive_structure_kernel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from .kernel import Kernel 4 | 5 | 6 | class AdditiveStructureKernel(Kernel): 7 | r""" 8 | A Kernel decorator for kernels with additive structure. If a kernel decomposes 9 | additively, then this module will be much more computationally efficient. 10 | 11 | A kernel function `k` decomposes additively if it can be written as 12 | 13 | .. math:: 14 | 15 | \begin{equation*} 16 | k(\mathbf{x_1}, \mathbf{x_2}) = k'(x_1^{(1)}, x_2^{(1)}) + \ldots + k'(x_1^{(d)}, x_2^{(d)}) 17 | \end{equation*} 18 | 19 | for some kernel :math:`k'` that operates on a subset of dimensions. 20 | 21 | Given a `b x n x d` input, `AdditiveStructureKernel` computes `d` one-dimensional kernels 22 | (using the supplied base_kernel), and then adds the component kernels together. 23 | Unlike :class:`~gpytorch.kernels.AdditiveKernel`, `AdditiveStructureKernel` computes each 24 | of the additive terms in batch, making it very fast. 25 | 26 | Args: 27 | :attr:`base_kernel` (Kernel): 28 | The kernel to approximate with KISS-GP 29 | :attr:`num_dims` (int): 30 | The dimension of the input data. 31 | :attr:`active_dims` (tuple of ints, optional): 32 | Passed down to the `base_kernel`. 33 | """ 34 | 35 | @property 36 | def is_stationary(self) -> bool: 37 | """ 38 | Kernel is stationary if the base kernel is stationary. 39 | """ 40 | return self.base_kernel.is_stationary 41 | 42 | def __init__(self, base_kernel, num_dims, active_dims=None): 43 | super(AdditiveStructureKernel, self).__init__(active_dims=active_dims) 44 | self.base_kernel = base_kernel 45 | self.num_dims = num_dims 46 | 47 | def forward(self, x1, x2, diag=False, last_dim_is_batch=False, **params): 48 | if last_dim_is_batch: 49 | raise RuntimeError("AdditiveStructureKernel does not accept the last_dim_is_batch argument.") 50 | 51 | res = self.base_kernel(x1, x2, diag=diag, last_dim_is_batch=True, **params) 52 | res = res.sum(-2 if diag else -3) 53 | return res 54 | 55 | def prediction_strategy(self, train_inputs, train_prior_dist, train_labels, likelihood): 56 | return self.base_kernel.prediction_strategy(train_inputs, train_prior_dist, train_labels, likelihood) 57 | 58 | def num_outputs_per_input(self, x1, x2): 59 | return self.base_kernel.num_outputs_per_input(x1, x2) 60 | -------------------------------------------------------------------------------- /gpytorch/kernels/keops/__init__.py: -------------------------------------------------------------------------------- 1 | from .matern_kernel import MaternKernel 2 | from .rbf_kernel import RBFKernel 3 | 4 | __all__ = ["MaternKernel", "RBFKernel"] 5 | -------------------------------------------------------------------------------- /gpytorch/kernels/keops/keops_kernel.py: -------------------------------------------------------------------------------- 1 | from abc import abstractmethod 2 | 3 | import torch 4 | 5 | from ..kernel import Kernel 6 | 7 | try: 8 | from pykeops.torch import LazyTensor as KEOLazyTensor 9 | 10 | class KeOpsKernel(Kernel): 11 | @abstractmethod 12 | def covar_func(self, x1: torch.Tensor, x2: torch.Tensor) -> KEOLazyTensor: 13 | raise NotImplementedError("KeOpsKernels must define a covar_func method") 14 | 15 | 16 | except ImportError: 17 | 18 | class KeOpsKernel(Kernel): 19 | def __init__(self, *args, **kwargs): 20 | raise RuntimeError("You must have KeOps installed to use a KeOpsKernel") 21 | -------------------------------------------------------------------------------- /gpytorch/kernels/keops/rbf_kernel.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from ...lazy import KeOpsLazyTensor 4 | from ..rbf_kernel import postprocess_rbf 5 | from .keops_kernel import KeOpsKernel 6 | 7 | try: 8 | from pykeops.torch import LazyTensor as KEOLazyTensor 9 | 10 | class RBFKernel(KeOpsKernel): 11 | """ 12 | Implements the RBF kernel using KeOps as a driver for kernel matrix multiplies. 13 | 14 | This class can be used as a drop in replacement for gpytorch.kernels.RBFKernel in most cases, and supports 15 | the same arguments. There are currently a few limitations, for example a lack of batch mode support. However, 16 | most other features like ARD will work. 17 | """ 18 | 19 | has_lengthscale = True 20 | 21 | def covar_func(self, x1, x2, diag=False): 22 | # TODO: x1 / x2 size checks are a work around for a very minor bug in KeOps. 23 | # This bug is fixed on KeOps master, and we'll remove that part of the check 24 | # when they cut a new release. 25 | if diag or x1.size(-2) == 1 or x2.size(-2) == 1: 26 | return self.covar_dist( 27 | x1, x2, square_dist=True, diag=diag, dist_postprocess_func=postprocess_rbf, postprocess=True 28 | ) 29 | else: 30 | with torch.autograd.enable_grad(): 31 | x1_ = KEOLazyTensor(x1[..., :, None, :]) 32 | x2_ = KEOLazyTensor(x2[..., None, :, :]) 33 | 34 | K = (-((x1_ - x2_) ** 2).sum(-1) / 2).exp() 35 | 36 | return K 37 | 38 | def forward(self, x1, x2, diag=False, **params): 39 | x1_ = x1.div(self.lengthscale) 40 | x2_ = x2.div(self.lengthscale) 41 | if diag: 42 | return self.covar_func(x1_, x2_, diag=True) 43 | 44 | covar_func = lambda x1, x2, diag=False: self.covar_func(x1, x2, diag) 45 | return KeOpsLazyTensor(x1_, x2_, covar_func) 46 | 47 | 48 | except ImportError: 49 | 50 | class RBFKernel(KeOpsKernel): 51 | def __init__(self, *args, **kwargs): 52 | super().__init__() 53 | -------------------------------------------------------------------------------- /gpytorch/kernels/lcm_kernel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from copy import deepcopy 4 | 5 | from torch.nn import ModuleList 6 | 7 | from .kernel import Kernel 8 | from .multitask_kernel import MultitaskKernel 9 | 10 | 11 | class LCMKernel(Kernel): 12 | """ 13 | This kernel supports the LCM kernel. It allows the user to specify a list of 14 | base kernels to use, and individual `MultitaskKernel` objects are fit to each 15 | of them. The final kernel is the linear sum of the Kronecker product of all 16 | these base kernels with their respective `MultitaskKernel` objects. 17 | 18 | The returned object is of type :obj:`gpytorch.lazy.KroneckerProductLazyTensor`. 19 | """ 20 | 21 | def __init__(self, base_kernels, num_tasks, rank=1, task_covar_prior=None): 22 | """ 23 | Args: 24 | base_kernels (:type: list of `Kernel` objects): A list of base kernels. 25 | num_tasks (int): The number of output tasks to fit. 26 | rank (int): Rank of index kernel to use for task covariance matrix for each 27 | of the base kernels. 28 | task_covar_prior (:obj:`gpytorch.priors.Prior`): Prior to use for each 29 | task kernel. See :class:`gpytorch.kernels.IndexKernel` for details. 30 | """ 31 | if len(base_kernels) < 1: 32 | raise ValueError("At least one base kernel must be provided.") 33 | for k in base_kernels: 34 | if not isinstance(k, Kernel): 35 | raise ValueError("base_kernels must only contain Kernel objects") 36 | super(LCMKernel, self).__init__() 37 | self.covar_module_list = ModuleList( 38 | [ 39 | MultitaskKernel(base_kernel, num_tasks=num_tasks, rank=rank, task_covar_prior=task_covar_prior) 40 | for base_kernel in base_kernels 41 | ] 42 | ) 43 | 44 | def forward(self, x1, x2, **params): 45 | res = self.covar_module_list[0].forward(x1, x2, **params) 46 | for m in self.covar_module_list[1:]: 47 | res += m.forward(x1, x2, **params) 48 | return res 49 | 50 | def num_outputs_per_input(self, x1, x2): 51 | """ 52 | Given `n` data points `x1` and `m` datapoints `x2`, this multitask kernel 53 | returns an `(n*num_tasks) x (m*num_tasks)` covariance matrix. 54 | """ 55 | return self.covar_module_list[0].num_outputs_per_input(x1, x2) 56 | 57 | def __getitem__(self, index): 58 | new_kernel = deepcopy(self) 59 | new_kernel.covar_module_list = ModuleList( 60 | [base_kernel.__getitem__(index) for base_kernel in self.covar_module_list] 61 | ) 62 | return new_kernel 63 | -------------------------------------------------------------------------------- /gpytorch/kernels/multitask_kernel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from ..lazy import KroneckerProductLazyTensor, lazify 4 | from .index_kernel import IndexKernel 5 | from .kernel import Kernel 6 | 7 | 8 | class MultitaskKernel(Kernel): 9 | """ 10 | Kernel supporting Kronecker style multitask Gaussian processes (where every data point is evaluated at every 11 | task) using :class:`gpytorch.kernels.IndexKernel` as a basic multitask kernel. 12 | 13 | Given a base covariance module to be used for the data, :math:`K_{XX}`, this kernel computes a task kernel of 14 | specified size :math:`K_{TT}` and returns :math:`K = K_{TT} \otimes K_{XX}`. as an 15 | :obj:`gpytorch.lazy.KroneckerProductLazyTensor`. 16 | 17 | Args: 18 | data_covar_module (:obj:`gpytorch.kernels.Kernel`): 19 | Kernel to use as the data kernel. 20 | num_tasks (int): 21 | Number of tasks 22 | batch_size (int, optional): 23 | Set if the MultitaskKernel is operating on batches of data (and you want different 24 | parameters for each batch) 25 | rank (int): 26 | Rank of index kernel to use for task covariance matrix. 27 | task_covar_prior (:obj:`gpytorch.priors.Prior`): 28 | Prior to use for task kernel. See :class:`gpytorch.kernels.IndexKernel` for details. 29 | """ 30 | 31 | def __init__(self, data_covar_module, num_tasks, rank=1, task_covar_prior=None, **kwargs): 32 | """ 33 | """ 34 | super(MultitaskKernel, self).__init__(**kwargs) 35 | self.task_covar_module = IndexKernel( 36 | num_tasks=num_tasks, batch_shape=self.batch_shape, rank=rank, prior=task_covar_prior 37 | ) 38 | self.data_covar_module = data_covar_module 39 | self.num_tasks = num_tasks 40 | 41 | def forward(self, x1, x2, diag=False, last_dim_is_batch=False, **params): 42 | if last_dim_is_batch: 43 | raise RuntimeError("MultitaskKernel does not accept the last_dim_is_batch argument.") 44 | covar_i = self.task_covar_module.covar_matrix 45 | if len(x1.shape[:-2]): 46 | covar_i = covar_i.repeat(*x1.shape[:-2], 1, 1) 47 | covar_x = lazify(self.data_covar_module.forward(x1, x2, **params)) 48 | res = KroneckerProductLazyTensor(covar_x, covar_i) 49 | return res.diag() if diag else res 50 | 51 | def num_outputs_per_input(self, x1, x2): 52 | """ 53 | Given `n` data points `x1` and `m` datapoints `x2`, this multitask 54 | kernel returns an `(n*num_tasks) x (m*num_tasks)` covariancn matrix. 55 | """ 56 | return self.num_tasks 57 | -------------------------------------------------------------------------------- /gpytorch/lazy/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from .added_diag_lazy_tensor import AddedDiagLazyTensor 4 | from .batch_repeat_lazy_tensor import BatchRepeatLazyTensor 5 | from .block_diag_lazy_tensor import BlockDiagLazyTensor 6 | from .block_interleaved_lazy_tensor import BlockInterleavedLazyTensor 7 | from .block_lazy_tensor import BlockLazyTensor 8 | from .cached_cg_lazy_tensor import CachedCGLazyTensor, ExtraComputationWarning 9 | from .cat_lazy_tensor import CatLazyTensor, cat 10 | from .chol_lazy_tensor import CholLazyTensor 11 | from .constant_mul_lazy_tensor import ConstantMulLazyTensor 12 | from .diag_lazy_tensor import DiagLazyTensor 13 | from .interpolated_lazy_tensor import InterpolatedLazyTensor 14 | from .keops_lazy_tensor import KeOpsLazyTensor 15 | from .kronecker_product_lazy_tensor import KroneckerProductLazyTensor 16 | from .lazy_evaluated_kernel_tensor import LazyEvaluatedKernelTensor 17 | from .lazy_tensor import LazyTensor, delazify 18 | from .matmul_lazy_tensor import MatmulLazyTensor 19 | from .mul_lazy_tensor import MulLazyTensor 20 | from .non_lazy_tensor import NonLazyTensor, lazify 21 | from .psd_sum_lazy_tensor import PsdSumLazyTensor 22 | from .root_lazy_tensor import RootLazyTensor 23 | from .sum_batch_lazy_tensor import SumBatchLazyTensor 24 | from .sum_lazy_tensor import SumLazyTensor 25 | from .toeplitz_lazy_tensor import ToeplitzLazyTensor 26 | from .zero_lazy_tensor import ZeroLazyTensor 27 | 28 | __all__ = [ 29 | "delazify", 30 | "lazify", 31 | "cat", 32 | "LazyTensor", 33 | "LazyEvaluatedKernelTensor", 34 | "AddedDiagLazyTensor", 35 | "BatchRepeatLazyTensor", 36 | "BlockLazyTensor", 37 | "BlockDiagLazyTensor", 38 | "BlockInterleavedLazyTensor", 39 | "CachedCGLazyTensor", 40 | "CatLazyTensor", 41 | "CholLazyTensor", 42 | "ConstantMulLazyTensor", 43 | "DiagLazyTensor", 44 | "ExtraComputationWarning", 45 | "InterpolatedLazyTensor", 46 | "KeOpsLazyTensor", 47 | "KroneckerProductLazyTensor", 48 | "MatmulLazyTensor", 49 | "MulLazyTensor", 50 | "NonLazyTensor", 51 | "PsdSumLazyTensor", 52 | "RootLazyTensor", 53 | "SumLazyTensor", 54 | "SumBatchLazyTensor", 55 | "ToeplitzLazyTensor", 56 | "ZeroLazyTensor", 57 | ] 58 | -------------------------------------------------------------------------------- /gpytorch/lazy/lazy_tensor_representation_tree.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | 4 | class LazyTensorRepresentationTree(object): 5 | def __init__(self, lazy_tsr): 6 | self._cls = lazy_tsr.__class__ 7 | self._kwargs = lazy_tsr._kwargs 8 | 9 | counter = 0 10 | self.children = [] 11 | for arg in lazy_tsr._args: 12 | if hasattr(arg, "representation") and callable(arg.representation): # Is it a lazy tensor? 13 | representation_size = len(arg.representation()) 14 | self.children.append((slice(counter, counter + representation_size, None), arg.representation_tree())) 15 | counter += representation_size 16 | else: 17 | self.children.append((counter, None)) 18 | counter += 1 19 | 20 | def __call__(self, *flattened_representation): 21 | unflattened_representation = [] 22 | 23 | for index, subtree in self.children: 24 | if subtree is None: 25 | unflattened_representation.append(flattened_representation[index]) 26 | else: 27 | sub_representation = flattened_representation[index] 28 | unflattened_representation.append(subtree(*sub_representation)) 29 | 30 | return self._cls(*unflattened_representation, **self._kwargs) 31 | -------------------------------------------------------------------------------- /gpytorch/lazy/psd_sum_lazy_tensor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from .sum_lazy_tensor import SumLazyTensor 4 | 5 | 6 | class PsdSumLazyTensor(SumLazyTensor): 7 | """ 8 | A SumLazyTensor, but where every component of the sum is positive semi-definite 9 | """ 10 | 11 | def zero_mean_mvn_samples(self, num_samples): 12 | return sum(lazy_tensor.zero_mean_mvn_samples(num_samples) for lazy_tensor in self.lazy_tensors) 13 | -------------------------------------------------------------------------------- /gpytorch/lazy/sum_batch_lazy_tensor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import torch 4 | 5 | from ..utils.broadcasting import _pad_with_singletons 6 | from ..utils.getitem import _noop_index 7 | from .block_lazy_tensor import BlockLazyTensor 8 | 9 | 10 | class SumBatchLazyTensor(BlockLazyTensor): 11 | """ 12 | Represents a lazy tensor that is actually the sum of several lazy tensors blocks. 13 | The :attr:`block_dim` attribute specifies which dimension of the base LazyTensor 14 | specifies the blocks. 15 | For example, (with `block_dim=-3` a `k x n x n` tensor represents `k` `n x n` blocks (a `n x n` matrix). 16 | A `b x k x n x n` tensor represents `k` `b x n x n` blocks (a `b x n x n` batch matrix). 17 | 18 | Args: 19 | :attr:`base_lazy_tensor` (LazyTensor): 20 | A `k x n x n` LazyTensor, or a `b x k x n x n` LazyTensor. 21 | :attr:`block_dim` (int): 22 | The dimension that specifies the blocks. 23 | """ 24 | 25 | def _add_batch_dim(self, other): 26 | shape = list(other.shape) 27 | expand_shape = list(other.shape) 28 | shape.insert(-2, 1) 29 | expand_shape.insert(-2, self.base_lazy_tensor.size(-3)) 30 | other = other.reshape(*shape).expand(*expand_shape) 31 | return other 32 | 33 | def _get_indices(self, row_index, col_index, *batch_indices): 34 | # Create an extra index for the summed dimension 35 | sum_index = torch.arange(0, self.base_lazy_tensor.size(-3), device=self.device) 36 | sum_index = _pad_with_singletons(sum_index, row_index.dim(), 0) 37 | row_index = row_index.unsqueeze(-1) 38 | col_index = col_index.unsqueeze(-1) 39 | batch_indices = [index.unsqueeze(-1) for index in batch_indices] 40 | 41 | res = self.base_lazy_tensor._get_indices(row_index, col_index, *batch_indices, sum_index) 42 | return res.sum(-1) 43 | 44 | def _getitem(self, row_index, col_index, *batch_indices): 45 | res = self.base_lazy_tensor._getitem(row_index, col_index, *batch_indices, _noop_index) 46 | return self.__class__(res, **self._kwargs) 47 | 48 | def _remove_batch_dim(self, other): 49 | return other.sum(-3) 50 | 51 | def _size(self): 52 | shape = list(self.base_lazy_tensor.shape) 53 | del shape[-3] 54 | return torch.Size(shape) 55 | 56 | def diag(self): 57 | diag = self.base_lazy_tensor.diag().sum(-2) 58 | return diag 59 | 60 | def evaluate(self): 61 | return self.base_lazy_tensor.evaluate().sum(dim=-3) # BlockLazyTensors always use dim3 for the block_dim 62 | -------------------------------------------------------------------------------- /gpytorch/lazy/toeplitz_lazy_tensor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import torch 4 | 5 | from ..utils.toeplitz import sym_toeplitz_derivative_quadratic_form, sym_toeplitz_matmul 6 | from .lazy_tensor import LazyTensor 7 | 8 | 9 | class ToeplitzLazyTensor(LazyTensor): 10 | def __init__(self, column): 11 | """ 12 | Args: 13 | :attr: `column` (Tensor) 14 | If `column` is a 1D Tensor of length `n`, this represents a 15 | Toeplitz matrix with `column` as its first column. 16 | If `column` is `b_1 x b_2 x ... x b_k x n`, then this represents a batch 17 | `b_1 x b_2 x ... x b_k` of Toeplitz matrices. 18 | """ 19 | super(ToeplitzLazyTensor, self).__init__(column) 20 | self.column = column 21 | 22 | def _expand_batch(self, batch_shape): 23 | return self.__class__(self.column.expand(*batch_shape, self.column.size(-1))) 24 | 25 | def _get_indices(self, row_index, col_index, *batch_indices): 26 | toeplitz_indices = (row_index - col_index).fmod(self.size(-1)).abs().long() 27 | return self.column[(*batch_indices, toeplitz_indices)] 28 | 29 | def _matmul(self, rhs): 30 | return sym_toeplitz_matmul(self.column, rhs) 31 | 32 | def _t_matmul(self, rhs): 33 | # Matrix is symmetric 34 | return self._matmul(rhs) 35 | 36 | def _quad_form_derivative(self, left_vecs, right_vecs): 37 | if left_vecs.ndimension() == 1: 38 | left_vecs = left_vecs.unsqueeze(1) 39 | right_vecs = right_vecs.unsqueeze(1) 40 | 41 | res = sym_toeplitz_derivative_quadratic_form(left_vecs, right_vecs) 42 | 43 | # Collapse any expanded broadcast dimensions 44 | if res.dim() > self.column.dim(): 45 | res = res.view(-1, *self.column.shape).sum(0) 46 | 47 | return (res,) 48 | 49 | def _size(self): 50 | return torch.Size((*self.column.shape, self.column.size(-1))) 51 | 52 | def _transpose_nonbatch(self): 53 | return ToeplitzLazyTensor(self.column) 54 | 55 | def add_jitter(self, jitter_val=1e-3): 56 | jitter = torch.zeros_like(self.column) 57 | jitter.narrow(-1, 0, 1).fill_(jitter_val) 58 | return ToeplitzLazyTensor(self.column.add(jitter)) 59 | 60 | def diag(self): 61 | """ 62 | Gets the diagonal of the Toeplitz matrix wrapped by this object. 63 | """ 64 | diag_term = self.column[..., 0] 65 | if self.column.ndimension() > 1: 66 | diag_term = diag_term.unsqueeze(-1) 67 | return diag_term.expand(*self.column.size()) 68 | -------------------------------------------------------------------------------- /gpytorch/likelihoods/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from .bernoulli_likelihood import BernoulliLikelihood 4 | from .gaussian_likelihood import FixedNoiseGaussianLikelihood, GaussianLikelihood, _GaussianLikelihoodBase 5 | from .likelihood import Likelihood, _OneDimensionalLikelihood 6 | from .likelihood_list import LikelihoodList 7 | from .multitask_gaussian_likelihood import ( 8 | MultitaskGaussianLikelihood, 9 | MultitaskGaussianLikelihoodKronecker, 10 | _MultitaskGaussianLikelihoodBase, 11 | ) 12 | from .noise_models import HeteroskedasticNoise 13 | from .softmax_likelihood import SoftmaxLikelihood 14 | 15 | __all__ = [ 16 | "_GaussianLikelihoodBase", 17 | "_OneDimensionalLikelihood", 18 | "_MultitaskGaussianLikelihoodBase", 19 | "BernoulliLikelihood", 20 | "FixedNoiseGaussianLikelihood", 21 | "GaussianLikelihood", 22 | "HeteroskedasticNoise", 23 | "Likelihood", 24 | "LikelihoodList", 25 | "MultitaskGaussianLikelihood", 26 | "MultitaskGaussianLikelihoodKronecker", 27 | "SoftmaxLikelihood", 28 | ] 29 | -------------------------------------------------------------------------------- /gpytorch/likelihoods/bernoulli_likelihood.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import warnings 4 | 5 | import torch 6 | 7 | from ..distributions import base_distributions 8 | from ..functions import log_normal_cdf 9 | from .likelihood import _OneDimensionalLikelihood 10 | 11 | 12 | class BernoulliLikelihood(_OneDimensionalLikelihood): 13 | r""" 14 | Implements the Bernoulli likelihood used for GP classification, using 15 | Probit regression (i.e., the latent function is warped to be in [0,1] 16 | using the standard Normal CDF \Phi(x)). Given the identity \Phi(-x) = 17 | 1-\Phi(x), we can write the likelihood compactly as: 18 | 19 | .. math:: 20 | \begin{equation*} 21 | p(Y=y|f)=\Phi(yf) 22 | \end{equation*} 23 | """ 24 | 25 | def forward(self, function_samples, **kwargs): 26 | output_probs = base_distributions.Normal(0, 1).cdf(function_samples) 27 | return base_distributions.Bernoulli(probs=output_probs) 28 | 29 | def log_marginal(self, observations, function_dist, *args, **kwargs): 30 | marginal = self.marginal(function_dist, *args, **kwargs) 31 | return marginal.log_prob(observations) 32 | 33 | def marginal(self, function_dist, **kwargs): 34 | mean = function_dist.mean 35 | var = function_dist.variance 36 | link = mean.div(torch.sqrt(1 + var)) 37 | output_probs = base_distributions.Normal(0, 1).cdf(link) 38 | return base_distributions.Bernoulli(probs=output_probs) 39 | 40 | def expected_log_prob(self, observations, function_dist, *params, **kwargs): 41 | if torch.any(observations.eq(-1)): 42 | # Remove after 1.0 43 | warnings.warn( 44 | "BernoulliLikelihood.expected_log_prob expects observations with labels in {0, 1}. " 45 | "Observations with labels in {-1, 1} are deprecated.", 46 | DeprecationWarning, 47 | ) 48 | else: 49 | observations = observations.mul(2).sub(1) 50 | # Custom function here so we can use log_normal_cdf rather than Normal.cdf 51 | # This is going to be less prone to overflow errors 52 | log_prob_lambda = lambda function_samples: log_normal_cdf(function_samples.mul(observations)) 53 | log_prob = self.quadrature(log_prob_lambda, function_dist) 54 | return log_prob 55 | -------------------------------------------------------------------------------- /gpytorch/likelihoods/likelihood_list.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | from torch.nn import ModuleList 4 | 5 | from gpytorch.likelihoods import Likelihood 6 | 7 | 8 | def _get_tuple_args_(*args): 9 | for arg in args: 10 | if isinstance(arg, tuple): 11 | yield arg 12 | else: 13 | yield (arg,) 14 | 15 | 16 | class LikelihoodList(Likelihood): 17 | def __init__(self, *likelihoods): 18 | super().__init__() 19 | self.likelihoods = ModuleList(likelihoods) 20 | 21 | def expected_log_prob(self, *args, **kwargs): 22 | return [ 23 | likelihood.expected_log_prob(*args_, **kwargs) 24 | for likelihood, args_ in zip(self.likelihoods, _get_tuple_args_(*args)) 25 | ] 26 | 27 | def forward(self, *args, **kwargs): 28 | if "noise" in kwargs: 29 | noise = kwargs.pop("noise") 30 | # if noise kwarg is passed, assume it's an iterable of noise tensors 31 | return [ 32 | likelihood.forward(*args_, {**kwargs, "noise": noise_}) 33 | for likelihood, args_, noise_ in zip(self.likelihoods, _get_tuple_args_(*args), noise) 34 | ] 35 | else: 36 | return [ 37 | likelihood.forward(*args_, **kwargs) 38 | for likelihood, args_ in zip(self.likelihoods, _get_tuple_args_(*args)) 39 | ] 40 | 41 | def pyro_sample_output(self, *args, **kwargs): 42 | return [ 43 | likelihood.pyro_sample_output(*args_, **kwargs) 44 | for likelihood, args_ in zip(self.likelihoods, _get_tuple_args_(*args)) 45 | ] 46 | 47 | def __call__(self, *args, **kwargs): 48 | if "noise" in kwargs: 49 | noise = kwargs.pop("noise") 50 | # if noise kwarg is passed, assume it's an iterable of noise tensors 51 | return [ 52 | likelihood(*args_, {**kwargs, "noise": noise_}) 53 | for likelihood, args_, noise_ in zip(self.likelihoods, _get_tuple_args_(*args), noise) 54 | ] 55 | else: 56 | return [ 57 | likelihood(*args_, **kwargs) for likelihood, args_ in zip(self.likelihoods, _get_tuple_args_(*args)) 58 | ] 59 | -------------------------------------------------------------------------------- /gpytorch/means/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from .constant_mean import ConstantMean 4 | from .constant_mean_grad import ConstantMeanGrad 5 | from .linear_mean import LinearMean 6 | from .mean import Mean 7 | from .multitask_mean import MultitaskMean 8 | from .zero_mean import ZeroMean 9 | 10 | __all__ = ["Mean", "ConstantMean", "ConstantMeanGrad", "LinearMean", "MultitaskMean", "ZeroMean"] 11 | -------------------------------------------------------------------------------- /gpytorch/means/constant_mean.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import torch 4 | 5 | from ..utils.broadcasting import _mul_broadcast_shape 6 | from .mean import Mean 7 | 8 | 9 | class ConstantMean(Mean): 10 | def __init__(self, prior=None, batch_shape=torch.Size(), **kwargs): 11 | super(ConstantMean, self).__init__() 12 | self.batch_shape = batch_shape 13 | self.register_parameter(name="constant", parameter=torch.nn.Parameter(torch.zeros(*batch_shape, 1))) 14 | if prior is not None: 15 | self.register_prior("mean_prior", prior, "constant") 16 | 17 | def forward(self, input): 18 | if input.shape[:-2] == self.batch_shape: 19 | return self.constant.expand(input.shape[:-1]) 20 | else: 21 | return self.constant.expand(_mul_broadcast_shape(input.shape[:-1], self.constant.shape)) 22 | -------------------------------------------------------------------------------- /gpytorch/means/constant_mean_grad.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import torch 4 | 5 | from ..utils.broadcasting import _mul_broadcast_shape 6 | from .mean import Mean 7 | 8 | 9 | class ConstantMeanGrad(Mean): 10 | def __init__(self, prior=None, batch_shape=torch.Size(), **kwargs): 11 | super(ConstantMeanGrad, self).__init__() 12 | self.batch_shape = batch_shape 13 | self.register_parameter(name="constant", parameter=torch.nn.Parameter(torch.zeros(*batch_shape, 1))) 14 | if prior is not None: 15 | self.register_prior("mean_prior", prior, "constant") 16 | 17 | def forward(self, input): 18 | batch_shape = _mul_broadcast_shape(self.batch_shape, input.shape[:-2]) 19 | mean = self.constant.unsqueeze(-1).expand(*batch_shape, input.size(-2), input.size(-1) + 1).contiguous() 20 | mean[..., 1:] = 0 21 | return mean 22 | -------------------------------------------------------------------------------- /gpytorch/means/linear_mean.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import torch 4 | 5 | from .mean import Mean 6 | 7 | 8 | class LinearMean(Mean): 9 | def __init__(self, input_size, batch_shape=torch.Size(), bias=True): 10 | super().__init__() 11 | self.register_parameter(name="weights", parameter=torch.nn.Parameter(torch.randn(*batch_shape, input_size, 1))) 12 | if bias: 13 | self.register_parameter(name="bias", parameter=torch.nn.Parameter(torch.randn(*batch_shape, 1))) 14 | else: 15 | self.bias = None 16 | 17 | def forward(self, x): 18 | res = x.matmul(self.weights).squeeze(-1) 19 | if self.bias is not None: 20 | res = res + self.bias 21 | return res 22 | -------------------------------------------------------------------------------- /gpytorch/means/mean.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from ..module import Module 4 | 5 | 6 | class Mean(Module): 7 | """ 8 | Mean function. 9 | """ 10 | 11 | def __init__(self): 12 | super().__init__() 13 | 14 | def forward(self, x): 15 | raise NotImplementedError() 16 | 17 | def __call__(self, x): 18 | # Add a last dimension 19 | if x.ndimension() == 1: 20 | x = x.unsqueeze(1) 21 | 22 | res = super(Mean, self).__call__(x) 23 | 24 | return res 25 | -------------------------------------------------------------------------------- /gpytorch/means/multitask_mean.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from copy import deepcopy 4 | 5 | import torch 6 | from torch.nn import ModuleList 7 | 8 | from .mean import Mean 9 | 10 | 11 | class MultitaskMean(Mean): 12 | """ 13 | Convenience :class:`gpytorch.means.Mean` implementation for defining a different mean for each task in a multitask 14 | model. Expects a list of `num_tasks` different mean functions, each of which is applied to the given data in 15 | :func:`~gpytorch.means.MultitaskMean.forward` and returned as an `n x t` matrix of means, one for each task. 16 | """ 17 | 18 | def __init__(self, base_means, num_tasks): 19 | """ 20 | Args: 21 | base_means (:obj:`list` or :obj:`gpytorch.means.Mean`): If a list, each mean is applied to the data. 22 | If a single mean (or a list containing a single mean), that mean is copied `t` times. 23 | num_tasks (int): Number of tasks. If base_means is a list, this should equal its length. 24 | """ 25 | super(MultitaskMean, self).__init__() 26 | 27 | if isinstance(base_means, Mean): 28 | base_means = [base_means] 29 | 30 | if not isinstance(base_means, list) or (len(base_means) != 1 and len(base_means) != num_tasks): 31 | raise RuntimeError("base_means should be a list of means of length either 1 or num_tasks") 32 | 33 | if len(base_means) == 1: 34 | base_means = base_means + [deepcopy(base_means[0]) for i in range(num_tasks - 1)] 35 | 36 | self.base_means = ModuleList(base_means) 37 | self.num_tasks = num_tasks 38 | 39 | def forward(self, input): 40 | """ 41 | Evaluate each mean in self.base_means on the input data, and return as an `n x t` matrix of means. 42 | """ 43 | return torch.cat([sub_mean(input).unsqueeze(-1) for sub_mean in self.base_means], dim=-1) 44 | -------------------------------------------------------------------------------- /gpytorch/means/zero_mean.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import torch 4 | 5 | from .mean import Mean 6 | 7 | 8 | class ZeroMean(Mean): 9 | def forward(self, input): 10 | return torch.zeros(input.shape[:-1], dtype=input.dtype, device=input.device) 11 | -------------------------------------------------------------------------------- /gpytorch/mlls/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import warnings 4 | 5 | from .added_loss_term import AddedLossTerm 6 | from .deep_approximate_mll import DeepApproximateMLL 7 | from .exact_marginal_log_likelihood import ExactMarginalLogLikelihood 8 | from .gamma_robust_variational_elbo import GammaRobustVariationalELBO 9 | from .inducing_point_kernel_added_loss_term import InducingPointKernelAddedLossTerm 10 | from .marginal_log_likelihood import MarginalLogLikelihood 11 | from .noise_model_added_loss_term import NoiseModelAddedLossTerm 12 | from .predictive_log_likelihood import PredictiveLogLikelihood 13 | from .sum_marginal_log_likelihood import SumMarginalLogLikelihood 14 | from .variational_elbo import VariationalELBO 15 | 16 | 17 | # Deprecated for 0.4 release 18 | class VariationalMarginalLogLikelihood(VariationalELBO): 19 | def __init__(self, *args, **kwargs): 20 | # Remove after 1.0 21 | warnings.warn( 22 | "VariationalMarginalLogLikelihood is deprecated. Please use VariationalELBO instead.", DeprecationWarning 23 | ) 24 | super().__init__(*args, **kwargs) 25 | 26 | 27 | class VariationalELBOEmpirical(VariationalELBO): 28 | def __init__(self, *args, **kwargs): 29 | # Remove after 1.0 30 | warnings.warn("VariationalELBOEmpirical is deprecated. Please use VariationalELBO instead.", DeprecationWarning) 31 | super().__init__(*args, **kwargs) 32 | 33 | 34 | __all__ = [ 35 | "AddedLossTerm", 36 | "DeepApproximateMLL", 37 | "ExactMarginalLogLikelihood", 38 | "InducingPointKernelAddedLossTerm", 39 | "MarginalLogLikelihood", 40 | "NoiseModelAddedLossTerm", 41 | "PredictiveLogLikelihood", 42 | "GammaRobustVariationalELBO", 43 | "SumMarginalLogLikelihood", 44 | "VariationalELBO", 45 | ] 46 | -------------------------------------------------------------------------------- /gpytorch/mlls/added_loss_term.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | 4 | class AddedLossTerm(object): 5 | def loss(self): 6 | raise NotImplementedError 7 | -------------------------------------------------------------------------------- /gpytorch/mlls/deep_approximate_mll.py: -------------------------------------------------------------------------------- 1 | from ._approximate_mll import _ApproximateMarginalLogLikelihood 2 | 3 | 4 | class DeepApproximateMLL(_ApproximateMarginalLogLikelihood): 5 | """ 6 | A wrapper to make a GPyTorch approximate marginal log likelihoods compatible with Deep GPs. 7 | 8 | Example: 9 | >>> deep_mll = gpytorch.mlls.DeepApproximateMLL( 10 | >>> gpytorch.mlls.VariationalELBO(likelihood, model, num_data=1000) 11 | >>> ) 12 | 13 | :param ~gpytorch.mlls._ApproximateMarginalLogLikelihood base_mll: The base 14 | approximate MLL 15 | """ 16 | 17 | def __init__(self, base_mll): 18 | if not base_mll.combine_terms: 19 | raise ValueError( 20 | "The base marginal log likelihood object should combine terms " 21 | "when used in conjunction with a DeepApproximateMLL." 22 | ) 23 | super().__init__(base_mll.likelihood, base_mll.model, num_data=base_mll.num_data, beta=base_mll.beta) 24 | self.base_mll = base_mll 25 | 26 | def _log_likelihood_term(self, approximate_dist_f, target, **kwargs): 27 | return self.base_mll._log_likelihood_term(approximate_dist_f, target, **kwargs).mean(0) 28 | 29 | def forward(self, approximate_dist_f, target, **kwargs): 30 | return self.base_mll.forward(approximate_dist_f, target, **kwargs).mean(0) 31 | -------------------------------------------------------------------------------- /gpytorch/mlls/inducing_point_kernel_added_loss_term.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from .added_loss_term import AddedLossTerm 4 | 5 | 6 | class InducingPointKernelAddedLossTerm(AddedLossTerm): 7 | def __init__(self, variational_dist, prior_dist, likelihood): 8 | self.prior_dist = prior_dist 9 | self.variational_dist = variational_dist 10 | self.likelihood = likelihood 11 | 12 | def loss(self, *params): 13 | prior_covar = self.prior_dist.lazy_covariance_matrix 14 | variational_covar = self.variational_dist.lazy_covariance_matrix 15 | diag = prior_covar.diag() - variational_covar.diag() 16 | shape = prior_covar.shape[:-1] 17 | noise_diag = self.likelihood._shaped_noise_covar(shape, *params).diag() 18 | return 0.5 * (diag / noise_diag).sum() 19 | -------------------------------------------------------------------------------- /gpytorch/mlls/marginal_log_likelihood.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from ..models import GP 4 | from ..module import Module 5 | 6 | 7 | class MarginalLogLikelihood(Module): 8 | r""" 9 | These are modules to compute (or approximate/bound) the marginal log likelihood 10 | (MLL) of the GP model when applied to data. I.e., given a GP :math:`f \sim 11 | \mathcal{GP}(\mu, K)`, and data :math:`\mathbf X, \mathbf y`, these modules 12 | compute/approximate 13 | 14 | .. math:: 15 | 16 | \begin{equation*} 17 | \mathcal{L} = p_f(\mathbf y \! \mid \! \mathbf X) 18 | = \int p \left( \mathbf y \! \mid \! f(\mathbf X) \right) \: p(f(\mathbf X) \! \mid \! \mathbf X) \: d f 19 | \end{equation*} 20 | 21 | This is computed exactly when the GP inference is computed exactly (e.g. regression w/ a Gaussian likelihood). 22 | It is approximated/bounded for GP models that use approximate inference. 23 | 24 | These models are typically used as the "loss" functions for GP models (though note that the output of 25 | these functions must be negated for optimization). 26 | """ 27 | 28 | def __init__(self, likelihood, model): 29 | super(MarginalLogLikelihood, self).__init__() 30 | if not isinstance(model, GP): 31 | raise RuntimeError( 32 | "All MarginalLogLikelihood objects must be given a GP object as a model. If you are " 33 | "using a more complicated model involving a GP, pass the underlying GP object as the " 34 | "model, not a full PyTorch module." 35 | ) 36 | self.likelihood = likelihood 37 | self.model = model 38 | 39 | def forward(self, output, target, **kwargs): 40 | """ 41 | Computes the MLL given :math:`p(\mathbf f)` and `\mathbf y` 42 | 43 | Args: 44 | :attr:`output` (:obj:`gpytorch.distributions.MultivariateNormal`): 45 | :math:`p(\mathbf f)` (or approximation) 46 | the outputs of the latent function (the :obj:`gpytorch.models.GP`) 47 | :attr:`target` (`torch.Tensor`): 48 | :math:`\mathbf y` The target values 49 | :attr:`**kwargs`: 50 | Additional arguments to pass to the likelihood's :attr:`forward` function. 51 | """ 52 | raise NotImplementedError 53 | 54 | def pyro_factor(self, output, target): 55 | """ 56 | As forward, but register the MLL with pyro using the pyro.factor primitive. 57 | """ 58 | raise NotImplementedError 59 | -------------------------------------------------------------------------------- /gpytorch/mlls/noise_model_added_loss_term.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from .added_loss_term import AddedLossTerm 4 | 5 | 6 | class NoiseModelAddedLossTerm(AddedLossTerm): 7 | def __init__(self, noise_model): 8 | from .exact_marginal_log_likelihood import ExactMarginalLogLikelihood 9 | 10 | self.noise_mll = ExactMarginalLogLikelihood(noise_model.likelihood, noise_model) 11 | 12 | def loss(self, *params): 13 | output = self.noise_mll.model(*params) 14 | targets = self.noise_mll.model.train_targets 15 | return self.noise_mll(output, targets) 16 | -------------------------------------------------------------------------------- /gpytorch/mlls/sum_marginal_log_likelihood.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | from torch.nn import ModuleList 4 | 5 | from gpytorch.mlls import ExactMarginalLogLikelihood, MarginalLogLikelihood 6 | 7 | 8 | class SumMarginalLogLikelihood(MarginalLogLikelihood): 9 | """Sum of marginal log likelihoods, to be used with Multi-Output models. 10 | 11 | Args: 12 | likelihood: A MultiOutputLikelihood 13 | model: A MultiOutputModel 14 | mll_cls: The Marginal Log Likelihood class (default: ExactMarginalLogLikelihood) 15 | 16 | In case the model outputs are independent, this provives the MLL of the multi-output model. 17 | 18 | """ 19 | 20 | def __init__(self, likelihood, model, mll_cls=ExactMarginalLogLikelihood): 21 | super().__init__(model.likelihood, model) 22 | self.mlls = ModuleList([mll_cls(mdl.likelihood, mdl) for mdl in model.models]) 23 | 24 | def forward(self, outputs, targets, *params): 25 | """ 26 | Args: 27 | outputs: (Iterable[MultivariateNormal]) - the outputs of the latent function 28 | targets: (Iterable[Tensor]) - the target values 29 | params: (Iterable[Iterable[Tensor]]) - the arguments to be passed through 30 | (e.g. parameters in case of heteroskedastic likelihoods) 31 | """ 32 | if len(params) == 0: 33 | sum_mll = sum(mll(output, target) for mll, output, target in zip(self.mlls, outputs, targets)) 34 | else: 35 | sum_mll = sum( 36 | mll(output, target, *iparams) 37 | for mll, output, target, iparams in zip(self.mlls, outputs, targets, params) 38 | ) 39 | return sum_mll.div_(len(self.mlls)) 40 | -------------------------------------------------------------------------------- /gpytorch/models/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import warnings 4 | 5 | from . import deep_gps, pyro 6 | from .approximate_gp import ApproximateGP 7 | from .exact_gp import ExactGP 8 | from .gp import GP 9 | from .model_list import AbstractModelList, IndependentModelList 10 | from .pyro import PyroGP 11 | 12 | # Alternative name for ApproximateGP 13 | VariationalGP = ApproximateGP 14 | 15 | 16 | # Deprecated for 0.4 release 17 | class AbstractVariationalGP(ApproximateGP): 18 | # Remove after 1.0 19 | def __init__(self, *args, **kwargs): 20 | warnings.warn("AbstractVariationalGP has been renamed to ApproximateGP.", DeprecationWarning) 21 | super().__init__(*args, **kwargs) 22 | 23 | 24 | # Deprecated for 0.4 release 25 | class PyroVariationalGP(ApproximateGP): 26 | # Remove after 1.0 27 | def __init__(self, *args, **kwargs): 28 | warnings.warn("PyroVariationalGP has been renamed to PyroGP.", DeprecationWarning) 29 | super().__init__(*args, **kwargs) 30 | 31 | 32 | __all__ = [ 33 | "AbstractModelList", 34 | "ApproximateGP", 35 | "ExactGP", 36 | "GP", 37 | "IndependentModelList", 38 | "PyroGP", 39 | "VariationalGP", 40 | "deep_gps", 41 | "pyro", 42 | ] 43 | -------------------------------------------------------------------------------- /gpytorch/models/deep_gps/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import warnings 4 | 5 | from .deep_gp import DeepGP, DeepGPLayer, DeepLikelihood 6 | 7 | 8 | # Deprecated for 1.0 release 9 | class AbstractDeepGP(DeepGP): 10 | def __init__(self, *args, **kwargs): 11 | warnings.warn("AbstractDeepGP has been renamed to DeepGP.", DeprecationWarning) 12 | super().__init__(*args, **kwargs) 13 | 14 | 15 | # Deprecated for 1.0 release 16 | class AbstractDeepGPLayer(DeepGPLayer): 17 | def __init__(self, *args, **kwargs): 18 | warnings.warn("AbstractDeepGPLayer has been renamed to DeepGPLayer.", DeprecationWarning) 19 | super().__init__(*args, **kwargs) 20 | 21 | 22 | __all__ = ["DeepGPLayer", "DeepGP", "AbstractDeepGPLayer", "AbstractDeepGP", "DeepLikelihood"] 23 | -------------------------------------------------------------------------------- /gpytorch/models/gp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from ..module import Module 4 | 5 | 6 | class GP(Module): 7 | pass 8 | -------------------------------------------------------------------------------- /gpytorch/models/pyro/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | try: 4 | from .pyro_gp import PyroGP 5 | from ._pyro_mixin import _PyroMixin 6 | except ImportError: 7 | 8 | class PyroGP(object): 9 | def __init__(self, *args, **kwargs): 10 | raise RuntimeError("Cannot use a PyroGP because you dont have Pyro installed.") 11 | 12 | class _PyroMixin(object): 13 | def pyro_factors(self, *args, **kwargs): 14 | raise RuntimeError("Cannot call `pyro_factors` because you dont have Pyro installed.") 15 | 16 | def pyro_guide(self, *args, **kwargs): 17 | raise RuntimeError("Cannot call `pyro_sample` because you dont have Pyro installed.") 18 | 19 | def pyro_model(self, *args, **kwargs): 20 | raise RuntimeError("Cannot call `pyro_sample` because you dont have Pyro installed.") 21 | 22 | 23 | __all__ = ["PyroGP", "_PyroMixin"] 24 | -------------------------------------------------------------------------------- /gpytorch/models/pyro/_pyro_mixin.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pyro 4 | import torch 5 | 6 | 7 | class _PyroMixin(object): 8 | def pyro_guide(self, input, beta=1.0, name_prefix=""): 9 | # Inducing values q(u) 10 | with pyro.poutine.scale(scale=beta): 11 | pyro.sample(name_prefix + ".u", self.variational_strategy.variational_distribution) 12 | 13 | # Draw samples from q(f) 14 | function_dist = self(input, prior=False) 15 | function_dist = pyro.distributions.Normal(loc=function_dist.mean, scale=function_dist.stddev).to_event( 16 | len(function_dist.event_shape) - 1 17 | ) 18 | return function_dist.mask(False) 19 | 20 | def pyro_model(self, input, beta=1.0, name_prefix=""): 21 | # Inducing values p(u) 22 | with pyro.poutine.scale(scale=beta): 23 | u_samples = pyro.sample(self.name_prefix + ".u", self.variational_strategy.prior_distribution) 24 | 25 | # Include term for GPyTorch priors 26 | log_prior = torch.tensor(0.0, dtype=u_samples.dtype, device=u_samples.device) 27 | for _, prior, closure, _ in self.named_priors(): 28 | log_prior.add_(prior.log_prob(closure()).sum().div(self.num_data)) 29 | pyro.factor(name_prefix + ".log_prior", log_prior) 30 | 31 | # Include factor for added loss terms 32 | added_loss = torch.tensor(0.0, dtype=u_samples.dtype, device=u_samples.device) 33 | for added_loss_term in self.added_loss_terms(): 34 | added_loss.add_(added_loss_term.loss()) 35 | pyro.factor(name_prefix + ".added_loss", added_loss) 36 | 37 | # Draw samples from p(f) 38 | function_dist = self(input, prior=True) 39 | function_dist = pyro.distributions.Normal(loc=function_dist.mean, scale=function_dist.stddev).to_event( 40 | len(function_dist.event_shape) - 1 41 | ) 42 | return function_dist.mask(False) 43 | -------------------------------------------------------------------------------- /gpytorch/priors/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from .horseshoe_prior import HorseshoePrior 4 | from .lkj_prior import LKJCholeskyFactorPrior, LKJCovariancePrior, LKJPrior 5 | from .prior import Prior 6 | from .smoothed_box_prior import SmoothedBoxPrior 7 | from .torch_priors import GammaPrior, LogNormalPrior, MultivariateNormalPrior, NormalPrior, UniformPrior 8 | 9 | # from .wishart_prior import InverseWishartPrior, WishartPrior 10 | 11 | 12 | __all__ = [ 13 | "Prior", 14 | "GammaPrior", 15 | "HorseshoePrior", 16 | "LKJPrior", 17 | "LKJCholeskyFactorPrior", 18 | "LKJCovariancePrior", 19 | "LogNormalPrior", 20 | "MultivariateNormalPrior", 21 | "NormalPrior", 22 | "SmoothedBoxPrior", 23 | "UniformPrior", 24 | # "InverseWishartPrior", 25 | # "WishartPrior", 26 | ] 27 | -------------------------------------------------------------------------------- /gpytorch/priors/horseshoe_prior.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import math 4 | from numbers import Number 5 | 6 | import torch 7 | from torch.distributions import HalfCauchy, Normal, constraints 8 | from torch.nn import Module as TModule 9 | 10 | from gpytorch.priors.prior import Prior 11 | 12 | 13 | class HorseshoePrior(Prior): 14 | """Horseshoe prior. 15 | 16 | There is no analytical form for the horeshoe prior's pdf, but it 17 | satisfies a tight bound of the form `lb(x) <= pdf(x) <= ub(x)`, where 18 | 19 | lb(x) = K/2 * log(1 + 4 * (scale / x) ** 2) 20 | ub(x) = K * log(1 + 2 * (scale / x) ** 2) 21 | 22 | with `K = 1 / sqrt(2 pi^3)`. Here, we simply use 23 | 24 | pdf(x) ~ (lb(x) + ub(x)) / 2 25 | 26 | Reference: C. M. Carvalho, N. G. Polson, and J. G. Scott. 27 | The horseshoe estimator for sparse signals. Biometrika, 2010. 28 | """ 29 | 30 | arg_constraints = {"scale": constraints.positive} 31 | support = constraints.real 32 | _validate_args = True 33 | 34 | def __init__(self, scale, validate_args=False, transform=None): 35 | TModule.__init__(self) 36 | if isinstance(scale, Number): 37 | scale = torch.tensor(float(scale)) 38 | self.K = 1 / math.sqrt(2 * math.pi ** 3) 39 | self.scale = scale 40 | super().__init__(scale.shape, validate_args=validate_args) 41 | # now need to delete to be able to register buffer 42 | del self.scale 43 | self.register_buffer("scale", scale) 44 | self._transform = transform 45 | 46 | def log_prob(self, X): 47 | A = (self.scale / self.transform(X)) ** 2 48 | lb = self.K / 2 * torch.log(1 + 4 * A) 49 | ub = self.K * torch.log(1 + 2 * A) 50 | return torch.log((lb + ub) / 2) 51 | 52 | def rsample(self, sample_shape=torch.Size([])): 53 | local_shrinkage = HalfCauchy(1).rsample(self.scale.shape) 54 | param_sample = Normal(0, local_shrinkage * self.scale).rsample(sample_shape) 55 | return param_sample 56 | 57 | def expand(self, expand_shape, _instance=None): 58 | batch_shape = torch.Size(expand_shape) 59 | return HorseshoePrior(self.scale.expand(batch_shape)) 60 | -------------------------------------------------------------------------------- /gpytorch/priors/prior.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from abc import ABC 4 | 5 | from torch.nn import Module 6 | 7 | from ..distributions import Distribution 8 | 9 | 10 | class Prior(Distribution, Module, ABC): 11 | """ 12 | Base class for Priors in GPyTorch. 13 | In GPyTorch, a parameter can be assigned a prior by passing it as the `prior` argument to 14 | :func:`~gpytorch.module.register_parameter`. GPyTorch performs internal bookkeeping of priors, 15 | and for each parameter with a registered prior includes the log probability of the parameter under its 16 | respective prior in computing the Marginal Log-Likelihood. 17 | """ 18 | 19 | def transform(self, x): 20 | return self._transform(x) if self._transform is not None else x 21 | 22 | def log_prob(self, x): 23 | """Returns the log-probability of the parameter value under the prior.""" 24 | return super(Prior, self).log_prob(self.transform(x)) 25 | -------------------------------------------------------------------------------- /gpytorch/priors/smoothed_box_prior.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import math 4 | from numbers import Number 5 | 6 | import torch 7 | from torch.distributions import constraints 8 | from torch.distributions.utils import broadcast_all 9 | from torch.nn import Module as TModule 10 | 11 | from .prior import Prior 12 | from .torch_priors import NormalPrior 13 | 14 | 15 | class SmoothedBoxPrior(Prior): 16 | r"""A smoothed approximation of a uniform prior. 17 | 18 | Has full support on the reals and is differentiable everywhere. 19 | 20 | .. math:: 21 | 22 | \begin{equation*} 23 | B = {x: a_i <= x_i <= b_i} 24 | d(x, B) = min_{x' in B} |x - x'| 25 | pdf(x) ~ exp(- d(x, B)**2 / sqrt(2 * sigma^2)) 26 | \end{equation*} 27 | 28 | """ 29 | 30 | arg_constraints = {"sigma": constraints.positive, "a": constraints.real, "b": constraints.real} 31 | support = constraints.real 32 | _validate_args = True 33 | 34 | def __init__(self, a, b, sigma=0.01, validate_args=False, transform=None): 35 | TModule.__init__(self) 36 | _a = torch.tensor(float(a)) if isinstance(a, Number) else a 37 | _a = _a.view(-1) if _a.dim() < 1 else _a 38 | _a, _b, _sigma = broadcast_all(_a, b, sigma) 39 | if not torch.all(constraints.less_than(_b).check(_a)): 40 | raise ValueError("must have that a < b (element-wise)") 41 | # TODO: Proper argument validation including broadcasting 42 | batch_shape, event_shape = _a.shape[:-1], _a.shape[-1:] 43 | # need to assign values before registering as buffers to make argument validation work 44 | self.a, self.b, self.sigma = _a, _b, _sigma 45 | super(SmoothedBoxPrior, self).__init__(batch_shape, event_shape, validate_args=validate_args) 46 | # now need to delete to be able to register buffer 47 | del self.a, self.b, self.sigma 48 | self.register_buffer("a", _a) 49 | self.register_buffer("b", _b) 50 | self.register_buffer("sigma", _sigma) 51 | self.tails = NormalPrior(torch.zeros_like(_a), _sigma, validate_args=validate_args) 52 | self._transform = transform 53 | 54 | @property 55 | def _c(self): 56 | return (self.a + self.b) / 2 57 | 58 | @property 59 | def _r(self): 60 | return (self.b - self.a) / 2 61 | 62 | @property 63 | def _M(self): 64 | # normalization factor to make this a probability distribution 65 | return torch.log(1 + (self.b - self.a) / (math.sqrt(2 * math.pi) * self.sigma)) 66 | 67 | def log_prob(self, x): 68 | return self._log_prob(self.transform(x)) 69 | 70 | def _log_prob(self, x): 71 | # x = "distances from box`" 72 | X = ((x - self._c).abs_() - self._r).clamp(min=0) 73 | return (self.tails.log_prob(X) - self._M).sum(-1) 74 | -------------------------------------------------------------------------------- /gpytorch/priors/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | 4 | def _bufferize_attributes(module, attributes): 5 | attr_clones = {attr: getattr(module, attr).clone() for attr in attributes} 6 | for attr, value in attr_clones.items(): 7 | delattr(module, attr) 8 | module.register_buffer(attr, value) 9 | 10 | 11 | def _del_attributes(module, attributes, raise_on_error=False): 12 | for attr in attributes: 13 | try: 14 | delattr(module, attr) 15 | except AttributeError as e: 16 | if raise_on_error: 17 | raise e 18 | return module 19 | -------------------------------------------------------------------------------- /gpytorch/test/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | -------------------------------------------------------------------------------- /gpytorch/test/base_mean_test_case.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from abc import abstractmethod 4 | 5 | import torch 6 | 7 | from .base_test_case import BaseTestCase 8 | 9 | 10 | class BaseMeanTestCase(BaseTestCase): 11 | batch_shape = None 12 | 13 | @abstractmethod 14 | def create_mean(self, **kwargs): 15 | raise NotImplementedError() 16 | 17 | def test_forward_vec(self): 18 | test_x = torch.randn(4) 19 | mean = self.create_mean() 20 | if self.__class__.batch_shape is None: 21 | self.assertEqual(mean(test_x).shape, torch.Size([4])) 22 | else: 23 | self.assertEqual(mean(test_x).shape, torch.Size([*self.__class__.batch_shape, 4])) 24 | 25 | def test_forward_mat(self): 26 | test_x = torch.randn(4, 3) 27 | mean = self.create_mean() 28 | if self.__class__.batch_shape is None: 29 | self.assertEqual(mean(test_x).shape, torch.Size([4])) 30 | else: 31 | self.assertEqual(mean(test_x).shape, torch.Size([*self.__class__.batch_shape, 4])) 32 | 33 | def test_forward_mat_batch(self): 34 | test_x = torch.randn(3, 4, 3) 35 | mean = self.create_mean() 36 | if self.__class__.batch_shape is None: 37 | self.assertEqual(mean(test_x).shape, torch.Size([3, 4])) 38 | else: 39 | self.assertEqual(mean(test_x).shape, torch.Size([*self.__class__.batch_shape, 4])) 40 | 41 | def test_forward_mat_multi_batch(self): 42 | test_x = torch.randn(2, 3, 4, 3) 43 | mean = self.create_mean() 44 | self.assertEqual(mean(test_x).shape, torch.Size([2, 3, 4])) 45 | -------------------------------------------------------------------------------- /gpytorch/test/base_test_case.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import random 5 | from abc import ABC 6 | 7 | import torch 8 | 9 | 10 | class BaseTestCase(ABC): 11 | def setUp(self): 12 | if hasattr(self.__class__, "seed"): 13 | seed = self.__class__.seed 14 | if os.getenv("UNLOCK_SEED") is None or os.getenv("UNLOCK_SEED").lower() == "false": 15 | self.rng_state = torch.get_rng_state() 16 | torch.manual_seed(seed) 17 | if torch.cuda.is_available(): 18 | torch.cuda.manual_seed_all(seed) 19 | random.seed(seed) 20 | 21 | def tearDown(self): 22 | if hasattr(self, "rng_state"): 23 | torch.set_rng_state(self.rng_state) 24 | 25 | def assertAllClose(self, tensor1, tensor2, rtol=1e-4, atol=1e-5, equal_nan=False): 26 | if not tensor1.shape == tensor2.shape: 27 | raise ValueError(f"tensor1 ({tensor1.shape}) and tensor2 ({tensor2.shape}) do not have the same shape.") 28 | 29 | if torch.allclose(tensor1, tensor2, rtol=rtol, atol=atol, equal_nan=equal_nan): 30 | return True 31 | 32 | if not equal_nan: 33 | if not torch.equal(tensor1, tensor1): 34 | raise AssertionError(f"tensor1 ({tensor1.shape}) contains NaNs") 35 | if not torch.equal(tensor2, tensor2): 36 | raise AssertionError(f"tensor2 ({tensor2.shape}) contains NaNs") 37 | 38 | rtol_diff = (torch.abs(tensor1 - tensor2) / torch.abs(tensor2)).view(-1) 39 | rtol_diff = rtol_diff[torch.isfinite(rtol_diff)] 40 | rtol_max = rtol_diff.max().item() 41 | 42 | atol_diff = (torch.abs(tensor1 - tensor2) - torch.abs(tensor2).mul(rtol)).view(-1) 43 | atol_diff = atol_diff[torch.isfinite(atol_diff)] 44 | atol_max = atol_diff.max().item() 45 | 46 | raise AssertionError( 47 | f"tensor1 ({tensor1.shape}) and tensor2 ({tensor2.shape}) are not close enough. \n" 48 | f"max rtol: {rtol_max:0.8f}\t\tmax atol: {atol_max:0.8f}" 49 | ) 50 | 51 | def assertEqual(self, item1, item2): 52 | if torch.is_tensor(item1) and torch.is_tensor(item2): 53 | if torch.equal(item1, item2): 54 | return True 55 | else: 56 | raise AssertionError(f"{item1} does not equal {item2}.") 57 | elif torch.is_tensor(item1) or torch.is_tensor(item2): 58 | raise AssertionError(f"item1 ({type(item1)}) and item2 ({type(item2)}) are not the same type.") 59 | elif item1 == item2: 60 | return True 61 | elif type(item1) != type(item2): 62 | raise AssertionError(f"item1 ({type(item1)}) and item2 ({type(item2)}) are not the same type.") 63 | else: 64 | raise AssertionError(f"tensor1 ({item1}) does not equal tensor2 ({item2}).") 65 | -------------------------------------------------------------------------------- /gpytorch/test/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from contextlib import contextmanager 4 | from typing import Generator 5 | 6 | import torch 7 | 8 | 9 | def approx_equal(self, other, epsilon=1e-4): 10 | """ 11 | Determines if two tensors are approximately equal 12 | Args: 13 | - self: tensor 14 | - other: tensor 15 | Returns: 16 | - bool 17 | """ 18 | if self.size() != other.size(): 19 | raise RuntimeError( 20 | "Size mismatch between self ({self}) and other ({other})".format(self=self.size(), other=other.size()) 21 | ) 22 | return torch.max((self - other).abs()) <= epsilon 23 | 24 | 25 | def get_cuda_max_memory_allocations() -> int: 26 | """Get the `max_memory_allocated` for each cuda device""" 27 | return torch.tensor([torch.cuda.max_memory_allocated(i) for i in range(torch.cuda.device_count())]) 28 | 29 | 30 | @contextmanager 31 | def least_used_cuda_device() -> Generator: 32 | """Contextmanager for automatically selecting the cuda device 33 | with the least allocated memory""" 34 | mem_allocs = get_cuda_max_memory_allocations() 35 | least_used_device = torch.argmin(mem_allocs).item() 36 | with torch.cuda.device(least_used_device): 37 | yield 38 | -------------------------------------------------------------------------------- /gpytorch/utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . import broadcasting, cholesky, fft, grid, interpolation, lanczos, pivoted_cholesky, quadrature, sparse 4 | from .linear_cg import linear_cg 5 | from .memoize import cached 6 | from .stochastic_lq import StochasticLQ 7 | 8 | 9 | def prod(items): 10 | """ 11 | """ 12 | if len(items): 13 | res = items[0] 14 | for item in items[1:]: 15 | res = res * item 16 | return res 17 | else: 18 | return 1 19 | 20 | 21 | __all__ = [ 22 | "broadcasting", 23 | "cached", 24 | "linear_cg", 25 | "StochasticLQ", 26 | "cholesky", 27 | "fft", 28 | "grid", 29 | "interpolation", 30 | "lanczos", 31 | "pivoted_cholesky", 32 | "quadrature", 33 | "sparse", 34 | ] 35 | -------------------------------------------------------------------------------- /gpytorch/utils/broadcasting.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import torch 4 | 5 | 6 | def _mul_broadcast_shape(*shapes, error_msg=None): 7 | """Compute dimension suggested by multiple tensor indices (supports broadcasting)""" 8 | 9 | # Pad each shape so they have the same number of dimensions 10 | num_dims = max(len(shape) for shape in shapes) 11 | shapes = tuple([1] * (num_dims - len(shape)) + list(shape) for shape in shapes) 12 | 13 | # Make sure that each dimension agrees in size 14 | final_size = [] 15 | for size_by_dim in zip(*shapes): 16 | non_singleton_sizes = tuple(size for size in size_by_dim if size != 1) 17 | if len(non_singleton_sizes): 18 | if any(size != non_singleton_sizes[0] for size in non_singleton_sizes): 19 | if error_msg is None: 20 | raise RuntimeError("Shapes are not broadcastable for mul operation") 21 | else: 22 | raise RuntimeError(error_msg) 23 | final_size.append(non_singleton_sizes[0]) 24 | # In this case - all dimensions are singleton sizes 25 | else: 26 | final_size.append(1) 27 | 28 | return torch.Size(final_size) 29 | 30 | 31 | def _matmul_broadcast_shape(shape_a, shape_b, error_msg=None): 32 | """Compute dimension of matmul operation on shapes (supports broadcasting)""" 33 | m, n, p = shape_a[-2], shape_a[-1], shape_b[-1] 34 | 35 | if len(shape_b) == 1: 36 | if n != p: 37 | if error_msg is None: 38 | raise RuntimeError(f"Incompatible dimensions for matmul: {shape_a} and {shape_b}") 39 | else: 40 | raise RuntimeError(error_msg) 41 | return shape_a[:-1] 42 | 43 | if n != shape_b[-2]: 44 | if error_msg is None: 45 | raise RuntimeError(f"Incompatible dimensions for matmul: {shape_a} and {shape_b}") 46 | else: 47 | raise RuntimeError(error_msg) 48 | 49 | tail_shape = torch.Size([m, p]) 50 | 51 | # Figure out batch shape 52 | batch_shape_a = shape_a[:-2] 53 | batch_shape_b = shape_b[:-2] 54 | if batch_shape_a == batch_shape_b: 55 | bc_shape = batch_shape_a 56 | else: 57 | bc_shape = _mul_broadcast_shape(batch_shape_a, batch_shape_b) 58 | return bc_shape + tail_shape 59 | 60 | 61 | def _pad_with_singletons(obj, num_singletons_before=0, num_singletons_after=0): 62 | """ 63 | Pad obj with singleton dimensions on the left and right 64 | 65 | Example: 66 | >>> x = torch.randn(10, 5) 67 | >>> _pad_width_singletons(x, 2, 3).shape 68 | >>> # [1, 1, 10, 5, 1, 1, 1] 69 | """ 70 | new_shape = [1] * num_singletons_before + list(obj.shape) + [1] * num_singletons_after 71 | return obj.view(*new_shape) 72 | -------------------------------------------------------------------------------- /gpytorch/utils/cholesky.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import warnings 4 | 5 | import torch 6 | 7 | from .errors import NanError 8 | 9 | 10 | def psd_safe_cholesky(A, upper=False, out=None, jitter=None): 11 | """Compute the Cholesky decomposition of A. If A is only p.s.d, add a small jitter to the diagonal. 12 | Args: 13 | :attr:`A` (Tensor): 14 | The tensor to compute the Cholesky decomposition of 15 | :attr:`upper` (bool, optional): 16 | See torch.cholesky 17 | :attr:`out` (Tensor, optional): 18 | See torch.cholesky 19 | :attr:`jitter` (float, optional): 20 | The jitter to add to the diagonal of A in case A is only p.s.d. If omitted, chosen 21 | as 1e-6 (float) or 1e-8 (double) 22 | """ 23 | try: 24 | L = torch.cholesky(A, upper=upper, out=out) 25 | return L 26 | except RuntimeError as e: 27 | isnan = torch.isnan(A) 28 | if isnan.any(): 29 | raise NanError( 30 | f"cholesky_cpu: {isnan.sum().item()} of {A.numel()} elements of the {A.shape} tensor are NaN." 31 | ) 32 | 33 | if jitter is None: 34 | jitter = 1e-6 if A.dtype == torch.float32 else 1e-8 35 | Aprime = A.clone() 36 | jitter_prev = 0 37 | for i in range(3): 38 | jitter_new = jitter * (10 ** i) 39 | Aprime.diagonal(dim1=-2, dim2=-1).add_(jitter_new - jitter_prev) 40 | jitter_prev = jitter_new 41 | try: 42 | L = torch.cholesky(Aprime, upper=upper, out=out) 43 | warnings.warn(f"A not p.d., added jitter of {jitter_new} to the diagonal", RuntimeWarning) 44 | return L 45 | except RuntimeError: 46 | continue 47 | raise e 48 | -------------------------------------------------------------------------------- /gpytorch/utils/deprecation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import functools 4 | import warnings 5 | from unittest.mock import MagicMock 6 | 7 | import torch 8 | 9 | # TODO: Use bool instead of uint8 dtype once pytorch #21113 is in stable release 10 | if isinstance(torch, MagicMock): 11 | bool_compat = torch.uint8 12 | else: 13 | bool_compat = (torch.ones(1) > 0).dtype 14 | 15 | 16 | class DeprecationError(Exception): 17 | pass 18 | 19 | 20 | def _deprecated_function_for(old_function_name, function): 21 | @functools.wraps(function) 22 | def _deprecated_function(*args, **kwargs): 23 | warnings.warn( 24 | "The `{}` function is deprecated. Use `{}` instead".format(old_function_name, function.__name__), 25 | DeprecationWarning, 26 | ) 27 | return function(*args, **kwargs) 28 | 29 | return _deprecated_function 30 | 31 | 32 | def _deprecate_kwarg(kwargs, old_kw, new_kw, new_kw_value): 33 | old_kwarg = kwargs.get(old_kw) 34 | if old_kwarg is not None: 35 | warnings.warn("The `{}` argument is deprecated. Use `{}` instead.".format(old_kw, new_kw), DeprecationWarning) 36 | if new_kw_value is not None: 37 | raise ValueError("Cannot set both `{}` and `{}`".format(old_kw, new_kw)) 38 | return old_kwarg 39 | return new_kw_value 40 | 41 | 42 | def _deprecate_kwarg_with_transform(kwargs, old_kw, new_kw, new_kw_value, transform): 43 | old_kwarg = kwargs.get(old_kw) 44 | if old_kwarg is not None: 45 | warnings.warn("The `{}` argument is deprecated. Use `{}` instead.".format(old_kw, new_kw), DeprecationWarning) 46 | return transform(old_kwarg) 47 | return new_kw_value 48 | 49 | 50 | def _deprecated_renamed_method(cls, old_method_name, new_method_name): 51 | def _deprecated_method(self, *args, **kwargs): 52 | warnings.warn( 53 | "The `{}` method is deprecated. Use `{}` instead".format(old_method_name, new_method_name), 54 | DeprecationWarning, 55 | ) 56 | return getattr(self, new_method_name)(*args, **kwargs) 57 | 58 | _deprecated_method.__name__ = old_method_name 59 | setattr(cls, old_method_name, _deprecated_method) 60 | return cls 61 | 62 | 63 | def _deprecate_renamed_methods(cls, **renamed_methods): 64 | for old_method_name, new_method_name in renamed_methods.items(): 65 | _deprecated_renamed_method(cls, old_method_name, new_method_name) 66 | return cls 67 | -------------------------------------------------------------------------------- /gpytorch/utils/errors.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | 4 | class NanError(RuntimeError): 5 | pass 6 | 7 | 8 | __all__ = ["NanError"] 9 | -------------------------------------------------------------------------------- /gpytorch/utils/fft.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import torch 4 | 5 | 6 | def fft1(input): 7 | complex_input = torch.stack((input, torch.zeros_like(input)), dim=-1) 8 | return complex_input.fft(1) 9 | 10 | 11 | def ifft1(input): 12 | complex_output = input.ifft(1) 13 | real_ind = torch.tensor(0, dtype=torch.long, device=input.device) 14 | return complex_output.index_select(-1, real_ind).squeeze(-1) 15 | -------------------------------------------------------------------------------- /gpytorch/utils/memoize.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import functools 4 | 5 | 6 | def add_to_cache(obj, name, val): 7 | """Add a result to the cache of an object.""" 8 | if not hasattr(obj, "_memoize_cache"): 9 | obj._memoize_cache = dict() 10 | obj._memoize_cache[name] = val 11 | return obj 12 | 13 | 14 | def get_from_cache(obj, name): 15 | """Get an item from the cache.""" 16 | if not is_in_cache(obj, name): 17 | raise RuntimeError("Object does not have item {} stored in cache.".format(name)) 18 | return obj._memoize_cache[name] 19 | 20 | 21 | def is_in_cache(obj, name): 22 | return hasattr(obj, "_memoize_cache") and name in obj._memoize_cache 23 | 24 | 25 | def cached(method=None, name=None): 26 | """A decorator allowing for specifying the name of a cache, allowing it to be modified elsewhere.""" 27 | if method is None: 28 | return functools.partial(cached, name=name) 29 | 30 | @functools.wraps(method) 31 | def g(self, *args, **kwargs): 32 | cache_name = name if name is not None else method 33 | if not is_in_cache(self, cache_name): 34 | add_to_cache(self, cache_name, method(self, *args, **kwargs)) 35 | return get_from_cache(self, cache_name) 36 | 37 | return g 38 | 39 | 40 | def is_cached(self, name): 41 | """ 42 | Determine if a cached item has been computed 43 | """ 44 | return hasattr(self, "_memoize_cache") and name in self._memoize_cache.keys() 45 | -------------------------------------------------------------------------------- /gpytorch/utils/transforms.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import torch 4 | 5 | 6 | def inv_softplus(x): 7 | return x + torch.log(-torch.expm1(-x)) 8 | 9 | 10 | def inv_sigmoid(x): 11 | return torch.log(x) - torch.log(1 - x) 12 | 13 | 14 | def _get_inv_param_transform(param_transform, inv_param_transform=None): 15 | reg_inv_tf = TRANSFORM_REGISTRY.get(param_transform, None) 16 | if reg_inv_tf is None: 17 | if inv_param_transform is None: 18 | raise RuntimeError("Must specify inv_param_transform for custom param_transforms") 19 | return inv_param_transform 20 | elif inv_param_transform is not None and reg_inv_tf != inv_param_transform: 21 | raise RuntimeError("TODO") 22 | return reg_inv_tf 23 | 24 | 25 | TRANSFORM_REGISTRY = {torch.exp: torch.log, torch.nn.functional.softplus: inv_softplus, torch.sigmoid: inv_sigmoid} 26 | -------------------------------------------------------------------------------- /gpytorch/variational/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from ._variational_distribution import _VariationalDistribution 4 | from ._variational_strategy import _VariationalStrategy 5 | from .additive_grid_interpolation_variational_strategy import AdditiveGridInterpolationVariationalStrategy 6 | from .cholesky_variational_distribution import CholeskyVariationalDistribution 7 | from .delta_variational_distribution import DeltaVariationalDistribution 8 | from .grid_interpolation_variational_strategy import GridInterpolationVariationalStrategy 9 | from .mean_field_variational_distribution import MeanFieldVariationalDistribution 10 | from .multitask_variational_strategy import MultitaskVariationalStrategy 11 | from .orthogonally_decoupled_variational_strategy import OrthogonallyDecoupledVariationalStrategy 12 | from .unwhitened_variational_strategy import UnwhitenedVariationalStrategy 13 | from .variational_strategy import VariationalStrategy 14 | from .whitened_variational_strategy import WhitenedVariationalStrategy 15 | 16 | __all__ = [ 17 | "_VariationalStrategy", 18 | "AdditiveGridInterpolationVariationalStrategy", 19 | "GridInterpolationVariationalStrategy", 20 | "MultitaskVariationalStrategy", 21 | "OrthogonallyDecoupledVariationalStrategy", 22 | "VariationalStrategy", 23 | "UnwhitenedVariationalStrategy", 24 | "WhitenedVariationalStrategy", 25 | "_VariationalDistribution", 26 | "CholeskyVariationalDistribution", 27 | "MeanFieldVariationalDistribution", 28 | "DeltaVariationalDistribution", 29 | ] 30 | -------------------------------------------------------------------------------- /gpytorch/variational/_variational_distribution.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import warnings 4 | from abc import ABC, abstractmethod 5 | 6 | import torch 7 | 8 | from ..module import Module 9 | 10 | 11 | class _VariationalDistribution(Module, ABC): 12 | r""" 13 | Abstract base class for all Variational Distributions. 14 | """ 15 | 16 | def __init__(self, num_inducing_points, batch_shape=torch.Size([]), mean_init_std=1e-3): 17 | super().__init__() 18 | self.num_inducing_points = num_inducing_points 19 | self.batch_shape = batch_shape 20 | self.mean_init_std = mean_init_std 21 | 22 | def forward(self): 23 | r""" 24 | Constructs and returns the variational distribution 25 | 26 | :rtype: :obj:`~gpytorch.distributions.MultivariateNormal` 27 | :return: The distribution :math:q(\mathbf u)" 28 | """ 29 | raise NotImplementedError 30 | 31 | @abstractmethod 32 | def initialize_variational_distribution(self, prior_dist): 33 | r""" 34 | Method for initializing the variational distribution, based on the prior distribution. 35 | 36 | :param ~gpytorch.distribution.Distribution prior_dist: The prior distribution :math:`p(\mathbf u)`. 37 | """ 38 | raise NotImplementedError 39 | 40 | def __call__(self): 41 | try: 42 | return self.forward() 43 | # Remove after 1.0 44 | except NotImplementedError: 45 | warnings.warn( 46 | "_VariationalDistribution.variational_distribution is deprecated. " 47 | "Please implement a `forward` method instead.", 48 | DeprecationWarning, 49 | ) 50 | return self.variational_distribution 51 | 52 | def __getattr__(self, attr): 53 | # Remove after 1.0 54 | if attr == "variational_distribution": 55 | warnings.warn( 56 | "_VariationalDistribution.variational_distribution is deprecated. " 57 | "To get q(u), call the _VariationalDistribution object instead.", 58 | DeprecationWarning, 59 | ) 60 | return self.forward() 61 | else: 62 | return super().__getattr__(attr) 63 | -------------------------------------------------------------------------------- /gpytorch/variational/delta_variational_distribution.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import torch 4 | 5 | from ..distributions import Delta 6 | from ._variational_distribution import _VariationalDistribution 7 | 8 | 9 | class DeltaVariationalDistribution(_VariationalDistribution): 10 | """ 11 | This :obj:`~gpytorch.variational._VariationalDistribution` object replaces a variational distribution 12 | with a single particle. It is equivalent to doing MAP inference. 13 | 14 | :param int num_inducing_points: Size of the variational distribution. This implies that the variational mean 15 | should be this size. 16 | :param torch.Size batch_shape: (Optional.) Specifies an optional batch size 17 | for the variational parameters. This is useful for example when doing additive variational inference. 18 | :param float mean_init_std: (default=1e-3) Standard deviation of gaussian noise to add to the mean initialization. 19 | """ 20 | 21 | def __init__(self, num_inducing_points, batch_shape=torch.Size([]), mean_init_std=1e-3, **kwargs): 22 | super().__init__(num_inducing_points=num_inducing_points, batch_shape=batch_shape, mean_init_std=mean_init_std) 23 | mean_init = torch.zeros(num_inducing_points) 24 | mean_init = mean_init.repeat(*batch_shape, 1) 25 | self.register_parameter(name="variational_mean", parameter=torch.nn.Parameter(mean_init)) 26 | 27 | def forward(self): 28 | return Delta(self.variational_mean) 29 | 30 | def initialize_variational_distribution(self, prior_dist): 31 | self.variational_mean.data.copy_(prior_dist.mean) 32 | self.variational_mean.data.add_(self.mean_init_std, torch.randn_like(prior_dist.mean)) 33 | -------------------------------------------------------------------------------- /gpytorch/variational/mean_field_variational_distribution.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import torch 4 | 5 | from ..distributions import MultivariateNormal 6 | from ..lazy import DiagLazyTensor 7 | from ._variational_distribution import _VariationalDistribution 8 | 9 | 10 | class MeanFieldVariationalDistribution(_VariationalDistribution): 11 | """ 12 | A :obj:`~gpytorch.variational._VariationalDistribution` that is defined to be a multivariate normal distribution 13 | with a diagonal covariance matrix. This will not be as flexible/expressive as a 14 | :obj:`~gpytorch.variational.CholeskyVariationalDistribution`. 15 | 16 | :param int num_inducing_points: Size of the variational distribution. This implies that the variational mean 17 | should be this size, and the variational covariance matrix should have this many rows and columns. 18 | :param torch.Size batch_shape: (Optional.) Specifies an optional batch size 19 | for the variational parameters. This is useful for example when doing additive variational inference. 20 | :param float mean_init_std: (default=1e-3) Standard deviation of gaussian noise to add to the mean initialization. 21 | """ 22 | 23 | def __init__(self, num_inducing_points, batch_shape=torch.Size([]), mean_init_std=1e-3, **kwargs): 24 | super().__init__(num_inducing_points=num_inducing_points, batch_shape=batch_shape, mean_init_std=mean_init_std) 25 | mean_init = torch.zeros(num_inducing_points) 26 | covar_init = torch.ones(num_inducing_points) 27 | mean_init = mean_init.repeat(*batch_shape, 1) 28 | covar_init = covar_init.repeat(*batch_shape, 1) 29 | 30 | self.register_parameter(name="variational_mean", parameter=torch.nn.Parameter(mean_init)) 31 | self.register_parameter(name="_variational_stddev", parameter=torch.nn.Parameter(covar_init)) 32 | 33 | @property 34 | def variational_stddev(self): 35 | return self._variational_stddev.abs().clamp_min(1e-8) 36 | 37 | def forward(self): 38 | variational_var = self.variational_stddev.pow(2) 39 | variational_covar = DiagLazyTensor(variational_var) 40 | return MultivariateNormal(self.variational_mean, variational_covar) 41 | 42 | def initialize_variational_distribution(self, prior_dist): 43 | self.variational_mean.data.copy_(prior_dist.mean) 44 | self.variational_mean.data.add_(self.mean_init_std, torch.randn_like(prior_dist.mean)) 45 | self.variational_stddev.data.copy_(prior_dist.stddev) 46 | -------------------------------------------------------------------------------- /gpytorch/variational/multitask_variational_strategy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from ..distributions import MultitaskMultivariateNormal 4 | from ..module import Module 5 | from ._variational_strategy import _VariationalStrategy 6 | 7 | 8 | class MultitaskVariationalStrategy(_VariationalStrategy): 9 | """ 10 | MultitaskVariationalStrategy wraps an existing :obj:`~gpytorch.variational.VariationalStrategy` 11 | to product a :obj:`~gpytorch.variational.MultitaskMultivariateNormal` distribution. 12 | This is useful for multi-output variational models. 13 | 14 | The base variational strategy is assumed to operate on a batch of GPs. One of the batch 15 | dimensions corresponds to the multiple tasks. 16 | 17 | :param ~gpytorch.variational.VariationalStrategy base_variational_strategy: Base variational strategy 18 | :param int task_dim: (default=-1) Which batch dimension is the task dimension 19 | """ 20 | 21 | def __init__(self, base_variational_strategy, num_tasks, task_dim=-1): 22 | Module.__init__(self) 23 | self.base_variational_strategy = base_variational_strategy 24 | self.task_dim = task_dim 25 | self.num_tasks = num_tasks 26 | 27 | @property 28 | def prior_distribution(self): 29 | return self.base_variational_strategy.prior_distribution 30 | 31 | @property 32 | def variational_distribution(self): 33 | return self.base_variational_strategy.variational_distribution 34 | 35 | @property 36 | def variational_params_initialized(self): 37 | return self.base_variational_strategy.variational_params_initialized 38 | 39 | def kl_divergence(self): 40 | return super().kl_divergence().sum(dim=-1) 41 | 42 | def __call__(self, x, prior=False): 43 | function_dist = self.base_variational_strategy(x, prior=prior) 44 | if ( 45 | self.task_dim > 0 46 | and self.task_dim > len(function_dist.batch_shape) 47 | or self.task_dim < 0 48 | and self.task_dim + len(function_dist.batch_shape) < 0 49 | ): 50 | return MultitaskMultivariateNormal.from_repeated_mvn(function_dist, num_tasks=self.num_tasks) 51 | else: 52 | function_dist = MultitaskMultivariateNormal.from_batch_mvn(function_dist, task_dim=self.task_dim) 53 | assert function_dist.event_shape[-1] == self.num_tasks 54 | return function_dist 55 | -------------------------------------------------------------------------------- /readthedocs.yml: -------------------------------------------------------------------------------- 1 | build: 2 | image: latest 3 | python: 4 | version: 3.6 -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch>=1.3 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal=1 3 | 4 | [pep8] 5 | max-line-length = 120 6 | 7 | [flake8] 8 | max-line-length = 120 9 | ignore = E203, F403, F405, E731, W503, W605 10 | exclude = 11 | build,examples 12 | 13 | [build_sphinx] 14 | all-files = 1 15 | source-dir = docs/source 16 | build-dir = docs/build 17 | warning-is-error = 1 18 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import io 4 | import os 5 | import re 6 | 7 | from setuptools import find_packages, setup 8 | 9 | 10 | # Get version 11 | def read(*names, **kwargs): 12 | with io.open(os.path.join(os.path.dirname(__file__), *names), encoding=kwargs.get("encoding", "utf8")) as fp: 13 | return fp.read() 14 | 15 | 16 | def find_version(*file_paths): 17 | version_file = read(*file_paths) 18 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) 19 | if version_match: 20 | return version_match.group(1) 21 | raise RuntimeError("Unable to find version string.") 22 | 23 | 24 | readme = open("README.md").read() 25 | version = find_version("gpytorch", "__init__.py") 26 | 27 | 28 | torch_min = "1.3" 29 | install_requires = [">=".join(["torch", torch_min])] 30 | # if recent dev version of PyTorch is installed, no need to install stable 31 | try: 32 | import torch 33 | 34 | if torch.__version__ >= torch_min: 35 | install_requires = [] 36 | except ImportError: 37 | pass 38 | 39 | 40 | # Run the setup 41 | setup( 42 | name="gpytorch", 43 | version=version, 44 | description="An implementation of Gaussian Processes in Pytorch", 45 | long_description=readme, 46 | long_description_content_type="text/markdown", 47 | author="Jake Gardner, Geoff Pleiss", 48 | url="https://gpytorch.ai", 49 | author_email="jrg365@cornell.edu, gpleiss@gmail.com", 50 | project_urls={ 51 | "Documentation": "https://gpytorch.readthedocs.io", 52 | "Source": "https://github.com/cornellius-gp/gpytorch/", 53 | }, 54 | license="MIT", 55 | classifiers=["Development Status :: 4 - Beta", "Programming Language :: Python :: 3"], 56 | packages=find_packages(), 57 | python_requires=">=3.6", 58 | install_requires=install_requires, 59 | extras_require={ 60 | "dev": ["black", "twine", "pre-commit"], 61 | "docs": ["ipython", "ipykernel", "sphinx<3.0.0", "sphinx_rtd_theme", "nbsphinx", "m2r"], 62 | "examples": ["ipython", "jupyter", "matplotlib", "scipy", "torchvision", "tqdm"], 63 | "pyro": ["pyro-ppl>=1.0.0"], 64 | "keops": ["pykeops>=1.1.1"], 65 | "test": ["flake8", "flake8-print", "pytest", "nbval"], 66 | }, 67 | ) 68 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | -------------------------------------------------------------------------------- /test/constraints/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | -------------------------------------------------------------------------------- /test/distributions/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | -------------------------------------------------------------------------------- /test/distributions/test_delta.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Mostly copied from https://raw.githubusercontent.com/pyro-ppl/pyro/dev/tests/distributions/test_delta.py 3 | 4 | import unittest 5 | 6 | import numpy as np 7 | import torch 8 | 9 | import gpytorch.distributions as dist 10 | from gpytorch.test.base_test_case import BaseTestCase 11 | 12 | 13 | class TestDelta(BaseTestCase, unittest.TestCase): 14 | def setUp(self): 15 | self.v = torch.tensor([3.0]) 16 | self.vs = torch.tensor([[0.0], [1.0], [2.0], [3.0]]) 17 | self.vs_expanded = self.vs.expand(4, 3) 18 | self.test_data = torch.tensor([[3.0], [3.0], [3.0]]) 19 | self.batch_test_data_1 = torch.arange(0.0, 4.0).unsqueeze(1).expand(4, 3) 20 | self.batch_test_data_2 = torch.arange(4.0, 8.0).unsqueeze(1).expand(4, 3) 21 | self.batch_test_data_3 = torch.Tensor([[3.0], [3.0], [3.0], [3.0]]) 22 | self.expected_support = [[[0.0], [1.0], [2.0], [3.0]]] 23 | self.expected_support_non_vec = [[3.0]] 24 | self.analytic_mean = 3.0 25 | self.analytic_var = 0.0 26 | self.n_samples = 10 27 | 28 | def test_log_prob_sum(self): 29 | log_px_torch = dist.Delta(self.v).log_prob(self.test_data).sum() 30 | self.assertEqual(log_px_torch.item(), 0) 31 | 32 | def test_batch_log_prob(self): 33 | log_px_torch = dist.Delta(self.vs_expanded).log_prob(self.batch_test_data_1).data 34 | self.assertEqual(log_px_torch.sum().item(), 0) 35 | log_px_torch = dist.Delta(self.vs_expanded).log_prob(self.batch_test_data_2).data 36 | self.assertEqual(log_px_torch.sum().item(), float("-inf")) 37 | 38 | def test_batch_log_prob_shape(self): 39 | assert dist.Delta(self.vs).log_prob(self.batch_test_data_3).size() == (4, 1) 40 | assert dist.Delta(self.v).log_prob(self.batch_test_data_3).size() == (4, 1) 41 | 42 | def test_mean_and_var(self): 43 | torch_samples = [dist.Delta(self.v).sample().detach().cpu().numpy() for _ in range(self.n_samples)] 44 | torch_mean = np.mean(torch_samples) 45 | torch_var = np.var(torch_samples) 46 | self.assertEqual(torch_mean, self.analytic_mean) 47 | self.assertEqual(torch_var, self.analytic_var) 48 | -------------------------------------------------------------------------------- /test/examples/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | -------------------------------------------------------------------------------- /test/examples/old_variational_strategy_model.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jwangjie/gpytorch/15979dacf1997af7daf0fdeddbdbfcef0730b007/test/examples/old_variational_strategy_model.pth -------------------------------------------------------------------------------- /test/functions/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | -------------------------------------------------------------------------------- /test/functions/test_log_normal_cdf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import math 4 | import unittest 5 | 6 | import torch 7 | from torch import nn 8 | 9 | import gpytorch 10 | 11 | 12 | class TestLogNormalCDF(unittest.TestCase): 13 | def test_forward(self): 14 | inputs = torch.tensor([-6, -5, -3, -1, 0, 1, 3, 5], dtype=torch.float) 15 | output = gpytorch.log_normal_cdf(inputs) 16 | 17 | # Answers should be reasonable for small values 18 | self.assertLess(math.fabs(output[0] + 20.7368), 1e-4) 19 | self.assertLess(math.fabs(output[1] + 15), 0.1) 20 | self.assertLess(math.fabs(output[2] + 6.6), 0.01) 21 | self.assertLess(math.fabs(output[3] + 1.841), 0.001) 22 | 23 | # Should be very accurate for positive values 24 | self.assertLess(math.fabs(output[4] + 0.693147), 1e-4) 25 | self.assertLess(math.fabs(output[5] + 0.1727), 1e-4) 26 | self.assertLess(math.fabs(output[6] + 0.00135081), 1e-4) 27 | self.assertLess(math.fabs(output[7] + 2.86652e-7), 1e-4) 28 | 29 | def test_backward(self): 30 | inputs = nn.Parameter(torch.tensor([-6, -5, -3, -1, 0, 1, 3, 5], dtype=torch.float)) 31 | output = gpytorch.log_normal_cdf(inputs) 32 | output.backward(torch.ones(8)) 33 | 34 | gradient = inputs.grad 35 | expected_gradient = torch.tensor( 36 | [6.1585, 5.1865, 3.2831, 1.5251, 0.7979, 0.2876, 0.0044, 0.0000], dtype=torch.float 37 | ) 38 | 39 | # Should be reasonable for small values 40 | for d in torch.abs(gradient[:3] - expected_gradient[:3]): 41 | self.assertLess(d, 5e-1) 42 | 43 | # Should be very accurate for larger ones 44 | for d in torch.abs(gradient[3:] - expected_gradient[3:]): 45 | self.assertLess(d, 5e-4) 46 | 47 | 48 | if __name__ == "__main__": 49 | unittest.main() 50 | -------------------------------------------------------------------------------- /test/functions/test_rbf_covariance.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | import gpytorch 8 | 9 | 10 | def sq_dist_func(x1, x2): 11 | dist_module = gpytorch.kernels.kernel.Distance() 12 | return dist_module._sq_dist(x1, x2, postprocess=torch.tensor(False)) 13 | 14 | 15 | class TestRBFCovariance(unittest.TestCase): 16 | def test_forward(self): 17 | batch_size = (3, 2, 4) 18 | x1 = torch.randn(*batch_size, 7, 9) 19 | x2 = torch.randn(*batch_size, 6, 9) 20 | # Doesn't support ARD 21 | lengthscale = torch.randn(*batch_size).view(*batch_size, 1, 1) ** 2 22 | res = gpytorch.functions.RBFCovariance().apply(x1, x2, lengthscale, sq_dist_func) 23 | actual = sq_dist_func(x1, x2).div(-2 * lengthscale ** 2).exp() 24 | self.assertTrue(torch.allclose(res, actual)) 25 | 26 | def test_backward(self): 27 | batch_size = (3, 2, 4) 28 | x1 = torch.randn(*batch_size, 7, 9, dtype=torch.float64) 29 | x2 = torch.randn(*batch_size, 6, 9, dtype=torch.float64) 30 | lengthscale = torch.randn(*batch_size, dtype=torch.float64, requires_grad=True).view(*batch_size, 1, 1) ** 2 31 | f = lambda x1, x2, l: gpytorch.functions.RBFCovariance().apply(x1, x2, l, sq_dist_func) 32 | try: 33 | torch.autograd.gradcheck(f, (x1, x2, lengthscale)) 34 | except RuntimeError: 35 | self.fail("Gradcheck failed") 36 | 37 | 38 | if __name__ == "__main__": 39 | unittest.main() 40 | -------------------------------------------------------------------------------- /test/functions/test_root_decomposition.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.lazy import NonLazyTensor 8 | from gpytorch.test.base_test_case import BaseTestCase 9 | 10 | 11 | class TestRootDecomposition(BaseTestCase, unittest.TestCase): 12 | seed = 0 13 | 14 | def _create_mat(self): 15 | mat = torch.randn(4, 4) 16 | mat = mat @ mat.transpose(-1, -2) 17 | mat.div_(5).add_(torch.eye(4)) 18 | return mat 19 | 20 | def test_root_decomposition(self): 21 | mat = self._create_mat().detach().requires_grad_(True) 22 | mat_clone = mat.detach().clone().requires_grad_(True) 23 | 24 | # Forward 25 | root = NonLazyTensor(mat).root_decomposition().root.evaluate() 26 | res = root.matmul(root.transpose(-1, -2)) 27 | self.assertAllClose(res, mat) 28 | 29 | # Backward 30 | sum([mat.trace() for mat in res.view(-1, mat.size(-2), mat.size(-1))]).backward() 31 | sum([mat.trace() for mat in mat_clone.view(-1, mat.size(-2), mat.size(-1))]).backward() 32 | self.assertAllClose(mat.grad, mat_clone.grad) 33 | 34 | def test_root_inv_decomposition(self): 35 | mat = self._create_mat().detach().requires_grad_(True) 36 | mat_clone = mat.detach().clone().requires_grad_(True) 37 | 38 | # Forward 39 | probe_vectors = torch.randn(*mat.shape[:-2], 4, 5) 40 | test_vectors = torch.randn(*mat.shape[:-2], 4, 5) 41 | root = NonLazyTensor(mat).root_inv_decomposition(probe_vectors, test_vectors).root.evaluate() 42 | res = root.matmul(root.transpose(-1, -2)) 43 | actual = mat_clone.inverse() 44 | self.assertAllClose(res, actual) 45 | 46 | # Backward 47 | sum([mat.trace() for mat in res.view(-1, mat.size(-2), mat.size(-1))]).backward() 48 | sum([mat.trace() for mat in actual.view(-1, mat.size(-2), mat.size(-1))]).backward() 49 | self.assertAllClose(mat.grad, mat_clone.grad) 50 | 51 | 52 | class TestRootDecompositionBatch(TestRootDecomposition): 53 | seed = 0 54 | 55 | def _create_mat(self): 56 | mat = torch.randn(3, 4, 4) 57 | mat = mat @ mat.transpose(-1, -2) 58 | mat.div_(5).add_(torch.eye(4).unsqueeze_(0)) 59 | return mat 60 | 61 | 62 | class TestRootDecompositionMultiBatch(TestRootDecomposition): 63 | seed = 0 64 | 65 | def _create_mat(self): 66 | mat = torch.randn(2, 3, 4, 4) 67 | mat = mat @ mat.transpose(-1, -2) 68 | mat.div_(5).add_(torch.eye(4).unsqueeze_(0)) 69 | return mat 70 | 71 | 72 | if __name__ == "__main__": 73 | unittest.main() 74 | -------------------------------------------------------------------------------- /test/kernels/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | -------------------------------------------------------------------------------- /test/kernels/keops/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jwangjie/gpytorch/15979dacf1997af7daf0fdeddbdbfcef0730b007/test/kernels/keops/__init__.py -------------------------------------------------------------------------------- /test/kernels/keops/test_rbf_kernel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.kernels import RBFKernel as GRBFKernel 8 | from gpytorch.kernels.keops import RBFKernel 9 | from gpytorch.test.base_kernel_test_case import BaseKernelTestCase 10 | 11 | try: 12 | import pykeops # noqa 13 | 14 | class TestRBFKeOpsBaseKernel(unittest.TestCase, BaseKernelTestCase): 15 | def create_kernel_no_ard(self, **kwargs): 16 | return RBFKernel(**kwargs) 17 | 18 | def create_kernel_ard(self, num_dims, **kwargs): 19 | return RBFKernel(ard_num_dims=num_dims, **kwargs) 20 | 21 | class TestRBFKeOpsKernel(unittest.TestCase): 22 | def test_forward_x1_eq_x2(self): 23 | if not torch.cuda.is_available(): 24 | return 25 | 26 | x1 = torch.randn(100, 3).cuda() 27 | 28 | kern1 = RBFKernel().cuda() 29 | kern2 = GRBFKernel().cuda() 30 | 31 | k1 = kern1(x1, x1).evaluate() 32 | k2 = kern2(x1, x1).evaluate() 33 | 34 | self.assertLess(torch.norm(k1 - k2), 1e-4) 35 | 36 | def test_forward_x1_neq_x2(self): 37 | if not torch.cuda.is_available(): 38 | return 39 | 40 | x1 = torch.randn(100, 3).cuda() 41 | x2 = torch.randn(50, 3).cuda() 42 | 43 | kern1 = RBFKernel().cuda() 44 | kern2 = GRBFKernel().cuda() 45 | 46 | k1 = kern1(x1, x2).evaluate() 47 | k2 = kern2(x1, x2).evaluate() 48 | 49 | self.assertLess(torch.norm(k1 - k2), 1e-4) 50 | 51 | def test_batch_matmul(self): 52 | if not torch.cuda.is_available(): 53 | return 54 | 55 | x1 = torch.randn(3, 2, 100, 3).cuda() 56 | kern1 = RBFKernel().cuda() 57 | kern2 = GRBFKernel().cuda() 58 | 59 | rhs = torch.randn(3, 2, 100, 1).cuda() 60 | res1 = kern1(x1, x1).matmul(rhs) 61 | res2 = kern2(x1, x1).matmul(rhs) 62 | 63 | self.assertLess(torch.norm(res1 - res2), 1e-4) 64 | 65 | 66 | except ImportError: 67 | pass 68 | 69 | if __name__ == "__main__": 70 | unittest.main() 71 | -------------------------------------------------------------------------------- /test/kernels/test_cylindrical_kernel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import math 4 | import unittest 5 | 6 | import torch 7 | 8 | from gpytorch.kernels import CylindricalKernel, MaternKernel 9 | from gpytorch.test.base_kernel_test_case import BaseKernelTestCase 10 | 11 | 12 | class TestCylindricalKernel(unittest.TestCase, BaseKernelTestCase): 13 | def create_kernel_no_ard(self, **kwargs): 14 | return CylindricalKernel(5, MaternKernel(nu=2.5), **kwargs) 15 | 16 | def create_data_no_batch(self): 17 | return torch.rand(50, 10) / math.sqrt(10) 18 | 19 | def create_data_single_batch(self): 20 | return torch.rand(2, 50, 2) / math.sqrt(2) 21 | 22 | def create_data_double_batch(self): 23 | return torch.rand(3, 2, 50, 2) / math.sqrt(2) 24 | 25 | 26 | if __name__ == "__main__": 27 | unittest.main() 28 | -------------------------------------------------------------------------------- /test/kernels/test_grid_interpolation_kernel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.kernels import GridInterpolationKernel, RBFKernel 8 | from gpytorch.lazy import InterpolatedLazyTensor 9 | 10 | 11 | class TestGridInterpolationKernel(unittest.TestCase): 12 | def test_standard(self): 13 | base_kernel = RBFKernel() 14 | kernel = GridInterpolationKernel(base_kernel, num_dims=2, grid_size=128, grid_bounds=[(-1.2, 1.2)] * 2) 15 | 16 | xs = torch.randn(5, 2).clamp(-1, 1) 17 | interp_covar = kernel(xs, xs).evaluate_kernel() 18 | self.assertIsInstance(interp_covar, InterpolatedLazyTensor) 19 | 20 | xs = torch.randn(5, 2).clamp(-1, 1) 21 | grid_eval = kernel(xs, xs).evaluate() 22 | actual_eval = base_kernel(xs, xs).evaluate() 23 | self.assertLess(torch.norm(grid_eval - actual_eval), 2e-5) 24 | 25 | xs = torch.randn(3, 5, 2).clamp(-1, 1) 26 | grid_eval = kernel(xs, xs).evaluate() 27 | actual_eval = base_kernel(xs, xs).evaluate() 28 | self.assertLess(torch.norm(grid_eval - actual_eval), 2e-5) 29 | 30 | def test_batch_base_kernel(self): 31 | base_kernel = RBFKernel(batch_shape=torch.Size([3])) 32 | kernel = GridInterpolationKernel(base_kernel, num_dims=2, grid_size=128, grid_bounds=[(-1.2, 1.2)] * 2) 33 | 34 | xs = torch.randn(5, 2).clamp(-1, 1) 35 | grid_eval = kernel(xs, xs).evaluate() 36 | actual_eval = base_kernel(xs, xs).evaluate() 37 | self.assertLess(torch.norm(grid_eval - actual_eval), 2e-5) 38 | 39 | xs = torch.randn(3, 5, 2).clamp(-1, 1) 40 | grid_eval = kernel(xs, xs).evaluate() 41 | actual_eval = base_kernel(xs, xs).evaluate() 42 | self.assertLess(torch.norm(grid_eval - actual_eval), 2e-5) 43 | 44 | xs = torch.randn(4, 3, 5, 2).clamp(-1, 1) 45 | grid_eval = kernel(xs, xs).evaluate() 46 | actual_eval = base_kernel(xs, xs).evaluate() 47 | self.assertLess(torch.norm(grid_eval - actual_eval), 2e-5) 48 | 49 | 50 | if __name__ == "__main__": 51 | unittest.main() 52 | -------------------------------------------------------------------------------- /test/kernels/test_grid_kernel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.kernels import GridKernel, LinearKernel, RBFKernel 8 | from gpytorch.lazy import KroneckerProductLazyTensor 9 | from gpytorch.utils.grid import create_data_from_grid 10 | 11 | grid = [torch.linspace(0, 1, 5), torch.linspace(0, 2, 3)] 12 | d = len(grid) 13 | grid_data = create_data_from_grid(grid) 14 | 15 | 16 | class TestGridKernel(unittest.TestCase): 17 | def test_grid_grid(self): 18 | base_kernel = RBFKernel() 19 | kernel = GridKernel(base_kernel, grid) 20 | grid_covar = kernel(grid_data, grid_data).evaluate_kernel() 21 | self.assertIsInstance(grid_covar, KroneckerProductLazyTensor) 22 | grid_eval = kernel(grid_data, grid_data).evaluate() 23 | actual_eval = base_kernel(grid_data, grid_data).evaluate() 24 | self.assertLess(torch.norm(grid_eval - actual_eval), 2e-5) 25 | 26 | def test_nongrid_grid(self): 27 | base_kernel = RBFKernel() 28 | data = torch.randn(5, d) 29 | kernel = GridKernel(base_kernel, grid) 30 | grid_eval = kernel(grid_data, data).evaluate() 31 | actual_eval = base_kernel(grid_data, data).evaluate() 32 | self.assertLess(torch.norm(grid_eval - actual_eval), 1e-5) 33 | 34 | def test_nongrid_nongrid(self): 35 | base_kernel = RBFKernel() 36 | data = torch.randn(5, d) 37 | kernel = GridKernel(base_kernel, grid) 38 | grid_eval = kernel(data, data).evaluate() 39 | actual_eval = base_kernel(data, data).evaluate() 40 | self.assertLess(torch.norm(grid_eval - actual_eval), 1e-5) 41 | 42 | def test_non_stationary_base(self): 43 | base_kernel = LinearKernel() 44 | with self.assertRaisesRegex(RuntimeError, "The base_kernel for GridKernel must be stationary."): 45 | GridKernel(base_kernel, grid) 46 | 47 | 48 | if __name__ == "__main__": 49 | unittest.main() 50 | -------------------------------------------------------------------------------- /test/kernels/test_periodic_kernel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import math 4 | import unittest 5 | 6 | import torch 7 | 8 | from gpytorch.kernels import PeriodicKernel 9 | 10 | 11 | class TestPeriodicKernel(unittest.TestCase): 12 | def test_computes_periodic_function(self): 13 | a = torch.tensor([4, 2, 8], dtype=torch.float).view(3, 1) 14 | b = torch.tensor([0, 2], dtype=torch.float).view(2, 1) 15 | lengthscale = 2 16 | period = 3 17 | kernel = PeriodicKernel().initialize(lengthscale=lengthscale, period_length=period) 18 | kernel.eval() 19 | 20 | actual = torch.zeros(3, 2) 21 | for i in range(3): 22 | for j in range(2): 23 | val = 2 * torch.pow(torch.sin(math.pi * (a[i] - b[j]) / 3), 2) / lengthscale 24 | actual[i, j] = torch.exp(-val).item() 25 | 26 | res = kernel(a, b).evaluate() 27 | self.assertLess(torch.norm(res - actual), 1e-5) 28 | 29 | def test_batch(self): 30 | a = torch.tensor([[4, 2, 8], [1, 2, 3]], dtype=torch.float).view(2, 3, 1) 31 | b = torch.tensor([[0, 2], [-1, 2]], dtype=torch.float).view(2, 2, 1) 32 | period = torch.tensor(1, dtype=torch.float).view(1, 1, 1) 33 | lengthscale = torch.tensor(2, dtype=torch.float).view(1, 1, 1) 34 | kernel = PeriodicKernel().initialize(lengthscale=lengthscale, period_length=period) 35 | kernel.eval() 36 | 37 | actual = torch.zeros(2, 3, 2) 38 | for k in range(2): 39 | for i in range(3): 40 | for j in range(2): 41 | val = 2 * torch.pow(torch.sin(math.pi * (a[k, i] - b[k, j]) / period), 2) / lengthscale 42 | actual[k, i, j] = torch.exp(-val).item() 43 | 44 | res = kernel(a, b).evaluate() 45 | self.assertLess(torch.norm(res - actual), 1e-5) 46 | 47 | def test_batch_separate(self): 48 | a = torch.tensor([[4, 2, 8], [1, 2, 3]], dtype=torch.float).view(2, 3, 1) 49 | b = torch.tensor([[0, 2], [-1, 2]], dtype=torch.float).view(2, 2, 1) 50 | period = torch.tensor([1, 2], dtype=torch.float).view(2, 1, 1) 51 | lengthscale = torch.tensor([2, 1], dtype=torch.float).view(2, 1, 1) 52 | kernel = PeriodicKernel(batch_shape=torch.Size([2])).initialize(lengthscale=lengthscale, period_length=period) 53 | kernel.eval() 54 | 55 | actual = torch.zeros(2, 3, 2) 56 | for k in range(2): 57 | for i in range(3): 58 | for j in range(2): 59 | val = 2 * torch.pow(torch.sin(math.pi * (a[k, i] - b[k, j]) / period[k]), 2) / lengthscale[k] 60 | actual[k, i, j] = torch.exp(-val).item() 61 | 62 | res = kernel(a, b).evaluate() 63 | self.assertLess(torch.norm(res - actual), 1e-5) 64 | 65 | 66 | if __name__ == "__main__": 67 | unittest.main() 68 | -------------------------------------------------------------------------------- /test/kernels/test_polynomial_kernel_grad.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | from gpytorch.kernels import PolynomialKernelGrad 6 | from gpytorch.test.base_kernel_test_case import BaseKernelTestCase 7 | 8 | 9 | class TestPolynomialKernel(unittest.TestCase, BaseKernelTestCase): 10 | def create_kernel_no_ard(self, **kwargs): 11 | return PolynomialKernelGrad(power=2, **kwargs) 12 | 13 | 14 | if __name__ == "__main__": 15 | unittest.main() 16 | -------------------------------------------------------------------------------- /test/lazy/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | -------------------------------------------------------------------------------- /test/lazy/test_batch_repeat_lazy_tensor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch import lazify 8 | from gpytorch.lazy import BatchRepeatLazyTensor, ToeplitzLazyTensor 9 | from gpytorch.test.lazy_tensor_test_case import LazyTensorTestCase, RectangularLazyTensorTestCase 10 | 11 | 12 | class TestBatchRepeatLazyTensor(LazyTensorTestCase, unittest.TestCase): 13 | seed = 0 14 | 15 | def create_lazy_tensor(self): 16 | toeplitz_column = torch.tensor([4, 0.1, 0.05, 0.01, 0.0], dtype=torch.float) 17 | toeplitz_column.detach_() 18 | return BatchRepeatLazyTensor(ToeplitzLazyTensor(toeplitz_column), torch.Size((3,))) 19 | 20 | def evaluate_lazy_tensor(self, lazy_tensor): 21 | evaluated = lazy_tensor.base_lazy_tensor.evaluate() 22 | return evaluated.repeat(*lazy_tensor.batch_repeat, 1, 1) 23 | 24 | 25 | class TestBatchRepeatLazyTensorNonSquare(RectangularLazyTensorTestCase, unittest.TestCase): 26 | seed = 0 27 | 28 | def create_lazy_tensor(self): 29 | rand_mat = torch.randn(25, 12, dtype=torch.float) 30 | rand_mat.detach_() 31 | return BatchRepeatLazyTensor(lazify(rand_mat), torch.Size((10,))) 32 | 33 | def evaluate_lazy_tensor(self, lazy_tensor): 34 | evaluated = lazy_tensor.base_lazy_tensor.evaluate() 35 | return evaluated.repeat(*lazy_tensor.batch_repeat, 1, 1) 36 | 37 | 38 | class TestBatchRepeatLazyTensorBatch(LazyTensorTestCase, unittest.TestCase): 39 | seed = 0 40 | 41 | def create_lazy_tensor(self): 42 | toeplitz_column = torch.tensor([[4, 0, 0, 1], [3, 0, -0.5, -1]], dtype=torch.float) 43 | toeplitz_column.detach_() 44 | return BatchRepeatLazyTensor(ToeplitzLazyTensor(toeplitz_column), torch.Size((3,))) 45 | return BatchRepeatLazyTensor(ToeplitzLazyTensor(toeplitz_column), torch.Size((3,))) 46 | 47 | def evaluate_lazy_tensor(self, lazy_tensor): 48 | evaluated = lazy_tensor.base_lazy_tensor.evaluate() 49 | return evaluated.repeat(*lazy_tensor.batch_repeat, 1, 1) 50 | 51 | 52 | class TestBatchRepeatLazyTensorMultiBatch(LazyTensorTestCase, unittest.TestCase): 53 | seed = 0 54 | # Because these LTs are large, we'll skil the big tests 55 | skip_slq_tests = True 56 | 57 | def create_lazy_tensor(self): 58 | toeplitz_column = torch.tensor( 59 | [[[4, 0, 0, 1], [3, 0, -0.5, -1]], [[2, 0.1, 0.01, 0.0], [3, 0, -0.1, -2]]], dtype=torch.float 60 | ) 61 | toeplitz_column.detach_() 62 | return BatchRepeatLazyTensor(ToeplitzLazyTensor(toeplitz_column), torch.Size((2, 3, 1, 4))) 63 | 64 | def evaluate_lazy_tensor(self, lazy_tensor): 65 | evaluated = lazy_tensor.base_lazy_tensor.evaluate() 66 | return evaluated.repeat(*lazy_tensor.batch_repeat, 1, 1) 67 | 68 | 69 | if __name__ == "__main__": 70 | unittest.main() 71 | -------------------------------------------------------------------------------- /test/lazy/test_block_diag_lazy_tensor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.lazy import BlockDiagLazyTensor, NonLazyTensor 8 | from gpytorch.test.lazy_tensor_test_case import LazyTensorTestCase 9 | 10 | 11 | class TestBlockDiagLazyTensor(LazyTensorTestCase, unittest.TestCase): 12 | seed = 0 13 | should_test_sample = True 14 | 15 | def create_lazy_tensor(self): 16 | blocks = torch.randn(8, 4, 4) 17 | blocks = blocks.matmul(blocks.transpose(-1, -2)) 18 | blocks.add_(torch.eye(4, 4).unsqueeze_(0)) 19 | return BlockDiagLazyTensor(NonLazyTensor(blocks)) 20 | 21 | def evaluate_lazy_tensor(self, lazy_tensor): 22 | blocks = lazy_tensor.base_lazy_tensor.tensor 23 | actual = torch.zeros(32, 32) 24 | for i in range(8): 25 | actual[i * 4 : (i + 1) * 4, i * 4 : (i + 1) * 4] = blocks[i] 26 | return actual 27 | 28 | 29 | class TestBlockDiagLazyTensorBatch(LazyTensorTestCase, unittest.TestCase): 30 | seed = 0 31 | should_test_sample = True 32 | 33 | def create_lazy_tensor(self): 34 | blocks = torch.randn(2, 6, 4, 4) 35 | blocks = blocks.matmul(blocks.transpose(-1, -2)) 36 | blocks.add_(torch.eye(4, 4)) 37 | return BlockDiagLazyTensor(NonLazyTensor(blocks), block_dim=2) 38 | 39 | def evaluate_lazy_tensor(self, lazy_tensor): 40 | blocks = lazy_tensor.base_lazy_tensor.tensor 41 | actual = torch.zeros(2, 24, 24) 42 | for i in range(2): 43 | for j in range(6): 44 | actual[i, j * 4 : (j + 1) * 4, j * 4 : (j + 1) * 4] = blocks[i, j] 45 | return actual 46 | 47 | 48 | class TestBlockDiagLazyTensorMultiBatch(LazyTensorTestCase, unittest.TestCase): 49 | seed = 0 50 | # Because these LTs are large, we'll skil the big tests 51 | should_test_sample = False 52 | skip_slq_tests = True 53 | 54 | def create_lazy_tensor(self): 55 | blocks = torch.randn(2, 6, 5, 4, 4) 56 | blocks = blocks.matmul(blocks.transpose(-1, -2)) 57 | blocks.add_(torch.eye(4, 4)) 58 | blocks.detach_() 59 | return BlockDiagLazyTensor(NonLazyTensor(blocks), block_dim=1) 60 | 61 | def evaluate_lazy_tensor(self, lazy_tensor): 62 | blocks = lazy_tensor.base_lazy_tensor.tensor 63 | actual = torch.zeros(2, 5, 24, 24) 64 | for i in range(2): 65 | for j in range(6): 66 | for k in range(5): 67 | actual[i, k, j * 4 : (j + 1) * 4, j * 4 : (j + 1) * 4] = blocks[i, k, j] 68 | return actual 69 | 70 | 71 | if __name__ == "__main__": 72 | unittest.main() 73 | -------------------------------------------------------------------------------- /test/lazy/test_chol_lazy_tensor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.lazy import CholLazyTensor 8 | from gpytorch.test.lazy_tensor_test_case import LazyTensorTestCase 9 | 10 | 11 | class TestCholLazyTensor(LazyTensorTestCase, unittest.TestCase): 12 | seed = 0 13 | should_test_sample = True 14 | should_call_cg = False 15 | should_call_lanczos = False 16 | 17 | def create_lazy_tensor(self): 18 | chol = torch.tensor( 19 | [[3, 0, 0, 0, 0], [-1, 2, 0, 0, 0], [1, 4, 1, 0, 0], [0, 2, 3, 2, 0], [-4, -2, 1, 3, 4]], 20 | dtype=torch.float, 21 | requires_grad=True, 22 | ) 23 | return CholLazyTensor(chol) 24 | 25 | def evaluate_lazy_tensor(self, lazy_tensor): 26 | chol = lazy_tensor.root.evaluate() 27 | return chol.matmul(chol.transpose(-1, -2)) 28 | 29 | 30 | class TestCholLazyTensorBatch(TestCholLazyTensor): 31 | seed = 0 32 | 33 | def create_lazy_tensor(self): 34 | chol = torch.tensor( 35 | [ 36 | [[3, 0, 0, 0, 0], [-1, 2, 0, 0, 0], [1, 4, 1, 0, 0], [0, 2, 3, 2, 0], [-4, -2, 1, 3, 4]], 37 | [[2, 0, 0, 0, 0], [3, 1, 0, 0, 0], [-2, 3, 2, 0, 0], [-2, 1, -1, 3, 0], [-4, -4, 5, 2, 3]], 38 | ], 39 | dtype=torch.float, 40 | ) 41 | chol.add_(torch.eye(5).unsqueeze(0)) 42 | chol.requires_grad_(True) 43 | return CholLazyTensor(chol) 44 | 45 | 46 | class TestCholLazyTensorMultiBatch(TestCholLazyTensor): 47 | seed = 0 48 | # Because these LTs are large, we'll skil the big tests 49 | should_test_sample = False 50 | skip_slq_tests = True 51 | 52 | def create_lazy_tensor(self): 53 | chol = torch.tensor( 54 | [ 55 | [[3, 0, 0, 0, 0], [-1, 2, 0, 0, 0], [1, 4, 1, 0, 0], [0, 2, 3, 2, 0], [-4, -2, 1, 3, 4]], 56 | [[2, 0, 0, 0, 0], [3, 1, 0, 0, 0], [-2, 3, 2, 0, 0], [-2, 1, -1, 3, 0], [-4, -4, 5, 2, 3]], 57 | ], 58 | dtype=torch.float, 59 | ) 60 | chol = chol.repeat(3, 1, 1, 1) 61 | chol[1].mul_(2) 62 | chol[2].mul_(0.5) 63 | chol.add_(torch.eye(5).unsqueeze_(0).unsqueeze_(0)) 64 | chol.requires_grad_(True) 65 | return CholLazyTensor(chol) 66 | 67 | 68 | if __name__ == "__main__": 69 | unittest.main() 70 | -------------------------------------------------------------------------------- /test/lazy/test_diag_lazy_tensor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.lazy import DiagLazyTensor 8 | from gpytorch.test.lazy_tensor_test_case import LazyTensorTestCase 9 | 10 | 11 | class TestDiagLazyTensor(LazyTensorTestCase, unittest.TestCase): 12 | seed = 0 13 | should_test_sample = True 14 | should_call_cg = False 15 | should_call_lanczos = False 16 | 17 | def create_lazy_tensor(self): 18 | diag = torch.tensor([1.0, 2.0, 4.0, 2.0, 3.0], requires_grad=True) 19 | return DiagLazyTensor(diag) 20 | 21 | def evaluate_lazy_tensor(self, lazy_tensor): 22 | diag = lazy_tensor._diag 23 | return diag.diag() 24 | 25 | 26 | class TestDiagLazyTensorBatch(TestDiagLazyTensor): 27 | seed = 0 28 | 29 | def create_lazy_tensor(self): 30 | diag = torch.tensor( 31 | [[1.0, 2.0, 4.0, 2.0, 3.0], [2.0, 1.0, 2.0, 1.0, 4.0], [1.0, 2.0, 2.0, 3.0, 4.0]], requires_grad=True 32 | ) 33 | return DiagLazyTensor(diag) 34 | 35 | def evaluate_lazy_tensor(self, lazy_tensor): 36 | diag = lazy_tensor._diag 37 | return torch.cat([diag[i].diag().unsqueeze(0) for i in range(3)]) 38 | 39 | 40 | class TestDiagLazyTensorMultiBatch(TestDiagLazyTensor): 41 | seed = 0 42 | # Because these LTs are large, we'll skil the big tests 43 | should_test_sample = True 44 | skip_slq_tests = True 45 | 46 | def create_lazy_tensor(self): 47 | diag = torch.randn(6, 3, 5).pow_(2) 48 | diag.requires_grad_(True) 49 | return DiagLazyTensor(diag) 50 | 51 | def evaluate_lazy_tensor(self, lazy_tensor): 52 | diag = lazy_tensor._diag 53 | flattened_diag = diag.view(-1, diag.size(-1)) 54 | res = torch.cat([flattened_diag[i].diag().unsqueeze(0) for i in range(18)]) 55 | return res.view(6, 3, 5, 5) 56 | 57 | 58 | if __name__ == "__main__": 59 | unittest.main() 60 | -------------------------------------------------------------------------------- /test/lazy/test_matmul_lazy_tensor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.lazy import MatmulLazyTensor 8 | from gpytorch.test.lazy_tensor_test_case import LazyTensorTestCase, RectangularLazyTensorTestCase 9 | 10 | 11 | class TestMatmulLazyTensor(LazyTensorTestCase, unittest.TestCase): 12 | seed = 1 13 | 14 | def create_lazy_tensor(self): 15 | lhs = torch.randn(5, 6, requires_grad=True) 16 | rhs = lhs.clone().detach().transpose(-1, -2) 17 | covar = MatmulLazyTensor(lhs, rhs) 18 | return covar 19 | 20 | def evaluate_lazy_tensor(self, lazy_tensor): 21 | return lazy_tensor.left_lazy_tensor.tensor.matmul(lazy_tensor.right_lazy_tensor.tensor) 22 | 23 | 24 | class TestMatmulLazyTensorBatch(LazyTensorTestCase, unittest.TestCase): 25 | seed = 3 26 | 27 | def create_lazy_tensor(self): 28 | lhs = torch.randn(5, 5, 6, requires_grad=True) 29 | rhs = lhs.clone().detach().transpose(-1, -2) 30 | covar = MatmulLazyTensor(lhs, rhs) 31 | return covar 32 | 33 | def evaluate_lazy_tensor(self, lazy_tensor): 34 | return lazy_tensor.left_lazy_tensor.tensor.matmul(lazy_tensor.right_lazy_tensor.tensor) 35 | 36 | 37 | class TestMatmulLazyTensorRectangular(RectangularLazyTensorTestCase, unittest.TestCase): 38 | def create_lazy_tensor(self): 39 | lhs = torch.randn(5, 3, requires_grad=True) 40 | rhs = torch.randn(3, 6, requires_grad=True) 41 | covar = MatmulLazyTensor(lhs, rhs) 42 | return covar 43 | 44 | def evaluate_lazy_tensor(self, lazy_tensor): 45 | return lazy_tensor.left_lazy_tensor.tensor.matmul(lazy_tensor.right_lazy_tensor.tensor) 46 | 47 | 48 | class TestMatmulLazyTensorRectangularMultiBatch(RectangularLazyTensorTestCase, unittest.TestCase): 49 | def create_lazy_tensor(self): 50 | lhs = torch.randn(2, 3, 5, 3, requires_grad=True) 51 | rhs = torch.randn(2, 3, 3, 6, requires_grad=True) 52 | covar = MatmulLazyTensor(lhs, rhs) 53 | return covar 54 | 55 | def evaluate_lazy_tensor(self, lazy_tensor): 56 | return lazy_tensor.left_lazy_tensor.tensor.matmul(lazy_tensor.right_lazy_tensor.tensor) 57 | 58 | 59 | if __name__ == "__main__": 60 | unittest.main() 61 | -------------------------------------------------------------------------------- /test/lazy/test_mul_lazy_tensor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.lazy import RootLazyTensor 8 | from gpytorch.test.lazy_tensor_test_case import LazyTensorTestCase 9 | 10 | 11 | def make_random_mat(size, rank, batch_shape=torch.Size(())): 12 | res = torch.randn(*batch_shape, size, rank) 13 | return res 14 | 15 | 16 | class TestMulLazyTensor(LazyTensorTestCase, unittest.TestCase): 17 | seed = 10 18 | 19 | def create_lazy_tensor(self): 20 | mat1 = make_random_mat(6, 6) 21 | mat2 = make_random_mat(6, 6) 22 | res = RootLazyTensor(mat1) * RootLazyTensor(mat2) 23 | return res.add_diag(torch.tensor(2.0)) 24 | 25 | def evaluate_lazy_tensor(self, lazy_tensor): 26 | diag_tensor = lazy_tensor._diag_tensor.evaluate() 27 | res = torch.mul( 28 | lazy_tensor._lazy_tensor.left_lazy_tensor.evaluate(), lazy_tensor._lazy_tensor.right_lazy_tensor.evaluate() 29 | ) 30 | res = res + diag_tensor 31 | return res 32 | 33 | 34 | class TestMulLazyTensorBatch(LazyTensorTestCase, unittest.TestCase): 35 | seed = 2 36 | 37 | def create_lazy_tensor(self): 38 | mat1 = make_random_mat(6, rank=6, batch_shape=torch.Size((2,))) 39 | mat2 = make_random_mat(6, rank=6, batch_shape=torch.Size((2,))) 40 | res = RootLazyTensor(mat1) * RootLazyTensor(mat2) 41 | return res.add_diag(torch.tensor(2.0)) 42 | 43 | def evaluate_lazy_tensor(self, lazy_tensor): 44 | diag_tensor = lazy_tensor._diag_tensor.evaluate() 45 | res = torch.mul( 46 | lazy_tensor._lazy_tensor.left_lazy_tensor.evaluate(), lazy_tensor._lazy_tensor.right_lazy_tensor.evaluate() 47 | ) 48 | res = res + diag_tensor 49 | return res 50 | 51 | 52 | class TestMulLazyTensorMultiBatch(LazyTensorTestCase, unittest.TestCase): 53 | seed = 1 54 | skip_slq_tests = True 55 | 56 | def create_lazy_tensor(self): 57 | mat1 = make_random_mat(6, rank=6, batch_shape=torch.Size((2, 3))) 58 | mat2 = make_random_mat(6, rank=6, batch_shape=torch.Size((2, 3))) 59 | res = RootLazyTensor(mat1) * RootLazyTensor(mat2) 60 | return res.add_diag(torch.tensor(0.5)) 61 | 62 | def evaluate_lazy_tensor(self, lazy_tensor): 63 | diag_tensor = lazy_tensor._diag_tensor.evaluate() 64 | res = torch.mul( 65 | lazy_tensor._lazy_tensor.left_lazy_tensor.evaluate(), lazy_tensor._lazy_tensor.right_lazy_tensor.evaluate() 66 | ) 67 | res = res + diag_tensor 68 | return res 69 | 70 | def test_inv_quad_logdet(self): 71 | pass 72 | 73 | 74 | if __name__ == "__main__": 75 | unittest.main() 76 | -------------------------------------------------------------------------------- /test/lazy/test_non_lazy_tensor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | import gpytorch 8 | from gpytorch.lazy import NonLazyTensor 9 | from gpytorch.test.lazy_tensor_test_case import LazyTensorTestCase 10 | 11 | 12 | class TestNonLazyTensor(LazyTensorTestCase, unittest.TestCase): 13 | seed = 0 14 | 15 | def create_lazy_tensor(self): 16 | mat = torch.randn(5, 6) 17 | mat = mat.matmul(mat.transpose(-1, -2)) 18 | mat.requires_grad_(True) 19 | return NonLazyTensor(mat) 20 | 21 | def evaluate_lazy_tensor(self, lazy_tensor): 22 | return lazy_tensor.tensor 23 | 24 | def test_root_decomposition_exact(self): 25 | lazy_tensor = self.create_lazy_tensor() 26 | test_mat = torch.randn(*lazy_tensor.batch_shape, lazy_tensor.size(-1), 5) 27 | with gpytorch.settings.fast_computations(covar_root_decomposition=False): 28 | root_approx = lazy_tensor.root_decomposition() 29 | res = root_approx.matmul(test_mat) 30 | actual = lazy_tensor.matmul(test_mat) 31 | self.assertLess(torch.norm(res - actual) / actual.norm(), 0.1) 32 | 33 | 34 | class TestNonLazyTensorBatch(LazyTensorTestCase, unittest.TestCase): 35 | seed = 0 36 | 37 | def create_lazy_tensor(self): 38 | mat = torch.randn(3, 5, 6) 39 | mat = mat.matmul(mat.transpose(-1, -2)) 40 | mat.requires_grad_(True) 41 | return NonLazyTensor(mat) 42 | 43 | def evaluate_lazy_tensor(self, lazy_tensor): 44 | return lazy_tensor.tensor 45 | 46 | def test_root_decomposition_exact(self): 47 | lazy_tensor = self.create_lazy_tensor() 48 | test_mat = torch.randn(*lazy_tensor.batch_shape, lazy_tensor.size(-1), 5) 49 | with gpytorch.settings.fast_computations(covar_root_decomposition=False): 50 | root_approx = lazy_tensor.root_decomposition() 51 | res = root_approx.matmul(test_mat) 52 | actual = lazy_tensor.matmul(test_mat) 53 | self.assertLess(torch.norm(res - actual) / actual.norm(), 0.1) 54 | 55 | 56 | class TestNonLazyTensorMultiBatch(LazyTensorTestCase, unittest.TestCase): 57 | seed = 0 58 | # Because these LTs are large, we'll skil the big tests 59 | should_test_sample = False 60 | skip_slq_tests = True 61 | 62 | def create_lazy_tensor(self): 63 | mat = torch.randn(2, 3, 5, 6) 64 | mat = mat.matmul(mat.transpose(-1, -2)) 65 | mat.requires_grad_(True) 66 | return NonLazyTensor(mat) 67 | 68 | def evaluate_lazy_tensor(self, lazy_tensor): 69 | return lazy_tensor.tensor 70 | -------------------------------------------------------------------------------- /test/lazy/test_psd_sum_lazy_tensor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.lazy import NonLazyTensor, PsdSumLazyTensor, ToeplitzLazyTensor 8 | from gpytorch.test.lazy_tensor_test_case import LazyTensorTestCase 9 | 10 | 11 | class TestPsdSumLazyTensor(LazyTensorTestCase, unittest.TestCase): 12 | seed = 0 13 | should_test_sample = True 14 | 15 | def create_lazy_tensor(self): 16 | c1 = torch.tensor([5, 1, 2, 0], dtype=torch.float, requires_grad=True) 17 | t1 = ToeplitzLazyTensor(c1) 18 | c2 = torch.tensor([6, 0, 1, -1], dtype=torch.float, requires_grad=True) 19 | t2 = ToeplitzLazyTensor(c2) 20 | return PsdSumLazyTensor(t1, t2) 21 | 22 | def evaluate_lazy_tensor(self, lazy_tensor): 23 | tensors = [lt.evaluate() for lt in lazy_tensor.lazy_tensors] 24 | return sum(tensors) 25 | 26 | 27 | class TestPsdSumLazyTensorBatch(LazyTensorTestCase, unittest.TestCase): 28 | seed = 0 29 | should_test_sample = True 30 | 31 | def create_lazy_tensor(self): 32 | c1 = torch.tensor([[2, 0.5, 0, 0], [5, 1, 2, 0]], dtype=torch.float, requires_grad=True) 33 | t1 = ToeplitzLazyTensor(c1) 34 | c2 = torch.tensor([[2, 0.5, 0, 0], [6, 0, 1, -1]], dtype=torch.float, requires_grad=True) 35 | t2 = ToeplitzLazyTensor(c2) 36 | return PsdSumLazyTensor(t1, t2) 37 | 38 | def evaluate_lazy_tensor(self, lazy_tensor): 39 | tensors = [lt.evaluate() for lt in lazy_tensor.lazy_tensors] 40 | return sum(tensors) 41 | 42 | 43 | class TestPsdSumLazyTensorMultiBatch(LazyTensorTestCase, unittest.TestCase): 44 | seed = 0 45 | # Because these LTs are large, we'll skil the big tests 46 | should_test_sample = False 47 | skip_slq_tests = True 48 | 49 | def create_lazy_tensor(self): 50 | mat1 = torch.randn(2, 3, 4, 4) 51 | lt1 = NonLazyTensor(mat1 @ mat1.transpose(-1, -2)) 52 | mat2 = torch.randn(2, 3, 4, 4) 53 | lt2 = NonLazyTensor(mat2 @ mat2.transpose(-1, -2)) 54 | return PsdSumLazyTensor(lt1, lt2) 55 | 56 | def evaluate_lazy_tensor(self, lazy_tensor): 57 | tensors = [lt.evaluate() for lt in lazy_tensor.lazy_tensors] 58 | return sum(tensors) 59 | 60 | 61 | if __name__ == "__main__": 62 | unittest.main() 63 | -------------------------------------------------------------------------------- /test/lazy/test_root_lazy_tensor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.lazy import RootLazyTensor 8 | from gpytorch.test.lazy_tensor_test_case import LazyTensorTestCase 9 | 10 | 11 | class TestRootLazyTensor(LazyTensorTestCase, unittest.TestCase): 12 | seed = 0 13 | should_test_sample = True 14 | should_call_lanczos = False 15 | 16 | def create_lazy_tensor(self): 17 | root = torch.randn(3, 5, requires_grad=True) 18 | return RootLazyTensor(root) 19 | 20 | def evaluate_lazy_tensor(self, lazy_tensor): 21 | root = lazy_tensor.root.tensor 22 | res = root.matmul(root.transpose(-1, -2)) 23 | return res 24 | 25 | 26 | class TestRootLazyTensorBatch(TestRootLazyTensor): 27 | seed = 1 28 | 29 | def create_lazy_tensor(self): 30 | root = torch.randn(3, 5, 5) 31 | root.add_(torch.eye(5).unsqueeze(0)) 32 | root.requires_grad_(True) 33 | return RootLazyTensor(root) 34 | 35 | 36 | class TestRootLazyTensorMultiBatch(TestRootLazyTensor): 37 | seed = 1 38 | # Because these LTs are large, we'll skil the big tests 39 | should_test_sample = False 40 | skip_slq_tests = True 41 | 42 | def create_lazy_tensor(self): 43 | root = torch.randn(4, 3, 5, 5) 44 | root.requires_grad_(True) 45 | return RootLazyTensor(root) 46 | 47 | 48 | if __name__ == "__main__": 49 | unittest.main() 50 | -------------------------------------------------------------------------------- /test/lazy/test_sum_batch_lazy_tensor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.lazy import NonLazyTensor, SumBatchLazyTensor 8 | from gpytorch.test.lazy_tensor_test_case import LazyTensorTestCase 9 | 10 | 11 | class TestSumBatchLazyTensor(LazyTensorTestCase, unittest.TestCase): 12 | seed = 6 13 | should_test_sample = True 14 | 15 | def create_lazy_tensor(self): 16 | blocks = torch.randn(12, 4, 4) 17 | blocks = blocks.transpose(-1, -2).matmul(blocks) 18 | blocks.requires_grad_(True) 19 | return SumBatchLazyTensor(NonLazyTensor(blocks)) 20 | 21 | def evaluate_lazy_tensor(self, lazy_tensor): 22 | blocks = lazy_tensor.base_lazy_tensor.tensor 23 | return blocks.sum(0) 24 | 25 | 26 | class TestSumBatchLazyTensorBatch(LazyTensorTestCase, unittest.TestCase): 27 | seed = 6 28 | should_test_sample = True 29 | 30 | def create_lazy_tensor(self): 31 | blocks = torch.randn(2, 6, 4, 4) 32 | blocks = blocks.transpose(-1, -2).matmul(blocks) 33 | blocks.requires_grad_(True) 34 | return SumBatchLazyTensor(NonLazyTensor(blocks)) 35 | 36 | def evaluate_lazy_tensor(self, lazy_tensor): 37 | blocks = lazy_tensor.base_lazy_tensor.tensor 38 | return blocks.view(2, 6, 4, 4).sum(1) 39 | 40 | 41 | class TestSumBatchLazyTensorMultiBatch(LazyTensorTestCase, unittest.TestCase): 42 | seed = 6 43 | # Because these LTs are large, we'll skil the big tests 44 | should_test_sample = False 45 | skip_slq_tests = True 46 | 47 | def create_lazy_tensor(self): 48 | blocks = torch.randn(2, 3, 6, 4, 4) 49 | blocks = blocks.transpose(-1, -2).matmul(blocks) 50 | blocks.detach_() 51 | return SumBatchLazyTensor(NonLazyTensor(blocks), block_dim=1) 52 | 53 | def evaluate_lazy_tensor(self, lazy_tensor): 54 | blocks = lazy_tensor.base_lazy_tensor.tensor 55 | return blocks.sum(-3) 56 | 57 | 58 | if __name__ == "__main__": 59 | unittest.main() 60 | -------------------------------------------------------------------------------- /test/lazy/test_toeplitz_lazy_tensor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | import gpytorch.utils.toeplitz as toeplitz 8 | from gpytorch.lazy import ToeplitzLazyTensor 9 | from gpytorch.test.lazy_tensor_test_case import LazyTensorTestCase 10 | 11 | 12 | class TestToeplitzLazyTensor(LazyTensorTestCase, unittest.TestCase): 13 | seed = 1 14 | 15 | def create_lazy_tensor(self): 16 | toeplitz_column = torch.tensor([4, 0.5, 0, 1], dtype=torch.float, requires_grad=True) 17 | return ToeplitzLazyTensor(toeplitz_column) 18 | 19 | def evaluate_lazy_tensor(self, lazy_tensor): 20 | return toeplitz.sym_toeplitz(lazy_tensor.column) 21 | 22 | 23 | class TestToeplitzLazyTensorBatch(LazyTensorTestCase, unittest.TestCase): 24 | seed = 0 25 | 26 | def create_lazy_tensor(self): 27 | toeplitz_column = torch.tensor([[2, -1, 0.5, 0.25], [4, 0.5, 0, 1]], dtype=torch.float, requires_grad=True) 28 | return ToeplitzLazyTensor(toeplitz_column) 29 | 30 | def evaluate_lazy_tensor(self, lazy_tensor): 31 | return torch.cat( 32 | [ 33 | toeplitz.sym_toeplitz(lazy_tensor.column[0]).unsqueeze(0), 34 | toeplitz.sym_toeplitz(lazy_tensor.column[1]).unsqueeze(0), 35 | ] 36 | ) 37 | 38 | 39 | class TestToeplitzLazyTensorMultiBatch(LazyTensorTestCase, unittest.TestCase): 40 | seed = 0 41 | 42 | def create_lazy_tensor(self): 43 | toeplitz_column = torch.tensor([[2, -1, 0.5, 0.25], [4, 0.5, 0, 1]], dtype=torch.float) 44 | toeplitz_column = toeplitz_column.repeat(3, 1, 1) 45 | toeplitz_column.requires_grad_(True) 46 | return ToeplitzLazyTensor(toeplitz_column) 47 | 48 | def evaluate_lazy_tensor(self, lazy_tensor): 49 | return torch.cat( 50 | [ 51 | toeplitz.sym_toeplitz(lazy_tensor.column[0, 0]).unsqueeze(0), 52 | toeplitz.sym_toeplitz(lazy_tensor.column[0, 1]).unsqueeze(0), 53 | toeplitz.sym_toeplitz(lazy_tensor.column[1, 0]).unsqueeze(0), 54 | toeplitz.sym_toeplitz(lazy_tensor.column[1, 1]).unsqueeze(0), 55 | toeplitz.sym_toeplitz(lazy_tensor.column[2, 0]).unsqueeze(0), 56 | toeplitz.sym_toeplitz(lazy_tensor.column[2, 1]).unsqueeze(0), 57 | ] 58 | ).view(3, 2, 4, 4) 59 | 60 | 61 | if __name__ == "__main__": 62 | unittest.main() 63 | -------------------------------------------------------------------------------- /test/likelihoods/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | -------------------------------------------------------------------------------- /test/likelihoods/test_bernoulli_likelihood.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.likelihoods import BernoulliLikelihood, _OneDimensionalLikelihood 8 | from gpytorch.test.base_likelihood_test_case import BaseLikelihoodTestCase 9 | 10 | 11 | class TestBernoulliLikelihood(BaseLikelihoodTestCase, unittest.TestCase): 12 | seed = 1 13 | 14 | def _create_targets(self, batch_shape=torch.Size([])): 15 | res = torch.randn(*batch_shape, 5).gt(0).float() 16 | return res 17 | 18 | def _test_log_marginal(self, batch_shape): 19 | # Overwriting this because use use probit, not logit 20 | # The values are close, but not exact 21 | # So we use looser checks 22 | likelihood = self.create_likelihood() 23 | input = self._create_marginal_input(batch_shape) 24 | target = self._create_targets(batch_shape) 25 | output = likelihood.log_marginal(target, input) 26 | 27 | self.assertTrue(torch.is_tensor(output)) 28 | self.assertEqual(output.shape, batch_shape + torch.Size([5])) 29 | default_log_prob = _OneDimensionalLikelihood.log_marginal(likelihood, target, input) 30 | self.assertAllClose(output.sum(-1), default_log_prob.sum(-1), rtol=0.25, atol=0.1) 31 | 32 | def _test_log_prob(self, batch_shape): 33 | # Overwriting this because use use probit, not logit 34 | # The values are close, but not exact 35 | # So we use looser checks 36 | likelihood = self.create_likelihood() 37 | input = self._create_marginal_input(batch_shape) 38 | target = self._create_targets(batch_shape) 39 | output = likelihood.expected_log_prob(target, input) 40 | 41 | self.assertTrue(torch.is_tensor(output)) 42 | self.assertEqual(output.shape, batch_shape + torch.Size([5])) 43 | default_log_prob = _OneDimensionalLikelihood.expected_log_prob(likelihood, target, input) 44 | self.assertAllClose(output.sum(-1), default_log_prob.sum(-1), rtol=0.25, atol=0.1) 45 | 46 | def create_likelihood(self): 47 | return BernoulliLikelihood() 48 | -------------------------------------------------------------------------------- /test/likelihoods/test_multitask_gaussian_likelihood.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.distributions import MultitaskMultivariateNormal 8 | from gpytorch.lazy import KroneckerProductLazyTensor, RootLazyTensor 9 | from gpytorch.likelihoods import MultitaskGaussianLikelihood 10 | from gpytorch.test.base_likelihood_test_case import BaseLikelihoodTestCase 11 | 12 | 13 | class TestMultitaskGaussianLikelihood(BaseLikelihoodTestCase, unittest.TestCase): 14 | seed = 2 15 | 16 | def _create_conditional_input(self, batch_shape=torch.Size([])): 17 | return torch.randn(*batch_shape, 5, 4) 18 | 19 | def _create_marginal_input(self, batch_shape=torch.Size([])): 20 | mat = torch.randn(*batch_shape, 5, 5) 21 | mat2 = torch.randn(*batch_shape, 4, 4) 22 | covar = KroneckerProductLazyTensor(RootLazyTensor(mat), RootLazyTensor(mat2)) 23 | return MultitaskMultivariateNormal(torch.randn(*batch_shape, 5, 4), covar) 24 | 25 | def _create_targets(self, batch_shape=torch.Size([])): 26 | return torch.randn(*batch_shape, 5, 4) 27 | 28 | def create_likelihood(self): 29 | return MultitaskGaussianLikelihood(num_tasks=4, rank=2) 30 | 31 | 32 | class TestMultitaskGaussianLikelihoodBatch(TestMultitaskGaussianLikelihood): 33 | seed = 0 34 | 35 | def create_likelihood(self): 36 | return MultitaskGaussianLikelihood(num_tasks=4, rank=2, batch_shape=torch.Size([3])) 37 | 38 | def test_nonbatch(self): 39 | pass 40 | -------------------------------------------------------------------------------- /test/likelihoods/test_softmax_likelihood.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | from torch.distributions import Distribution 7 | 8 | from gpytorch.distributions import MultivariateNormal 9 | from gpytorch.likelihoods import SoftmaxLikelihood 10 | from gpytorch.test.base_likelihood_test_case import BaseLikelihoodTestCase 11 | 12 | 13 | class TestSoftmaxLikelihood(BaseLikelihoodTestCase, unittest.TestCase): 14 | seed = 0 15 | 16 | def _create_conditional_input(self, batch_shape=torch.Size([])): 17 | return torch.randn(*batch_shape, 6, 5) 18 | 19 | def _create_marginal_input(self, batch_shape=torch.Size([])): 20 | mat = torch.randn(*batch_shape, 6, 5, 5) 21 | return MultivariateNormal(torch.randn(*batch_shape, 6, 5), mat @ mat.transpose(-1, -2)) 22 | 23 | def _create_targets(self, batch_shape=torch.Size([])): 24 | return torch.distributions.Categorical(probs=torch.tensor([0.25, 0.25, 0.25, 0.25])).sample( 25 | torch.Size([*batch_shape, 5]) 26 | ) 27 | 28 | def create_likelihood(self): 29 | return SoftmaxLikelihood(num_features=6, num_classes=4) 30 | 31 | def _test_conditional(self, batch_shape): 32 | likelihood = self.create_likelihood() 33 | input = self._create_conditional_input(batch_shape) 34 | output = likelihood(input) 35 | 36 | self.assertIsInstance(output, Distribution) 37 | self.assertEqual(output.sample().shape, torch.Size([*batch_shape, 5])) 38 | 39 | def _test_log_prob(self, batch_shape): 40 | likelihood = self.create_likelihood() 41 | input = self._create_marginal_input(batch_shape) 42 | target = self._create_targets(batch_shape) 43 | output = likelihood.expected_log_prob(target, input) 44 | 45 | self.assertTrue(torch.is_tensor(output)) 46 | self.assertEqual(output.shape, batch_shape + torch.Size([5])) 47 | 48 | def _test_marginal(self, batch_shape): 49 | likelihood = self.create_likelihood() 50 | input = self._create_marginal_input(batch_shape) 51 | output = likelihood(input) 52 | 53 | self.assertTrue(isinstance(output, Distribution)) 54 | self.assertEqual(output.sample().shape[-len(batch_shape) - 1 :], torch.Size([*batch_shape, 5])) 55 | 56 | 57 | class TestSoftmaxLikelihoodNoMixing(TestSoftmaxLikelihood): 58 | seed = 0 59 | 60 | def create_likelihood(self): 61 | return SoftmaxLikelihood(num_features=6, num_classes=6, mixing_weights=False) 62 | -------------------------------------------------------------------------------- /test/means/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | -------------------------------------------------------------------------------- /test/means/test_constant_mean.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.means import ConstantMean 8 | from gpytorch.test.base_mean_test_case import BaseMeanTestCase 9 | 10 | 11 | class TestConstantMean(BaseMeanTestCase, unittest.TestCase): 12 | def create_mean(self): 13 | return ConstantMean() 14 | 15 | 16 | class TestConstantMeanBatch(BaseMeanTestCase, unittest.TestCase): 17 | batch_shape = torch.Size([3]) 18 | 19 | def create_mean(self): 20 | return ConstantMean(batch_shape=self.__class__.batch_shape) 21 | 22 | 23 | class TestConstantMeanMultiBatch(BaseMeanTestCase, unittest.TestCase): 24 | batch_shape = torch.Size([2, 3]) 25 | 26 | def create_mean(self): 27 | return ConstantMean(batch_shape=self.__class__.batch_shape) 28 | -------------------------------------------------------------------------------- /test/means/test_constant_mean_grad.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.means import ConstantMeanGrad 8 | from gpytorch.test.base_mean_test_case import BaseMeanTestCase 9 | 10 | 11 | class TestConstantMeanGrad(BaseMeanTestCase, unittest.TestCase): 12 | batch_shape = None 13 | 14 | def create_mean(self): 15 | return ConstantMeanGrad(batch_shape=self.__class__.batch_shape or torch.Size()) 16 | 17 | def test_forward_vec(self): 18 | test_x = torch.randn(4) 19 | mean = self.create_mean() 20 | if self.__class__.batch_shape is None: 21 | self.assertEqual(mean(test_x).shape, torch.Size([4, 2])) 22 | else: 23 | self.assertEqual(mean(test_x).shape, torch.Size([*self.__class__.batch_shape, 4, 2])) 24 | self.assertEqual(mean(test_x)[..., 1:].norm().item(), 0) 25 | 26 | def test_forward_mat(self): 27 | test_x = torch.randn(4, 3) 28 | mean = self.create_mean() 29 | if self.__class__.batch_shape is None: 30 | self.assertEqual(mean(test_x).shape, torch.Size([4, 4])) 31 | else: 32 | self.assertEqual(mean(test_x).shape, torch.Size([*self.__class__.batch_shape, 4, 4])) 33 | self.assertEqual(mean(test_x)[..., 1:].norm().item(), 0) 34 | 35 | def test_forward_mat_batch(self): 36 | test_x = torch.randn(3, 4, 3) 37 | mean = self.create_mean() 38 | if self.__class__.batch_shape is None: 39 | self.assertEqual(mean(test_x).shape, torch.Size([3, 4, 4])) 40 | else: 41 | self.assertEqual(mean(test_x).shape, torch.Size([*self.__class__.batch_shape, 4, 4])) 42 | self.assertEqual(mean(test_x)[..., 1:].norm().item(), 0) 43 | 44 | def test_forward_mat_multi_batch(self): 45 | test_x = torch.randn(2, 3, 4, 3) 46 | mean = self.create_mean() 47 | self.assertEqual(mean(test_x).shape, torch.Size([2, 3, 4, 4])) 48 | self.assertEqual(mean(test_x)[..., 1:].norm().item(), 0) 49 | 50 | 51 | class TestConstantMeanGradBatch(TestConstantMeanGrad): 52 | batch_shape = torch.Size([3]) 53 | 54 | 55 | class TestConstantMeanGradMultiBatch(TestConstantMeanGrad): 56 | batch_shape = torch.Size([2, 3]) 57 | -------------------------------------------------------------------------------- /test/means/test_linear_mean.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.means import LinearMean 8 | from gpytorch.test.base_mean_test_case import BaseMeanTestCase 9 | 10 | 11 | class TestLinearMean(BaseMeanTestCase, unittest.TestCase): 12 | def create_mean(self, input_size=1, batch_shape=torch.Size(), bias=True, **kwargs): 13 | return LinearMean(input_size=input_size, batch_shape=batch_shape, bias=bias) 14 | 15 | def forward_vec(self): 16 | n = 4 17 | test_x = torch.randn(n) 18 | mean = self.create_mean(input_size=1) 19 | self.assertEqual(mean(test_x).shape, torch.Size([4])) 20 | 21 | def test_forward_mat(self): 22 | n, d = 4, 5 23 | test_x = torch.randn(n, d) 24 | mean = self.create_mean(d) 25 | self.assertEqual(mean(test_x).shape, torch.Size([n])) 26 | 27 | def test_forward_mat_batch(self): 28 | b, n, d = torch.Size([3]), 4, 5 29 | test_x = torch.randn(*b, n, d) 30 | mean = self.create_mean(d, b) 31 | self.assertEqual(mean(test_x).shape, torch.Size([*b, n])) 32 | 33 | def test_forward_mat_multi_batch(self): 34 | b, n, d = torch.Size([2, 3]), 4, 5 35 | test_x = torch.randn(*b, n, d) 36 | mean = self.create_mean(d, b) 37 | self.assertEqual(mean(test_x).shape, torch.Size([*b, n])) 38 | -------------------------------------------------------------------------------- /test/means/test_multitask_mean.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.means import ConstantMean, MultitaskMean, ZeroMean 8 | from gpytorch.test.base_mean_test_case import BaseMeanTestCase 9 | 10 | 11 | class TestMultitaskMean(BaseMeanTestCase, unittest.TestCase): 12 | def create_mean(self): 13 | return MultitaskMean([ConstantMean(), ZeroMean(), ZeroMean()], num_tasks=3) 14 | 15 | def test_forward_vec(self): 16 | test_x = torch.randn(4) 17 | mean = self.create_mean() 18 | self.assertEqual(mean(test_x).shape, torch.Size([4, 3])) 19 | self.assertEqual(mean(test_x)[..., 1:].norm().item(), 0) 20 | 21 | def test_forward_mat(self): 22 | test_x = torch.randn(4, 3) 23 | mean = self.create_mean() 24 | self.assertEqual(mean(test_x).shape, torch.Size([4, 3])) 25 | self.assertEqual(mean(test_x)[..., 1:].norm().item(), 0) 26 | 27 | def test_forward_mat_batch(self): 28 | test_x = torch.randn(3, 4, 3) 29 | mean = self.create_mean() 30 | self.assertEqual(mean(test_x).shape, torch.Size([3, 4, 3])) 31 | self.assertEqual(mean(test_x)[..., 1:].norm().item(), 0) 32 | 33 | def test_forward_mat_multi_batch(self): 34 | test_x = torch.randn(2, 3, 4, 3) 35 | mean = self.create_mean() 36 | self.assertEqual(mean(test_x).shape, torch.Size([2, 3, 4, 3])) 37 | self.assertEqual(mean(test_x)[..., 1:].norm().item(), 0) 38 | 39 | 40 | class TestMultitaskMeanBatch(TestMultitaskMean): 41 | def create_mean(self): 42 | return MultitaskMean([ConstantMean(batch_shape=torch.Size([3])), ZeroMean(), ZeroMean()], num_tasks=3) 43 | 44 | def test_forward_vec(self): 45 | pass 46 | 47 | def test_forward_mat(self): 48 | pass 49 | 50 | 51 | class TestMultitaskMeanMultiBatch(TestMultitaskMean): 52 | def create_mean(self): 53 | return MultitaskMean([ConstantMean(batch_shape=torch.Size([2, 3])), ZeroMean(), ZeroMean()], num_tasks=3) 54 | 55 | def test_forward_vec(self): 56 | pass 57 | 58 | def test_forward_mat(self): 59 | pass 60 | 61 | def test_forward_mat_batch(self): 62 | pass 63 | -------------------------------------------------------------------------------- /test/means/test_zero_mean.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | from gpytorch.means import ZeroMean 6 | from gpytorch.test.base_mean_test_case import BaseMeanTestCase 7 | 8 | 9 | class TestZeroMean(BaseMeanTestCase, unittest.TestCase): 10 | def create_mean(self): 11 | return ZeroMean() 12 | -------------------------------------------------------------------------------- /test/models/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | -------------------------------------------------------------------------------- /test/models/test_model_list.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.likelihoods import FixedNoiseGaussianLikelihood 8 | from gpytorch.models import IndependentModelList 9 | 10 | from .test_exact_gp import TestExactGP 11 | 12 | 13 | class TestModelListGP(unittest.TestCase): 14 | def create_model(self, fixed_noise=False): 15 | data = TestExactGP.create_test_data(self) 16 | likelihood, labels = TestExactGP.create_likelihood_and_labels(self) 17 | if fixed_noise: 18 | noise = 0.1 + 0.2 * torch.rand_like(labels) 19 | likelihood = FixedNoiseGaussianLikelihood(noise) 20 | return TestExactGP.create_model(self, data, labels, likelihood) 21 | 22 | def test_forward_eval(self): 23 | models = [self.create_model() for _ in range(2)] 24 | model = IndependentModelList(*models) 25 | model.eval() 26 | model(torch.rand(3)) 27 | 28 | def test_forward_eval_fixed_noise(self): 29 | models = [self.create_model(fixed_noise=True) for _ in range(2)] 30 | model = IndependentModelList(*models) 31 | model.eval() 32 | model(torch.rand(3)) 33 | 34 | def test_get_fantasy_model(self): 35 | models = [self.create_model() for _ in range(2)] 36 | model = IndependentModelList(*models) 37 | model.eval() 38 | model(torch.rand(3), torch.rand(3)) 39 | fant_x = [torch.randn(2), torch.randn(3)] 40 | fant_y = [torch.randn(2), torch.randn(3)] 41 | fmodel = model.get_fantasy_model(fant_x, fant_y) 42 | fmodel(torch.randn(4)) 43 | 44 | def test_get_fantasy_model_fixed_noise(self): 45 | models = [self.create_model(fixed_noise=True) for _ in range(2)] 46 | model = IndependentModelList(*models) 47 | model.eval() 48 | model(torch.rand(3), torch.rand(3)) 49 | fant_x = [torch.randn(2), torch.randn(3)] 50 | fant_y = [torch.randn(2), torch.randn(3)] 51 | fant_noise = [0.1 * torch.ones(2), 0.1 * torch.ones(3)] 52 | fmodel = model.get_fantasy_model(fant_x, fant_y, noise=fant_noise) 53 | fmodel(torch.randn(4)) 54 | 55 | 56 | if __name__ == "__main__": 57 | unittest.main() 58 | -------------------------------------------------------------------------------- /test/priors/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | -------------------------------------------------------------------------------- /test/utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | -------------------------------------------------------------------------------- /test/utils/test_getitem.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | from itertools import product 5 | 6 | import torch 7 | 8 | from gpytorch.utils.getitem import _compute_getitem_size, _convert_indices_to_tensors 9 | 10 | 11 | class TestGetitem(unittest.TestCase): 12 | def test_compute_getitem_size(self): 13 | a = torch.tensor(0.0).expand(5, 5, 5, 5, 5) 14 | 15 | for indices in product([torch.tensor([0, 1, 1, 0]), slice(None, None, None), 1, slice(0, 2, None)], repeat=5): 16 | res = _compute_getitem_size(a, indices) 17 | actual = a[indices].shape 18 | self.assertEqual(res, actual) 19 | 20 | def test_convert_indices_to_tensors(self): 21 | a = torch.randn(5, 5, 5, 5, 5) 22 | 23 | for indices in product([torch.tensor([0, 1, 1, 0]), slice(None, None, None), 1, slice(0, 2, None)], repeat=5): 24 | if not any(torch.is_tensor(index) for index in indices): 25 | continue 26 | new_indices = _convert_indices_to_tensors(a, indices) 27 | self.assertTrue(all(torch.is_tensor(index) for index in new_indices)) 28 | self.assertTrue(torch.equal(a[indices], a[new_indices])) 29 | -------------------------------------------------------------------------------- /test/utils/test_grid.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | import gpytorch 8 | 9 | 10 | class TestGrid(unittest.TestCase): 11 | def test_scale_to_bounds(self): 12 | """ 13 | """ 14 | x = torch.randn(100) * 50 15 | res = gpytorch.utils.grid.scale_to_bounds(x, -1, 1) 16 | self.assertGreater(res.min().item(), -1) 17 | self.assertLess(res.max().item(), 1) 18 | 19 | def test_choose_grid_size(self): 20 | """ 21 | """ 22 | x = torch.randn(100) 23 | grid_size = gpytorch.utils.grid.choose_grid_size(x, ratio=2.0) 24 | self.assertEqual(grid_size, 200) 25 | 26 | x = torch.randn(100, 1) 27 | grid_size = gpytorch.utils.grid.choose_grid_size(x, ratio=2.0) 28 | self.assertEqual(grid_size, 200) 29 | 30 | x = torch.randn(10000, 2) 31 | grid_size = gpytorch.utils.grid.choose_grid_size(x, ratio=2.0) 32 | self.assertEqual(grid_size, 200) 33 | 34 | x = torch.randn(16, 10000, 4) 35 | grid_size = gpytorch.utils.grid.choose_grid_size(x, ratio=2.0) 36 | self.assertEqual(grid_size, 20) 37 | -------------------------------------------------------------------------------- /test/utils/test_lanczos.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.test.utils import approx_equal 8 | from gpytorch.utils.lanczos import lanczos_tridiag 9 | 10 | 11 | class TestLanczos(unittest.TestCase): 12 | def test_lanczos(self): 13 | size = 100 14 | matrix = torch.randn(size, size) 15 | matrix = matrix.matmul(matrix.transpose(-1, -2)) 16 | matrix.div_(matrix.norm()) 17 | matrix.add_(torch.ones(matrix.size(-1)).mul(1e-6).diag()) 18 | q_mat, t_mat = lanczos_tridiag( 19 | matrix.matmul, max_iter=size, dtype=matrix.dtype, device=matrix.device, matrix_shape=matrix.shape 20 | ) 21 | 22 | approx = q_mat.matmul(t_mat).matmul(q_mat.transpose(-1, -2)) 23 | self.assertTrue(approx_equal(approx, matrix)) 24 | 25 | 26 | if __name__ == "__main__": 27 | unittest.main() 28 | -------------------------------------------------------------------------------- /test/utils/test_sparse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch.utils.sparse import sparse_eye, sparse_getitem, to_sparse 8 | 9 | 10 | class TestSparse(unittest.TestCase): 11 | def setUp(self): 12 | self.indices = torch.tensor([[0, 1, 2, 3, 4], [2, 1, 0, 0, 1]], dtype=torch.long) 13 | self.values = torch.tensor([3, 4, 5, 2, 6], dtype=torch.float) 14 | self.sparse = torch.sparse.FloatTensor(self.indices, self.values, torch.Size((5, 3))) 15 | self.dense = self.sparse.to_dense() 16 | 17 | def test_sparse_eye(self): 18 | res = sparse_eye(5) 19 | actual = torch.eye(5) 20 | self.assertTrue(torch.equal(res.to_dense(), actual)) 21 | 22 | def test_sparse_getitem_one_dim_int(self): 23 | actual = self.dense[3] 24 | res = sparse_getitem(self.sparse, 3) 25 | self.assertTrue(torch.equal(actual, res.to_dense())) 26 | 27 | def test_sparse_getitem_one_dim_slice(self): 28 | actual = self.dense[2:4] 29 | res = sparse_getitem(self.sparse, slice(2, 4)) 30 | self.assertTrue(torch.equal(actual, res.to_dense())) 31 | 32 | def test_sparse_getitem_two_dim_int(self): 33 | actual = self.dense[2, 1] 34 | res = sparse_getitem(self.sparse, (2, 1)) 35 | self.assertEqual(actual, res) 36 | 37 | def test_sparse_getitem_two_dim_int_slice(self): 38 | actual = self.dense[:, 1] 39 | res = sparse_getitem(self.sparse, (slice(None, None, None), 1)) 40 | self.assertTrue(torch.equal(actual, res.to_dense())) 41 | 42 | actual = self.dense[1, :] 43 | res = sparse_getitem(self.sparse, (1, slice(None, None, None))) 44 | self.assertTrue(torch.equal(actual, res.to_dense())) 45 | 46 | def test_sparse_getitem_two_dim_slice(self): 47 | actual = self.dense[2:4, 1:3] 48 | res = sparse_getitem(self.sparse, (slice(2, 4), slice(1, 3))) 49 | self.assertTrue(torch.equal(actual, res.to_dense())) 50 | 51 | def test_to_sparse(self): 52 | actual = self.sparse 53 | res = to_sparse(self.sparse.to_dense()) 54 | self.assertTrue(torch.equal(actual.to_dense(), res.to_dense())) 55 | 56 | 57 | if __name__ == "__main__": 58 | unittest.main() 59 | -------------------------------------------------------------------------------- /test/utils/test_toeplitz.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | from gpytorch import utils 8 | from gpytorch.test.utils import approx_equal 9 | 10 | 11 | class TestToeplitz(unittest.TestCase): 12 | def test_sym_toeplitz_constructs_tensor_from_vector(self): 13 | c = torch.tensor([1, 6, 4, 5], dtype=torch.float) 14 | 15 | res = utils.toeplitz.sym_toeplitz(c) 16 | actual = torch.tensor([[1, 6, 4, 5], [6, 1, 6, 4], [4, 6, 1, 6], [5, 4, 6, 1]], dtype=torch.float) 17 | 18 | self.assertTrue(torch.equal(res, actual)) 19 | 20 | def test_toeplitz_matmul(self): 21 | col = torch.tensor([1, 6, 4, 5], dtype=torch.float) 22 | row = torch.tensor([1, 2, 1, 1], dtype=torch.float) 23 | rhs_mat = torch.randn(4, 2) 24 | 25 | # Actual 26 | lhs_mat = utils.toeplitz.toeplitz(col, row) 27 | actual = torch.matmul(lhs_mat, rhs_mat) 28 | 29 | # Fast toeplitz 30 | res = utils.toeplitz.toeplitz_matmul(col, row, rhs_mat) 31 | self.assertTrue(approx_equal(res, actual)) 32 | 33 | def test_toeplitz_matmul_batch(self): 34 | cols = torch.tensor([[1, 6, 4, 5], [2, 3, 1, 0], [1, 2, 3, 1]], dtype=torch.float) 35 | rows = torch.tensor([[1, 2, 1, 1], [2, 0, 0, 1], [1, 5, 1, 0]], dtype=torch.float) 36 | 37 | rhs_mats = torch.randn(3, 4, 2) 38 | 39 | # Actual 40 | lhs_mats = torch.zeros(3, 4, 4) 41 | for i, (col, row) in enumerate(zip(cols, rows)): 42 | lhs_mats[i].copy_(utils.toeplitz.toeplitz(col, row)) 43 | actual = torch.matmul(lhs_mats, rhs_mats) 44 | 45 | # Fast toeplitz 46 | res = utils.toeplitz.toeplitz_matmul(cols, rows, rhs_mats) 47 | self.assertTrue(approx_equal(res, actual)) 48 | 49 | def test_toeplitz_matmul_batchmat(self): 50 | col = torch.tensor([1, 6, 4, 5], dtype=torch.float) 51 | row = torch.tensor([1, 2, 1, 1], dtype=torch.float) 52 | rhs_mat = torch.randn(3, 4, 2) 53 | 54 | # Actual 55 | lhs_mat = utils.toeplitz.toeplitz(col, row) 56 | actual = torch.matmul(lhs_mat.unsqueeze(0), rhs_mat) 57 | 58 | # Fast toeplitz 59 | res = utils.toeplitz.toeplitz_matmul(col.unsqueeze(0), row.unsqueeze(0), rhs_mat) 60 | self.assertTrue(approx_equal(res, actual)) 61 | 62 | 63 | if __name__ == "__main__": 64 | unittest.main() 65 | -------------------------------------------------------------------------------- /test/variational/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jwangjie/gpytorch/15979dacf1997af7daf0fdeddbdbfcef0730b007/test/variational/__init__.py -------------------------------------------------------------------------------- /test/variational/test_orthogonally_decoupled_variational_strategy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | import gpytorch 8 | from gpytorch.test.variational_test_case import VariationalTestCase 9 | 10 | 11 | def likelihood_cls(): 12 | return gpytorch.likelihoods.GaussianLikelihood() 13 | 14 | 15 | def strategy_cls(model, inducing_points, variational_distribution, learn_inducing_locations): 16 | base_inducing_points = torch.randn(8, inducing_points.size(-1), device=inducing_points.device) 17 | base_variational_distribution = gpytorch.variational.CholeskyVariationalDistribution(8) 18 | return gpytorch.variational.OrthogonallyDecoupledVariationalStrategy( 19 | gpytorch.variational.VariationalStrategy( 20 | model, base_inducing_points, base_variational_distribution, learn_inducing_locations 21 | ), 22 | inducing_points, 23 | variational_distribution, 24 | ) 25 | 26 | 27 | class TestOrthogonallyDecoupledVariationalGP(VariationalTestCase, unittest.TestCase): 28 | @property 29 | def batch_shape(self): 30 | return torch.Size([]) 31 | 32 | @property 33 | def distribution_cls(self): 34 | return gpytorch.variational.DeltaVariationalDistribution 35 | 36 | @property 37 | def likelihood_cls(self): 38 | return likelihood_cls 39 | 40 | @property 41 | def mll_cls(self): 42 | return gpytorch.mlls.VariationalELBO 43 | 44 | @property 45 | def strategy_cls(self): 46 | return strategy_cls 47 | 48 | def test_training_iteration(self, *args, **kwargs): 49 | cg_mock, cholesky_mock = super().test_training_iteration(*args, **kwargs) 50 | self.assertFalse(cg_mock.called) 51 | self.assertEqual(cholesky_mock.call_count, 3) # One for each forward pass, and for computing prior dist 52 | 53 | def test_eval_iteration(self, *args, **kwargs): 54 | cg_mock, cholesky_mock = super().test_eval_iteration(*args, **kwargs) 55 | self.assertFalse(cg_mock.called) 56 | self.assertEqual(cholesky_mock.call_count, 1) # One to compute cache, that's it! 57 | 58 | 59 | class TestOrthogonallyDecoupledPredictiveGP(TestOrthogonallyDecoupledVariationalGP): 60 | @property 61 | def mll_cls(self): 62 | return gpytorch.mlls.PredictiveLogLikelihood 63 | 64 | 65 | class TestOrthogonallyDecoupledRobustVGP(TestOrthogonallyDecoupledVariationalGP): 66 | @property 67 | def mll_cls(self): 68 | return gpytorch.mlls.GammaRobustVariationalELBO 69 | 70 | 71 | if __name__ == "__main__": 72 | unittest.main() 73 | -------------------------------------------------------------------------------- /test/variational/test_variational_strategy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import unittest 4 | 5 | import torch 6 | 7 | import gpytorch 8 | from gpytorch.test.variational_test_case import VariationalTestCase 9 | 10 | 11 | class TestVariationalGP(VariationalTestCase, unittest.TestCase): 12 | @property 13 | def batch_shape(self): 14 | return torch.Size([]) 15 | 16 | @property 17 | def distribution_cls(self): 18 | return gpytorch.variational.CholeskyVariationalDistribution 19 | 20 | @property 21 | def mll_cls(self): 22 | return gpytorch.mlls.VariationalELBO 23 | 24 | @property 25 | def strategy_cls(self): 26 | return gpytorch.variational.VariationalStrategy 27 | 28 | def test_training_iteration(self, *args, **kwargs): 29 | cg_mock, cholesky_mock = super().test_training_iteration(*args, **kwargs) 30 | self.assertFalse(cg_mock.called) 31 | self.assertEqual(cholesky_mock.call_count, 2) # One for each forward pass 32 | 33 | def test_eval_iteration(self, *args, **kwargs): 34 | cg_mock, cholesky_mock = super().test_eval_iteration(*args, **kwargs) 35 | self.assertFalse(cg_mock.called) 36 | self.assertEqual(cholesky_mock.call_count, 1) # One to compute cache, that's it! 37 | 38 | 39 | class TestPredictiveGP(TestVariationalGP): 40 | @property 41 | def mll_cls(self): 42 | return gpytorch.mlls.PredictiveLogLikelihood 43 | 44 | 45 | class TestRobustVGP(TestVariationalGP): 46 | @property 47 | def mll_cls(self): 48 | return gpytorch.mlls.GammaRobustVariationalELBO 49 | 50 | 51 | class TestMeanFieldVariationalGP(TestVariationalGP): 52 | @property 53 | def distribution_cls(self): 54 | return gpytorch.variational.MeanFieldVariationalDistribution 55 | 56 | 57 | class TestMeanFieldPredictiveGP(TestPredictiveGP): 58 | @property 59 | def distribution_cls(self): 60 | return gpytorch.variational.MeanFieldVariationalDistribution 61 | 62 | 63 | class TestMeanFieldRobustVGP(TestRobustVGP): 64 | @property 65 | def distribution_cls(self): 66 | return gpytorch.variational.MeanFieldVariationalDistribution 67 | 68 | 69 | class TestDeltaVariationalGP(TestVariationalGP): 70 | @property 71 | def distribution_cls(self): 72 | return gpytorch.variational.DeltaVariationalDistribution 73 | 74 | 75 | class TestDeltaPredictiveGP(TestPredictiveGP): 76 | @property 77 | def distribution_cls(self): 78 | return gpytorch.variational.DeltaVariationalDistribution 79 | 80 | 81 | class TestDeltaRobustVGP(TestRobustVGP): 82 | @property 83 | def distribution_cls(self): 84 | return gpytorch.variational.DeltaVariationalDistribution 85 | 86 | 87 | if __name__ == "__main__": 88 | unittest.main() 89 | --------------------------------------------------------------------------------