├── .gitignore ├── LICENSE ├── README.md ├── activetesting ├── __init__.py ├── acquisition.py ├── datasets.py ├── experiment.py ├── hoover.py ├── loss.py ├── models │ ├── __init__.py │ ├── cnn │ │ ├── .gitignore │ │ ├── models.py │ │ ├── resnet_custom.py │ │ └── wide_resnet.py │ ├── efficient_models.py │ ├── radial_bnn │ │ ├── __init__.py │ │ ├── bnn_models.py │ │ ├── consistent_mc_dropout.py │ │ └── radial_layers │ │ │ ├── __init__.py │ │ │ ├── distributions.py │ │ │ ├── loss.py │ │ │ └── variational_bayes.py │ ├── sk2torch.py │ ├── skmodels.py │ └── torchmodels.py ├── plotting │ ├── __init__.py │ ├── paths.py │ └── utils.py ├── risk_estimators.py ├── utils │ └── maps.py └── visualize.py ├── conf ├── config.yaml ├── hydra │ └── default.yaml └── paper │ ├── LargeCIFAR100ResNet.yaml │ ├── LargeCIFAR100WideResNet.yaml │ ├── LargeCIFAR10ResNet.yaml │ ├── LargeCIFAR10ResNetAccuracy.yaml │ ├── LargeFMNISTBNN.yaml │ ├── LargeFMNISTResNet.yaml │ ├── LargeMNISTBNN.yaml │ ├── SmallFMNISTResNet.yaml │ ├── SmallMNISTBNN.yaml │ ├── SyntheticGPGP.yaml │ ├── SyntheticQuadraticLinear.yaml │ └── SyntheticTwoMoonsRF.yaml ├── main.py ├── notebooks ├── explore_experiment.ipynb └── plots_paper.ipynb ├── outputs └── animation.gif ├── reproduce └── experiments │ ├── figure-123.sh │ ├── figure-4.sh │ ├── figure-5.sh │ ├── figure-6.sh │ └── figure-7.sh └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/README.md -------------------------------------------------------------------------------- /activetesting/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /activetesting/acquisition.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/acquisition.py -------------------------------------------------------------------------------- /activetesting/datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/datasets.py -------------------------------------------------------------------------------- /activetesting/experiment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/experiment.py -------------------------------------------------------------------------------- /activetesting/hoover.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/hoover.py -------------------------------------------------------------------------------- /activetesting/loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/loss.py -------------------------------------------------------------------------------- /activetesting/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/models/__init__.py -------------------------------------------------------------------------------- /activetesting/models/cnn/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/models/cnn/.gitignore -------------------------------------------------------------------------------- /activetesting/models/cnn/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/models/cnn/models.py -------------------------------------------------------------------------------- /activetesting/models/cnn/resnet_custom.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/models/cnn/resnet_custom.py -------------------------------------------------------------------------------- /activetesting/models/cnn/wide_resnet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/models/cnn/wide_resnet.py -------------------------------------------------------------------------------- /activetesting/models/efficient_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/models/efficient_models.py -------------------------------------------------------------------------------- /activetesting/models/radial_bnn/__init__.py: -------------------------------------------------------------------------------- 1 | from .bnn_models import RadialBNN 2 | -------------------------------------------------------------------------------- /activetesting/models/radial_bnn/bnn_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/models/radial_bnn/bnn_models.py -------------------------------------------------------------------------------- /activetesting/models/radial_bnn/consistent_mc_dropout.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/models/radial_bnn/consistent_mc_dropout.py -------------------------------------------------------------------------------- /activetesting/models/radial_bnn/radial_layers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /activetesting/models/radial_bnn/radial_layers/distributions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/models/radial_bnn/radial_layers/distributions.py -------------------------------------------------------------------------------- /activetesting/models/radial_bnn/radial_layers/loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/models/radial_bnn/radial_layers/loss.py -------------------------------------------------------------------------------- /activetesting/models/radial_bnn/radial_layers/variational_bayes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/models/radial_bnn/radial_layers/variational_bayes.py -------------------------------------------------------------------------------- /activetesting/models/sk2torch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/models/sk2torch.py -------------------------------------------------------------------------------- /activetesting/models/skmodels.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/models/skmodels.py -------------------------------------------------------------------------------- /activetesting/models/torchmodels.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/models/torchmodels.py -------------------------------------------------------------------------------- /activetesting/plotting/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/plotting/__init__.py -------------------------------------------------------------------------------- /activetesting/plotting/paths.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/plotting/paths.py -------------------------------------------------------------------------------- /activetesting/plotting/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/plotting/utils.py -------------------------------------------------------------------------------- /activetesting/risk_estimators.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/risk_estimators.py -------------------------------------------------------------------------------- /activetesting/utils/maps.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/utils/maps.py -------------------------------------------------------------------------------- /activetesting/visualize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/activetesting/visualize.py -------------------------------------------------------------------------------- /conf/config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/conf/config.yaml -------------------------------------------------------------------------------- /conf/hydra/default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/conf/hydra/default.yaml -------------------------------------------------------------------------------- /conf/paper/LargeCIFAR100ResNet.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/conf/paper/LargeCIFAR100ResNet.yaml -------------------------------------------------------------------------------- /conf/paper/LargeCIFAR100WideResNet.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/conf/paper/LargeCIFAR100WideResNet.yaml -------------------------------------------------------------------------------- /conf/paper/LargeCIFAR10ResNet.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/conf/paper/LargeCIFAR10ResNet.yaml -------------------------------------------------------------------------------- /conf/paper/LargeCIFAR10ResNetAccuracy.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/conf/paper/LargeCIFAR10ResNetAccuracy.yaml -------------------------------------------------------------------------------- /conf/paper/LargeFMNISTBNN.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/conf/paper/LargeFMNISTBNN.yaml -------------------------------------------------------------------------------- /conf/paper/LargeFMNISTResNet.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/conf/paper/LargeFMNISTResNet.yaml -------------------------------------------------------------------------------- /conf/paper/LargeMNISTBNN.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/conf/paper/LargeMNISTBNN.yaml -------------------------------------------------------------------------------- /conf/paper/SmallFMNISTResNet.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/conf/paper/SmallFMNISTResNet.yaml -------------------------------------------------------------------------------- /conf/paper/SmallMNISTBNN.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/conf/paper/SmallMNISTBNN.yaml -------------------------------------------------------------------------------- /conf/paper/SyntheticGPGP.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/conf/paper/SyntheticGPGP.yaml -------------------------------------------------------------------------------- /conf/paper/SyntheticQuadraticLinear.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/conf/paper/SyntheticQuadraticLinear.yaml -------------------------------------------------------------------------------- /conf/paper/SyntheticTwoMoonsRF.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/conf/paper/SyntheticTwoMoonsRF.yaml -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/main.py -------------------------------------------------------------------------------- /notebooks/explore_experiment.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/notebooks/explore_experiment.ipynb -------------------------------------------------------------------------------- /notebooks/plots_paper.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/notebooks/plots_paper.ipynb -------------------------------------------------------------------------------- /outputs/animation.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/outputs/animation.gif -------------------------------------------------------------------------------- /reproduce/experiments/figure-123.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/reproduce/experiments/figure-123.sh -------------------------------------------------------------------------------- /reproduce/experiments/figure-4.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/reproduce/experiments/figure-4.sh -------------------------------------------------------------------------------- /reproduce/experiments/figure-5.sh: -------------------------------------------------------------------------------- 1 | python main.py +paper=LargeCIFAR100ResNet -------------------------------------------------------------------------------- /reproduce/experiments/figure-6.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/reproduce/experiments/figure-6.sh -------------------------------------------------------------------------------- /reproduce/experiments/figure-7.sh: -------------------------------------------------------------------------------- 1 | python main.py +paper=LargeFMNISTBNN -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jlko/active-testing/HEAD/requirements.txt --------------------------------------------------------------------------------