├── .gitignore
├── LICENSE
├── README.md
├── configs
├── cifar10
│ ├── al
│ │ ├── RESNET18.yaml
│ │ ├── RESNET18_ENS.yaml
│ │ └── RESNET18_IM.yaml
│ ├── evaluate
│ │ └── RESNET18.yaml
│ └── train
│ │ ├── RESNET18.yaml
│ │ └── RESNET18_ENS.yaml
├── cifar100
│ └── al
│ │ └── RESNET18.yaml
├── mnist
│ └── al
│ │ └── RESNET18.yaml
├── template.yaml
└── tinyimagenet
│ └── al
│ └── RESNET18.yaml
├── docs
├── AL_results.png
└── GETTING_STARTED.md
├── output
├── CIFAR10
│ └── resnet18
│ │ └── ENT_1
│ │ ├── Episodes_vs_Test Accuracy.png
│ │ ├── episode_0
│ │ ├── Epochs_vs_Loss.png
│ │ ├── Epochs_vs_Validation Accuracy.png
│ │ ├── Iterations_vs_Loss.png
│ │ ├── activeSet.npy
│ │ ├── lSet.npy
│ │ ├── plot_epoch_xvalues.txt
│ │ ├── plot_epoch_yvalues.txt
│ │ ├── plot_it_x_values.txt
│ │ ├── plot_it_y_values.txt
│ │ ├── uSet.npy
│ │ ├── val_acc_epochs_x.txt
│ │ └── val_acc_epochs_y.txt
│ │ ├── episode_1
│ │ ├── Epochs_vs_Loss.png
│ │ ├── Epochs_vs_Validation Accuracy.png
│ │ ├── Iterations_vs_Loss.png
│ │ ├── activeSet.npy
│ │ ├── lSet.npy
│ │ ├── plot_epoch_xvalues.txt
│ │ ├── plot_epoch_yvalues.txt
│ │ ├── plot_it_x_values.txt
│ │ ├── plot_it_y_values.txt
│ │ ├── uSet.npy
│ │ ├── val_acc_epochs_x.txt
│ │ └── val_acc_epochs_y.txt
│ │ ├── episode_2
│ │ ├── Epochs_vs_Loss.png
│ │ ├── Epochs_vs_Validation Accuracy.png
│ │ ├── Iterations_vs_Loss.png
│ │ ├── activeSet.npy
│ │ ├── lSet.npy
│ │ ├── plot_epoch_xvalues.txt
│ │ ├── plot_epoch_yvalues.txt
│ │ ├── plot_it_x_values.txt
│ │ ├── plot_it_y_values.txt
│ │ ├── uSet.npy
│ │ ├── val_acc_epochs_x.txt
│ │ └── val_acc_epochs_y.txt
│ │ ├── episode_3
│ │ ├── Epochs_vs_Loss.png
│ │ ├── Epochs_vs_Validation Accuracy.png
│ │ ├── Iterations_vs_Loss.png
│ │ ├── activeSet.npy
│ │ ├── lSet.npy
│ │ ├── plot_epoch_xvalues.txt
│ │ ├── plot_epoch_yvalues.txt
│ │ ├── plot_it_x_values.txt
│ │ ├── plot_it_y_values.txt
│ │ ├── uSet.npy
│ │ ├── val_acc_epochs_x.txt
│ │ └── val_acc_epochs_y.txt
│ │ ├── episode_4
│ │ ├── Epochs_vs_Loss.png
│ │ ├── Epochs_vs_Validation Accuracy.png
│ │ ├── Iterations_vs_Loss.png
│ │ ├── activeSet.npy
│ │ ├── lSet.npy
│ │ ├── plot_epoch_xvalues.txt
│ │ ├── plot_epoch_yvalues.txt
│ │ ├── plot_it_x_values.txt
│ │ ├── plot_it_y_values.txt
│ │ ├── uSet.npy
│ │ ├── val_acc_epochs_x.txt
│ │ └── val_acc_epochs_y.txt
│ │ ├── episode_5
│ │ ├── Epochs_vs_Loss.png
│ │ ├── Epochs_vs_Validation Accuracy.png
│ │ ├── Iterations_vs_Loss.png
│ │ ├── plot_epoch_xvalues.txt
│ │ ├── plot_epoch_yvalues.txt
│ │ ├── plot_it_x_values.txt
│ │ ├── plot_it_y_values.txt
│ │ ├── val_acc_epochs_x.txt
│ │ └── val_acc_epochs_y.txt
│ │ ├── lSet.npy
│ │ ├── plot_episode_xvalues.txt
│ │ ├── plot_episode_yvalues.txt
│ │ ├── stdout.log
│ │ ├── uSet.npy
│ │ └── valSet.npy
└── results_aggregator.ipynb
├── pycls
├── __init__.py
├── al
│ ├── ActiveLearning.py
│ ├── Sampling.py
│ ├── __init__.py
│ └── vaal_util.py
├── core
│ ├── __init__.py
│ ├── builders.py
│ ├── config.py
│ ├── losses.py
│ ├── net.py
│ └── optimizer.py
├── datasets
│ ├── __init__.py
│ ├── augment.py
│ ├── custom_datasets.py
│ ├── data.py
│ ├── imbalanced_cifar.py
│ ├── randaugment.py
│ ├── sampler.py
│ ├── simclr_augment.py
│ ├── tiny_imagenet.py
│ └── utils
│ │ ├── __init__.py
│ │ ├── gaussian_blur.py
│ │ └── helpers.py
├── models
│ ├── __init__.py
│ ├── alexnet.py
│ ├── resnet.py
│ ├── vaal_model.py
│ └── vgg.py
└── utils
│ ├── __init__.py
│ ├── benchmark.py
│ ├── checkpoint.py
│ ├── distributed.py
│ ├── io.py
│ ├── logging.py
│ ├── meters.py
│ ├── metrics.py
│ ├── net.py
│ ├── plotting.py
│ └── timer.py
├── requirements.txt
└── tools
├── __init__.py
├── ensemble_al.py
├── ensemble_train.py
├── test_model.py
├── train.py
└── train_al.py
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # Shared objects
7 | *.so
8 |
9 | # Distribution / packaging
10 | build/
11 | *.egg-info/
12 | *.egg
13 |
14 | # Temporary files
15 | *.swn
16 | *.swo
17 | *.swp
18 |
19 | # PyCharm
20 | .idea/
21 |
22 | # Mac
23 | .DS_STORE
24 |
25 | # Data symlinks
26 | pycls/datasets/data/
27 |
28 | # Other
29 | logs/
30 | scratch*
31 |
32 | data/
33 | output/
34 | *.ipynb_checkpoints/
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright 2021 (c) Akshay L Chandra and Vineeth N Balasubramanian
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Deep Active Learning Toolkit for Image Classification in PyTorch
2 |
3 | This is a code base for deep active learning for image classification written in [PyTorch](https://pytorch.org/). It is build on top of FAIR's [pycls](https://github.com/facebookresearch/pycls/). I want to emphasize that this is a derivative of the toolkit originally shared with me via email by Prateek Munjal _et al._, the authors of the paper _"Towards Robust and Reproducible Active Learning using Neural Networks"_, paper available [here](https://arxiv.org/abs/2002.09564).
4 |
5 | ## Introduction
6 |
7 | The goal of this repository is to provide a simple and flexible codebase for deep active learning. It is designed to support rapid implementation and evaluation of research ideas. We also provide a results on CIFAR10 below.
8 |
9 | The codebase currently only supports single-machine single-gpu training. We will soon scale it to single-machine multi-gpu training, powered by the PyTorch distributed package.
10 |
11 | ## Using the Toolkit
12 |
13 | Please see [`GETTING_STARTED`](docs/GETTING_STARTED.md) for brief instructions on installation, adding new datasets, basic usage examples, etc.
14 |
15 | ## Active Learning Methods Supported
16 | * Uncertainty Sampling
17 | * Least Confidence
18 | * Min-Margin
19 | * Max-Entropy
20 | * Deep Bayesian Active Learning (DBAL) [1]
21 | * Bayesian Active Learning by Disagreement (BALD) [1]
22 | * Diversity Sampling
23 | * Coreset (greedy) [2]
24 | * Variational Adversarial Active Learning (VAAL) [3]
25 | * Query-by-Committee Sampling
26 | * Ensemble Variation Ratio (Ens-varR) [4]
27 |
28 |
29 | ## Datasets Supported
30 | * [CIFAR10/100](https://www.cs.toronto.edu/~kriz/cifar.html)
31 | * [MNIST](http://yann.lecun.com/exdb/mnist/)
32 | * [SVHN](http://ufldl.stanford.edu/housenumbers/)
33 | * [Tiny ImageNet](https://www.kaggle.com/c/tiny-imagenet) (Download the zip file [here](http://cs231n.stanford.edu/tiny-imagenet-200.zip))
34 | * Long-Tail CIFAR-10/100
35 |
36 | Follow the instructions in [`GETTING_STARTED`](docs/GETTING_STARTED.md) to add a new dataset.
37 |
38 | ## Results on CIFAR10 and CIFAR100
39 |
40 | The following are the results on CIFAR10 and CIFAR100, trained with hyperameters present in `configs/cifar10/al/RESNET18.yaml` and `configs/cifar100/al/RESNET18.yaml` respectively. All results were averaged over 3 runs.
41 |
42 |
43 |
44 |
45 |
46 |
[dash=train, solid=test]",
95 | xaxis={"title": "epoch", "titlefont": titlefont},
96 | yaxis={"title": metric, "titlefont": titlefont},
97 | showlegend=True,
98 | hoverlabel={"namelength": -1},
99 | updatemenus=[
100 | {
101 | "buttons": buttons,
102 | "direction": "down",
103 | "showactive": True,
104 | "x": 1.02,
105 | "xanchor": "left",
106 | "y": 1.08,
107 | "yanchor": "top",
108 | }
109 | ],
110 | )
111 | # Create plotly plot
112 | offline.plot({"data": data, "layout": layout}, filename=filename)
113 |
114 |
115 | def plot_error_curves_pyplot(log_files, names, filename=None, metric="top1_err"):
116 | """Plot error curves using matplotlib.pyplot and save to file."""
117 | plot_data = prepare_plot_data(log_files, names, metric)
118 | colors = get_plot_colors(len(names))
119 | for ind, d in enumerate(plot_data):
120 | c, lbl = colors[ind], d["test_label"]
121 | plt.plot(d["x_train"], d["y_train"], "--", c=c, alpha=0.8)
122 | plt.plot(d["x_test"], d["y_test"], "-", c=c, alpha=0.8, label=lbl)
123 | plt.title(metric + " vs. epoch\n[dash=train, solid=test]", fontsize=14)
124 | plt.xlabel("epoch", fontsize=14)
125 | plt.ylabel(metric, fontsize=14)
126 | plt.grid(alpha=0.4)
127 | plt.legend()
128 | if filename:
129 | plt.savefig(filename)
130 | plt.clf()
131 | else:
132 | plt.show()
133 |
--------------------------------------------------------------------------------
/pycls/utils/timer.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | # Copyright (c) Facebook, Inc. and its affiliates.
4 | #
5 | # This source code is licensed under the MIT license found in the
6 | # LICENSE file in the root directory of this source tree.
7 |
8 | """Timer."""
9 |
10 | import time
11 |
12 |
13 | class Timer(object):
14 | """A simple timer (adapted from Detectron)."""
15 |
16 | def __init__(self):
17 | self.total_time = None
18 | self.calls = None
19 | self.start_time = None
20 | self.diff = None
21 | self.average_time = None
22 | self.reset()
23 |
24 | def tic(self):
25 | # using time.time as time.clock does not normalize for multithreading
26 | self.start_time = time.time()
27 |
28 | def toc(self):
29 | self.diff = time.time() - self.start_time
30 | self.total_time += self.diff
31 | self.calls += 1
32 | self.average_time = self.total_time / self.calls
33 |
34 | def reset(self):
35 | self.total_time = 0.0
36 | self.calls = 0
37 | self.start_time = 0.0
38 | self.diff = 0.0
39 | self.average_time = 0.0
40 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | black==19.3b0
2 | flake8==3.8.4
3 | isort==4.3.21
4 | matplotlib==3.3.4
5 | numpy
6 | opencv-python==4.2.0.34
7 | torch==1.7.1
8 | torchvision==0.8.2
9 | parameterized
10 | setuptools
11 | simplejson
12 | yacs
--------------------------------------------------------------------------------
/tools/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/acl21/deep-active-learning-pytorch/637fd507235632903bcf84ed841ff524d847b94e/tools/__init__.py
--------------------------------------------------------------------------------
/tools/test_model.py:
--------------------------------------------------------------------------------
1 | import os
2 | import sys
3 | from datetime import datetime
4 | import argparse
5 | import numpy as np
6 | from tqdm import tqdm
7 |
8 | import torch
9 | import torch.nn as nn
10 | import torch.nn.functional as F
11 |
12 | # local
13 |
14 | def add_path(path):
15 | if path not in sys.path:
16 | sys.path.insert(0, path)
17 |
18 | add_path(os.path.abspath('..'))
19 |
20 | import pycls.core.builders as model_builder
21 | from pycls.core.config import cfg, dump_cfg
22 | from pycls.datasets.data import Data
23 | import pycls.utils.checkpoint as cu
24 | import pycls.utils.logging as lu
25 | import pycls.utils.metrics as mu
26 | import pycls.utils.net as nu
27 | from pycls.utils.meters import TestMeter
28 |
29 | logger = lu.get_logger(__name__)
30 |
31 | def argparser():
32 | parser = argparse.ArgumentParser(description='Passive Learning - Image Classification')
33 | parser.add_argument('--cfg', dest='cfg_file', help='Config file', required=True, type=str)
34 |
35 | return parser
36 |
37 | def plot_arrays(x_vals, y_vals, x_name, y_name, dataset_name, out_dir, isDebug=False):
38 | # if not du.is_master_proc():
39 | # return
40 |
41 | import matplotlib.pyplot as plt
42 | temp_name = "{}_vs_{}".format(x_name, y_name)
43 | plt.xlabel(x_name)
44 | plt.ylabel(y_name)
45 | plt.title("Dataset: {}; {}".format(dataset_name, temp_name))
46 | plt.plot(x_vals, y_vals)
47 |
48 | if isDebug: print("plot_saved at : {}".format(os.path.join(out_dir, temp_name+'.png')))
49 |
50 | plt.savefig(os.path.join(out_dir, temp_name+".png"))
51 | plt.close()
52 |
53 | def save_plot_values(temp_arrays, temp_names, out_dir, isParallel=True, saveInTextFormat=False, isDebug=True):
54 |
55 | """ Saves arrays provided in the list in npy format """
56 | # Return if not master process
57 | # if isParallel:
58 | # if not du.is_master_proc():
59 | # return
60 |
61 | for i in range(len(temp_arrays)):
62 | temp_arrays[i] = np.array(temp_arrays[i])
63 | temp_dir = out_dir
64 | # if cfg.TRAIN.TRANSFER_EXP:
65 | # temp_dir += os.path.join("transfer_experiment",cfg.MODEL.TRANSFER_MODEL_TYPE+"_depth_"+str(cfg.MODEL.TRANSFER_MODEL_DEPTH))+"/"
66 |
67 | if not os.path.exists(temp_dir):
68 | os.makedirs(temp_dir)
69 | if saveInTextFormat:
70 | # if isDebug: print(f"Saving {temp_names[i]} at {temp_dir+temp_names[i]}.txt in text format!!")
71 | np.savetxt(temp_dir+'/'+temp_names[i]+".txt", temp_arrays[i], fmt="%d")
72 | else:
73 | # if isDebug: print(f"Saving {temp_names[i]} at {temp_dir+temp_names[i]}.npy in numpy format!!")
74 | np.save(temp_dir+'/'+temp_names[i]+".npy", temp_arrays[i])
75 |
76 | def is_eval_epoch(cur_epoch):
77 | """Determines if the model should be evaluated at the current epoch."""
78 | return (
79 | (cur_epoch + 1) % cfg.TRAIN.EVAL_PERIOD == 0 or
80 | (cur_epoch + 1) == cfg.OPTIM.MAX_EPOCH
81 | )
82 |
83 |
84 | def main(cfg):
85 |
86 | # Setting up GPU args
87 | use_cuda = (cfg.NUM_GPUS > 0) and torch.cuda.is_available()
88 | device = torch.device("cuda" if use_cuda else "cpu")
89 | kwargs = {'num_workers': cfg.DATA_LOADER.NUM_WORKERS, 'pin_memory': cfg.DATA_LOADER.PIN_MEMORY} if use_cuda else {}
90 |
91 | # Using specific GPU
92 | # os.environ['NVIDIA_VISIBLE_DEVICES'] = str(cfg.GPU_ID)
93 | # os.environ['CUDA_VISIBLE_DEVICES'] = '0'
94 | # print("Using GPU : {}.\n".format(cfg.GPU_ID))
95 |
96 | # Getting the output directory ready (default is "/output")
97 | cfg.OUT_DIR = os.path.join(os.path.abspath('..'), cfg.OUT_DIR)
98 | if not os.path.exists(cfg.OUT_DIR):
99 | os.mkdir(cfg.OUT_DIR)
100 | # Create "DATASET/MODEL TYPE" specific directory
101 | dataset_out_dir = os.path.join(cfg.OUT_DIR, cfg.DATASET.NAME, cfg.MODEL.TYPE)
102 | if not os.path.exists(dataset_out_dir):
103 | os.makedirs(dataset_out_dir)
104 | # Creating the experiment directory inside the dataset specific directory
105 | # all logs, labeled, unlabeled, validation sets are stroed here
106 | # E.g., output/CIFAR10/resnet18/{timestamp or cfg.EXP_NAME based on arguments passed}
107 | if cfg.EXP_NAME == 'auto':
108 | now = datetime.now()
109 | exp_dir = f'{now.year}_{now.month}_{now.day}_{now.hour}{now.minute}{now.second}'
110 | else:
111 | exp_dir = cfg.EXP_NAME
112 |
113 | exp_dir = os.path.join(dataset_out_dir, exp_dir)
114 | if not os.path.exists(exp_dir):
115 | os.mkdir(exp_dir)
116 | print("Experiment Directory is {}.\n".format(exp_dir))
117 | else:
118 | print("Experiment Directory Already Exists: {}. Reusing it may lead to loss of old logs in the directory.\n".format(exp_dir))
119 | cfg.EXP_DIR = exp_dir
120 |
121 | # Save the config file in EXP_DIR
122 | dump_cfg(cfg)
123 |
124 | # Setup Logger
125 | lu.setup_logging(cfg)
126 |
127 | # Dataset preparing steps
128 | print("\n======== PREPARING TEST DATA ========\n")
129 | cfg.DATASET.ROOT_DIR = os.path.join(os.path.abspath('..'), cfg.DATASET.ROOT_DIR)
130 | data_obj = Data(cfg)
131 | test_data, test_size = data_obj.getDataset(save_dir=cfg.DATASET.ROOT_DIR, isTrain=False, isDownload=True)
132 |
133 | print("\nDataset {} Loaded Sucessfully. Total Test Size: {}\n".format(cfg.DATASET.NAME, test_size))
134 | logger.info("Dataset {} Loaded Sucessfully. Total Test Size: {}\n".format(cfg.DATASET.NAME, test_size))
135 |
136 | # Preparing dataloaders for testing
137 | test_loader = data_obj.getTestLoader(data=test_data, test_batch_size=cfg.TRAIN.BATCH_SIZE, seed_id=cfg.RNG_SEED)
138 |
139 | print("======== TESTING ========\n")
140 | logger.info("======== TESTING ========\n")
141 | test_acc = test_model(test_loader, os.path.join(os.path.abspath('..'), cfg.TEST.MODEL_PATH), cfg)
142 | print("Test Accuracy: {}.\n".format(round(test_acc, 4)))
143 | logger.info("Test Accuracy {}.\n".format(test_acc))
144 |
145 | print('Check the test accuracy inside {}/stdout.log'.format(cfg.EXP_DIR))
146 |
147 | print("================================\n\n")
148 | logger.info("================================\n\n")
149 |
150 |
151 | def test_model(test_loader, checkpoint_file, cfg, cur_episode=0):
152 |
153 | test_meter = TestMeter(len(test_loader))
154 |
155 | model = model_builder.build_model(cfg)
156 | model = cu.load_checkpoint(checkpoint_file, model)
157 |
158 | test_err = test_epoch(test_loader, model, test_meter, cur_episode)
159 | test_acc = 100. - test_err
160 |
161 | return test_acc
162 |
163 |
164 | @torch.no_grad()
165 | def test_epoch(test_loader, model, test_meter, cur_epoch):
166 | """Evaluates the model on the test set."""
167 |
168 | if torch.cuda.is_available():
169 | model.cuda()
170 |
171 | # Enable eval mode
172 | model.eval()
173 | test_meter.iter_tic()
174 |
175 | misclassifications = 0.
176 | totalSamples = 0.
177 |
178 | for cur_iter, (inputs, labels) in enumerate(tqdm(test_loader, desc="Test Data")):
179 | with torch.no_grad():
180 | # Transfer the data to the current GPU device
181 | inputs, labels = inputs.cuda(), labels.cuda(non_blocking=True)
182 | inputs = inputs.type(torch.cuda.FloatTensor)
183 | # Compute the predictions
184 | preds = model(inputs)
185 | # Compute the errors
186 | top1_err, top5_err = mu.topk_errors(preds, labels, [1, 5])
187 | # Combine the errors across the GPUs
188 | # if cfg.NUM_GPUS > 1:
189 | # top1_err = du.scaled_all_reduce([top1_err])
190 | # #as above returns a list
191 | # top1_err = top1_err[0]
192 | # Copy the errors from GPU to CPU (sync point)
193 | top1_err = top1_err.item()
194 | # Multiply by Number of GPU's as top1_err is scaled by 1/Num_GPUs
195 | misclassifications += top1_err * inputs.size(0) * cfg.NUM_GPUS
196 | totalSamples += inputs.size(0)*cfg.NUM_GPUS
197 | test_meter.iter_toc()
198 | # Update and log stats
199 | test_meter.update_stats(
200 | top1_err=top1_err, mb_size=inputs.size(0) * cfg.NUM_GPUS
201 | )
202 | test_meter.log_iter_stats(cur_epoch, cur_iter)
203 | test_meter.iter_tic()
204 | # Log epoch stats
205 | test_meter.log_epoch_stats(cur_epoch)
206 | test_meter.reset()
207 |
208 | return misclassifications/totalSamples
209 |
210 |
211 | if __name__ == "__main__":
212 | cfg.merge_from_file(argparser().parse_args().cfg_file)
213 | main(cfg)
214 |
--------------------------------------------------------------------------------