├── .gitignore ├── README.md ├── main.py ├── models ├── __init__.py ├── inferences │ ├── __init__.py │ ├── bp.py │ ├── mf.py │ └── utils.py └── learnings │ ├── BP │ ├── __init__.py │ ├── arguments.py │ ├── dataset.py │ ├── model.py │ └── train.py │ ├── BayesDGC │ ├── __init__.py │ ├── arguments.py │ ├── dataset.py │ ├── model.py │ └── train.py │ ├── CL │ ├── __init__.py │ ├── arguments.py │ ├── dataset.py │ ├── model.py │ └── train.py │ ├── MF │ ├── __init__.py │ ├── arguments.py │ ├── dataset.py │ ├── model.py │ └── train.py │ ├── MV │ ├── __init__.py │ ├── arguments.py │ ├── dataset.py │ ├── model.py │ └── train.py │ ├── base │ ├── __init__.py │ ├── dataset.py │ ├── model.py │ └── train.py │ ├── deepBP │ ├── __init__.py │ ├── arguments.py │ ├── dataset.py │ ├── model.py │ └── train.py │ └── deepMF │ ├── __init__.py │ ├── arguments.py │ ├── dataset.py │ ├── model.py │ └── train.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.npy 3 | *.h5 4 | wandb/ 5 | logs/ 6 | checkpoints/ 7 | ckpts/ 8 | config*.json 9 | *.sh 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Robust Deep Learning from Crowds with Belief Propagation 2 | This repository is the official implementation of ["Robust Deep Learning from Crowds with Belief Propagation"](https://arxiv.org/abs/2111.00734) accepted by AISTATS 2022. 3 | 4 | ## Abstract 5 | Crowdsourcing systems enable us to collect large-scale dataset, but inherently suffer from noisy labels of low-paid workers. We address the inference and learning problems using such a crowdsourced dataset with noise. Due to the nature of sparsity in crowdsourcing, it is critical to exploit both probabilistic model to capture worker prior and neural network to extract task feature despite risks from wrong prior and overfitted feature in practice. We hence establish a neural-powered Bayesian framework, from which we devise deepMF and deepBP with different choice of variational approximation methods, mean field (MF) and belief propagation (BP), respectively. This provides a unified view of existing methods, which are special cases of deepMF with different priors. In addition, our empirical study suggests that deepBP is a new approach, which is more robust against wrong prior, feature overfitting and extreme workers thanks to the more sophisticated BP than MF. 6 | 7 | ## Usages 8 | You can reproduce the experiments from our paper using the following command: 9 | ``` 10 | > python main.py --model deepBP --help 11 | usage: main.py [-h] --model {MV,BP,MF,CL,BayesDGC,deepBP,deepMF} [--seed SEED] [--n_epochs N_EPOCHS] [--test_interval TEST_INTERVAL] [--device DEVICE] [--batch_size BATCH_SIZE] 12 | [--lr LR] [--given_alpha GIVEN_ALPHA] [--given_beta GIVEN_BETA] [--clipping CLIPPING] [--n_iters N_ITERS] [--n_samples N_SAMPLES] --data_path DATA_PATH 13 | [--blur BLUR] [--n_tasks N_TASKS] [--true_alpha TRUE_ALPHA] [--true_beta TRUE_BETA] [--n_tasks_per_worker N_TASKS_PER_WORKER] 14 | [--n_workers_per_task N_WORKERS_PER_TASK] [--n_extreme_spammers N_EXTREME_SPAMMERS] 15 | 16 | optional arguments: 17 | -h, --help show this help message and exit 18 | --model {MV,BP,MF,CL,BayesDGC,deepBP,deepMF} 19 | 20 | Training: 21 | --seed SEED 22 | --n_epochs N_EPOCHS 23 | --test_interval TEST_INTERVAL 24 | --device DEVICE 25 | --batch_size BATCH_SIZE 26 | --lr LR 27 | 28 | Model: 29 | --given_alpha GIVEN_ALPHA 30 | --given_beta GIVEN_BETA 31 | --clipping CLIPPING 32 | --n_iters N_ITERS 33 | --n_samples N_SAMPLES 34 | 35 | Dataset: 36 | --data_path DATA_PATH 37 | --blur BLUR 38 | --n_tasks N_TASKS 39 | --true_alpha TRUE_ALPHA 40 | --true_beta TRUE_BETA 41 | --n_tasks_per_worker N_TASKS_PER_WORKER 42 | --n_workers_per_task N_WORKERS_PER_TASK 43 | --n_extreme_spammers N_EXTREME_SPAMMERS 44 | ``` 45 | Because the hyperparameters differs for each model, you should check the hyperparameters for your desired model using the above command. 46 | 47 | ### Models 48 | For the models, we implemented `MV`, `BP`, `MF`, `CL`, `BayesDGC` and our methods `deepBP` and `deepMF`. 49 | - `MV`: Learn a classifier with majority-voted labels. 50 | - `BP` and `MF`: Learn a classifier with the labels obtained by denoising the crowdsourced data using BP and MF [(Liu et al., 2012)](https://papers.nips.cc/paper/2012/hash/cd00692c3bfe59267d5ecfac5310286c-Abstract.html). 51 | - `CL`: Implementation of [Rodrigues and Pereira, 2018](https://arxiv.org/abs/1709.01779) with a regularizer proposed in [Tanno et al. 2019](https://arxiv.org/abs/1902.03680). 52 | - `BayesDGC`: Implementation of [Li et al., 2021](http://scis.scichina.com/en/2021/130104.pdf). 53 | - `deepBP` and `deepMF`: Implementation of our methods. 54 | 55 | ## Cite 56 | Please cite our paper if you use the model or this code in your own work: 57 | ``` 58 | @inproceedings{ho2022crowds, 59 | title={Robust Deep Learning from Crowds with Belief Propagation}, 60 | author={Hoyoung Kim* and Seunghyuk Cho* and Dongwoo Kim and Jungseul Ok}, 61 | booktitle=AISTATS, 62 | year={2022}, 63 | url={https://arxiv.org/abs/2111.00734} 64 | } 65 | ``` 66 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import random 3 | import argparse 4 | import importlib 5 | import numpy as np 6 | from torch.utils.data import DataLoader 7 | 8 | models = ['MV', 'BP', 'MF', 'CL', 'BayesDGC', 'deepBP', 'deepMF'] 9 | 10 | 11 | def initial_parser(): 12 | parser = argparse.ArgumentParser(add_help=False) 13 | parser.add_argument('--model', type=str, choices=models, required=True) 14 | return parser 15 | 16 | 17 | if __name__ == '__main__': 18 | parser = initial_parser() 19 | model_name = parser.parse_known_args()[0].model 20 | module = importlib.import_module(f'models.learnings.{model_name}') 21 | 22 | parser = argparse.ArgumentParser() 23 | parser.add_argument('--model', type=str, choices=models, required=True) 24 | getattr(module, 'add_args')(parser) 25 | args = parser.parse_args() 26 | 27 | random.seed(args.seed) 28 | np.random.seed(args.seed) 29 | torch.set_num_threads(2) 30 | torch.manual_seed(args.seed) 31 | torch.cuda.manual_seed(args.seed) 32 | torch.backends.cudnn.benchmark = False 33 | torch.backends.cudnn.deterministic = True 34 | 35 | print('Loading data...') 36 | train_data = getattr(module, 'Dataset')(args, is_train=True) 37 | train_loader = DataLoader(train_data, batch_size=args.batch_size, shuffle=True) 38 | test_data = getattr(module, 'Dataset')(args, is_train=False) 39 | test_loader = DataLoader(test_data, batch_size=args.batch_size, shuffle=False) 40 | 41 | print('Start training!') 42 | model = getattr(module, 'Model')(args).to(args.device) 43 | getattr(module, 'train')(args, model, train_loader, test_loader) 44 | 45 | -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ml-postech/robust-deep-learning-from-crowds-with-belief-propagation/9d9a524e746a15fd7c75de8434c9dddcb4a0cf89/models/__init__.py -------------------------------------------------------------------------------- /models/inferences/__init__.py: -------------------------------------------------------------------------------- 1 | from .bp import run_bp 2 | from .mf import run_mf 3 | -------------------------------------------------------------------------------- /models/inferences/bp.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.distributions import Dirichlet 3 | 4 | from .utils import get_stats 5 | 6 | def get_init_messages(ann, n_classes=2, device='cuda:0'): 7 | n_tasks, n_workers = get_stats(ann) 8 | m_tasks = torch.ones((n_workers, n_tasks, n_classes)).to(device) * (ann.transpose(0, 1) != -1).unsqueeze(dim=-1) * 0.5 9 | return m_tasks 10 | 11 | 12 | def compute_marginal(ann, m_tasks, m_features=None): 13 | m_tasks = m_tasks + (ann.transpose(0, 1) == -1).int().unsqueeze(dim=-1) 14 | probs = m_tasks.prod(dim=0) 15 | if m_features is not None: 16 | probs = probs * m_features 17 | 18 | return probs 19 | 20 | 21 | def bp_iteration(ann, m_tasks, q_prior, m_features=None, device='cuda:0', n_samples=400): 22 | # Update worker messages 23 | m_tasks_tmp = m_tasks + (ann.transpose(0, 1) == -1).int().unsqueeze(dim=-1) 24 | m_prods = m_tasks_tmp.prod(dim=0) 25 | if m_features is not None: 26 | m_prods = m_prods * m_features 27 | m_workers_new = m_prods.unsqueeze(dim=0) / m_tasks_tmp 28 | m_workers_new = m_workers_new.transpose(0, 1) 29 | m_workers_new = m_workers_new / m_workers_new.sum(dim=-1).unsqueeze(dim=-1) 30 | 31 | # Sample worker abilities 32 | q_dist = Dirichlet(q_prior.transpose(0, 1)) 33 | qs = q_dist.sample((n_samples,)).to(device) 34 | torch.cuda.empty_cache() 35 | 36 | # Update task messages 37 | m_workers_sum = m_workers_new.unsqueeze(dim=0) * qs[:, :, ann].permute(0, 2, 3, 1) 38 | m_workers_sum = m_workers_sum.sum(dim=-1) 39 | m_workers_sum = m_workers_sum * (ann != -1).unsqueeze(dim=0) 40 | m_workers_sum = m_workers_sum + (ann == -1).int().unsqueeze(dim=0) 41 | m_prods = m_workers_sum.log().sum(dim=1).unsqueeze(dim=1) - m_workers_sum.log() 42 | m_prods = m_prods.unsqueeze(dim=-1) + qs[:, :, ann].permute(0, 2, 3, 1).log() 43 | m_prods = m_prods - m_prods.amax(dim=(0, 1, 3)).reshape((1, 1, m_prods.size(2), 1)) 44 | m_tasks_new = m_prods.exp().sum(dim=0) 45 | m_tasks_new = m_tasks_new.transpose(0, 1) 46 | m_tasks_new = m_tasks_new / m_tasks_new.sum(dim=-1).unsqueeze(dim=-1) 47 | m_tasks_new = m_tasks_new * (ann.transpose(0, 1) != -1).unsqueeze(dim=-1) 48 | 49 | return m_tasks_new 50 | 51 | 52 | def run_bp(ann, n_iters, q_prior, m_features=None, device='cuda:0', n_samples=400, n_classes=2): 53 | m_tasks_prev = get_init_messages(ann, n_classes, device) 54 | 55 | for _ in range(n_iters): 56 | m_tasks = bp_iteration(ann, m_tasks_prev, q_prior, m_features, device, n_samples) 57 | err = (m_tasks[:, :, 0] - m_tasks_prev[:, :, 0]).abs().max() 58 | if err < 0.03: 59 | break 60 | m_tasks_prev = m_tasks 61 | z_bp = compute_marginal(ann, m_tasks, m_features) 62 | z_bp = z_bp / z_bp.sum(dim=1).unsqueeze(dim=-1) 63 | 64 | return z_bp 65 | 66 | -------------------------------------------------------------------------------- /models/inferences/mf.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | 4 | from .utils import get_workers_of_task, get_tasks_of_worker, get_stats 5 | 6 | 7 | def mf_iteration(ann, p_tasks, prior): 8 | softmax = nn.Softmax(dim=1) 9 | n_tasks, n_workers = get_stats(ann) 10 | 11 | qs = torch.FloatTensor([prior for _ in range(n_workers)]) 12 | 13 | for u in range(n_workers): 14 | tasks_of_worker = get_tasks_of_worker(ann, u) 15 | 16 | for t in tasks_of_worker: 17 | qs[u, :, ann[t, u]] += p_tasks[t] 18 | 19 | for t in range(n_tasks): 20 | workers_of_task = get_workers_of_task(ann, t) 21 | 22 | tmp = -qs[workers_of_task].sum(dim=2).digamma().sum(dim=0) 23 | for u in workers_of_task: 24 | tmp += qs[u, :, ann[t, u]].digamma() 25 | p_tasks[t] = tmp 26 | 27 | p_tasks = softmax(p_tasks) 28 | return p_tasks 29 | 30 | 31 | def run_mf(ann, n_iters, prior, n_classes=2): 32 | n_tasks, _ = get_stats(ann) 33 | p_tasks = torch.ones(n_tasks, n_classes) * 0.5 34 | 35 | for _ in range(n_iters): 36 | p_tasks = mf_iteration(ann, p_tasks, prior) 37 | 38 | return p_tasks 39 | 40 | -------------------------------------------------------------------------------- /models/inferences/utils.py: -------------------------------------------------------------------------------- 1 | def get_stats(ann): 2 | n_tasks = ann.size(0) 3 | n_workers = ann.size(1) 4 | return n_tasks, n_workers 5 | 6 | 7 | def get_tasks_of_worker(ann, u): 8 | return (ann[:, u].view(-1) != -1).nonzero().view(-1) 9 | 10 | 11 | def get_workers_of_task(ann, t): 12 | return (ann[t].view(-1) != -1).nonzero().view(-1) 13 | 14 | -------------------------------------------------------------------------------- /models/learnings/BP/__init__.py: -------------------------------------------------------------------------------- 1 | from .arguments import add_args 2 | from .model import Model 3 | from .dataset import Dataset 4 | from .train import train 5 | -------------------------------------------------------------------------------- /models/learnings/BP/arguments.py: -------------------------------------------------------------------------------- 1 | from .model import add_model_args 2 | from .dataset import add_dataset_args 3 | from .train import add_train_args 4 | 5 | 6 | def add_args(parser): 7 | add_train_args(parser) 8 | add_model_args(parser) 9 | add_dataset_args(parser) 10 | 11 | -------------------------------------------------------------------------------- /models/learnings/BP/dataset.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from ...inferences import run_bp 3 | 4 | from ..base import BaseDataset, add_base_dataset_args 5 | 6 | 7 | def add_dataset_args(parser): 8 | group = parser.add_argument_group('Dataset') 9 | add_base_dataset_args(group) 10 | group.add_argument('--given_alpha', type=float, default=2.) 11 | group.add_argument('--given_beta', type=float, default=1.) 12 | group.add_argument('--n_iters', type=int, default=100) 13 | group.add_argument('--n_samples', type=int, default=400) 14 | 15 | 16 | class Dataset(BaseDataset): 17 | def __init__(self, args, is_train): 18 | super().__init__(args, is_train) 19 | 20 | def preprocess(self, ann): 21 | ann = torch.LongTensor(ann).to(self.args.device) 22 | prior = torch.FloatTensor([ 23 | [self.args.given_alpha, self.args.given_beta], 24 | [self.args.given_beta, self.args.given_alpha] 25 | ]) 26 | 27 | labels_bp = run_bp(ann, self.args.n_iters, prior, device=self.args.device, n_samples=self.args.n_samples) 28 | labels_bp = labels_bp.cpu().numpy() 29 | return labels_bp 30 | 31 | -------------------------------------------------------------------------------- /models/learnings/BP/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from ..base import Classifier 6 | 7 | 8 | def add_model_args(parser): 9 | parser.add_argument_group('Model') 10 | 11 | 12 | class Model(nn.Module): 13 | def __init__(self, args): 14 | super(Model, self).__init__() 15 | 16 | self.classifier = Classifier() 17 | self.criterion = nn.KLDivLoss(reduction='sum') 18 | 19 | 20 | def forward(self, x, z_bp): 21 | z_pred = F.softmax(self.classifier(x), dim=-1) 22 | loss = self.criterion(z_pred.log(), z_bp) 23 | 24 | return loss 25 | 26 | -------------------------------------------------------------------------------- /models/learnings/BP/train.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.optim import Adam 3 | from ..base import add_base_train_args 4 | 5 | 6 | def add_train_args(parser): 7 | group = parser.add_argument_group('Training') 8 | add_base_train_args(group) 9 | group.add_argument('--lr', type=float, default=1e-3) 10 | 11 | 12 | def test(args, model, train_loader, test_loader): 13 | correct_train, correct_test = 0, 0 14 | with torch.no_grad(): 15 | model.eval() 16 | for x, z, _ in train_loader: 17 | x, z = x.to(args.device), z.to(args.device) 18 | z_pred = model.classifier(x).argmax(dim=1) 19 | correct_train += z_pred.eq(z).sum().item() 20 | 21 | for x, z in test_loader: 22 | x, z = x.to(args.device), z.to(args.device) 23 | z_pred = model.classifier(x).argmax(dim=1) 24 | correct_test += z_pred.eq(z).sum().item() 25 | 26 | correct_train /= len(train_loader.dataset) 27 | correct_test /= len(test_loader.dataset) 28 | 29 | print(f'======> Inference acc: {correct_train * 100:.2f}') 30 | print(f'======> Learning acc: {correct_test * 100:.2f}') 31 | 32 | 33 | def train_epoch(args, epoch, model, train_loader, opt): 34 | loss_total = 0.0 35 | 36 | model.train() 37 | for x, z, z_mv in train_loader: 38 | model.zero_grad() 39 | x, z, z_mv = x.to(args.device), z.to(args.device), z_mv.to(args.device) 40 | loss = model(x, z_mv) 41 | loss.backward() 42 | opt.step() 43 | 44 | loss_total += loss.item() 45 | 46 | print(f'Epoch {epoch:4d} | Loss: {loss_total:.3f}') 47 | 48 | 49 | def train(args, model, train_loader, test_loader): 50 | opt = Adam(model.classifier.parameters(), lr=args.lr) 51 | 52 | for e in range(1, args.n_epochs + 1): 53 | train_epoch(args, e, model, train_loader, opt) 54 | if e % args.test_interval == 0 or e == args.n_epochs: 55 | test(args, model, train_loader, test_loader) 56 | 57 | -------------------------------------------------------------------------------- /models/learnings/BayesDGC/__init__.py: -------------------------------------------------------------------------------- 1 | from .arguments import add_args 2 | from .model import Model 3 | from .dataset import Dataset 4 | from .train import train 5 | -------------------------------------------------------------------------------- /models/learnings/BayesDGC/arguments.py: -------------------------------------------------------------------------------- 1 | from .model import add_model_args 2 | from .dataset import add_dataset_args 3 | from .train import add_train_args 4 | 5 | 6 | def add_args(parser): 7 | add_train_args(parser) 8 | add_model_args(parser) 9 | add_dataset_args(parser) 10 | 11 | -------------------------------------------------------------------------------- /models/learnings/BayesDGC/dataset.py: -------------------------------------------------------------------------------- 1 | from ..base import BaseDataset, add_base_dataset_args 2 | 3 | 4 | def add_dataset_args(parser): 5 | group = parser.add_argument_group('Dataset') 6 | add_base_dataset_args(group) 7 | 8 | 9 | class Dataset(BaseDataset): 10 | def __init__(self, args, is_train): 11 | super().__init__(args, is_train) 12 | -------------------------------------------------------------------------------- /models/learnings/BayesDGC/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from ..base import Classifier 6 | 7 | 8 | def add_model_args(parser): 9 | group = parser.add_argument_group('Model') 10 | group.add_argument('--given_alpha', type=float, default=2.) 11 | group.add_argument('--given_beta', type=float, default=1.) 12 | group.add_argument('--clipping', type=float, default=1.0) 13 | group.add_argument('--lr_confusion', type=float, default=1e-3) 14 | 15 | 16 | class Model(nn.Module): 17 | def __init__(self, args): 18 | super(Model, self).__init__() 19 | 20 | self.n_classes = 2 21 | self.given_alpha = args.given_alpha 22 | self.given_beta = args.given_beta 23 | self.clipping = args.clipping 24 | self.lr_confusion = args.lr_confusion 25 | self.n_worker = args.n_tasks * args.n_workers_per_task // args.n_tasks_per_worker 26 | self.classifier = Classifier() 27 | self.criterion = nn.KLDivLoss(reduction='sum') 28 | 29 | prior = [[self.given_alpha, self.given_beta], [self.given_beta, self.given_alpha]] 30 | self.confusion = torch.FloatTensor([prior for _ in range(self.n_worker)]) 31 | 32 | def inference(self, x, ann): 33 | z_learning = F.softmax(self.classifier(x), dim=-1) 34 | z_learning = torch.clamp(z_learning, min=1 - self.clipping, max=self.clipping) + 1e-9 35 | z_learning = z_learning.cpu() 36 | 37 | with torch.no_grad(): 38 | z_inference = z_learning.log() 39 | 40 | for t in range(x.size(0)): 41 | workers_of_task = (ann[t].view(-1) != -1).nonzero().view(-1) 42 | 43 | tmp = -self.confusion[workers_of_task].sum(dim=2).digamma().sum(dim=0) 44 | for u in workers_of_task: 45 | tmp += self.confusion[u, :, ann[t, u]].digamma() 46 | z_inference[t] += tmp 47 | 48 | z_inference = F.softmax(z_inference, dim=-1) 49 | return z_inference, z_learning 50 | 51 | def forward(self, x, ann): 52 | z_inference, z_learning = self.inference(x, ann) 53 | 54 | for u in range(ann.size(1)): 55 | tasks_of_worker = (ann[:, u].view(-1) != -1).nonzero().view(-1) 56 | 57 | grad_confusion = torch.FloatTensor([[self.given_alpha, self.given_beta], [self.given_beta, self.given_alpha]]) 58 | for t in tasks_of_worker: 59 | grad_confusion[:, ann[t, u]] += z_inference[t] 60 | self.confusion[u] = self.confusion[u] - self.lr_confusion * (grad_confusion - self.confusion[u]) 61 | 62 | loss = self.criterion(z_learning.log(), z_inference) 63 | return loss 64 | 65 | -------------------------------------------------------------------------------- /models/learnings/BayesDGC/train.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.optim import Adam 3 | from ..base import add_base_train_args 4 | 5 | 6 | def add_train_args(parser): 7 | group = parser.add_argument_group('Training') 8 | add_base_train_args(group) 9 | group.add_argument('--lr', type=float, default=1e-3) 10 | 11 | 12 | def test(args, model, train_loader, test_loader): 13 | correct_train, correct_test = 0, 0 14 | with torch.no_grad(): 15 | model.eval() 16 | for x, z, ann in train_loader: 17 | x = x.to(args.device) 18 | z_pred, _ = model.inference(x, ann) 19 | z_pred = z_pred.argmax(dim=-1) 20 | correct_train += z_pred.eq(z).sum().item() 21 | 22 | for x, z in test_loader: 23 | x, z = x.to(args.device), z.to(args.device) 24 | z_pred = model.classifier(x).argmax(dim=1) 25 | correct_test += z_pred.eq(z).sum().item() 26 | 27 | correct_train /= len(train_loader.dataset) 28 | correct_test /= len(test_loader.dataset) 29 | 30 | print(f'======> Inference acc: {correct_train * 100:.2f}') 31 | print(f'======> Learning acc: {correct_test * 100:.2f}') 32 | 33 | 34 | def train_epoch(args, epoch, model, train_loader, opt): 35 | loss_total = 0.0 36 | 37 | model.train() 38 | for x, z, ann in train_loader: 39 | model.zero_grad() 40 | x, z, ann = x.to(args.device), z.to(args.device), ann.to(args.device) 41 | loss = model(x, ann) 42 | loss.backward() 43 | opt.step() 44 | 45 | loss_total += loss.item() 46 | 47 | print(f'Epoch {epoch:4d} | Loss: {loss_total:.3f}') 48 | 49 | 50 | def train(args, model, train_loader, test_loader): 51 | opt = Adam(model.classifier.parameters(), lr=args.lr) 52 | 53 | for e in range(1, args.n_epochs + 1): 54 | train_epoch(args, e, model, train_loader, opt) 55 | if e % args.test_interval == 0 or e == args.n_epochs: 56 | test(args, model, train_loader, test_loader) 57 | 58 | -------------------------------------------------------------------------------- /models/learnings/CL/__init__.py: -------------------------------------------------------------------------------- 1 | from .arguments import add_args 2 | from .model import Model 3 | from .dataset import Dataset 4 | from .train import train 5 | -------------------------------------------------------------------------------- /models/learnings/CL/arguments.py: -------------------------------------------------------------------------------- 1 | from .model import add_model_args 2 | from .dataset import add_dataset_args 3 | from .train import add_train_args 4 | 5 | 6 | def add_args(parser): 7 | add_train_args(parser) 8 | add_model_args(parser) 9 | add_dataset_args(parser) 10 | 11 | -------------------------------------------------------------------------------- /models/learnings/CL/dataset.py: -------------------------------------------------------------------------------- 1 | from ..base import BaseDataset, add_base_dataset_args 2 | 3 | 4 | def add_dataset_args(parser): 5 | group = parser.add_argument_group('Dataset') 6 | add_base_dataset_args(group) 7 | 8 | 9 | class Dataset(BaseDataset): 10 | def __init__(self, args, is_train): 11 | super().__init__(args, is_train) 12 | -------------------------------------------------------------------------------- /models/learnings/CL/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from ..base import Classifier 6 | 7 | 8 | def add_model_args(parser): 9 | group = parser.add_argument_group('Model') 10 | group.add_argument('--init', type=float, default=1.) 11 | group.add_argument('--lam', type=float, default=0.) 12 | 13 | 14 | class Model(nn.Module): 15 | def __init__(self, args): 16 | super(Model, self).__init__() 17 | 18 | self.n_classes = 2 19 | self.init = args.init 20 | self.lam = args.lam 21 | self.n_worker = args.n_tasks * args.n_workers_per_task // args.n_tasks_per_worker 22 | self.classifier = Classifier() 23 | self.criterion = nn.CrossEntropyLoss(ignore_index=-1, reduction='sum') 24 | 25 | w_weight = [torch.eye(self.n_classes) * self.init for _ in range(self.n_worker)] 26 | self.confusion = nn.parameter.Parameter(torch.stack(w_weight), requires_grad=True) 27 | 28 | def forward(self, x, ann): 29 | z_pred = F.softmax(self.classifier(x), dim=1) 30 | pm = F.softmax(self.confusion, dim=2) 31 | ann_pred = torch.einsum('ik,jkl->ijl', z_pred, pm).view((-1, self.n_classes)) 32 | 33 | reg = torch.zeros(1, device=x.device) 34 | for i in range(self.n_worker): 35 | reg += pm[i, 0, 0].log() 36 | 37 | ann = ann.view(-1) 38 | loss = self.criterion(ann_pred, ann) + self.lam * reg 39 | 40 | return loss 41 | 42 | -------------------------------------------------------------------------------- /models/learnings/CL/train.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.optim import Adam 3 | from ..base import add_base_train_args 4 | 5 | 6 | def add_train_args(parser): 7 | group = parser.add_argument_group('Training') 8 | add_base_train_args(group) 9 | group.add_argument('--lr', type=float, default=1e-3) 10 | group.add_argument('--lr_confusion', type=float, default=1e-3) 11 | 12 | 13 | def test(args, model, train_loader, test_loader): 14 | correct_train, correct_test = 0, 0 15 | with torch.no_grad(): 16 | model.eval() 17 | for x, z, _ in train_loader: 18 | x, z = x.to(args.device), z.to(args.device) 19 | z_pred = model.classifier(x).argmax(dim=1) 20 | correct_train += z_pred.eq(z).sum().item() 21 | 22 | for x, z in test_loader: 23 | x, z = x.to(args.device), z.to(args.device) 24 | z_pred = model.classifier(x).argmax(dim=1) 25 | correct_test += z_pred.eq(z).sum().item() 26 | 27 | correct_train /= len(train_loader.dataset) 28 | correct_test /= len(test_loader.dataset) 29 | 30 | print(f'======> Inference acc: {correct_train * 100:.2f}') 31 | print(f'======> Learning acc: {correct_test * 100:.2f}') 32 | 33 | 34 | def train_epoch(args, epoch, model, train_loader, opt_1, opt_2): 35 | loss_total = 0.0 36 | 37 | model.train() 38 | for x, z, ann in train_loader: 39 | model.zero_grad() 40 | x, z, ann = x.to(args.device), z.to(args.device), ann.to(args.device) 41 | loss = model(x, ann) 42 | loss.backward() 43 | opt_1.step() 44 | opt_2.step() 45 | 46 | loss_total += loss.item() 47 | 48 | print(f'Epoch {epoch:4d} | Loss: {loss_total:.3f}') 49 | 50 | 51 | def train(args, model, train_loader, test_loader): 52 | opt_classifier = Adam(model.classifier.parameters(), lr=args.lr) 53 | opt_workers = Adam([model.confusion], lr=args.lr_confusion) 54 | 55 | for e in range(1, args.n_epochs + 1): 56 | train_epoch(args, e, model, train_loader, opt_classifier, opt_workers) 57 | if e % args.test_interval == 0 or e == args.n_epochs: 58 | test(args, model, train_loader, test_loader) 59 | 60 | -------------------------------------------------------------------------------- /models/learnings/MF/__init__.py: -------------------------------------------------------------------------------- 1 | from .arguments import add_args 2 | from .model import Model 3 | from .dataset import Dataset 4 | from .train import train 5 | -------------------------------------------------------------------------------- /models/learnings/MF/arguments.py: -------------------------------------------------------------------------------- 1 | from .model import add_model_args 2 | from .dataset import add_dataset_args 3 | from .train import add_train_args 4 | 5 | 6 | def add_args(parser): 7 | add_train_args(parser) 8 | add_model_args(parser) 9 | add_dataset_args(parser) 10 | 11 | -------------------------------------------------------------------------------- /models/learnings/MF/dataset.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from ...inferences import run_mf 3 | 4 | from ..base import BaseDataset, add_base_dataset_args 5 | 6 | 7 | def add_dataset_args(parser): 8 | group = parser.add_argument_group('Dataset') 9 | add_base_dataset_args(group) 10 | group.add_argument('--given_alpha', type=float, default=2.) 11 | group.add_argument('--given_beta', type=float, default=1.) 12 | group.add_argument('--n_iters', type=int, default=100) 13 | group.add_argument('--n_samples', type=int, default=400) 14 | 15 | 16 | class Dataset(BaseDataset): 17 | def __init__(self, args, is_train): 18 | super().__init__(args, is_train) 19 | 20 | def preprocess(self, ann): 21 | ann = torch.LongTensor(ann).to(self.args.device) 22 | prior = [ 23 | [self.args.given_alpha, self.args.given_beta], 24 | [self.args.given_beta, self.args.given_alpha] 25 | ] 26 | 27 | labels_mf = run_mf(ann, self.args.n_iters, prior) 28 | labels_mf = labels_mf.cpu().numpy() 29 | return labels_mf 30 | 31 | -------------------------------------------------------------------------------- /models/learnings/MF/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from ..base import Classifier 6 | 7 | 8 | def add_model_args(parser): 9 | parser.add_argument_group('Model') 10 | 11 | 12 | class Model(nn.Module): 13 | def __init__(self, args): 14 | super(Model, self).__init__() 15 | 16 | self.classifier = Classifier() 17 | self.criterion = nn.KLDivLoss(reduction='sum') 18 | 19 | 20 | def forward(self, x, z_mf): 21 | z_pred = F.softmax(self.classifier(x), dim=-1) 22 | loss = self.criterion(z_pred.log(), z_mf) 23 | 24 | return loss 25 | 26 | -------------------------------------------------------------------------------- /models/learnings/MF/train.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.optim import Adam 3 | from ..base import add_base_train_args 4 | 5 | 6 | def add_train_args(parser): 7 | group = parser.add_argument_group('Training') 8 | add_base_train_args(group) 9 | group.add_argument('--lr', type=float, default=1e-3) 10 | 11 | 12 | def test(args, model, train_loader, test_loader): 13 | correct_train, correct_test = 0, 0 14 | with torch.no_grad(): 15 | model.eval() 16 | for x, z, _ in train_loader: 17 | x, z = x.to(args.device), z.to(args.device) 18 | z_pred = model.classifier(x).argmax(dim=1) 19 | correct_train += z_pred.eq(z).sum().item() 20 | 21 | for x, z in test_loader: 22 | x, z = x.to(args.device), z.to(args.device) 23 | z_pred = model.classifier(x).argmax(dim=1) 24 | correct_test += z_pred.eq(z).sum().item() 25 | 26 | correct_train /= len(train_loader.dataset) 27 | correct_test /= len(test_loader.dataset) 28 | 29 | print(f'======> Inference acc: {correct_train * 100:.2f}') 30 | print(f'======> Learning acc: {correct_test * 100:.2f}') 31 | 32 | 33 | def train_epoch(args, epoch, model, train_loader, opt): 34 | loss_total = 0.0 35 | 36 | model.train() 37 | for x, z, z_mv in train_loader: 38 | model.zero_grad() 39 | x, z, z_mv = x.to(args.device), z.to(args.device), z_mv.to(args.device) 40 | loss = model(x, z_mv) 41 | loss.backward() 42 | opt.step() 43 | 44 | loss_total += loss.item() 45 | 46 | print(f'Epoch {epoch:4d} | Loss: {loss_total:.3f}') 47 | 48 | 49 | def train(args, model, train_loader, test_loader): 50 | opt = Adam(model.classifier.parameters(), lr=args.lr) 51 | 52 | for e in range(1, args.n_epochs + 1): 53 | train_epoch(args, e, model, train_loader, opt) 54 | if e % args.test_interval == 0 or e == args.n_epochs: 55 | test(args, model, train_loader, test_loader) 56 | 57 | -------------------------------------------------------------------------------- /models/learnings/MV/__init__.py: -------------------------------------------------------------------------------- 1 | from .arguments import add_args 2 | from .model import Model 3 | from .dataset import Dataset 4 | from .train import train 5 | -------------------------------------------------------------------------------- /models/learnings/MV/arguments.py: -------------------------------------------------------------------------------- 1 | from .model import add_model_args 2 | from .dataset import add_dataset_args 3 | from .train import add_train_args 4 | 5 | 6 | def add_args(parser): 7 | add_train_args(parser) 8 | add_model_args(parser) 9 | add_dataset_args(parser) 10 | 11 | -------------------------------------------------------------------------------- /models/learnings/MV/dataset.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from ..base import BaseDataset, add_base_dataset_args 4 | 5 | 6 | def add_dataset_args(parser): 7 | group = parser.add_argument_group('Dataset') 8 | add_base_dataset_args(group) 9 | 10 | 11 | class Dataset(BaseDataset): 12 | def __init__(self, args, is_train): 13 | super().__init__(args, is_train) 14 | 15 | def preprocess(self, ann): 16 | labels_mv = np.zeros(self.args.n_tasks) 17 | for t in range(ann.shape[0]): 18 | cnt = np.array([0, 0]) 19 | for label in ann[t]: 20 | if label != -1: 21 | cnt[label] += 1 22 | labels_mv[t] = cnt.argmax(axis=0) 23 | 24 | labels_mv = labels_mv.astype(np.int64) 25 | return labels_mv 26 | 27 | -------------------------------------------------------------------------------- /models/learnings/MV/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from ..base import Classifier 6 | 7 | 8 | def add_model_args(parser): 9 | parser.add_argument_group('Model') 10 | 11 | 12 | class Model(nn.Module): 13 | def __init__(self, args): 14 | super(Model, self).__init__() 15 | 16 | self.classifier = Classifier() 17 | self.criterion = nn.CrossEntropyLoss(reduction='sum') 18 | 19 | 20 | def forward(self, x, z_mv): 21 | z_pred = self.classifier(x) 22 | loss = self.criterion(z_pred, z_mv) 23 | 24 | return loss 25 | 26 | -------------------------------------------------------------------------------- /models/learnings/MV/train.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.optim import Adam 3 | from ..base import add_base_train_args 4 | 5 | 6 | def add_train_args(parser): 7 | group = parser.add_argument_group('Training') 8 | add_base_train_args(group) 9 | group.add_argument('--lr', type=float, default=1e-3) 10 | 11 | 12 | def test(args, model, train_loader, test_loader): 13 | correct_train, correct_test = 0, 0 14 | with torch.no_grad(): 15 | model.eval() 16 | for x, z, _ in train_loader: 17 | x, z = x.to(args.device), z.to(args.device) 18 | z_pred = model.classifier(x).argmax(dim=1) 19 | correct_train += z_pred.eq(z).sum().item() 20 | 21 | for x, z in test_loader: 22 | x, z = x.to(args.device), z.to(args.device) 23 | z_pred = model.classifier(x).argmax(dim=1) 24 | correct_test += z_pred.eq(z).sum().item() 25 | 26 | correct_train /= len(train_loader.dataset) 27 | correct_test /= len(test_loader.dataset) 28 | 29 | print(f'======> Inference acc: {correct_train * 100:.2f}') 30 | print(f'======> Learning acc: {correct_test * 100:.2f}') 31 | 32 | 33 | def train_epoch(args, epoch, model, train_loader, opt): 34 | loss_total = 0.0 35 | 36 | model.train() 37 | for x, z, z_mv in train_loader: 38 | model.zero_grad() 39 | x, z, z_mv = x.to(args.device), z.to(args.device), z_mv.to(args.device) 40 | loss = model(x, z_mv) 41 | loss.backward() 42 | opt.step() 43 | 44 | loss_total += loss.item() 45 | 46 | print(f'Epoch {epoch:4d} | Loss: {loss_total:.3f}') 47 | 48 | 49 | def train(args, model, train_loader, test_loader): 50 | opt = Adam(model.classifier.parameters(), lr=args.lr) 51 | 52 | for e in range(1, args.n_epochs + 1): 53 | train_epoch(args, e, model, train_loader, opt) 54 | if e % args.test_interval == 0 or e == args.n_epochs: 55 | test(args, model, train_loader, test_loader) 56 | 57 | -------------------------------------------------------------------------------- /models/learnings/base/__init__.py: -------------------------------------------------------------------------------- 1 | from .model import Classifier 2 | from .dataset import add_base_dataset_args, BaseDataset 3 | from .train import add_base_train_args 4 | -------------------------------------------------------------------------------- /models/learnings/base/dataset.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import random 3 | import numpy as np 4 | from pathlib import Path 5 | from PIL import Image, ImageFilter 6 | from torchvision import transforms 7 | from torch.utils.data import Dataset 8 | 9 | 10 | def add_base_dataset_args(parser): 11 | parser.add_argument('--data_path', type=str, required=True) 12 | parser.add_argument('--blur', type=int) 13 | parser.add_argument('--n_tasks', type=int, default=1000) 14 | parser.add_argument('--true_alpha', type=float, default=2.) 15 | parser.add_argument('--true_beta', type=float, default=1.) 16 | parser.add_argument('--n_tasks_per_worker', type=int, default=4) 17 | parser.add_argument('--n_workers_per_task', type=int, default=3) 18 | parser.add_argument('--n_extreme_spammers', type=int, default=0) 19 | 20 | 21 | class BaseDataset(Dataset): 22 | def __init__(self, args, is_train=True): 23 | self.args = args 24 | self.is_train = is_train 25 | 26 | root_dir = Path(args.data_path) / ('train' if self.is_train else 'test') 27 | self.images = np.load(root_dir / 'data.npy') 28 | self.labels = np.load(root_dir / 'labels.npy') 29 | if self.is_train: 30 | self.images = self.images[86:86 + args.n_tasks] 31 | self.labels = self.labels[86:86 + args.n_tasks] 32 | 33 | if args.blur is not None: 34 | blur = ImageFilter.GaussianBlur(args.blur) 35 | for idx in range(self.images.shape[0]): 36 | img = Image.fromarray(self.images[idx]) 37 | img = img.filter(blur) 38 | self.images[idx] = np.array(img) 39 | 40 | self.transforms = transforms.Compose([ 41 | transforms.ToPILImage(), 42 | transforms.ToTensor(), 43 | transforms.Normalize( 44 | (0.4880633112757274, 0.45490764498668607, 0.45490764498668607), 45 | (0.26311969928021317, 0.25659017730169825, 0.259108662261481) 46 | ) 47 | ]) 48 | 49 | if self.is_train: 50 | ann, _ = generate_labels(self.labels, args.true_alpha, args.true_beta, args.n_tasks_per_worker, args.n_workers_per_task, args.n_extreme_spammers) 51 | self.annotations = self.preprocess(ann) 52 | 53 | def __len__(self): 54 | return self.images.shape[0] 55 | 56 | def preprocess(self, ann): 57 | return ann 58 | 59 | def __getitem__(self, idx): 60 | x = self.images[idx] 61 | x = self.transforms(x) 62 | y = self.labels[idx] 63 | 64 | if self.is_train: 65 | ann = self.annotations[idx] 66 | return x, y, ann 67 | else: 68 | return x, y 69 | 70 | 71 | def generate_labels(ground_truth, alpha, beta, n_tasks_per_worker, n_workers_per_task, n_extreme_spammers=0): 72 | n_tasks = ground_truth.shape[0] 73 | solved = [0] * n_tasks 74 | n_workers = int(n_tasks * n_workers_per_task / n_tasks_per_worker) 75 | n_classes = 2 76 | 77 | labels = [] 78 | new_qs = [] 79 | 80 | for _ in range(n_extreme_spammers): 81 | worker_labels = random.choices([0, 1], k=n_tasks) 82 | labels.append(worker_labels) 83 | 84 | dist = torch.distributions.Beta(torch.FloatTensor([alpha]), torch.FloatTensor([beta])) 85 | for _ in range(n_workers): 86 | q = dist.sample().numpy()[0] 87 | q = [[q, 1 - q], [1 - q, q]] 88 | new_qs.append(q) 89 | 90 | worker_labels = [-1] * n_tasks 91 | task_indices = list(range(n_tasks)) 92 | random.shuffle(task_indices) 93 | 94 | cnt = 0 95 | for t in task_indices: 96 | if solved[t] == n_workers_per_task: 97 | continue 98 | worker_labels[t] = np.random.choice(list(range(n_classes)), p=q[ground_truth[t]]) 99 | cnt += 1 100 | solved[t] += 1 101 | 102 | if cnt == n_tasks_per_worker: 103 | break 104 | if cnt != 0: 105 | labels.append(worker_labels) 106 | 107 | labels = np.array(labels).transpose() 108 | return labels, new_qs 109 | 110 | 111 | -------------------------------------------------------------------------------- /models/learnings/base/model.py: -------------------------------------------------------------------------------- 1 | from torch import nn 2 | 3 | 4 | class Classifier(nn.Module): 5 | def __init__(self): 6 | super().__init__() 7 | 8 | n_classes = 2 9 | self.conv1 = nn.Conv2d(3, 32, kernel_size=(3,3)) 10 | self.pool1 = nn.MaxPool2d(2) 11 | self.bn1 = nn.BatchNorm2d(32) 12 | 13 | self.conv2 = nn.Conv2d(32, 32, kernel_size=(3,3)) 14 | self.pool2 = nn.MaxPool2d(2) 15 | 16 | self.conv3 = nn.Conv2d(32, 64, kernel_size=(3,3)) 17 | self.pool3 = nn.MaxPool2d(2) 18 | 19 | self.conv4 = nn.Conv2d(64, 64, kernel_size=(3,3)) 20 | self.pool4 = nn.MaxPool2d(2) 21 | 22 | self.fc1 = nn.Linear(64 * 7 * 7, 128) 23 | self.drop1 = nn.Dropout(0.5) 24 | self.fc2 = nn.Linear(128, n_classes) 25 | 26 | self.relu = nn.ReLU() 27 | 28 | def forward(self, x): 29 | x = self.relu(self.conv1(x)) 30 | x = self.pool1(x) 31 | x = self.bn1(x) 32 | x = self.relu(self.conv2(x)) 33 | x = self.pool2(x) 34 | x = self.relu(self.conv3(x)) 35 | x = self.pool3(x) 36 | x = self.relu(self.conv4(x)) 37 | x = self.pool4(x) 38 | x = x.view(x.size(0), -1) 39 | x = self.relu(self.fc1(x)) 40 | x = self.drop1(x) 41 | x = self.fc2(x) 42 | 43 | return x 44 | 45 | -------------------------------------------------------------------------------- /models/learnings/base/train.py: -------------------------------------------------------------------------------- 1 | def add_base_train_args(parser): 2 | parser.add_argument('--seed', type=int, default=7777) 3 | parser.add_argument('--n_epochs', type=int, default=200) 4 | parser.add_argument('--test_interval', type=int, default=200) 5 | parser.add_argument('--device', type=str, default='cuda:0') 6 | parser.add_argument('--batch_size', type=int, default=1024) 7 | 8 | -------------------------------------------------------------------------------- /models/learnings/deepBP/__init__.py: -------------------------------------------------------------------------------- 1 | from .arguments import add_args 2 | from .model import Model 3 | from .dataset import Dataset 4 | from .train import train 5 | -------------------------------------------------------------------------------- /models/learnings/deepBP/arguments.py: -------------------------------------------------------------------------------- 1 | from .model import add_model_args 2 | from .dataset import add_dataset_args 3 | from .train import add_train_args 4 | 5 | 6 | def add_args(parser): 7 | add_train_args(parser) 8 | add_model_args(parser) 9 | add_dataset_args(parser) 10 | 11 | -------------------------------------------------------------------------------- /models/learnings/deepBP/dataset.py: -------------------------------------------------------------------------------- 1 | from ..base import BaseDataset, add_base_dataset_args 2 | 3 | 4 | def add_dataset_args(parser): 5 | group = parser.add_argument_group('Dataset') 6 | add_base_dataset_args(group) 7 | 8 | 9 | class Dataset(BaseDataset): 10 | def __init__(self, args, is_train): 11 | super().__init__(args, is_train) 12 | -------------------------------------------------------------------------------- /models/learnings/deepBP/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from ..base import Classifier 6 | from ...inferences import run_bp 7 | 8 | def add_model_args(parser): 9 | group = parser.add_argument_group('Model') 10 | group.add_argument('--given_alpha', type=float, default=2.) 11 | group.add_argument('--given_beta', type=float, default=1.) 12 | group.add_argument('--clipping', type=float, default=1.0) 13 | group.add_argument('--n_iters', type=int, default=100) 14 | group.add_argument('--n_samples', type=int, default=400) 15 | 16 | 17 | class Model(nn.Module): 18 | def __init__(self, args): 19 | super(Model, self).__init__() 20 | 21 | self.n_classes = 2 22 | self.given_alpha = args.given_alpha 23 | self.given_beta = args.given_beta 24 | self.clipping = args.clipping 25 | self.n_iters = args.n_iters 26 | self.n_samples = args.n_samples 27 | self.classifier = Classifier() 28 | 29 | self.prior = torch.FloatTensor([[self.given_alpha, self.given_beta], [self.given_beta, self.given_alpha]]) 30 | 31 | def inference(self, x, ann): 32 | z_learning = F.softmax(self.classifier(x), dim=-1) 33 | z_learning = torch.clamp(z_learning, min=1 - self.clipping, max=self.clipping) + 1e-9 34 | 35 | with torch.no_grad(): 36 | z_inference = run_bp(ann, self.n_iters, self.prior, z_learning, x.device, self.n_samples) 37 | 38 | return z_inference, z_learning 39 | 40 | def forward(self, x, ann): 41 | z_inference, z_learning = self.inference(x, ann) 42 | loss = -(z_inference * z_learning.log()).sum() 43 | return loss 44 | 45 | -------------------------------------------------------------------------------- /models/learnings/deepBP/train.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.optim import Adam 3 | from ..base import add_base_train_args 4 | 5 | 6 | def add_train_args(parser): 7 | group = parser.add_argument_group('Training') 8 | add_base_train_args(group) 9 | group.add_argument('--lr', type=float, default=1e-3) 10 | 11 | 12 | def test(args, model, train_loader, test_loader): 13 | correct_train, correct_test = 0, 0 14 | with torch.no_grad(): 15 | model.eval() 16 | for x, z, ann in train_loader: 17 | x, z, ann = x.to(args.device), z.to(args.device), ann.to(args.device) 18 | z_pred, _ = model.inference(x, ann) 19 | z_pred = z_pred.argmax(dim=-1) 20 | correct_train += z_pred.eq(z).sum().item() 21 | 22 | for x, z in test_loader: 23 | x, z = x.to(args.device), z.to(args.device) 24 | z_pred = model.classifier(x).argmax(dim=1) 25 | correct_test += z_pred.eq(z).sum().item() 26 | 27 | correct_train /= len(train_loader.dataset) 28 | correct_test /= len(test_loader.dataset) 29 | 30 | print(f'======> Inference acc: {correct_train * 100:.2f}') 31 | print(f'======> Learning acc: {correct_test * 100:.2f}') 32 | 33 | 34 | def train_epoch(args, epoch, model, train_loader, opt): 35 | loss_total = 0.0 36 | 37 | model.train() 38 | for x, z, ann in train_loader: 39 | model.zero_grad() 40 | x, z, ann = x.to(args.device), z.to(args.device), ann.to(args.device) 41 | loss = model(x, ann) 42 | loss.backward() 43 | opt.step() 44 | 45 | loss_total += loss.item() 46 | 47 | print(f'Epoch {epoch:4d} | Loss: {loss_total:.3f}') 48 | 49 | 50 | def train(args, model, train_loader, test_loader): 51 | opt = Adam(model.classifier.parameters(), lr=args.lr) 52 | 53 | for e in range(1, args.n_epochs + 1): 54 | train_epoch(args, e, model, train_loader, opt) 55 | if e % args.test_interval == 0 or e == args.n_epochs: 56 | test(args, model, train_loader, test_loader) 57 | 58 | -------------------------------------------------------------------------------- /models/learnings/deepMF/__init__.py: -------------------------------------------------------------------------------- 1 | from .arguments import add_args 2 | from .model import Model 3 | from .dataset import Dataset 4 | from .train import train 5 | -------------------------------------------------------------------------------- /models/learnings/deepMF/arguments.py: -------------------------------------------------------------------------------- 1 | from .model import add_model_args 2 | from .dataset import add_dataset_args 3 | from .train import add_train_args 4 | 5 | 6 | def add_args(parser): 7 | add_train_args(parser) 8 | add_model_args(parser) 9 | add_dataset_args(parser) 10 | 11 | -------------------------------------------------------------------------------- /models/learnings/deepMF/dataset.py: -------------------------------------------------------------------------------- 1 | from ..base import BaseDataset, add_base_dataset_args 2 | 3 | 4 | def add_dataset_args(parser): 5 | group = parser.add_argument_group('Dataset') 6 | add_base_dataset_args(group) 7 | 8 | 9 | class Dataset(BaseDataset): 10 | def __init__(self, args, is_train): 11 | super().__init__(args, is_train) 12 | -------------------------------------------------------------------------------- /models/learnings/deepMF/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from ..base import Classifier 6 | 7 | 8 | def add_model_args(parser): 9 | group = parser.add_argument_group('Model') 10 | group.add_argument('--given_alpha', type=float, default=2.) 11 | group.add_argument('--given_beta', type=float, default=1.) 12 | group.add_argument('--clipping', type=float, default=1.0) 13 | 14 | 15 | class Model(nn.Module): 16 | def __init__(self, args): 17 | super(Model, self).__init__() 18 | 19 | self.n_classes = 2 20 | self.given_alpha = args.given_alpha 21 | self.given_beta = args.given_beta 22 | self.clipping = args.clipping 23 | self.n_worker = args.n_tasks * args.n_workers_per_task // args.n_tasks_per_worker 24 | self.classifier = Classifier() 25 | self.criterion = nn.KLDivLoss(reduction='sum') 26 | 27 | prior = [[self.given_alpha, self.given_beta], [self.given_beta, self.given_alpha]] 28 | self.confusion = torch.FloatTensor([prior for _ in range(self.n_worker)]) 29 | 30 | def inference(self, x, ann): 31 | z_learning = F.softmax(self.classifier(x), dim=-1) 32 | z_learning = torch.clamp(z_learning, min=1 - self.clipping, max=self.clipping) + 1e-9 33 | z_learning = z_learning.cpu() 34 | 35 | with torch.no_grad(): 36 | z_inference = z_learning.log() 37 | 38 | for t in range(x.size(0)): 39 | workers_of_task = (ann[t].view(-1) != -1).nonzero().view(-1) 40 | 41 | tmp = -self.confusion[workers_of_task].sum(dim=2).digamma().sum(dim=0) 42 | for u in workers_of_task: 43 | tmp += self.confusion[u, :, ann[t, u]].digamma() 44 | z_inference[t] += tmp 45 | 46 | z_inference = F.softmax(z_inference, dim=-1) 47 | return z_inference, z_learning 48 | 49 | def forward(self, x, ann): 50 | z_inference, z_learning = self.inference(x, ann) 51 | 52 | for u in range(ann.size(1)): 53 | tasks_of_worker = (ann[:, u].view(-1) != -1).nonzero().view(-1) 54 | 55 | self.confusion[u] = torch.FloatTensor([[self.given_alpha, self.given_beta], [self.given_beta, self.given_alpha]]) 56 | for t in tasks_of_worker: 57 | self.confusion[u, :, ann[t, u]] += z_inference[t] 58 | 59 | loss = self.criterion(z_learning.log(), z_inference) 60 | return loss 61 | 62 | -------------------------------------------------------------------------------- /models/learnings/deepMF/train.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.optim import Adam 3 | from ..base import add_base_train_args 4 | 5 | 6 | def add_train_args(parser): 7 | group = parser.add_argument_group('Training') 8 | add_base_train_args(group) 9 | group.add_argument('--lr', type=float, default=1e-3) 10 | 11 | 12 | def test(args, model, train_loader, test_loader): 13 | correct_train, correct_test = 0, 0 14 | with torch.no_grad(): 15 | model.eval() 16 | for x, z, ann in train_loader: 17 | x = x.to(args.device) 18 | z_pred, _ = model.inference(x, ann) 19 | z_pred = z_pred.argmax(dim=-1) 20 | correct_train += z_pred.eq(z).sum().item() 21 | 22 | for x, z in test_loader: 23 | x, z = x.to(args.device), z.to(args.device) 24 | z_pred = model.classifier(x).argmax(dim=1) 25 | correct_test += z_pred.eq(z).sum().item() 26 | 27 | correct_train /= len(train_loader.dataset) 28 | correct_test /= len(test_loader.dataset) 29 | 30 | print(f'======> Inference acc: {correct_train * 100:.2f}') 31 | print(f'======> Learning acc: {correct_test * 100:.2f}') 32 | 33 | 34 | def train_epoch(args, epoch, model, train_loader, opt): 35 | loss_total = 0.0 36 | 37 | model.train() 38 | for x, z, ann in train_loader: 39 | model.zero_grad() 40 | x, z, ann = x.to(args.device), z.to(args.device), ann.to(args.device) 41 | loss = model(x, ann) 42 | loss.backward() 43 | opt.step() 44 | 45 | loss_total += loss.item() 46 | 47 | print(f'Epoch {epoch:4d} | Loss: {loss_total:.3f}') 48 | 49 | 50 | def train(args, model, train_loader, test_loader): 51 | opt = Adam(model.classifier.parameters(), lr=args.lr) 52 | 53 | for e in range(1, args.n_epochs + 1): 54 | train_epoch(args, e, model, train_loader, opt) 55 | if e % args.test_interval == 0 or e == args.n_epochs: 56 | test(args, model, train_loader, test_loader) 57 | 58 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | torch 3 | torchvision 4 | Pillow 5 | --------------------------------------------------------------------------------