├── .gitignore ├── Makefile ├── README.md ├── dev-requirements.in ├── dev-requirements.txt ├── experiments ├── __init__.py ├── baseline.py ├── click_log.py ├── dataset.py ├── eval_model.py ├── evaluate.py ├── models.py ├── plots │ ├── __init__.py │ ├── batch_sizes.py │ ├── etas.py │ ├── optimizers.py │ ├── plot.py │ ├── toy_sample.py │ └── toy_sample_animation.py ├── simulate_clicks.py ├── tables │ ├── __init__.py │ ├── batch_sizes.py │ ├── distribution.py │ ├── etas.py │ └── optimizers.py ├── train.py └── util.py ├── makescripts ├── baselines.mk ├── clicklogs.mk ├── directories.mk ├── istella_batch_sizes.mk ├── istella_etas.mk ├── istella_optimizers.mk ├── plots.mk ├── tables.mk ├── yahoo_batch_sizes.mk ├── yahoo_etas.mk └── yahoo_optimizers.mk ├── requirements.in └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # IPython 79 | profile_default/ 80 | ipython_config.py 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # pipenv 86 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 87 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 88 | # having no cross-platform support, pipenv may install dependencies that don’t work, or not 89 | # install all needed dependencies. 90 | #Pipfile.lock 91 | 92 | # celery beat schedule file 93 | celerybeat-schedule 94 | 95 | # SageMath parsed files 96 | *.sage.py 97 | 98 | # Environments 99 | .env 100 | .venv 101 | env/ 102 | venv/ 103 | ENV/ 104 | env.bak/ 105 | venv.bak/ 106 | 107 | # Spyder project settings 108 | .spyderproject 109 | .spyproject 110 | 111 | # Rope project settings 112 | .ropeproject 113 | 114 | # mkdocs documentation 115 | /site 116 | 117 | # mypy 118 | .mypy_cache/ 119 | .dmypy.json 120 | dmypy.json 121 | 122 | # Pyre type checker 123 | .pyre/ 124 | 125 | # custom 126 | cache/ 127 | Pipfile.lock 128 | 129 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Variables, customize to your environment 2 | YAHOO_DIR ?= /Users/rolfjagerman/Datasets/Yahoo/set1 3 | ISTELLA_DIR ?= /Users/rolfjagerman/Datasets/istella-s 4 | BUILD ?= build 5 | TRAIN_ARGS ?= 6 | 7 | # Default make target runs the entire pipeline to generate plots and tables 8 | all: plots tables 9 | 10 | # Directories 11 | include makescripts/directories.mk 12 | 13 | # Baselines 14 | include makescripts/baselines.mk 15 | 16 | # Clicklogs 17 | include makescripts/clicklogs.mk 18 | 19 | # Results 20 | include makescripts/yahoo_optimizers.mk 21 | include makescripts/yahoo_batch_sizes.mk 22 | include makescripts/yahoo_etas.mk 23 | include makescripts/istella_optimizers.mk 24 | include makescripts/istella_batch_sizes.mk 25 | include makescripts/istella_etas.mk 26 | include makescripts/plots.mk 27 | include makescripts/tables.mk 28 | 29 | # Phony target for easier running of just experiments 30 | experiments: yahoo_batch_sizes_repeat_5 istella_batch_sizes_repeat_5 yahoo_optimizers_repeat_5 istella_optimizers_repeat_5 yahoo_etas_repeat_5 istella_etas_repeat_5 $(BUILD)/skylines/yahoo.json $(BUILD)/skylines/istella.json 31 | .PHONY: all experiments 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Accelerated Convergence for Counterfactual Learning to Rank 2 | 3 | :information_source: This software uses PyTorchLTR. Check it out [here](http://github.com/rjagerman/pytorchltr)! 4 | 5 | Source code with our SIGIR2020 paper. 6 | To run all experiments, build the plots and create the tables that appear in the paper run the following: 7 | 8 | export YAHOO_DIR=/path/to/yahoo/set1/ 9 | export ISTELLA_DIR=/path/to/istella/sample/ 10 | make -j$(nproc) 11 | 12 | Or, if you are on macOS: 13 | 14 | export YAHOO_DIR=/path/to/yahoo/set1/ 15 | export ISTELLA_DIR=/path/to/istella/sample/ 16 | make -j$(sysctl -n hw.logicalcpu) 17 | 18 | This can take significant amount of time depending on the number of CPU cores available. 19 | 20 | -------------------------------------------------------------------------------- /dev-requirements.in: -------------------------------------------------------------------------------- 1 | ipython 2 | -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile dev-requirements.in 6 | # 7 | appnope==0.1.0 # via ipython 8 | backcall==0.1.0 # via ipython 9 | decorator==4.4.2 # via ipython, traitlets 10 | ipython-genutils==0.2.0 # via traitlets 11 | ipython==7.13.0 # via -r dev-requirements.in 12 | jedi==0.17.0 # via ipython 13 | parso==0.7.0 # via jedi 14 | pexpect==4.8.0 # via ipython 15 | pickleshare==0.7.5 # via ipython 16 | prompt-toolkit==3.0.5 # via ipython 17 | ptyprocess==0.6.0 # via pexpect 18 | pygments==2.6.1 # via ipython 19 | six==1.14.0 # via traitlets 20 | traitlets==4.3.3 # via ipython 21 | wcwidth==0.1.9 # via prompt-toolkit 22 | 23 | # The following packages are considered to be unsafe in a requirements file: 24 | # setuptools 25 | -------------------------------------------------------------------------------- /experiments/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rjagerman/sigir2020/5218fc2c4acd96208caad9a64afc8d06aeafca55/experiments/__init__.py -------------------------------------------------------------------------------- /experiments/baseline.py: -------------------------------------------------------------------------------- 1 | """Trains a baseline ranker on a fraction of supervised training data.""" 2 | import logging 3 | from argparse import ArgumentParser 4 | 5 | import torch 6 | from pytorchltr.dataset.svmrank import create_svmranking_collate_fn 7 | from pytorchltr.loss.pairwise import AdditivePairwiseLoss 8 | from pytorchltr.evaluation.dcg import ndcg 9 | from pytorchltr.evaluation.arp import arp 10 | 11 | from experiments.evaluate import evaluate 12 | from experiments.dataset import load_ranking_dataset 13 | 14 | 15 | LOGGER = logging.getLogger(__name__) 16 | 17 | 18 | def get_parser(): 19 | """Gets the parser to create arguments for `main`.""" 20 | parser = ArgumentParser() 21 | parser.add_argument("--train_data", type=str, required=True) 22 | parser.add_argument("--output", type=str, required=True) 23 | parser.add_argument("--vali_data", type=str, default=None) 24 | parser.add_argument('--optimizer', default="sgd", 25 | choices=['sgd', 'adam', 'adagrad']) 26 | parser.add_argument("--fraction", type=float, default=0.01) 27 | parser.add_argument("--seed", type=int, default=42) 28 | parser.add_argument("--batch_size", type=int, default=16) 29 | parser.add_argument("--epochs", type=int, default=50) 30 | parser.add_argument("--lr", type=float, default=0.001) 31 | parser.add_argument("--loss", choices=["rank", "dcg"], default="rank") 32 | return parser 33 | 34 | 35 | def main(args): 36 | """Trains the baseline ranker using given arguments.""" 37 | torch.manual_seed(args.seed) 38 | 39 | LOGGER.info("Starting baseline run with args: %s", f"{args}") 40 | 41 | LOGGER.info("Loading train data %s", args.train_data) 42 | train = load_ranking_dataset(args.train_data, normalize=True) 43 | 44 | if args.vali_data is not None: 45 | LOGGER.info("Loading vali data %s", args.vali_data) 46 | vali = load_ranking_dataset( 47 | args.vali_data, normalize=True, filter_queries=True) 48 | 49 | LOGGER.info("Subsampling train data by %.3f", args.fraction) 50 | indices = torch.randperm(len(train))[0:int(args.fraction * len(train))] 51 | train = torch.utils.data.Subset(train, indices) 52 | 53 | LOGGER.info("Creating linear model") 54 | linear_model = torch.nn.Linear(train[0]["features"].shape[1], 1) 55 | 56 | LOGGER.info("Creating optimizer and loss function") 57 | optimizer = { 58 | "sgd": lambda: torch.optim.SGD(linear_model.parameters(), args.lr), 59 | "adam": lambda: torch.optim.Adam(linear_model.parameters(), args.lr), 60 | "adagrad": lambda: torch.optim.Adagrad(linear_model.parameters(), args.lr) 61 | }[args.optimizer]() 62 | loss_fn = AdditivePairwiseLoss(args.loss) 63 | 64 | LOGGER.info("Start training") 65 | for e in range(1, 1 + args.epochs): 66 | loader = torch.utils.data.DataLoader( 67 | train, batch_size=args.batch_size, shuffle=True, 68 | collate_fn=create_svmranking_collate_fn()) 69 | for i, batch in enumerate(loader): 70 | linear_model.train() 71 | xs, ys, n = batch["features"], batch["relevance"], batch["n"] 72 | scores = linear_model(xs) 73 | loss = torch.mean(loss_fn(scores, ys, n)) 74 | 75 | optimizer.zero_grad() 76 | loss.backward() 77 | optimizer.step() 78 | 79 | LOGGER.info("Finished epoch %d", e) 80 | 81 | if args.vali_data is not None: 82 | results = evaluate(vali, linear_model, { 83 | "arp": lambda scores, ys, n: arp(scores, ys, n), 84 | "ndcg@10": lambda scores, ys, n: ndcg(scores, ys, n, k=10) 85 | }) 86 | LOGGER.info("arp : %.4f", results["arp"]) 87 | LOGGER.info("ndcg@10: %.4f", results["ndcg@10"]) 88 | 89 | LOGGER.info("Saving model to %s", args.output) 90 | torch.save(linear_model, args.output) 91 | LOGGER.info("Done") 92 | 93 | 94 | if __name__ == "__main__": 95 | logging.basicConfig( 96 | format="[%(asctime)s, %(levelname)s, %(module)s] %(message)s", 97 | level=logging.INFO) 98 | main(get_parser().parse_args()) 99 | -------------------------------------------------------------------------------- /experiments/click_log.py: -------------------------------------------------------------------------------- 1 | """Utility for loading click logs.""" 2 | import logging 3 | import pickle 4 | 5 | import numpy as np 6 | import torch 7 | from pytorchltr.dataset.svmrank import create_svmranking_collate_fn 8 | 9 | 10 | class ClicklogDataset(torch.utils.data.Dataset): 11 | """A click log dataset.""" 12 | def __init__(self, ranking_dataset, click_log, clip=None, n_clicks=None): 13 | self._ranking_dataset = ranking_dataset 14 | self._clicked_docs = click_log["clicked_docs"] 15 | self._qids = click_log["qids"] 16 | self._propensities = click_log["propensities"] 17 | if clip is not None: 18 | self._propensities = np.clip( 19 | self._propensities, a_min=clip, a_max=None) 20 | self._clip = clip 21 | if n_clicks is not None: 22 | self._clicked_docs = self._clicked_docs[:n_clicks] 23 | self._qids = self._qids[:n_clicks] 24 | self._propensities = self._propensities[:n_clicks] 25 | 26 | @property 27 | def propensities(self): 28 | return self._propensities 29 | 30 | def __len__(self): 31 | return self._clicked_docs.shape[0] 32 | 33 | def __getitem__(self, index): 34 | qid = self._qids[index] 35 | rd_index = self._ranking_dataset.get_index(qid) 36 | sample = self._ranking_dataset[rd_index] 37 | sample["clicks"] = torch.zeros_like(sample["relevance"]) 38 | sample["clicks"][self._clicked_docs[index]] = 1 39 | sample["propensity"] = self._propensities[index] 40 | return sample 41 | 42 | 43 | def create_clicklog_collate_fn(rng=np.random.RandomState(42), 44 | max_list_size=None): 45 | """Creates a collate_fn for click log datasets.""" 46 | svmrank_collate_fn = create_svmranking_collate_fn( 47 | rng, max_list_size) 48 | def _collate_fn(batch): 49 | out = svmrank_collate_fn(batch) 50 | out["clicks"] = torch.zeros_like(out["relevance"]) 51 | for i, sample in enumerate(batch): 52 | out["clicks"][i, 0:sample["clicks"].shape[0]] = sample["clicks"] 53 | out["propensity"] = torch.FloatTensor( 54 | [sample["propensity"] for sample in batch]) 55 | return out 56 | return _collate_fn 57 | 58 | 59 | def clicklog_dataset(ranking_dataset, click_log_file_path, clip=None, 60 | n_clicks=None): 61 | """Loads a click log dataset from given file path. 62 | 63 | Arguments: 64 | ranking_dataset: The svmranking dataset that was used to generate 65 | clicks. 66 | click_log_file_path: Path to the generated click log file. 67 | clip: Value to clip propensities at (if None, apply no clipping). 68 | n_clicks: The number of clicks to limit. 69 | 70 | Returns: 71 | A ClicklogDataset used for ranking experiments. 72 | """ 73 | with open(click_log_file_path, "rb") as f: 74 | click_log = pickle.load(f) 75 | return ClicklogDataset(ranking_dataset, click_log, clip, n_clicks) 76 | -------------------------------------------------------------------------------- /experiments/dataset.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | from pytorchltr.dataset.svmrank import svmranking_dataset as _load 4 | from joblib.memory import Memory as _Memory 5 | 6 | from experiments.click_log import clicklog_dataset 7 | from experiments.click_log import create_clicklog_collate_fn 8 | 9 | 10 | load_ranking_dataset = _Memory(".cache", compress=6).cache(_load) 11 | 12 | 13 | def load_click_dataset(data_file, click_log_file, ips_strategy, ips_clip=None, 14 | n_clicks=None, batch_size=50, max_list_size=None, 15 | normalize=True, filter_queries=False): 16 | """Loads a click dataset from file. 17 | 18 | Arguments: 19 | data_file: The data file to load. 20 | click_log_file: The corresponding click log to load. 21 | ips_strategy: A string indicating the IPS strategy to use. 22 | ips_clip: (Optional) whether to apply IPS clipping. 23 | n_clicks: (Optional) number of clicks to include (None includes all). 24 | batch_size: (Optional) the batch size to use (default: 50). 25 | max_list_size: (Optional) a cut-off for list size padding, default: 26 | None. 27 | normalize: (Optional) whether to normalize input data, default: True 28 | filter_queries: (Optional) whether to filter queries that have no 29 | relevant documents from the training data (default: False). 30 | 31 | Returns: 32 | A tuple containing a `torch.utils.data.DataLoader` for loading data and 33 | an int indicating the dimensionality of input data. 34 | """ 35 | # Load training data. 36 | data = load_ranking_dataset( 37 | data_file, normalize=True, filter_queries=filter_queries) 38 | 39 | # Load click log from file. 40 | click_log = clicklog_dataset(data, click_log_file, clip=ips_clip, 41 | n_clicks=n_clicks) 42 | 43 | # Construct sampling strategy. 44 | if ips_strategy == "sample": 45 | propensities = torch.FloatTensor(click_log.propensities) 46 | weights = (1.0 / propensities) 47 | bias_correction_factor = float(torch.mean(weights)) 48 | probabilities = weights / torch.sum(weights) 49 | sampler = torch.utils.data.sampler.WeightedRandomSampler( 50 | probabilities, len(click_log)) 51 | else: 52 | bias_correction_factor = 1.0 53 | sampler = torch.utils.data.sampler.RandomSampler(click_log) 54 | 55 | # Return a data loader. 56 | return (torch.utils.data.DataLoader( 57 | click_log, batch_size=batch_size, sampler=sampler, 58 | collate_fn=create_clicklog_collate_fn(max_list_size=max_list_size)), 59 | data[0]["features"].shape[1], 60 | bias_correction_factor) 61 | -------------------------------------------------------------------------------- /experiments/eval_model.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | import logging 3 | import json 4 | 5 | import torch 6 | from pytorchltr.dataset.svmrank import create_svmranking_collate_fn 7 | from experiments.dataset import load_ranking_dataset 8 | from experiments.evaluate import evaluate 9 | from experiments.evaluate import ARP 10 | from experiments.evaluate import NDCG 11 | from experiments.train import create_ltr_evaluator 12 | from experiments.util import get_torch_device 13 | 14 | 15 | LOGGER = logging.getLogger(__name__) 16 | 17 | 18 | def get_parser(): 19 | parser = ArgumentParser() 20 | parser.add_argument("--data", type=str, required=True) 21 | parser.add_argument("--model", type=str, required=True) 22 | parser.add_argument("--output", type=str, required=True) 23 | parser.add_argument("--eval_batch_size", type=int, default=500) 24 | parser.add_argument("--seed", type=int, default=42) 25 | return parser 26 | 27 | 28 | def main(args): 29 | torch.manual_seed(args.seed) 30 | device = get_torch_device(enable_cuda=False) 31 | 32 | LOGGER.info("Loading dataset") 33 | data_loader = torch.utils.data.DataLoader( 34 | load_ranking_dataset(args.data, normalize=True, filter_queries=True), 35 | shuffle=False, batch_size=args.eval_batch_size, 36 | collate_fn=create_svmranking_collate_fn()) 37 | 38 | LOGGER.info("Loading model") 39 | model = torch.load(args.model) 40 | 41 | LOGGER.info("Starting evaluation") 42 | metrics = {"ndcg@10": NDCG(k=10), "arp": ARP()} 43 | evaluator = create_ltr_evaluator(model, device, metrics) 44 | evaluator.run(data_loader) 45 | with open(args.output, "wt") as f: 46 | json.dump(evaluator.state.metrics, f) 47 | 48 | LOGGER.info("Done") 49 | 50 | 51 | if __name__ == "__main__": 52 | logging.basicConfig( 53 | format="[%(asctime)s, %(levelname)s, %(module)s] %(message)s", 54 | level=logging.INFO) 55 | main(get_parser().parse_args()) 56 | -------------------------------------------------------------------------------- /experiments/evaluate.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from pytorchltr.dataset.svmrank import create_svmranking_collate_fn 3 | from pytorchltr.evaluation.dcg import ndcg 4 | from pytorchltr.evaluation.arp import arp 5 | from ignite.metrics.metric import Metric 6 | from ignite.exceptions import NotComputableError 7 | 8 | 9 | def evaluate(dataset, model, metrics, 10 | collate_fn=create_svmranking_collate_fn(), batch_size=16, 11 | device=None): 12 | model.eval() 13 | out_metrics = { 14 | metric: [] 15 | for metric in metrics.keys() 16 | } 17 | with torch.no_grad(): 18 | eval_loader = torch.utils.data.DataLoader( 19 | dataset, batch_size=batch_size, shuffle=False, 20 | collate_fn=collate_fn) 21 | for batch in eval_loader: 22 | xs, ys, n = batch["features"], batch["relevance"], batch["n"] 23 | xs, ys, n = xs.to(device), ys.to(device), n.to(device) 24 | scores = model(xs) 25 | for metric in metrics.keys(): 26 | values = metrics[metric](scores, ys, n) 27 | out_metrics[metric].extend(values.tolist()) 28 | return { 29 | metric: float(torch.mean(torch.FloatTensor(values))) 30 | for metric, values in out_metrics.items() 31 | } 32 | 33 | 34 | class OnlineAverageMetric(Metric): 35 | """ 36 | Calculates an online average metric. 37 | """ 38 | def __init__(self, output_transform=lambda x: x): 39 | super().__init__(output_transform) 40 | self._avg = 0.0 41 | 42 | def reset(self): 43 | self._avg = 0.0 44 | self._n = 0 45 | 46 | def update(self, output): 47 | for e in output: 48 | self._avg = (self._avg * self._n + e.item()) / (self._n + 1) 49 | self._n += 1 50 | 51 | def compute(self): 52 | if self._n == 0: 53 | raise NotComputableError("OnlineAverageMetric must have at least one example before it can be computed.") 54 | return self._avg 55 | 56 | 57 | class LTRMetric(OnlineAverageMetric): 58 | """ 59 | Calculates an LTR metric online. 60 | """ 61 | def __init__(self, ltr_metric_fn, output_transform=lambda x: x): 62 | super().__init__(self._compute_metric) 63 | self._ltr_metric_fn = ltr_metric_fn 64 | self._output_fn = output_transform 65 | 66 | def _compute_metric(self, output): 67 | scores, ys, n = self._output_fn(output) 68 | return self._ltr_metric_fn(scores, ys, n) 69 | 70 | 71 | class NDCG(LTRMetric): 72 | def __init__(self, k=10, exp=True, output_transform=lambda x: x): 73 | super().__init__(lambda scores, ys, n: ndcg(scores, ys, n, k, exp), 74 | output_transform=output_transform) 75 | 76 | 77 | class ARP(LTRMetric): 78 | def __init__(self, output_transform=lambda x: x): 79 | super().__init__(arp, output_transform=output_transform) 80 | -------------------------------------------------------------------------------- /experiments/models.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class DeepScorer(torch.nn.Module): 5 | """A deep feed-forward neural network with non-linear activations.""" 6 | def __init__(self, input_size, layers=[], activation_fn=torch.nn.ReLU): 7 | super().__init__() 8 | current_size = input_size 9 | self.layers = torch.nn.ModuleList([]) 10 | self.activations = torch.nn.ModuleList([]) 11 | for layer in layers: 12 | self.layers.append(torch.nn.Linear(current_size, layer)) 13 | self.activations.append(activation_fn()) 14 | current_size = layer 15 | self.output = torch.nn.Linear(current_size, 1) 16 | 17 | def forward(self, xs): 18 | current = xs 19 | for layer, activation in zip(self.layers, self.activations): 20 | current = activation(layer(current)) 21 | return self.output(current) 22 | 23 | 24 | class LinearScorer(torch.nn.Module): 25 | """A linear model.""" 26 | def __init__(self, input_size): 27 | super().__init__() 28 | self.output = torch.nn.Linear(input_size, 1) 29 | 30 | def forward(self, xs): 31 | return self.output(xs) 32 | -------------------------------------------------------------------------------- /experiments/plots/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rjagerman/sigir2020/5218fc2c4acd96208caad9a64afc8d06aeafca55/experiments/plots/__init__.py -------------------------------------------------------------------------------- /experiments/plots/batch_sizes.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import os 4 | from copy import deepcopy 5 | from time import sleep 6 | from argparse import ArgumentParser 7 | from argparse import FileType 8 | 9 | import numpy as np 10 | import matplotlib 11 | matplotlib.rcParams['text.latex.preamble'] = '\\usepackage{biolinum}\n\\usepackage[T1]{fontenc}\n\\usepackage[libertine]{newtxmath}' 12 | matplotlib.rcParams['text.usetex'] = True 13 | matplotlib.rcParams.update({'font.size': 13}) 14 | import tikzplotlib 15 | from matplotlib import pyplot as plt 16 | from matplotlib import animation as anim 17 | from cycler import cycler 18 | plt.rcParams["figure.figsize"] = [12,3] 19 | 20 | 21 | def get_parser(): 22 | """Gets the parser to create arguments for `main`.""" 23 | parser = ArgumentParser() 24 | parser.add_argument("--json_files", type=FileType("rt"), nargs="+") 25 | parser.add_argument("--color_by", type=str, required=False, default="ips_strategy", 26 | choices=["none", "ips_strategy", "optimizer"]) 27 | parser.add_argument("--out", type=FileType("wb"), required=False, default=None) 28 | parser.add_argument("--dataset", type=str, required=False, default="vali") 29 | parser.add_argument("--model", type=str, required=False, default="avgmodel") 30 | parser.add_argument("--metric", type=str, default="ndcg@10") 31 | parser.add_argument("--legend", action="store_true", default=False) 32 | parser.add_argument("--width", type=float, default=12.0) 33 | parser.add_argument("--height", type=float, default=3.0) 34 | parser.add_argument("--format", type=str, default=None) 35 | parser.add_argument("--points", type=int, default=None) 36 | parser.add_argument("--darkstyle", action="store_true", default=False) 37 | parser.add_argument("--datasets", type=str, nargs="+", default=["yahoo", "istella"]) 38 | return parser 39 | 40 | 41 | def color_by_ips_strategy(name): 42 | if "none" in name: 43 | return "C0" 44 | elif "weight" in name: 45 | return "C1" 46 | elif "sample" in name: 47 | return "C2" 48 | else: 49 | return None 50 | 51 | 52 | def color_by_optimizer(name): 53 | if "adam" in name: 54 | return "C0" 55 | elif "sgd" in name: 56 | return "C1" 57 | elif "adagrad" in name: 58 | return "C2" 59 | else: 60 | return None 61 | 62 | 63 | def compress_list_of_arrays(list_of_arrays): 64 | min_shape = np.min([np.array(a).shape for a in list_of_arrays]) 65 | mean = np.mean([a[:min_shape] for a in list_of_arrays], axis=0) 66 | std = np.std([a[:min_shape] for a in list_of_arrays], axis=0) 67 | n = len(list_of_arrays) 68 | return mean, std, n 69 | 70 | 71 | def filter_arg(dictionary, key, value): 72 | return {k: v for k, v in dictionary.items() if v["args"][key] == value} 73 | 74 | 75 | def sample_points(xs, points): 76 | if points is None: 77 | return xs 78 | else: 79 | # Plot the first 10% of the points perfectly (where convergence mostly takes place), 80 | # subsample the remainder of the points to prevent very slow PDF viewing 81 | start_points = 0 # points // 10 82 | end_points = points - start_points 83 | sample = np.hstack([ 84 | np.arange(start_points, dtype=np.int32), 85 | np.linspace(start_points, xs.shape[0] - 1, num=end_points, dtype=np.int32)]) 86 | return xs[sample] 87 | 88 | 89 | def main(args): 90 | 91 | if args.darkstyle: 92 | plt.style.use('dark_background') 93 | matplotlib.rcParams['axes.prop_cycle'] = cycler(color=[ 94 | 'lightgrey', 95 | 'orange', 96 | 'deepskyblue' 97 | ]) 98 | 99 | fig = plt.figure(figsize=(args.width, args.height)) 100 | 101 | def plot_data(args, fig): 102 | fig.clear() 103 | data = {} 104 | for json_file in args.json_files: 105 | data[os.path.basename(json_file.name)] = json.load(json_file) 106 | json_file.seek(0) 107 | 108 | color = { 109 | "optimizer": color_by_optimizer, 110 | "ips_strategy": color_by_ips_strategy, 111 | "none": lambda x: None 112 | }[args.color_by] 113 | 114 | # Compute error bars for identical runs with multiple seeds 115 | avg_runs = {} 116 | for name, results in data.items(): 117 | if args.dataset in results and args.model in results[args.dataset]: 118 | x = results[args.dataset][args.model]["iteration"] 119 | y = results[args.dataset][args.model][args.metric] 120 | a = deepcopy(results['args']) 121 | del a["seed"] 122 | del a["output"] 123 | key = tuple(sorted(a.items())) 124 | if key not in avg_runs: 125 | avg_runs[key] = { 126 | 'name': name, 127 | 'results': {args.dataset: {args.model: {args.metric: [], "iteration": []}}} 128 | } 129 | avg_runs[key]['results'][args.dataset][args.model][args.metric].append(y) 130 | avg_runs[key]['results'][args.dataset][args.model]["iteration"].append(x) 131 | avg_runs[key]['args'] = results['args'] 132 | data = {} 133 | for run in avg_runs.values(): 134 | ys_mean, ys_std, ys_n = compress_list_of_arrays(run['results'][args.dataset][args.model][args.metric]) 135 | xs_mean, _, _ = compress_list_of_arrays(run['results'][args.dataset][args.model]["iteration"]) 136 | data[run['name']] = { 137 | args.dataset: {args.model: { 138 | args.metric: ys_mean, 139 | f"{args.metric}/std": ys_std, 140 | f"{args.metric}/n": ys_n, 141 | "iteration": xs_mean 142 | }}, 143 | "args": run['args'] 144 | } 145 | 146 | # Plot actual results 147 | labels = { 148 | "none": "Biased-SGD", 149 | "weight": "IPS-SGD", 150 | "sample": "\\textsc{CounterSample}" 151 | } 152 | inv_labels = {k: v for v, k in labels.items()} 153 | sorting = { 154 | "none": 3, 155 | "weight": 2, 156 | "sample": 1 157 | } 158 | metrics = { 159 | "ndcg@10": "nDCG@10", 160 | "arp": "ARP" 161 | } 162 | markers = { 163 | "none": "s", 164 | "weight": "o", 165 | "sample": "^" 166 | } 167 | for j, dataset in enumerate(args.datasets): 168 | # Compute y limits 169 | max_last_y = 0.0 170 | min_last_y = 1e30 171 | for name, results in data.items(): 172 | if args.dataset in results and args.model in results[args.dataset] and dataset in results["args"]["train_data"]: 173 | y = results[args.dataset][args.model][args.metric][-1] 174 | min_last_y = min(y, min_last_y) 175 | max_last_y = max(y, max_last_y) 176 | 177 | # Plot subplots 178 | for i, batch_size in enumerate([10, 20, 50]): 179 | ax = fig.add_subplot(len(args.datasets), 3, 1 + i + 3 * j) 180 | for name, results in filter_arg(data, "batch_size", batch_size).items(): 181 | if args.dataset in results and args.model in results[args.dataset] and dataset in results["args"]["train_data"]: 182 | ys = np.array(results[args.dataset][args.model][args.metric]) 183 | xs = np.array(results[args.dataset][args.model]["iteration"]) 184 | xs = xs / 100_000 185 | xs = sample_points(xs, args.points) 186 | ys = sample_points(ys, args.points) 187 | label = f"{results['args']['ips_strategy']}" 188 | ax.plot(xs, ys, label=labels[label], color=color(name), marker=markers[label], markevery=0.1, markersize=4.5) 189 | if f"{args.metric}/std" in results[args.dataset][args.model]: 190 | ys_std = np.array(results[args.dataset][args.model][f"{args.metric}/std"]) 191 | ys_std = sample_points(ys_std, args.points) 192 | ax.fill_between(xs, ys - ys_std, ys + ys_std, color=color(name), alpha=0.35) 193 | ax.set_ylim([0.99 * min_last_y, 1.01 * max_last_y]) 194 | if j == 0: 195 | ax.set_title(f"Batch size = {batch_size}") 196 | if i == 0: 197 | ax.set_ylabel(metrics[args.metric]) 198 | else: 199 | ax.set_yticklabels([]) 200 | if j == len(args.datasets) - 1: 201 | ax.set_xlabel(f"Iterations ($\\times 10^5$)") 202 | else: 203 | ax.set_xticklabels([]) 204 | #ax.set_xticks(np.arange(0, 5 + 1, step=1.0)) 205 | 206 | # Legend 207 | if args.legend: 208 | handles, labels = plt.gca().get_legend_handles_labels() 209 | by_label = dict(zip(labels, handles)) 210 | keys = sorted(by_label.keys(), key=lambda key: sorting[inv_labels[key]]) 211 | values = [by_label[key] for key in keys] 212 | fig.legend(values, keys, loc='upper center', 213 | bbox_to_anchor=(0.5 + 0.02, 1.0 + 0.05), ncol=3) 214 | 215 | plot_data(args, fig) 216 | plt.tight_layout() 217 | 218 | if args.out is not None: 219 | if args.out.name.endswith(".tex"): 220 | tikzplotlib.save(args.out.name) 221 | else: 222 | plt.savefig(args.out, bbox_inches="tight", format=args.format) 223 | else: 224 | plt.show() 225 | 226 | 227 | if __name__ == "__main__": 228 | logging.basicConfig( 229 | format="[%(asctime)s, %(levelname)s, %(module)s, %(threadName)s] %(message)s", 230 | level=logging.INFO) 231 | main(get_parser().parse_args()) 232 | -------------------------------------------------------------------------------- /experiments/plots/etas.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import os 4 | from copy import deepcopy 5 | from time import sleep 6 | from argparse import ArgumentParser 7 | from argparse import FileType 8 | 9 | import numpy as np 10 | import matplotlib 11 | matplotlib.rcParams['text.latex.preamble'] = '\\usepackage{biolinum}\n\\usepackage[T1]{fontenc}\n\\usepackage[libertine]{newtxmath}' 12 | matplotlib.rcParams['text.usetex'] = True 13 | matplotlib.rcParams.update({'font.size': 13}) 14 | import tikzplotlib 15 | from matplotlib import pyplot as plt 16 | from matplotlib import animation as anim 17 | from cycler import cycler 18 | plt.rcParams["figure.figsize"] = [12,3] 19 | 20 | 21 | def get_parser(): 22 | """Gets the parser to create arguments for `main`.""" 23 | parser = ArgumentParser() 24 | parser.add_argument("--json_files", type=FileType("rt"), nargs="+") 25 | parser.add_argument("--color_by", type=str, required=False, default="ips_strategy", 26 | choices=["none", "ips_strategy", "optimizer"]) 27 | parser.add_argument("--out", type=FileType("wb"), required=False, default=None) 28 | parser.add_argument("--dataset", type=str, required=False, default="vali") 29 | parser.add_argument("--model", type=str, required=False, default="avgmodel") 30 | parser.add_argument("--metric", type=str, default="ndcg@10") 31 | parser.add_argument("--legend", action="store_true", default=False) 32 | parser.add_argument("--width", type=float, default=12.0) 33 | parser.add_argument("--height", type=float, default=3.0) 34 | parser.add_argument("--format", type=str, default=None) 35 | parser.add_argument("--points", type=int, default=None) 36 | parser.add_argument("--etas", type=str, nargs="+", default=["0.5", "0.75", "1.0", "1.25", "1.5"]) 37 | parser.add_argument("--darkstyle", action="store_true", default=False) 38 | parser.add_argument("--datasets", type=str, nargs="+", default=["yahoo", "istella"]) 39 | return parser 40 | 41 | 42 | def color_by_ips_strategy(name): 43 | if "none" in name: 44 | return "C0" 45 | elif "weight" in name: 46 | return "C1" 47 | elif "sample" in name: 48 | return "C2" 49 | else: 50 | return None 51 | 52 | 53 | def color_by_optimizer(name): 54 | if "adam" in name: 55 | return "C0" 56 | elif "sgd" in name: 57 | return "C1" 58 | elif "adagrad" in name: 59 | return "C2" 60 | else: 61 | return None 62 | 63 | 64 | def compress_list_of_arrays(list_of_arrays): 65 | min_shape = np.min([np.array(a).shape for a in list_of_arrays]) 66 | mean = np.mean([a[:min_shape] for a in list_of_arrays], axis=0) 67 | std = np.std([a[:min_shape] for a in list_of_arrays], axis=0) 68 | n = len(list_of_arrays) 69 | return mean, std, n 70 | 71 | 72 | def filter_arg(dictionary, key, func): 73 | return {k: v for k, v in dictionary.items() if func(v["args"][key])} 74 | 75 | 76 | def sample_points(xs, points): 77 | if points is None: 78 | return xs 79 | else: 80 | # Plot the first 10% of the points perfectly (where convergence mostly takes place), 81 | # subsample the remainder of the points to prevent very slow PDF viewing 82 | start_points = 0 # points // 10 83 | end_points = points - start_points 84 | sample = np.hstack([ 85 | np.arange(start_points, dtype=np.int32), 86 | np.linspace(start_points, xs.shape[0] - 1, num=end_points, dtype=np.int32)]) 87 | return xs[sample] 88 | 89 | 90 | def main(args): 91 | 92 | if args.darkstyle: 93 | plt.style.use('dark_background') 94 | matplotlib.rcParams['axes.prop_cycle'] = cycler(color=[ 95 | 'lightgrey', 96 | 'orange', 97 | 'deepskyblue' 98 | ]) 99 | 100 | fig = plt.figure(figsize=(args.width, args.height)) 101 | 102 | def plot_data(args, fig): 103 | fig.clear() 104 | data = {} 105 | for json_file in args.json_files: 106 | data[os.path.basename(json_file.name)] = json.load(json_file) 107 | json_file.seek(0) 108 | 109 | color = { 110 | "optimizer": color_by_optimizer, 111 | "ips_strategy": color_by_ips_strategy, 112 | "none": lambda x: None 113 | }[args.color_by] 114 | 115 | # Compute error bars for identical runs with multiple seeds 116 | avg_runs = {} 117 | for name, results in data.items(): 118 | if args.dataset in results and args.model in results[args.dataset]: 119 | x = results[args.dataset][args.model]["iteration"] 120 | y = results[args.dataset][args.model][args.metric] 121 | a = deepcopy(results['args']) 122 | del a["seed"] 123 | del a["output"] 124 | key = tuple(sorted(a.items())) 125 | if key not in avg_runs: 126 | avg_runs[key] = { 127 | 'name': name, 128 | 'results': {args.dataset: {args.model: {args.metric: [], "iteration": []}}} 129 | } 130 | avg_runs[key]['results'][args.dataset][args.model][args.metric].append(y) 131 | avg_runs[key]['results'][args.dataset][args.model]["iteration"].append(x) 132 | avg_runs[key]['args'] = results['args'] 133 | data = {} 134 | for run in avg_runs.values(): 135 | ys_mean, ys_std, ys_n = compress_list_of_arrays(run['results'][args.dataset][args.model][args.metric]) 136 | xs_mean, _, _ = compress_list_of_arrays(run['results'][args.dataset][args.model]["iteration"]) 137 | data[run['name']] = { 138 | args.dataset: {args.model: { 139 | args.metric: ys_mean, 140 | f"{args.metric}/std": ys_std, 141 | f"{args.metric}/n": ys_n, 142 | "iteration": xs_mean 143 | }}, 144 | "args": run['args'] 145 | } 146 | 147 | # Plot actual results 148 | labels = { 149 | "none": "Biased-SGD", 150 | "weight": "IPS-SGD", 151 | "sample": "\\textsc{CounterSample}" 152 | } 153 | inv_labels = {k: v for v, k in labels.items()} 154 | sorting = { 155 | "none": 3, 156 | "weight": 2, 157 | "sample": 1 158 | } 159 | metrics = { 160 | "ndcg@10": "nDCG@10", 161 | "arp": "ARP" 162 | } 163 | markers = { 164 | "none": "s", 165 | "weight": "o", 166 | "sample": "^" 167 | } 168 | optimizers = { 169 | "sgd": "SGD", 170 | "adam": "Adam", 171 | "adagrad": "Adagrad" 172 | } 173 | for j, dataset in enumerate(args.datasets): 174 | # Compute y limits 175 | max_last_y = 0.0 176 | min_last_y = 1e30 177 | for name, results in data.items(): 178 | if args.dataset in results and args.model in results[args.dataset] and dataset in results["args"]["train_data"]: 179 | y = results[args.dataset][args.model][args.metric][-1] 180 | min_last_y = min(y, min_last_y) 181 | max_last_y = max(y, max_last_y) 182 | 183 | # Plot subplots 184 | for i, eta in enumerate(args.etas): 185 | ax = fig.add_subplot(len(args.datasets), len(args.etas), 1 + i + len(args.etas) * j) 186 | for name, results in filter_arg(data, "click_log", lambda v: eta in v).items(): 187 | if args.dataset in results and args.model in results[args.dataset] and dataset in results["args"]["train_data"]: 188 | ys = np.array(results[args.dataset][args.model][args.metric]) 189 | xs = np.array(results[args.dataset][args.model]["iteration"]) 190 | xs = (xs * int(results['args']['batch_size'])) / 1_000_000 191 | xs = sample_points(xs, args.points) 192 | ys = sample_points(ys, args.points) 193 | label = f"{results['args']['ips_strategy']}" 194 | ax.plot(xs, ys, label=labels[label], color=color(name), marker=markers[label], markevery=0.1, markersize=4.5) 195 | if f"{args.metric}/std" in results[args.dataset][args.model]: 196 | ys_std = np.array(results[args.dataset][args.model][f"{args.metric}/std"]) 197 | ys_std = sample_points(ys_std, args.points) 198 | ax.fill_between(xs, ys - ys_std, ys + ys_std, color=color(name), alpha=0.35) 199 | ax.set_ylim([0.98 * min_last_y, 1.01 * max_last_y]) 200 | if j == 0: 201 | ax.set_title(f"$\\gamma$ = {eta}") 202 | if i == 0: 203 | ax.set_ylabel(metrics[args.metric]) 204 | else: 205 | ax.set_yticklabels([]) 206 | if j == len(args.datasets) - 1: 207 | ax.set_xlabel(r"Iterations ($\times 10^6$)") 208 | else: 209 | ax.set_xticklabels([]) 210 | ax.set_xticks(np.arange(0, 5 + 1, step=1.0)) 211 | 212 | # Legend 213 | if args.legend: 214 | handles, labels = plt.gca().get_legend_handles_labels() 215 | by_label = dict(zip(labels, handles)) 216 | keys = sorted(by_label.keys(), key=lambda key: sorting[inv_labels[key]]) 217 | values = [by_label[key] for key in keys] 218 | fig.legend(values, keys, loc='upper center', 219 | bbox_to_anchor=(0.5 + 0.02, 1.0 + 0.05), ncol=3) 220 | 221 | plot_data(args, fig) 222 | plt.tight_layout() 223 | 224 | if args.out is not None: 225 | if args.out.name.endswith(".tex"): 226 | tikzplotlib.save(args.out.name) 227 | else: 228 | plt.savefig(args.out, bbox_inches="tight", format=args.format) 229 | else: 230 | plt.show() 231 | 232 | 233 | if __name__ == "__main__": 234 | logging.basicConfig( 235 | format="[%(asctime)s, %(levelname)s, %(module)s, %(threadName)s] %(message)s", 236 | level=logging.INFO) 237 | main(get_parser().parse_args()) 238 | -------------------------------------------------------------------------------- /experiments/plots/optimizers.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import os 4 | from copy import deepcopy 5 | from time import sleep 6 | from argparse import ArgumentParser 7 | from argparse import FileType 8 | 9 | import numpy as np 10 | import matplotlib 11 | matplotlib.rcParams['text.latex.preamble'] = '\\usepackage{biolinum}\n\\usepackage[T1]{fontenc}\n\\usepackage[libertine]{newtxmath}' 12 | matplotlib.rcParams['text.usetex'] = True 13 | matplotlib.rcParams.update({'font.size': 13}) 14 | import tikzplotlib 15 | from matplotlib import pyplot as plt 16 | from matplotlib import animation as anim 17 | from cycler import cycler 18 | plt.rcParams["figure.figsize"] = [12,3] 19 | 20 | 21 | def get_parser(): 22 | """Gets the parser to create arguments for `main`.""" 23 | parser = ArgumentParser() 24 | parser.add_argument("--json_files", type=FileType("rt"), nargs="+") 25 | parser.add_argument("--color_by", type=str, required=False, default="ips_strategy", 26 | choices=["none", "ips_strategy", "optimizer"]) 27 | parser.add_argument("--out", type=FileType("wb"), required=False, default=None) 28 | parser.add_argument("--dataset", type=str, required=False, default="vali") 29 | parser.add_argument("--model", type=str, required=False, default="avgmodel") 30 | parser.add_argument("--metric", type=str, default="ndcg@10") 31 | parser.add_argument("--legend", action="store_true", default=False) 32 | parser.add_argument("--width", type=float, default=12.0) 33 | parser.add_argument("--height", type=float, default=3.0) 34 | parser.add_argument("--format", type=str, default=None) 35 | parser.add_argument("--points", type=int, default=None) 36 | parser.add_argument("--darkstyle", action="store_true", default=False) 37 | parser.add_argument("--datasets", type=str, nargs="+", default=["yahoo", "istella"]) 38 | return parser 39 | 40 | 41 | def color_by_ips_strategy(name): 42 | if "none" in name: 43 | return "C0" 44 | elif "weight" in name: 45 | return "C1" 46 | elif "sample" in name: 47 | return "C2" 48 | else: 49 | return None 50 | 51 | 52 | def color_by_optimizer(name): 53 | if "adam" in name: 54 | return "C0" 55 | elif "sgd" in name: 56 | return "C1" 57 | elif "adagrad" in name: 58 | return "C2" 59 | else: 60 | return None 61 | 62 | 63 | def compress_list_of_arrays(list_of_arrays): 64 | min_shape = np.min([np.array(a).shape for a in list_of_arrays]) 65 | mean = np.mean([a[:min_shape] for a in list_of_arrays], axis=0) 66 | std = np.std([a[:min_shape] for a in list_of_arrays], axis=0) 67 | n = len(list_of_arrays) 68 | return mean, std, n 69 | 70 | 71 | def filter_arg(dictionary, key, value): 72 | return {k: v for k, v in dictionary.items() if v["args"][key] == value} 73 | 74 | 75 | def sample_points(xs, points): 76 | if points is None: 77 | return xs 78 | else: 79 | # Plot the first 10% of the points perfectly (where convergence mostly takes place), 80 | # subsample the remainder of the points to prevent very slow PDF viewing 81 | start_points = 0 # points // 10 82 | end_points = points - start_points 83 | sample = np.hstack([ 84 | np.arange(start_points, dtype=np.int32), 85 | np.linspace(start_points, xs.shape[0] - 1, num=end_points, dtype=np.int32)]) 86 | return xs[sample] 87 | 88 | 89 | def main(args): 90 | 91 | if args.darkstyle: 92 | plt.style.use('dark_background') 93 | matplotlib.rcParams['axes.prop_cycle'] = cycler(color=[ 94 | 'lightgrey', 95 | 'orange', 96 | 'deepskyblue' 97 | ]) 98 | 99 | fig = plt.figure(figsize=(args.width, args.height)) 100 | 101 | def plot_data(args, fig): 102 | fig.clear() 103 | data = {} 104 | for json_file in args.json_files: 105 | data[os.path.basename(json_file.name)] = json.load(json_file) 106 | json_file.seek(0) 107 | 108 | color = { 109 | "optimizer": color_by_optimizer, 110 | "ips_strategy": color_by_ips_strategy, 111 | "none": lambda x: None 112 | }[args.color_by] 113 | 114 | # Compute error bars for identical runs with multiple seeds 115 | avg_runs = {} 116 | for name, results in data.items(): 117 | if args.dataset in results and args.model in results[args.dataset]: 118 | x = results[args.dataset][args.model]["iteration"] 119 | y = results[args.dataset][args.model][args.metric] 120 | a = deepcopy(results['args']) 121 | del a["seed"] 122 | del a["output"] 123 | key = tuple(sorted(a.items())) 124 | if key not in avg_runs: 125 | avg_runs[key] = { 126 | 'name': name, 127 | 'results': {args.dataset: {args.model: {args.metric: [], "iteration": []}}} 128 | } 129 | avg_runs[key]['results'][args.dataset][args.model][args.metric].append(y) 130 | avg_runs[key]['results'][args.dataset][args.model]["iteration"].append(x) 131 | avg_runs[key]['args'] = results['args'] 132 | data = {} 133 | for run in avg_runs.values(): 134 | ys_mean, ys_std, ys_n = compress_list_of_arrays(run['results'][args.dataset][args.model][args.metric]) 135 | xs_mean, _, _ = compress_list_of_arrays(run['results'][args.dataset][args.model]["iteration"]) 136 | data[run['name']] = { 137 | args.dataset: {args.model: { 138 | args.metric: ys_mean, 139 | f"{args.metric}/std": ys_std, 140 | f"{args.metric}/n": ys_n, 141 | "iteration": xs_mean 142 | }}, 143 | "args": run['args'] 144 | } 145 | 146 | # Plot actual results 147 | labels = { 148 | "none": "Biased-SGD", 149 | "weight": "IPS-SGD", 150 | "sample": "\\textsc{CounterSample}" 151 | } 152 | inv_labels = {k: v for v, k in labels.items()} 153 | sorting = { 154 | "none": 3, 155 | "weight": 2, 156 | "sample": 1 157 | } 158 | metrics = { 159 | "ndcg@10": "nDCG@10", 160 | "arp": "ARP" 161 | } 162 | markers = { 163 | "none": "s", 164 | "weight": "o", 165 | "sample": "^" 166 | } 167 | optimizers = { 168 | "sgd": "SGD", 169 | "adam": "Adam", 170 | "adagrad": "Adagrad" 171 | } 172 | for j, dataset in enumerate(args.datasets): 173 | # Compute y limits 174 | max_last_y = 0.0 175 | min_last_y = 1e30 176 | for name, results in data.items(): 177 | if args.dataset in results and args.model in results[args.dataset] and dataset in results["args"]["train_data"]: 178 | y = results[args.dataset][args.model][args.metric][-1] 179 | min_last_y = min(y, min_last_y) 180 | max_last_y = max(y, max_last_y) 181 | 182 | # Plot subplots 183 | for i, optimizer in enumerate(["sgd", "adam", "adagrad"]): 184 | ax = fig.add_subplot(len(args.datasets), 3, 1 + i + j * 3) 185 | for name, results in filter_arg(data, "optimizer", optimizer).items(): 186 | if args.dataset in results and args.model in results[args.dataset] and dataset in results["args"]["train_data"]: 187 | ys = np.array(results[args.dataset][args.model][args.metric]) 188 | xs = np.array(results[args.dataset][args.model]["iteration"]) 189 | xs = (xs * int(results['args']['batch_size'])) / 1_000_000 190 | xs = sample_points(xs, args.points) 191 | ys = sample_points(ys, args.points) 192 | label = f"{results['args']['ips_strategy']}" 193 | ax.plot(xs, ys, label=labels[label], color=color(name), marker=markers[label], markevery=0.1, markersize=4.5) 194 | if f"{args.metric}/std" in results[args.dataset][args.model]: 195 | ys_std = np.array(results[args.dataset][args.model][f"{args.metric}/std"]) 196 | ys_std = sample_points(ys_std, args.points) 197 | ax.fill_between(xs, ys - ys_std, ys + ys_std, color=color(name), alpha=0.35) 198 | ax.set_ylim([0.99 * min_last_y, 1.01 * max_last_y]) 199 | if j == 0: 200 | ax.set_title(f"Optimizer = {optimizers[optimizer]}") 201 | if i == 0: 202 | ax.set_ylabel(metrics[args.metric]) 203 | else: 204 | ax.set_yticklabels([]) 205 | if j == len(args.datasets) - 1: 206 | ax.set_xlabel(r"Iterations ($\times 10^6$)") 207 | else: 208 | ax.set_xticklabels([]) 209 | ax.set_xticks(np.arange(0, 5 + 1, step=1.0)) 210 | 211 | # Legend 212 | if args.legend: 213 | handles, labels = plt.gca().get_legend_handles_labels() 214 | by_label = dict(zip(labels, handles)) 215 | keys = sorted(by_label.keys(), key=lambda key: sorting[inv_labels[key]]) 216 | values = [by_label[key] for key in keys] 217 | fig.legend(values, keys, loc='upper center', 218 | bbox_to_anchor=(0.5 + 0.02, 1.0 + 0.05), ncol=3) 219 | 220 | plot_data(args, fig) 221 | plt.tight_layout() 222 | 223 | if args.out is not None: 224 | if args.out.name.endswith(".tex"): 225 | tikzplotlib.save(args.out.name) 226 | else: 227 | plt.savefig(args.out, bbox_inches="tight", format=args.format) 228 | else: 229 | plt.show() 230 | 231 | 232 | if __name__ == "__main__": 233 | logging.basicConfig( 234 | format="[%(asctime)s, %(levelname)s, %(module)s, %(threadName)s] %(message)s", 235 | level=logging.INFO) 236 | main(get_parser().parse_args()) 237 | -------------------------------------------------------------------------------- /experiments/plots/plot.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import os 4 | from copy import deepcopy 5 | from time import sleep 6 | from argparse import ArgumentParser 7 | from argparse import FileType 8 | 9 | import numpy as np 10 | import matplotlib 11 | from matplotlib import pyplot as plt 12 | from matplotlib import animation as anim 13 | plt.rcParams["figure.figsize"] = [12,6] 14 | 15 | 16 | def get_parser(): 17 | """Gets the parser to create arguments for `main`.""" 18 | parser = ArgumentParser() 19 | parser.add_argument("--json_files", type=FileType("rt"), nargs="+") 20 | parser.add_argument("--color_by", type=str, required=False, default="ips_strategy", 21 | choices=["none", "ips_strategy", "optimizer"]) 22 | parser.add_argument("--out", type=FileType("wb"), required=False, default=None) 23 | parser.add_argument("--dataset", type=str, required=False, default="vali") 24 | parser.add_argument("--model", type=str, required=False, default="avgmodel") 25 | parser.add_argument("--metric", type=str, default="ndcg@10") 26 | parser.add_argument("--select_best_lr", action="store_true", default=False) 27 | parser.add_argument("--seed_error_bars", action="store_true", default=False) 28 | return parser 29 | 30 | 31 | def color_by_ips_strategy(name): 32 | if "none" in name: 33 | return "C0" 34 | elif "weight" in name: 35 | return "C1" 36 | elif "sample" in name: 37 | return "C2" 38 | else: 39 | return None 40 | 41 | 42 | def color_by_optimizer(name): 43 | if "adam" in name: 44 | return "C0" 45 | elif "sgd" in name: 46 | return "C1" 47 | elif "adagrad" in name: 48 | return "C2" 49 | else: 50 | return None 51 | 52 | 53 | def compress_list_of_arrays(list_of_arrays): 54 | min_shape = np.min([np.array(a).shape for a in list_of_arrays]) 55 | mean = np.mean([a[:min_shape] for a in list_of_arrays], axis=0) 56 | std = np.std([a[:min_shape] for a in list_of_arrays], axis=0) 57 | n = len(list_of_arrays) 58 | return mean, std, n 59 | 60 | 61 | def main(args): 62 | 63 | fig = plt.figure() 64 | 65 | def plot_data(args, fig): 66 | fig.clear() 67 | ax = fig.add_subplot(111) 68 | data = {} 69 | for json_file in args.json_files: 70 | data[os.path.basename(json_file.name)] = json.load(json_file) 71 | json_file.seek(0) 72 | 73 | color = { 74 | "optimizer": color_by_optimizer, 75 | "ips_strategy": color_by_ips_strategy, 76 | "none": lambda x: None 77 | }[args.color_by] 78 | 79 | # Print all loaded results 80 | print_results = [] 81 | for name, results in data.items(): 82 | if args.dataset in results and args.model in results[args.dataset]: 83 | y = results[args.dataset][args.model][args.metric][-1] 84 | print_results.append((y, name)) 85 | for value, name in sorted(print_results, key=lambda e: e[0]): 86 | print(f"{name:12s} {value:.4f}") 87 | 88 | # Select only best LR per setting 89 | if args.select_best_lr: 90 | print("============== selecting only best LR ============") 91 | best = {} 92 | for name, results in data.items(): 93 | if args.dataset in results and args.model in results[args.dataset]: 94 | y = np.mean(results[args.dataset][args.model][args.metric]) 95 | if args.metric == "arp": 96 | y = -y 97 | a = deepcopy(results['args']) 98 | del a["lr"] 99 | del a["output"] 100 | key = tuple(sorted(a.items())) 101 | if key not in best or best[key]['y'] < y: 102 | best[key] = {} 103 | best[key]['y'] = y 104 | best[key]['name'] = name 105 | best[key]['results'] = results 106 | data = {v['name']: v['results'] for v in best.values()} 107 | print_results = [] 108 | for name, results in data.items(): 109 | if args.dataset in results and args.model in results[args.dataset]: 110 | y = np.mean(results[args.dataset][args.model][args.metric]) 111 | x = results[args.dataset][args.model]["iteration"][-1] 112 | print_results.append((y, x, name)) 113 | for value, x, name in sorted(print_results, key=lambda e: e[0]): 114 | print(f"{name:12s} {value:.4f} [{x:6d}]") 115 | 116 | # Compute error bars for identical runs with multiple seeds 117 | if args.seed_error_bars: 118 | avg_runs = {} 119 | for name, results in data.items(): 120 | if args.dataset in results and args.model in results[args.dataset]: 121 | x = results[args.dataset][args.model]["iteration"] 122 | y = results[args.dataset][args.model][args.metric] 123 | a = deepcopy(results['args']) 124 | del a["seed"] 125 | del a["output"] 126 | key = tuple(sorted(a.items())) 127 | if key not in avg_runs: 128 | avg_runs[key] = { 129 | 'name': name, 130 | 'results': {args.dataset: {args.model: {args.metric: [], "iteration": []}}} 131 | } 132 | avg_runs[key]['results'][args.dataset][args.model][args.metric].append(y) 133 | avg_runs[key]['results'][args.dataset][args.model]["iteration"].append(x) 134 | avg_runs[key]['args'] = results['args'] 135 | data = {} 136 | for run in avg_runs.values(): 137 | ys_mean, ys_std, ys_n = compress_list_of_arrays(run['results'][args.dataset][args.model][args.metric]) 138 | xs_mean, _, _ = compress_list_of_arrays(run['results'][args.dataset][args.model]["iteration"]) 139 | data[run['name']] = { 140 | args.dataset: {args.model: { 141 | args.metric: ys_mean, 142 | f"{args.metric}/std": ys_std, 143 | f"{args.metric}/n": ys_n, 144 | "iteration": xs_mean 145 | }}, 146 | "args": run['args'] 147 | } 148 | 149 | # Compute y limits 150 | max_last_y = 0.0 151 | min_last_y = 1e30 152 | for name, results in data.items(): 153 | if args.dataset in results and args.model in results[args.dataset]: 154 | y = results[args.dataset][args.model][args.metric][-1] 155 | min_last_y = min(y, min_last_y) 156 | max_last_y = max(y, max_last_y) 157 | 158 | # Plot actual results 159 | for name, results in data.items(): 160 | if args.dataset in results and args.model in results[args.dataset]: 161 | ys = np.array(results[args.dataset][args.model][args.metric]) 162 | xs = np.array(results[args.dataset][args.model]["iteration"]) 163 | xs = (xs * int(results['args']['batch_size'])) / 1_000_000 164 | ax.plot(xs, ys, label=name, color=color(name)) 165 | if f"{args.metric}/std" in results[args.dataset][args.model]: 166 | ys_std = np.array(results[args.dataset][args.model][f"{args.metric}/std"]) 167 | ax.fill_between(xs, ys - ys_std, ys + ys_std, color=color(name), alpha=0.35) 168 | ax.set_ylim([0.96 * max_last_y, 1.01 * max_last_y]) 169 | ax.set_ylabel(args.metric) 170 | ax.set_xlabel("Epochs") 171 | fig.legend() 172 | 173 | plot_data(args, fig) 174 | 175 | if args.out is not None: 176 | plt.savefig(args.out) 177 | else: 178 | plt.show() 179 | 180 | 181 | if __name__ == "__main__": 182 | logging.basicConfig( 183 | format="[%(asctime)s, %(levelname)s, %(module)s, %(threadName)s] %(message)s", 184 | level=logging.INFO) 185 | main(get_parser().parse_args()) 186 | -------------------------------------------------------------------------------- /experiments/plots/toy_sample.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | from argparse import FileType 3 | 4 | import numpy as np 5 | import matplotlib 6 | matplotlib.rcParams['text.latex.preamble'] = '\\usepackage{biolinum}\n\\usepackage{sfmath}\n\\usepackage[T1]{fontenc}' #\\usepackage{libertine}\n 7 | matplotlib.rcParams['text.usetex'] = True 8 | matplotlib.rcParams.update({'font.size': 14}) 9 | import torch 10 | from torchcontrib.optim import SWA 11 | from matplotlib import pyplot as plt 12 | from matplotlib.legend_handler import HandlerPatch 13 | from sklearn.datasets import load_iris 14 | from sklearn.linear_model import LinearRegression 15 | from sklearn.preprocessing import normalize 16 | import matplotlib.patches as mpatches 17 | 18 | 19 | def get_parser(): 20 | """Gets the parser to create arguments for `main`.""" 21 | parser = ArgumentParser() 22 | parser.add_argument("--out", type=FileType("wb"), required=False, default=None) 23 | parser.add_argument("--format", type=str, default=None) 24 | return parser 25 | 26 | 27 | def main(args): 28 | # Parameters for toy data and experiments 29 | plt.figure(figsize=(6.4,3.4)) 30 | rng_seed = 42 31 | np.random.seed(rng_seed) 32 | 33 | wstar = torch.tensor([[0.973, 1.144]], dtype=torch.float) 34 | xs = torch.tensor(np.random.randn(50, 2), dtype=torch.float) 35 | labels = torch.mm(xs, wstar.T) 36 | 37 | p = torch.tensor(np.random.uniform(0.05, 1.0, xs.shape[0]), dtype=torch.float) 38 | ips = 1.0 / p 39 | n_iters = 50 40 | plot_every = n_iters // 10 41 | arrow_width = 0.012 42 | legends = {} 43 | 44 | # The loss function we want to optimize 45 | def loss_fn(out, y, mult): 46 | l2loss = (out - y) ** 2.0 47 | logl2loss = torch.log(1.0 + (out - y) ** 2.0) 48 | return torch.mean(mult * l2loss) 49 | 50 | # IPS-weighted approach 51 | for color_index, lr in enumerate([0.01, 0.02, 0.03, 0.05, 0.1]): # 0.01, 0.03, 0.05, 0.1, 0.3 52 | color = "C%d" % (color_index + 2) if color_index > 0 else "C1" 53 | model = torch.nn.Linear(2, 1) 54 | optimizer = torch.optim.SGD(model.parameters(), lr=lr) 55 | optimizer = SWA(optimizer, swa_start=0, swa_freq=1, swa_lr=lr) 56 | with torch.no_grad(): 57 | model.bias.zero_() 58 | model.weight.zero_() 59 | old_weights = np.copy(model.weight.data.numpy()) 60 | np.random.seed(rng_seed + color_index + 1) 61 | for t in range(n_iters): 62 | i = np.random.randint(xs.shape[0]) 63 | x = xs[i, :] 64 | y = labels[i] 65 | optimizer.zero_grad() 66 | o = model(x) 67 | l = loss_fn(o, y, ips[i]) 68 | l.backward() 69 | optimizer.step() 70 | if t % plot_every == 0: 71 | optimizer.swap_swa_sgd() 72 | x, y = model.weight.data.numpy()[0] 73 | optimizer.swap_swa_sgd() 74 | ox, oy = old_weights[0] 75 | label = f"IPS-SGD ($\\eta={lr}$)" 76 | arr = plt.arrow(ox, oy, x - ox, y - oy, width=arrow_width, length_includes_head=True, 77 | color=color, label=label) 78 | optimizer.swap_swa_sgd() 79 | old_weights = np.copy(model.weight.data.numpy()) 80 | optimizer.swap_swa_sgd() 81 | legends[label] = arr 82 | 83 | # Sample based approach 84 | for lr in [10.0]: 85 | # lr = 3.0 # 1.0 86 | model = torch.nn.Linear(2, 1) 87 | optimizer = torch.optim.SGD(model.parameters(), lr=lr) 88 | optimizer = SWA(optimizer, swa_start=0, swa_freq=1, swa_lr=lr) 89 | with torch.no_grad(): 90 | model.bias.zero_() 91 | model.weight.zero_() 92 | old_weights = np.copy(model.weight.data.numpy()) 93 | sample_probs = np.array(ips / torch.sum(ips)) 94 | Mbar = float(np.mean(sample_probs)) 95 | np.random.seed(rng_seed - 1) 96 | for t in range(n_iters): 97 | i = np.argwhere(np.random.multinomial(1, sample_probs) == 1.0)[0, 0] 98 | x = xs[i, :] 99 | y = labels[i] 100 | optimizer.zero_grad() 101 | o = model(x) 102 | l = loss_fn(o, y, Mbar) 103 | l.backward() 104 | optimizer.step() 105 | if t % plot_every == 0: 106 | optimizer.swap_swa_sgd() 107 | x, y = model.weight.data.numpy()[0] 108 | optimizer.swap_swa_sgd() 109 | ox, oy = old_weights[0] 110 | label = f"\\textsc{{CounterSample}} ($\\eta={lr}$)" 111 | arr = plt.arrow(ox, oy, x - ox, y - oy, width=arrow_width, length_includes_head=True, 112 | color="C2", label=label) 113 | 114 | optimizer.swap_swa_sgd() 115 | old_weights = np.copy(model.weight.data.numpy()) 116 | optimizer.swap_swa_sgd() 117 | legends[label] = arr 118 | 119 | # True IPS-weighted loss over all datapoints, used for plotting contour 120 | def f(x1, x2): 121 | w = torch.tensor([[x1], [x2]], dtype=torch.float) 122 | o = torch.mm(xs, w) 123 | return float(loss_fn(o, torch.mm(xs, wstar.reshape((2, 1))), ips)) 124 | 125 | # Compute all useful combinations of weights and compute true loss for each one 126 | # This will be used to compute a contour plot 127 | true_x1 = np.linspace(float(wstar[0, 0]) - 1.5, float(wstar[0, 0]) + 0.8) # - 1.5 / + 1.0 128 | true_x2 = np.linspace(float(wstar[0, 1]) - 1.5, float(wstar[0, 1]) + 1.2) # - 1.5 / + 1.0 129 | true_x1, true_x2 = np.meshgrid(true_x1, true_x2) 130 | true_y = np.array([ 131 | [f(true_x1[i1, i2], true_x2[i1, i2]) for i2 in range(len(true_x2))] 132 | for i1 in range(len(true_x1)) 133 | ]) 134 | 135 | # Contour plot with optimum 136 | plt.plot(wstar[0, 0], wstar[0, 1], marker='o', markersize=3, color="black") 137 | plt.contour(true_x1, true_x2, true_y, 10, colors="black", alpha=0.35) 138 | 139 | # Generate legends from arrows and make figure 140 | def make_legend_arrow(legend, orig_handle, 141 | xdescent, ydescent, 142 | width, height, fontsize): 143 | p = mpatches.FancyArrow(0, 0.5*height, width, 0, length_includes_head=True, 144 | head_width=0.75*height) 145 | return p 146 | 147 | def sort_op(key): 148 | if "CounterSample" in key: 149 | return "" 150 | else: 151 | return key 152 | 153 | labels = [key for key in sorted(legends.keys(), key=sort_op)] 154 | arrows = [legends[key] for key in sorted(legends.keys(), key=sort_op)] 155 | plt.legend(arrows, labels, ncol=2, loc='upper center', framealpha=0.95, handler_map={mpatches.FancyArrow : HandlerPatch(patch_func=make_legend_arrow)}, 156 | bbox_to_anchor=(0.5, 1.0 + 0.03)) 157 | plt.xlabel("$w_1$") 158 | plt.ylabel("$w_2$") 159 | plt.tight_layout() 160 | plt.savefig(args.out, format=args.format) 161 | 162 | if __name__ == "__main__": 163 | main(get_parser().parse_args()) 164 | -------------------------------------------------------------------------------- /experiments/plots/toy_sample_animation.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | from argparse import FileType 3 | 4 | import json 5 | import numpy as np 6 | import matplotlib 7 | matplotlib.rcParams['text.latex.preamble'] = '\\usepackage{biolinum}\n\\usepackage{sfmath}\n\\usepackage[T1]{fontenc}' #\\usepackage{libertine}\n 8 | matplotlib.rcParams['text.usetex'] = True 9 | matplotlib.rcParams.update({'font.size': 14}) 10 | import torch 11 | from torchcontrib.optim import SWA 12 | from matplotlib import pyplot as plt 13 | from matplotlib.legend_handler import HandlerPatch 14 | from sklearn.datasets import load_iris 15 | from sklearn.linear_model import LinearRegression 16 | from sklearn.preprocessing import normalize 17 | import matplotlib.patches as mpatches 18 | from mpl_toolkits.mplot3d import Axes3D 19 | import matplotlib.animation as animation 20 | 21 | 22 | def get_parser(): 23 | """Gets the parser to create arguments for `main`.""" 24 | parser = ArgumentParser() 25 | parser.add_argument("--out", type=str, required=True, default=None) 26 | parser.add_argument("--format", type=str, default=None) 27 | return parser 28 | 29 | 30 | def main(args): 31 | 32 | # Parameters for toy data and experiments 33 | plt.style.use('dark_background') 34 | rng_seed = 4200 35 | np.random.seed(rng_seed) 36 | 37 | wstar = torch.tensor([[0.973, 1.144]], dtype=torch.float) 38 | xs = torch.tensor(np.random.randn(50, 2), dtype=torch.float) 39 | labels = torch.mm(xs, wstar.T) 40 | 41 | p = torch.tensor(np.random.uniform(0.05, 1.0, xs.shape[0]), dtype=torch.float) 42 | ips = 1.0 / p 43 | n_iters = 500 44 | plot_every = n_iters // 10 45 | arrow_width = 0.012 * 0.01 46 | legend = {} 47 | data = {} 48 | colors = {} 49 | interpsize = 30 50 | 51 | # Figure 52 | fig = plt.figure(figsize=(8.4,5.4), dpi=150) 53 | ax = fig.add_subplot(111) 54 | ax.set_xlim([-0.3, 1.8]) 55 | ax.set_ylim([-0.3, 1.8]) 56 | 57 | # The loss function we want to optimize 58 | def loss_fn(out, y, mult): 59 | l2loss = (out - y) ** 2.0 60 | logl2loss = torch.log(1.0 + (out - y) ** 2.0) 61 | return torch.mean(mult * l2loss) 62 | 63 | # True IPS-weighted loss over all datapoints, used for plotting contour 64 | def f(x1, x2): 65 | w = torch.tensor([[x1], [x2]], dtype=torch.float) 66 | o = torch.mm(xs, w) 67 | return float(loss_fn(o, torch.mm(xs, wstar.reshape((2, 1))), ips)) 68 | 69 | # Plot contour 70 | true_x1 = np.linspace(float(wstar[0, 0]) - 1.5, float(wstar[0, 0]) + 0.8) # - 1.5 / + 1.0 71 | true_x2 = np.linspace(float(wstar[0, 1]) - 1.5, float(wstar[0, 1]) + 1.2) # - 1.5 / + 1.0 72 | true_x1, true_x2 = np.meshgrid(true_x1, true_x2) 73 | true_y = np.array([ 74 | [f(true_x1[i1, i2], true_x2[i1, i2]) for i2 in range(len(true_x2))] 75 | for i1 in range(len(true_x1)) 76 | ]) 77 | ax.contour(true_x1, true_x2, true_y, levels=10, colors='white', alpha=0.45) 78 | plt.plot(0.0, 0.0, marker='o', markersize=6, color="white") 79 | plt.plot(wstar[0, 0], wstar[0, 1], marker='*', markersize=6, color="white") 80 | 81 | # IPS-weighted approach 82 | for color_index, lr in enumerate([0.1, 0.01, 0.03]): # 0.01, 0.03, 0.05, 0.1, 0.3 83 | if lr == 0.01: 84 | color = "violet" 85 | elif lr == 0.03: 86 | color = "orange" 87 | else: 88 | color = "lightgreen" 89 | #color = "C%d" % (color_index + 2) 90 | model = torch.nn.Linear(2, 1) 91 | optimizer = torch.optim.SGD(model.parameters(), lr=lr) 92 | optimizer = SWA(optimizer, swa_start=0, swa_freq=1, swa_lr=lr) 93 | with torch.no_grad(): 94 | model.bias.zero_() 95 | model.weight.zero_() 96 | old_weights = np.copy(model.weight.data.numpy()) 97 | np.random.seed(rng_seed + color_index + 1)# + color_index + 1) 98 | label = f"IPS-SGD ($\\eta={lr}$)" 99 | data[label] = np.zeros((3, n_iters * interpsize + 1)) 100 | colors[label] = color 101 | for t in range(n_iters): 102 | i = np.random.randint(xs.shape[0]) 103 | x = xs[i, :] 104 | y = labels[i] 105 | optimizer.zero_grad() 106 | o = model(x) 107 | l = loss_fn(o, y, ips[i]) 108 | l.backward() 109 | optimizer.step() 110 | 111 | # Record current iteration performance and location 112 | optimizer.swap_swa_sgd() 113 | x, y = model.weight.data.numpy()[0] 114 | optimizer.swap_swa_sgd() 115 | old_x, old_y, old_z = data[label][:, t * interpsize] 116 | xr = np.linspace(old_x, x, num=interpsize) 117 | yr = np.linspace(old_y, y, num=interpsize) 118 | for i in range(interpsize): 119 | data[label][:, 1 + t * interpsize + i] = np.array([ 120 | xr[i], yr[i], f(xr[i], yr[i])]) 121 | #data[label][:, t] = np.array([x, y, f(x, y)]) 122 | 123 | 124 | # Sample based approach 125 | lr = 10.0 # 1.0 126 | model = torch.nn.Linear(2, 1) 127 | optimizer = torch.optim.SGD(model.parameters(), lr=lr) 128 | optimizer = SWA(optimizer, swa_start=0, swa_freq=1, swa_lr=lr) 129 | with torch.no_grad(): 130 | model.bias.zero_() 131 | model.weight.zero_() 132 | old_weights = np.copy(model.weight.data.numpy()) 133 | sample_probs = np.array(ips / torch.sum(ips)) 134 | Mbar = float(np.mean(sample_probs)) 135 | np.random.seed(rng_seed - 1) 136 | label = f"\\textsc{{CounterSample}} ($\\eta={lr}$)" 137 | data[label] = np.zeros((3, n_iters * interpsize + 1)) 138 | for t in range(n_iters): 139 | i = np.argwhere(np.random.multinomial(1, sample_probs) == 1.0)[0, 0] 140 | x = xs[i, :] 141 | y = labels[i] 142 | optimizer.zero_grad() 143 | o = model(x) 144 | l = loss_fn(o, y, Mbar) 145 | l.backward() 146 | optimizer.step() 147 | 148 | # Record current iteration location and performance 149 | optimizer.swap_swa_sgd() 150 | x, y = model.weight.data.numpy()[0] 151 | optimizer.swap_swa_sgd() 152 | old_x, old_y, old_z = data[label][:, t * interpsize] 153 | xr = np.linspace(old_x, x, num=interpsize) 154 | yr = np.linspace(old_y, y, num=interpsize) 155 | for i in range(interpsize): 156 | data[label][:, 1 + t * interpsize + i] = np.array([ 157 | xr[i], yr[i], f(xr[i], yr[i])]) 158 | colors[label] = "deepskyblue" 159 | 160 | # Print summary to quickly find performance at convergence 161 | for label in data.keys(): 162 | print(f"{label}: {data[label][2, -1]}") 163 | 164 | # Create legend 165 | lines = {} 166 | for label in data.keys(): 167 | line = data[label] 168 | lines[label] = ax.plot(line[:, 0], line[:, 1], color=colors[label], label=label, linewidth=2.0) 169 | legend[colors[label]] = label 170 | 171 | ax.set_xlabel('$w_1$') 172 | ax.set_ylabel('$w_2$') 173 | legend_lines = [ 174 | lines[legend["deepskyblue"]][0], 175 | lines[legend["orange"]][0], 176 | lines[legend["violet"]][0], 177 | lines[legend["lightgreen"]][0]] 178 | legend_labels = [ 179 | "\\textsc{CounterSample}", 180 | "IPS-SGD (best learning rate)", 181 | "IPS-SGD (learning rate too small)", 182 | "IPS-SGD (learning rate too large)"] 183 | legend_artist = ax.legend(legend_lines, legend_labels, loc='lower right') #, bbox_to_anchor=(1.0 - 0.3, 0.25)) 184 | 185 | # Update function for animation 186 | n_frames = n_iters * 2 187 | n_data = n_iters * interpsize 188 | from math import floor 189 | def transform_num(num): 190 | x = 3 * ((1.0 * num) / n_frames) 191 | y = (((x + 0.5)**2 - 0.5**2) / 12.0) 192 | return floor(y * n_data) 193 | 194 | def update_lines(num): 195 | print(f"frame {num:4d} / {n_frames:4d} [{num / n_frames * 100:.0f}%]", end="\r") 196 | num = transform_num(num) 197 | out = [legend_artist] 198 | for label in data.keys(): 199 | line = lines[label][0] 200 | d = data[label] 201 | line.set_data(d[0:2, :num]) 202 | out.append(line) 203 | return out 204 | 205 | # Write animation to file 206 | line_ani = animation.FuncAnimation(fig, update_lines, n_frames, interval=100, blit=False, repeat_delay=3000) 207 | writer = animation.FFMpegWriter(fps=60, codec='h264') # for keynote 208 | line_ani.save(args.out, writer=writer) 209 | print("\033[K\n") 210 | 211 | if __name__ == "__main__": 212 | main(get_parser().parse_args()) 213 | -------------------------------------------------------------------------------- /experiments/simulate_clicks.py: -------------------------------------------------------------------------------- 1 | """Simulates a click log from supervised ranking data.""" 2 | import logging 3 | import pickle 4 | from argparse import ArgumentParser 5 | 6 | import numpy as np 7 | import torch 8 | from pytorchltr.click_simulation import simulate_perfect 9 | from pytorchltr.click_simulation import simulate_position 10 | from pytorchltr.click_simulation import simulate_nearrandom 11 | from pytorchltr.dataset.svmrank import create_svmranking_collate_fn 12 | from pytorchltr.util import rank_by_score 13 | 14 | from experiments.dataset import load_ranking_dataset 15 | 16 | 17 | LOGGER = logging.getLogger(__name__) 18 | 19 | 20 | def get_parser(): 21 | """Gets the parser to create arguments for `main`.""" 22 | parser = ArgumentParser() 23 | parser.add_argument("--input_data", type=str, required=True) 24 | parser.add_argument("--ranker", type=str, required=True) 25 | parser.add_argument("--output_log", type=str, required=True) 26 | parser.add_argument("--vali_data", type=str, default=None) 27 | parser.add_argument('--behavior', default="perfect", 28 | choices=["perfect", "position", "nearrandom"]) 29 | parser.add_argument("--cutoff", type=int, default=None) 30 | parser.add_argument("--eta", type=float, default=1.0) 31 | parser.add_argument("--seed", type=int, default=42) 32 | parser.add_argument("--sessions", type=int, default=1_000_000) 33 | parser.add_argument("--max_clicks", type=int, default=None) 34 | return parser 35 | 36 | 37 | def main(args): 38 | """Runs the click simulation with given arguments.""" 39 | torch.manual_seed(args.seed) 40 | LOGGER.info("Loading input data") 41 | dataset = load_ranking_dataset(args.input_data, normalize=True) 42 | indices = torch.randint(len(dataset), (args.sessions,)) 43 | dataset = torch.utils.data.Subset(dataset, indices) 44 | loader = torch.utils.data.DataLoader( 45 | dataset, batch_size=1000, collate_fn=create_svmranking_collate_fn(), 46 | drop_last=False, shuffle=False) 47 | 48 | LOGGER.info("Loading ranker") 49 | ranker = torch.load(args.ranker) 50 | 51 | simulator = { 52 | "perfect": lambda rankings, n, ys: simulate_perfect( 53 | rankings, n, ys, cutoff=args.cutoff), 54 | "position": lambda rankings, n, ys: simulate_position( 55 | rankings, n, ys, cutoff=args.cutoff, eta=args.eta), 56 | "nearrandom": lambda rankings, n, ys: simulate_nearrandom( 57 | rankings, n, ys, cutoff=args.cutoff, eta=args.eta) 58 | }[args.behavior] 59 | 60 | LOGGER.info("Generating rankings and clicks (%d sessions)", args.sessions) 61 | count = 0 62 | clicked_docs = [] 63 | propensities = [] 64 | qids = [] 65 | for batch in loader: 66 | xs, ys, qid, n = ( 67 | batch["features"], batch["relevance"], batch["qid"], batch["n"]) 68 | scores = ranker(xs) 69 | rankings = rank_by_score(scores, n) 70 | clicks, obs_probs = simulator(rankings, ys, n) 71 | count += xs.shape[0] 72 | for row, col in zip(*torch.where(clicks == 1)): 73 | if args.max_clicks is None or len(clicked_docs) < args.max_clicks: 74 | clicked_docs.append(int(col)) 75 | propensities.append(float(obs_probs[row, col])) 76 | qids.append(int(qid[row])) 77 | LOGGER.info( 78 | "%d clicks, %d sessions (+%d), current list_size=%d", 79 | len(clicked_docs), count, xs.shape[0], xs.shape[1]) 80 | 81 | if args.max_clicks is not None and len(clicked_docs) >= args.max_clicks: 82 | LOGGER.info("reached %d maximum clicks, stopping simulation.", 83 | args.max_clicks) 84 | break 85 | 86 | LOGGER.info("Compressing click log to numpy arrays") 87 | click_log = { 88 | "args": args, 89 | "clicked_docs": np.array(clicked_docs, dtype=np.int32), 90 | "propensities": np.array(propensities, dtype=np.float32), 91 | "qids": np.array(qids, dtype=np.int32) 92 | } 93 | 94 | LOGGER.info("Writing click log to file") 95 | with open(args.output_log, "wb") as f: 96 | pickle.dump(click_log, f, protocol=pickle.HIGHEST_PROTOCOL) 97 | 98 | 99 | if __name__ == "__main__": 100 | logging.basicConfig( 101 | format="[%(asctime)s, %(levelname)s, %(module)s] %(message)s", 102 | level=logging.INFO) 103 | main(get_parser().parse_args()) 104 | -------------------------------------------------------------------------------- /experiments/tables/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rjagerman/sigir2020/5218fc2c4acd96208caad9a64afc8d06aeafca55/experiments/tables/__init__.py -------------------------------------------------------------------------------- /experiments/tables/batch_sizes.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import os 4 | from collections import defaultdict 5 | from copy import deepcopy 6 | from time import sleep 7 | from argparse import ArgumentParser 8 | from argparse import FileType 9 | from scipy.stats import ttest_ind_from_stats 10 | 11 | import numpy as np 12 | 13 | 14 | def get_parser(): 15 | """Gets the parser to create arguments for `main`.""" 16 | parser = ArgumentParser() 17 | parser.add_argument("--json_files", type=FileType("rt"), nargs="+") 18 | parser.add_argument("--skylines", type=FileType("rt"), nargs="+") 19 | parser.add_argument("--out", type=FileType("wb"), required=False, default=None) 20 | parser.add_argument("--dataset", type=str, required=False, default="vali") 21 | parser.add_argument("--model", type=str, required=False, default="avgmodel") 22 | parser.add_argument("--metric", type=str, default="ndcg@10") 23 | return parser 24 | 25 | 26 | def compress_list_of_arrays(list_of_arrays): 27 | min_shape = np.min([np.array(a).shape for a in list_of_arrays]) 28 | mean = np.mean([a[:min_shape] for a in list_of_arrays], axis=0) 29 | std = np.std([a[:min_shape] for a in list_of_arrays], axis=0) 30 | n = len(list_of_arrays) 31 | return mean, std, n 32 | 33 | 34 | def filter_arg(dictionary, key, value): 35 | return {k: v for k, v in dictionary.items() if v["args"][key] == value} 36 | 37 | 38 | def main(args): 39 | 40 | def create_table(args): 41 | data = {} 42 | skylines = {} 43 | for json_file in args.json_files: 44 | data[os.path.basename(json_file.name)] = json.load(json_file) 45 | json_file.seek(0) 46 | 47 | for json_file in args.skylines: 48 | dataset = "istella" if "istella" in json_file.name else "yahoo" 49 | skylines[dataset] = json.load(json_file) 50 | json_file.seek(0) 51 | 52 | # Compute error bars for identical runs with multiple seeds 53 | max_y = defaultdict(float) 54 | avg_runs = {} 55 | for name, results in data.items(): 56 | if args.dataset in results and args.model in results[args.dataset]: 57 | dataset = "istella" if "istella" in results["args"]["train_data"] else "yahoo" 58 | x = results[args.dataset][args.model]["iteration"] 59 | y = results[args.dataset][args.model][args.metric] 60 | for e in y: 61 | max_y[dataset] = max(e, max_y[dataset]) 62 | a = deepcopy(results['args']) 63 | del a["seed"] 64 | del a["output"] 65 | key = tuple(sorted(a.items())) 66 | if key not in avg_runs: 67 | avg_runs[key] = { 68 | 'name': name, 69 | 'results': {args.dataset: {args.model: {args.metric: [], "iteration": []}}} 70 | } 71 | avg_runs[key]['results'][args.dataset][args.model][args.metric].append(y) 72 | avg_runs[key]['results'][args.dataset][args.model]["iteration"].append(x) 73 | avg_runs[key]['args'] = results['args'] 74 | data = {} 75 | for run in avg_runs.values(): 76 | dataset = "istella" if "istella" in run["args"]["train_data"] else "yahoo" 77 | regret = np.mean(skylines[dataset][args.metric] - np.array(run['results'][args.dataset][args.model][args.metric]), axis=1) 78 | ys_mean, ys_std, ys_n = np.mean(regret), np.std(regret), regret.shape[0] 79 | method = run["args"]["ips_strategy"] 80 | batch_size = run["args"]["batch_size"] 81 | if method not in data: 82 | data[method] = {} 83 | if dataset not in data[method]: 84 | data[method][dataset] = {} 85 | 86 | data[method][dataset][batch_size] ={ 87 | "mean": ys_mean, 88 | "std": ys_std, 89 | "n": ys_n 90 | } 91 | 92 | methods = { 93 | "none": r"\BiasedSGD{}", 94 | "weight": r"\IPSSGD{}", 95 | "sample": r"\OurMethod{}" 96 | } 97 | datasets = { 98 | "yahoo": r"\Yahoo{}", 99 | "istella": r"\Istella{}" 100 | } 101 | # print(r"\begin{tabular}{l@{\hspace{6mm}}rrr@{\hspace{6mm}}rrr}") 102 | # print(r"\toprule") 103 | # print(r" & \multicolumn{3}{c}{\Yahoo{}} & \multicolumn{3}{c}{\Istella{}} \\") 104 | # print(r" & 10 & 20 & 50 & 10 & 20 & 50 \\") 105 | # print(r"\midrule") 106 | # for method in ["none", "weight", "sample"]: 107 | # result_row = [] 108 | # print(f"{methods[method]}", end = " & ") 109 | # for dataset in ["yahoo", "istella"]: 110 | # for batch_size in [10, 20, 50]: 111 | # statsig = "" 112 | # if method != "weight": 113 | # d1 = data[method][dataset][batch_size] 114 | # d2 = data["weight"][dataset][batch_size] 115 | # res = ttest_ind_from_stats(d1["mean"], d1["std"], d1["n"], d2["mean"], d2["std"], d2["n"]) 116 | # if res.pvalue < 0.01: 117 | # statsig = r"\rlap{$^{\triangledown}$}" if res.statistic < 0.0 else r"\rlap{$^{\triangle}$}" 118 | # result_row.append(f"{100.0 * data[method][dataset][batch_size]['mean']:.2f}{statsig}") 119 | # print(" & ".join(result_row) + r" \\") 120 | # print(r"\bottomrule") 121 | # print(r"\end{tabular}") 122 | print(r"%!TEX root = ../sigir2020.tex") 123 | print(r"\begin{tabular}{l@{\hspace{6mm}}rrr}") 124 | print(r"\toprule") 125 | print(r"Batch size: & 10 & 20 & 50 \\") 126 | for dataset in ["yahoo", "istella"]: 127 | print(r"\midrule") 128 | print(f"{datasets[dataset]} \\\\") 129 | for method in ["none", "weight", "sample"]: 130 | result_row = [] 131 | print(f"\quad{methods[method]}", end = " & ") 132 | for batch_size in [10, 20, 50]: 133 | statsig = "" 134 | if method != "weight": 135 | d1 = data[method][dataset][batch_size] 136 | d2 = data["weight"][dataset][batch_size] 137 | res = ttest_ind_from_stats(d1["mean"], d1["std"], d1["n"], d2["mean"], d2["std"], d2["n"]) 138 | if res.pvalue < 0.01: 139 | statsig = r"\rlap{$^{\triangledown}$}" if res.statistic < 0.0 else r"\rlap{$^{\triangle}$}" 140 | result_row.append(f"{100.0 * data[method][dataset][batch_size]['mean']:.2f}{statsig}") 141 | print(" & ".join(result_row) + r" \\") 142 | print(r"\bottomrule") 143 | print(r"\end{tabular}") 144 | 145 | create_table(args) 146 | 147 | 148 | if __name__ == "__main__": 149 | logging.basicConfig( 150 | format="[%(asctime)s, %(levelname)s, %(module)s, %(threadName)s] %(message)s", 151 | level=logging.INFO) 152 | main(get_parser().parse_args()) 153 | -------------------------------------------------------------------------------- /experiments/tables/distribution.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import os 4 | from argparse import ArgumentParser 5 | from argparse import FileType 6 | from scipy.stats import ttest_ind_from_stats 7 | 8 | import numpy as np 9 | 10 | from experiments.dataset import load_ranking_dataset 11 | from experiments.click_log import clicklog_dataset 12 | 13 | 14 | def get_parser(): 15 | """Gets the parser to create arguments for `main`.""" 16 | parser = ArgumentParser() 17 | parser.add_argument("--istella_click_logs", type=FileType("rt"), nargs="+") 18 | parser.add_argument("--istella_dataset", type=FileType("rt")) 19 | parser.add_argument("--yahoo_click_logs", type=FileType("rt"), nargs="+") 20 | parser.add_argument("--yahoo_dataset", type=FileType("rt")) 21 | return parser 22 | 23 | 24 | def load_propensity_stats(dataset, click_logs): 25 | r = load_ranking_dataset(dataset.name) 26 | clicklogs = [] 27 | for c in click_logs: 28 | clicklogs.append(clicklog_dataset(r, c.name)) 29 | 30 | out = {} 31 | for c, clicklog in zip(click_logs, clicklogs): 32 | out[c.name] = { 33 | 'max': np.max(1.0 / clicklog.propensities), 34 | 'mean': np.mean(1.0 / clicklog.propensities) 35 | } 36 | return out 37 | 38 | 39 | def main(args): 40 | # Load propensity distribution 41 | datasets = { 42 | "yahoo": load_propensity_stats(args.yahoo_dataset, args.yahoo_click_logs), 43 | "istella": load_propensity_stats(args.istella_dataset, args.istella_click_logs) 44 | } 45 | 46 | # Print 47 | latex_metrics = { 48 | "mean": r"$\bar{M}$", 49 | "max": r"$M$" 50 | } 51 | latex_datasets = { 52 | "yahoo": r"\Yahoo{}", 53 | "istella": r"\Istella{}" 54 | } 55 | print(r"%!TEX root = ../sigir2020.tex") 56 | print(r"\begin{tabular}{l@{\hspace{6mm}}rrrrr}") 57 | print(r"\toprule") 58 | print(r"Position bias ($\gamma$): & 0.5 & 0.75 & 1.0 & 1.25 & 1.5 \\") 59 | for dataset in datasets.keys(): 60 | print(r"\midrule") 61 | for metric in ["mean", "max"]: 62 | print(f"{latex_datasets[dataset]}: {latex_metrics[metric]} ", end='') 63 | # row dataset+metric 64 | for eta in ["0.5", "0.75", "1.0", "1.25", "1.5"]: 65 | # col eta 66 | d = list(filter(lambda d: ("_eta_" + eta) in d, datasets[dataset].keys()))[0] 67 | print(f"& {datasets[dataset][d][metric]:.2f}", end='') 68 | print(r" \\") 69 | print(r"\bottomrule") 70 | print(r"\end{tabular}") 71 | 72 | if __name__ == "__main__": 73 | main(get_parser().parse_args()) 74 | -------------------------------------------------------------------------------- /experiments/tables/etas.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import os 4 | from collections import defaultdict 5 | from copy import deepcopy 6 | from time import sleep 7 | from argparse import ArgumentParser 8 | from argparse import FileType 9 | from scipy.stats import ttest_ind_from_stats 10 | 11 | import numpy as np 12 | 13 | 14 | def get_parser(): 15 | """Gets the parser to create arguments for `main`.""" 16 | parser = ArgumentParser() 17 | parser.add_argument("--json_files", type=FileType("rt"), nargs="+") 18 | parser.add_argument("--skylines", type=FileType("rt"), nargs="+") 19 | parser.add_argument("--out", type=FileType("wb"), required=False, default=None) 20 | parser.add_argument("--dataset", type=str, required=False, default="vali") 21 | parser.add_argument("--model", type=str, required=False, default="avgmodel") 22 | parser.add_argument("--metric", type=str, default="ndcg@10") 23 | return parser 24 | 25 | 26 | def compress_list_of_arrays(list_of_arrays): 27 | min_shape = np.min([np.array(a).shape for a in list_of_arrays]) 28 | mean = np.mean([a[:min_shape] for a in list_of_arrays], axis=0) 29 | std = np.std([a[:min_shape] for a in list_of_arrays], axis=0) 30 | n = len(list_of_arrays) 31 | return mean, std, n 32 | 33 | 34 | def filter_arg(dictionary, key, value): 35 | return {k: v for k, v in dictionary.items() if v["args"][key] == value} 36 | 37 | 38 | def main(args): 39 | 40 | def create_table(args): 41 | data = {} 42 | skylines = {} 43 | for json_file in args.json_files: 44 | data[os.path.basename(json_file.name)] = json.load(json_file) 45 | json_file.seek(0) 46 | 47 | for json_file in args.skylines: 48 | dataset = "istella" if "istella" in json_file.name else "yahoo" 49 | skylines[dataset] = json.load(json_file) 50 | json_file.seek(0) 51 | 52 | # Compute error bars for identical runs with multiple seeds 53 | max_y = defaultdict(float) 54 | avg_runs = {} 55 | for name, results in data.items(): 56 | if args.dataset in results and args.model in results[args.dataset]: 57 | dataset = "istella" if "istella" in results["args"]["train_data"] else "yahoo" 58 | x = results[args.dataset][args.model]["iteration"] 59 | y = results[args.dataset][args.model][args.metric] 60 | for e in y: 61 | max_y[dataset] = max(e, max_y[dataset]) 62 | a = deepcopy(results['args']) 63 | del a["seed"] 64 | del a["output"] 65 | key = tuple(sorted(a.items())) 66 | if key not in avg_runs: 67 | avg_runs[key] = { 68 | 'name': name, 69 | 'results': {args.dataset: {args.model: {args.metric: [], "iteration": []}}} 70 | } 71 | avg_runs[key]['results'][args.dataset][args.model][args.metric].append(y) 72 | avg_runs[key]['results'][args.dataset][args.model]["iteration"].append(x) 73 | avg_runs[key]['args'] = results['args'] 74 | data = {} 75 | for run in avg_runs.values(): 76 | dataset = "istella" if "istella" in run["args"]["train_data"] else "yahoo" 77 | regret = np.mean(skylines[dataset][args.metric] - np.array(run['results'][args.dataset][args.model][args.metric]), axis=1) 78 | ys_mean, ys_std, ys_n = np.mean(regret), np.std(regret), regret.shape[0] 79 | method = run["args"]["ips_strategy"] 80 | eta = None 81 | for t_eta in ["0.5", "0.75", "1.0", "1.25", "1.5"]: 82 | if f"{t_eta}" in run["args"]["click_log"]: 83 | eta = t_eta 84 | 85 | if method not in data: 86 | data[method] = {} 87 | if dataset not in data[method]: 88 | data[method][dataset] = {} 89 | 90 | data[method][dataset][eta] ={ 91 | "mean": ys_mean, 92 | "std": ys_std, 93 | "n": ys_n 94 | } 95 | 96 | methods = { 97 | "none": r"\BiasedSGD{}", 98 | "weight": r"\IPSSGD{}", 99 | "sample": r"\OurMethod{}" 100 | } 101 | datasets = { 102 | "yahoo": r"\Yahoo{}", 103 | "istella": r"\Istella{}" 104 | } 105 | print(r"%!TEX root = ../sigir2020.tex") 106 | print(r"\begin{tabular}{l@{\hspace{6mm}}rrrrr}") 107 | print(r"\toprule") 108 | print(r"Position bias ($\gamma$): & 0.5 & 0.75 & 1.0 & 1.25 & 1.5 \\") 109 | for dataset in ["yahoo", "istella"]: 110 | print(r"\midrule") 111 | print(f"{datasets[dataset]} \\\\") 112 | for method in ["none", "weight", "sample"]: 113 | result_row = [] 114 | print(f"\quad{methods[method]}", end = " & ") 115 | for eta in ["0.5", "0.75", "1.0", "1.25", "1.5"]: 116 | statsig = "" 117 | if method != "weight": 118 | d1 = data[method][dataset][eta] 119 | d2 = data["weight"][dataset][eta] 120 | res = ttest_ind_from_stats(d1["mean"], d1["std"], d1["n"], d2["mean"], d2["std"], d2["n"]) 121 | if res.pvalue < 0.01: 122 | statsig = r"\rlap{$^{\triangledown}$}" if res.statistic < 0.0 else r"\rlap{$^{\triangle}$}" 123 | result_row.append(f"{100.0 * data[method][dataset][eta]['mean']:.2f}{statsig}") 124 | print(" & ".join(result_row) + r" \\") 125 | print(r"\bottomrule") 126 | print(r"\end{tabular}") 127 | 128 | create_table(args) 129 | 130 | 131 | if __name__ == "__main__": 132 | logging.basicConfig( 133 | format="[%(asctime)s, %(levelname)s, %(module)s, %(threadName)s] %(message)s", 134 | level=logging.INFO) 135 | main(get_parser().parse_args()) 136 | -------------------------------------------------------------------------------- /experiments/tables/optimizers.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import os 4 | from collections import defaultdict 5 | from copy import deepcopy 6 | from time import sleep 7 | from argparse import ArgumentParser 8 | from argparse import FileType 9 | from scipy.stats import ttest_ind_from_stats 10 | 11 | import numpy as np 12 | 13 | 14 | def get_parser(): 15 | """Gets the parser to create arguments for `main`.""" 16 | parser = ArgumentParser() 17 | parser.add_argument("--json_files", type=FileType("rt"), nargs="+") 18 | parser.add_argument("--skylines", type=FileType("rt"), nargs="+") 19 | parser.add_argument("--out", type=FileType("wb"), required=False, default=None) 20 | parser.add_argument("--dataset", type=str, required=False, default="vali") 21 | parser.add_argument("--model", type=str, required=False, default="avgmodel") 22 | parser.add_argument("--metric", type=str, default="ndcg@10") 23 | return parser 24 | 25 | 26 | def compress_list_of_arrays(list_of_arrays): 27 | min_shape = np.min([np.array(a).shape for a in list_of_arrays]) 28 | mean = np.mean([a[:min_shape] for a in list_of_arrays], axis=0) 29 | std = np.std([a[:min_shape] for a in list_of_arrays], axis=0) 30 | n = len(list_of_arrays) 31 | return mean, std, n 32 | 33 | 34 | def filter_arg(dictionary, key, value): 35 | return {k: v for k, v in dictionary.items() if v["args"][key] == value} 36 | 37 | 38 | def main(args): 39 | 40 | def create_table(args): 41 | data = {} 42 | skylines = {} 43 | for json_file in args.json_files: 44 | data[os.path.basename(json_file.name)] = json.load(json_file) 45 | json_file.seek(0) 46 | 47 | for json_file in args.skylines: 48 | dataset = "istella" if "istella" in json_file.name else "yahoo" 49 | skylines[dataset] = json.load(json_file) 50 | json_file.seek(0) 51 | 52 | # Compute error bars for identical runs with multiple seeds 53 | max_y = defaultdict(float) 54 | avg_runs = {} 55 | for name, results in data.items(): 56 | if args.dataset in results and args.model in results[args.dataset]: 57 | dataset = "istella" if "istella" in results["args"]["train_data"] else "yahoo" 58 | x = results[args.dataset][args.model]["iteration"] 59 | y = results[args.dataset][args.model][args.metric] 60 | for e in y: 61 | max_y[dataset] = max(e, max_y[dataset]) 62 | a = deepcopy(results['args']) 63 | del a["seed"] 64 | del a["output"] 65 | key = tuple(sorted(a.items())) 66 | if key not in avg_runs: 67 | avg_runs[key] = { 68 | 'name': name, 69 | 'results': {args.dataset: {args.model: {args.metric: [], "iteration": []}}} 70 | } 71 | avg_runs[key]['results'][args.dataset][args.model][args.metric].append(y) 72 | avg_runs[key]['results'][args.dataset][args.model]["iteration"].append(x) 73 | avg_runs[key]['args'] = results['args'] 74 | data = {} 75 | for run in avg_runs.values(): 76 | dataset = "istella" if "istella" in run["args"]["train_data"] else "yahoo" 77 | regret = np.mean(skylines[dataset][args.metric] - np.array(run['results'][args.dataset][args.model][args.metric]), axis=1) 78 | ys_mean, ys_std, ys_n = np.mean(regret), np.std(regret), regret.shape[0] 79 | method = run["args"]["ips_strategy"] 80 | optimizer = run["args"]["optimizer"] 81 | if method not in data: 82 | data[method] = {} 83 | if dataset not in data[method]: 84 | data[method][dataset] = {} 85 | 86 | data[method][dataset][optimizer] ={ 87 | "mean": ys_mean, 88 | "std": ys_std, 89 | "n": ys_n 90 | } 91 | 92 | methods = { 93 | "none": r"\BiasedSGD{}", 94 | "weight": r"\IPSSGD{}", 95 | "sample": r"\OurMethod{}" 96 | } 97 | datasets = { 98 | "yahoo": r"\Yahoo{}", 99 | "istella": r"\Istella{}" 100 | } 101 | print(r"%!TEX root = ../sigir2020.tex") 102 | print(r"\begin{tabular}{l@{\hspace{6mm}}rrr}") 103 | print(r"\toprule") 104 | print(r"Optimizer: & SGD & \textsc{Adam} & \textsc{Adagrad} \\") 105 | for dataset in ["yahoo", "istella"]: 106 | print(r"\midrule") 107 | print(f"{datasets[dataset]} \\\\") 108 | for method in ["none", "weight", "sample"]: 109 | result_row = [] 110 | print(f"\quad{methods[method]}", end = " & ") 111 | for optimizer in ["sgd", "adam", "adagrad"]: 112 | statsig = "" 113 | if method != "weight": 114 | d1 = data[method][dataset][optimizer] 115 | d2 = data["weight"][dataset][optimizer] 116 | res = ttest_ind_from_stats(d1["mean"], d1["std"], d1["n"], d2["mean"], d2["std"], d2["n"]) 117 | if res.pvalue < 0.01: 118 | statsig = r"\rlap{$^{\triangledown}$}" if res.statistic < 0.0 else r"\rlap{$^{\triangle}$}" 119 | result_row.append(f"{100.0 * data[method][dataset][optimizer]['mean']:.2f}{statsig}") 120 | print(" & ".join(result_row) + r" \\") 121 | print(r"\bottomrule") 122 | print(r"\end{tabular}") 123 | 124 | create_table(args) 125 | 126 | 127 | if __name__ == "__main__": 128 | logging.basicConfig( 129 | format="[%(asctime)s, %(levelname)s, %(module)s, %(threadName)s] %(message)s", 130 | level=logging.INFO) 131 | main(get_parser().parse_args()) 132 | -------------------------------------------------------------------------------- /experiments/train.py: -------------------------------------------------------------------------------- 1 | """Counterfactual Learning to Rank training procedure.""" 2 | import logging 3 | import json 4 | from argparse import ArgumentParser 5 | 6 | import torch 7 | import random 8 | import time 9 | from torchcontrib.optim import SWA 10 | from ignite.engine import Engine 11 | from ignite.engine import Events 12 | from pytorchltr.loss.pairwise import AdditivePairwiseLoss 13 | from pytorchltr.dataset.svmrank import create_svmranking_collate_fn 14 | 15 | from experiments.evaluate import evaluate 16 | from experiments.evaluate import NDCG 17 | from experiments.evaluate import ARP 18 | from experiments.click_log import clicklog_dataset 19 | from experiments.click_log import create_clicklog_collate_fn 20 | from experiments.dataset import load_click_dataset 21 | from experiments.dataset import load_ranking_dataset 22 | from experiments.models import LinearScorer 23 | from experiments.models import DeepScorer 24 | from experiments.util import get_torch_device 25 | from experiments.util import JsonLogger 26 | from experiments.util import every_n_iteration 27 | 28 | 29 | LOGGER = logging.getLogger(__name__) 30 | 31 | 32 | def get_parser(): 33 | """Gets the parser to create arguments for `main`.""" 34 | parser = ArgumentParser() 35 | parser.add_argument("--train_data", type=str, required=True) 36 | parser.add_argument("--click_log", type=str, required=True) 37 | parser.add_argument("--output", type=str, default=None) 38 | parser.add_argument("--vali_data", type=str, default=None) 39 | parser.add_argument("--test_data", type=str, default=None) 40 | parser.add_argument("--optimizer", default="sgd", 41 | choices=["sgd", "adam", "adagrad"]) 42 | parser.add_argument("--ips_strategy", default="none", 43 | choices=["none", "weight", "sample"]) 44 | parser.add_argument("--ips_clip", type=float, default=None) 45 | parser.add_argument("--objective", default="rank", 46 | choices=["rank", "normrank", "dcg"]) 47 | parser.add_argument("--seed", type=int, default=42) 48 | parser.add_argument("--enable_cuda", action="store_true", default=False) 49 | parser.add_argument("--disable_swa", action="store_true", default=False) 50 | parser.add_argument("--batch_size", type=int, default=50) 51 | parser.add_argument("--eval_batch_size", type=int, default=500) 52 | parser.add_argument("--n_clicks", type=int, default=None) 53 | parser.add_argument("--epochs", type=int, default=50) 54 | parser.add_argument("--lr", type=float, default=0.001) 55 | parser.add_argument("--momentum", type=float, default=0.0) 56 | parser.add_argument("--layers", type=int, nargs="+", default=None) 57 | parser.add_argument("--max_list_size", type=int, default=None) 58 | parser.add_argument("--log_every", type=int, default=None) 59 | parser.add_argument("--eval_every", type=int, default=None) 60 | parser.add_argument("--delay_start", action="store_true", default=False) 61 | return parser 62 | 63 | 64 | def create_pairwise_loss(objective, ips_strategy, bcf=1.0): 65 | """Creates a pairwise loss objective 66 | 67 | Arguments: 68 | objective: A string indicating the AdditivePairwiseLoss objective. 69 | ips_strategy: A string indicating the IPS strategy to use. 70 | bcf: Bias correction factor for sample-based learning. 71 | 72 | Returns: 73 | A loss function that takes 4 arguments: (scores, ys, n, p). 74 | """ 75 | pairwise_loss_fn = AdditivePairwiseLoss(objective) 76 | if ips_strategy == "weight": 77 | def _loss_fn(scores, ys, n, p): 78 | return pairwise_loss_fn(scores, ys, n) / p 79 | else: 80 | def _loss_fn(scores, ys, n, p): 81 | return pairwise_loss_fn(scores, ys, n) * bcf 82 | return _loss_fn 83 | 84 | 85 | def create_cfltr_trainer(optimizer, loss_fn, model, device, metrics={}): 86 | """Creates a training `Engine` for counterfactual LTR. 87 | 88 | Arguments: 89 | optimizer: The optimizer to use. 90 | loss_fn: The loss function to call. 91 | model: The model to predict scores with. 92 | device: The device to move batches to. 93 | metrics: Metrics to compute per step. 94 | 95 | Returns: 96 | An `Engine` used for training. 97 | """ 98 | def _update_fn(engine, batch): 99 | model.train() 100 | xs, clicks, n, relevance, p = ( 101 | batch["features"], batch["clicks"], batch["n"], batch["relevance"], 102 | batch["propensity"]) 103 | xs, clicks, n, relevance, p = ( 104 | xs.to(device), clicks.to(device), n.to(device), 105 | relevance.to(device), p.to(device)) 106 | model.train() 107 | optimizer.zero_grad() 108 | scores = model(xs) 109 | loss = torch.mean(loss_fn(scores, clicks, n, p)) 110 | loss.backward() 111 | optimizer.step() 112 | return scores, relevance, n 113 | 114 | engine = Engine(_update_fn) 115 | for name, metric in metrics.items(): 116 | metric.attach(engine, name) 117 | 118 | return engine 119 | 120 | 121 | def create_ltr_evaluator(model, device, metrics): 122 | """Creates an evaluation `Engine` for LTR. 123 | 124 | Arguments: 125 | model: The model to predict scores with. 126 | device: The device to move batches to. 127 | metrics: Metrics to compute per step. 128 | 129 | Returns: 130 | An `Engine` used for evaluation. 131 | """ 132 | def _inference_fn(engine, batch): 133 | model.eval() 134 | with torch.no_grad(): 135 | xs, relevance, n = batch["features"], batch["relevance"], batch["n"] 136 | xs, relevance, n = xs.to(device), relevance.to(device), n.to(device) 137 | scores = model(xs) 138 | return scores, relevance, n 139 | 140 | engine = Engine(_inference_fn) 141 | for name, metric in metrics.items(): 142 | metric.attach(engine, name) 143 | 144 | return engine 145 | 146 | 147 | def main(args): 148 | """Trains the baseline ranker using given arguments.""" 149 | 150 | if args.delay_start: 151 | seconds = random.randint(0, 120) 152 | LOGGER.info("Delaying start of execution by %d seconds", seconds) 153 | time.sleep(seconds) 154 | 155 | LOGGER.info("Setting device and seeding RNG") 156 | device = get_torch_device(args.enable_cuda) 157 | torch.manual_seed(args.seed) 158 | 159 | LOGGER.info("Setting evaluation and logging frequency") 160 | if args.eval_every is None and args.log_every is None: 161 | args.eval_every = 10_000 // args.batch_size 162 | args.log_every = 10_000 // args.batch_size 163 | elif args.eval_every is None: 164 | args.eval_every = args.log_every 165 | elif args.log_every is None: 166 | args.log_every = args.eval_every 167 | 168 | LOGGER.info("Loading click log for training") 169 | train_data_loader, input_dimensionality, bcf = load_click_dataset( 170 | args.train_data, args.click_log, args.ips_strategy, args.ips_clip, 171 | args.n_clicks, args.batch_size, args.max_list_size) 172 | 173 | eval_data_loaders = {} 174 | if args.vali_data is not None: 175 | LOGGER.info("Loading vali data") 176 | eval_data_loaders["vali"] = torch.utils.data.DataLoader( 177 | load_ranking_dataset(args.vali_data, normalize=True, 178 | filter_queries=True), 179 | shuffle=False, batch_size=args.eval_batch_size, 180 | collate_fn=create_svmranking_collate_fn()) 181 | 182 | if args.test_data is not None: 183 | LOGGER.info("Loading test data") 184 | eval_data_loaders["test"] = torch.utils.data.DataLoader( 185 | load_ranking_dataset(args.test_data, normalize=True, 186 | filter_queries=True), 187 | shuffle=False, batch_size=args.eval_batch_size, 188 | collate_fn=create_svmranking_collate_fn()) 189 | 190 | if args.layers is None: 191 | LOGGER.info("Creating linear model") 192 | model = LinearScorer(input_dimensionality) 193 | else: 194 | LOGGER.info("Creating deep model") 195 | model = DeepScorer(input_dimensionality, args.layers) 196 | model = model.to(device=device) 197 | 198 | LOGGER.info("Creating loss function (bcf=%f)", bcf) 199 | loss_fn = create_pairwise_loss(args.objective, args.ips_strategy, bcf) 200 | args.bias_correction_factor = bcf 201 | 202 | LOGGER.info("Creating optimizer") 203 | optimizer = { 204 | "sgd": lambda: torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum), 205 | "adam": lambda: torch.optim.Adam(model.parameters(), args.lr), 206 | "adagrad": lambda: torch.optim.Adagrad(model.parameters(), args.lr) 207 | }[args.optimizer]() 208 | 209 | if not args.disable_swa: 210 | optimizer = SWA(optimizer, swa_start=0, swa_freq=1, swa_lr=args.lr) 211 | 212 | if args.output is not None: 213 | LOGGER.info("Creating result logger") 214 | json_logger = JsonLogger(args.output, indent=1, args=args) 215 | 216 | LOGGER.info("Setup training engine") 217 | trainer = create_cfltr_trainer( 218 | optimizer, loss_fn, model, device) 219 | 220 | for eval_name, eval_data_loader in eval_data_loaders.items(): 221 | LOGGER.info("Setup %s engine", eval_name) 222 | metrics = {"ndcg@10": NDCG(k=10), "arp": ARP()} 223 | evaluator = create_ltr_evaluator(model, device, metrics) 224 | if not args.disable_swa: 225 | swa_evaluator = create_ltr_evaluator(model, device, metrics) 226 | 227 | # Run evaluation 228 | def run_evaluation(trainer): 229 | if not args.disable_swa: 230 | optim_state = optimizer.state_dict() 231 | if "swa_state" in optim_state and len(optim_state["swa_state"]) > 0: 232 | optimizer.swap_swa_sgd() 233 | swa_evaluator.run(eval_data_loader) 234 | if "swa_state" in optim_state and len(optim_state["swa_state"]) > 0: 235 | optimizer.swap_swa_sgd() 236 | evaluator.run(eval_data_loader) 237 | 238 | trainer.add_event_handler(Events.STARTED, run_evaluation) 239 | trainer.add_event_handler( 240 | Events.ITERATION_COMPLETED, 241 | every_n_iteration(trainer, args.eval_every, run_evaluation)) 242 | 243 | # Write results to file when evaluation finishes. 244 | if args.output is not None: 245 | if not args.disable_swa: 246 | @swa_evaluator.on(Events.COMPLETED) 247 | def log_results(evaluator): 248 | json_logger.append_all( 249 | "%s/avgmodel" % eval_name, trainer.state.iteration, 250 | swa_evaluator.state.metrics) 251 | json_logger.write_to_disk() 252 | 253 | @evaluator.on(Events.COMPLETED) 254 | def log_results(evaluator): 255 | json_logger.append_all( 256 | "%s/latestmodel" % eval_name, trainer.state.iteration, 257 | evaluator.state.metrics) 258 | json_logger.write_to_disk() 259 | 260 | LOGGER.info("Run train loop") 261 | trainer.run(train_data_loader, args.epochs) 262 | 263 | LOGGER.info("Writing final results to disk.") 264 | json_logger.write_to_disk() 265 | 266 | LOGGER.info("Done") 267 | 268 | 269 | if __name__ == "__main__": 270 | logging.basicConfig( 271 | format="[%(asctime)s, %(levelname)s, %(module)s] %(message)s", 272 | level=logging.INFO) 273 | main(get_parser().parse_args()) 274 | -------------------------------------------------------------------------------- /experiments/util.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import shutil 5 | from tempfile import NamedTemporaryFile 6 | from time import time 7 | from time import sleep 8 | 9 | import torch 10 | from pytorchltr.dataset.svmrank import svmranking_dataset as _load 11 | from joblib.memory import Memory as _Memory 12 | from ignite.engine import Events 13 | 14 | load_dataset = _Memory(".cache", compress=6).cache(_load) 15 | 16 | 17 | LOGGER = logging.getLogger(__name__) 18 | 19 | 20 | def get_torch_device(enable_cuda): 21 | """Gets the torch device to use. 22 | 23 | Arguments: 24 | enable_cuda: Boolean to indicate whether to use CUDA. 25 | 26 | Returns: 27 | The torch device. 28 | """ 29 | if enable_cuda and torch.cuda.is_available(): 30 | out = torch.device("cuda") 31 | else: 32 | out = torch.device("cpu") 33 | LOGGER.info("Using device %s", out) 34 | return out 35 | 36 | 37 | def every_n_iteration(engine, n, callback): 38 | def _callback_fn(_engine): 39 | if engine.state.iteration % n == 0: 40 | return callback(_engine) 41 | return _callback_fn 42 | 43 | 44 | class JsonLogger: 45 | def __init__(self, output_file, indent=None, args=None, nr_of_tries=3, timeout=10.0): 46 | self._output_file = output_file 47 | self._output = {} 48 | self._indent = indent 49 | if args is not None: 50 | self._output["args"] = vars(args) 51 | self._nr_of_tries = nr_of_tries 52 | self._timeout = timeout 53 | 54 | def append(self, key, value): 55 | next_dict = self._output 56 | keys = key.split("/") 57 | for k in keys[:-1]: 58 | if k not in next_dict: 59 | next_dict[k] = {} 60 | next_dict = next_dict[k] 61 | if keys[-1] not in next_dict: 62 | next_dict[keys[-1]] = [] 63 | next_dict[keys[-1]].append(value) 64 | 65 | def write_to_disk(self): 66 | success = False 67 | tries = 0 68 | last_error = None 69 | while tries < self._nr_of_tries and not success: 70 | tries += 1 71 | try: 72 | # Performs an atomic write to file. 73 | dirpath, filename = os.path.split(self._output_file) 74 | tempname = "" 75 | with NamedTemporaryFile(dir=dirpath, prefix=filename, mode='wt', 76 | suffix='.tmp', delete=False) as f: 77 | json.dump(self._output, f, indent=self._indent) 78 | f.flush() 79 | os.fsync(f) 80 | f.close() 81 | shutil.move(f.name, self._output_file) 82 | success = True 83 | except IOError as e: 84 | LOGGER.warn("IOError when writing JSON log: %s", e) 85 | LOGGER.warn("Retrying attempt %d/%d in %d seconds", tries, self._nr_of_tries, self._timeout) 86 | last_error = e 87 | sleep(self._timeout) 88 | except Exception as e: 89 | LOGGER.error("Exception when writing JSON log: %s", e) 90 | raise e 91 | if not success and last_error is not None: 92 | LOGGER.error("IOError when writing JSON log (exhausted %d attempts): %s", self._nr_of_tries, last_error) 93 | raise last_error 94 | 95 | def append_all(self, top_level_key, iteration, metrics): 96 | self.append("%s/iteration" % top_level_key, iteration) 97 | self.append("%s/time" % top_level_key, time()) 98 | for metric, value in metrics.items(): 99 | self.append("%s/%s" % (top_level_key, metric), value) 100 | -------------------------------------------------------------------------------- /makescripts/baselines.mk: -------------------------------------------------------------------------------- 1 | # Phony target for easier dependencies 2 | baselines: $(BUILD)/baselines/yahoo.pth $(BUILD)/baselines/istella.pth 3 | .PHONY: baselines 4 | 5 | # Baseline rankers trained on fractions of data 6 | $(BUILD)/baselines/yahoo.pth : | $(BUILD)/baselines/ 7 | python -m experiments.baseline --train_data $(YAHOO_DIR)/train.txt \ 8 | --output $@.tmp \ 9 | --optimizer sgd \ 10 | --lr 0.0001 \ 11 | --fraction 0.001 12 | mv $@.tmp $@ 13 | 14 | $(BUILD)/baselines/yahoo.json : $(BUILD)/baselines/yahoo.pth | $(BUILD)/baselines/ 15 | python -m experiments.eval_model --data $(YAHOO_DIR)/test.txt \ 16 | --output $@.tmp \ 17 | --model $(BUILD)/baselines/yahoo.pth 18 | mv $@.tmp $@ 19 | 20 | $(BUILD)/baselines/istella.pth : | $(BUILD)/baselines/ 21 | python -m experiments.baseline --train_data $(ISTELLA_DIR)/train.txt \ 22 | --output $@.tmp \ 23 | --optimizer sgd \ 24 | --lr 0.001 \ 25 | --fraction 0.001 26 | mv $@.tmp $@ 27 | 28 | $(BUILD)/baselines/istella.json : $(BUILD)/baselines/istella.pth | $(BUILD)/baselines/ 29 | python -m experiments.eval_model --data $(ISTELLA_DIR)/test.txt \ 30 | --output $@.tmp \ 31 | --model $(BUILD)/baselines/istella.pth 32 | mv $@.tmp $@ 33 | 34 | 35 | # Skyline rankers trained on full supervision data 36 | $(BUILD)/skylines/yahoo.pth : | $(BUILD)/skylines/ 37 | python -m experiments.baseline --train_data $(YAHOO_DIR)/train.txt \ 38 | --output $@.tmp \ 39 | --optimizer sgd \ 40 | --lr 1e-03 \ 41 | --fraction 1.0 \ 42 | --epochs 10 \ 43 | --batch_size 32 44 | mv $@.tmp $@ 45 | 46 | $(BUILD)/skylines/yahoo.json : $(BUILD)/skylines/yahoo.pth | $(BUILD)/skylines/ 47 | python -m experiments.eval_model --data $(YAHOO_DIR)/test.txt \ 48 | --output $@.tmp \ 49 | --model $(BUILD)/skylines/yahoo.pth 50 | mv $@.tmp $@ 51 | 52 | $(BUILD)/skylines/istella.pth : | $(BUILD)/skylines/ 53 | python -m experiments.baseline --train_data $(ISTELLA_DIR)/train.txt \ 54 | --output $@.tmp \ 55 | --optimizer sgd \ 56 | --lr 3e-04 \ 57 | --fraction 1.0 \ 58 | --epochs 10 \ 59 | --batch_size 32 60 | mv $@.tmp $@ 61 | 62 | $(BUILD)/skylines/istella.json : $(BUILD)/skylines/istella.pth | $(BUILD)/skylines/ 63 | python -m experiments.eval_model --data $(ISTELLA_DIR)/test.txt \ 64 | --output $@.tmp \ 65 | --model $(BUILD)/skylines/istella.pth 66 | mv $@.tmp $@ 67 | -------------------------------------------------------------------------------- /makescripts/clicklogs.mk: -------------------------------------------------------------------------------- 1 | # Phony targets for easier dependencies 2 | yahoo_clicklogs: $(BUILD)/clicklogs/yahoo_1m_perfect.clog 3 | yahoo_clicklogs: $(BUILD)/clicklogs/yahoo_1m_position_eta_0.0.clog 4 | yahoo_clicklogs: $(BUILD)/clicklogs/yahoo_1m_position_eta_0.5.clog 5 | yahoo_clicklogs: $(BUILD)/clicklogs/yahoo_1m_position_eta_0.75.clog 6 | yahoo_clicklogs: $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog 7 | yahoo_clicklogs: $(BUILD)/clicklogs/yahoo_1m_position_eta_1.25.clog 8 | yahoo_clicklogs: $(BUILD)/clicklogs/yahoo_1m_position_eta_1.5.clog 9 | yahoo_clicklogs: $(BUILD)/clicklogs/yahoo_1m_position_eta_2.0.clog 10 | yahoo_clicklogs: $(BUILD)/clicklogs/yahoo_1m_nearrandom_eta_1.0.clog 11 | istella_clicklogs: $(BUILD)/clicklogs/istella_1m_perfect.clog 12 | istella_clicklogs: $(BUILD)/clicklogs/istella_1m_position_eta_0.0.clog 13 | istella_clicklogs: $(BUILD)/clicklogs/istella_1m_position_eta_0.5.clog 14 | istella_clicklogs: $(BUILD)/clicklogs/istella_1m_position_eta_0.75.clog 15 | istella_clicklogs: $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog 16 | istella_clicklogs: $(BUILD)/clicklogs/istella_1m_position_eta_1.25.clog 17 | istella_clicklogs: $(BUILD)/clicklogs/istella_1m_position_eta_1.5.clog 18 | istella_clicklogs: $(BUILD)/clicklogs/istella_1m_position_eta_2.0.clog 19 | istella_clicklogs: $(BUILD)/clicklogs/istella_1m_nearrandom_eta_1.0.clog 20 | clicklogs: yahoo_clicklogs istella_clicklogs 21 | .PHONY: clicklogs yahoo_clicklogs istella_clicklogs 22 | 23 | # Click logs generated by baseline rankers 24 | $(BUILD)/clicklogs/yahoo_1m_perfect.clog : $(BUILD)/baselines/yahoo.pth | $(BUILD)/clicklogs/ 25 | python -m experiments.simulate_clicks --input_data $(YAHOO_DIR)/train.txt \ 26 | --ranker $(BUILD)/baselines/yahoo.pth \ 27 | --output_log $@.tmp \ 28 | --sessions 10_000_000 \ 29 | --max_clicks 1_000_000 \ 30 | --behavior perfect 31 | mv $@.tmp $@ 32 | 33 | $(BUILD)/clicklogs/yahoo_1m_position_eta_0.0.clog : $(BUILD)/baselines/yahoo.pth | $(BUILD)/clicklogs/ 34 | python -m experiments.simulate_clicks --input_data $(YAHOO_DIR)/train.txt \ 35 | --ranker $(BUILD)/baselines/yahoo.pth \ 36 | --output_log $@.tmp \ 37 | --sessions 10_000_000 \ 38 | --max_clicks 1_000_000 \ 39 | --behavior position \ 40 | --eta 0.0 41 | mv $@.tmp $@ 42 | 43 | $(BUILD)/clicklogs/yahoo_1m_position_eta_0.5.clog : $(BUILD)/baselines/yahoo.pth | $(BUILD)/clicklogs/ 44 | python -m experiments.simulate_clicks --input_data $(YAHOO_DIR)/train.txt \ 45 | --ranker $(BUILD)/baselines/yahoo.pth \ 46 | --output_log $@.tmp \ 47 | --sessions 10_000_000 \ 48 | --max_clicks 1_000_000 \ 49 | --behavior position \ 50 | --eta 0.5 51 | mv $@.tmp $@ 52 | 53 | $(BUILD)/clicklogs/yahoo_1m_position_eta_0.75.clog : $(BUILD)/baselines/yahoo.pth | $(BUILD)/clicklogs/ 54 | python -m experiments.simulate_clicks --input_data $(YAHOO_DIR)/train.txt \ 55 | --ranker $(BUILD)/baselines/yahoo.pth \ 56 | --output_log $@.tmp \ 57 | --sessions 10_000_000 \ 58 | --max_clicks 1_000_000 \ 59 | --behavior position \ 60 | --eta 0.75 61 | mv $@.tmp $@ 62 | 63 | $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog : $(BUILD)/baselines/yahoo.pth | $(BUILD)/clicklogs/ 64 | python -m experiments.simulate_clicks --input_data $(YAHOO_DIR)/train.txt \ 65 | --ranker $(BUILD)/baselines/yahoo.pth \ 66 | --output_log $@.tmp \ 67 | --sessions 10_000_000 \ 68 | --max_clicks 1_000_000 \ 69 | --behavior position \ 70 | --eta 1.0 71 | mv $@.tmp $@ 72 | 73 | $(BUILD)/clicklogs/yahoo_1m_position_eta_1.25.clog : $(BUILD)/baselines/yahoo.pth | $(BUILD)/clicklogs/ 74 | python -m experiments.simulate_clicks --input_data $(YAHOO_DIR)/train.txt \ 75 | --ranker $(BUILD)/baselines/yahoo.pth \ 76 | --output_log $@.tmp \ 77 | --sessions 10_000_000 \ 78 | --max_clicks 1_000_000 \ 79 | --behavior position \ 80 | --eta 1.25 81 | mv $@.tmp $@ 82 | 83 | $(BUILD)/clicklogs/yahoo_1m_position_eta_1.5.clog : $(BUILD)/baselines/yahoo.pth | $(BUILD)/clicklogs/ 84 | python -m experiments.simulate_clicks --input_data $(YAHOO_DIR)/train.txt \ 85 | --ranker $(BUILD)/baselines/yahoo.pth \ 86 | --output_log $@.tmp \ 87 | --sessions 10_000_000 \ 88 | --max_clicks 1_000_000 \ 89 | --behavior position \ 90 | --eta 1.5 91 | mv $@.tmp $@ 92 | 93 | $(BUILD)/clicklogs/yahoo_1m_position_eta_2.0.clog : $(BUILD)/baselines/yahoo.pth | $(BUILD)/clicklogs/ 94 | python -m experiments.simulate_clicks --input_data $(YAHOO_DIR)/train.txt \ 95 | --ranker $(BUILD)/baselines/yahoo.pth \ 96 | --output_log $@.tmp \ 97 | --sessions 10_000_000 \ 98 | --max_clicks 1_000_000 \ 99 | --behavior position \ 100 | --eta 2.0 101 | mv $@.tmp $@ 102 | 103 | $(BUILD)/clicklogs/yahoo_1m_nearrandom_eta_1.0.clog : $(BUILD)/baselines/yahoo.pth | $(BUILD)/clicklogs/ 104 | python -m experiments.simulate_clicks --input_data $(YAHOO_DIR)/train.txt \ 105 | --ranker $(BUILD)/baselines/yahoo.pth \ 106 | --output_log $@.tmp \ 107 | --sessions 10_000_000 \ 108 | --max_clicks 1_000_000 \ 109 | --behavior nearrandom \ 110 | --eta 1.0 111 | mv $@.tmp $@ 112 | 113 | $(BUILD)/clicklogs/istella_1m_perfect.clog : $(BUILD)/baselines/istella.pth | $(BUILD)/clicklogs/ 114 | python -m experiments.simulate_clicks --input_data $(ISTELLA_DIR)/train.txt \ 115 | --ranker $(BUILD)/baselines/istella.pth \ 116 | --output_log $@.tmp \ 117 | --sessions 10_000_000 \ 118 | --max_clicks 1_000_000 \ 119 | --behavior perfect 120 | mv $@.tmp $@ 121 | 122 | $(BUILD)/clicklogs/istella_1m_position_eta_0.0.clog : $(BUILD)/baselines/istella.pth | $(BUILD)/clicklogs/ 123 | python -m experiments.simulate_clicks --input_data $(ISTELLA_DIR)/train.txt \ 124 | --ranker $(BUILD)/baselines/istella.pth \ 125 | --output_log $@.tmp \ 126 | --sessions 10_000_000 \ 127 | --max_clicks 1_000_000 \ 128 | --behavior position \ 129 | --eta 0.0 130 | mv $@.tmp $@ 131 | 132 | $(BUILD)/clicklogs/istella_1m_position_eta_0.5.clog : $(BUILD)/baselines/istella.pth | $(BUILD)/clicklogs/ 133 | python -m experiments.simulate_clicks --input_data $(ISTELLA_DIR)/train.txt \ 134 | --ranker $(BUILD)/baselines/istella.pth \ 135 | --output_log $@.tmp \ 136 | --sessions 10_000_000 \ 137 | --max_clicks 1_000_000 \ 138 | --behavior position \ 139 | --eta 0.5 140 | mv $@.tmp $@ 141 | 142 | $(BUILD)/clicklogs/istella_1m_position_eta_0.75.clog : $(BUILD)/baselines/istella.pth | $(BUILD)/clicklogs/ 143 | python -m experiments.simulate_clicks --input_data $(ISTELLA_DIR)/train.txt \ 144 | --ranker $(BUILD)/baselines/istella.pth \ 145 | --output_log $@.tmp \ 146 | --sessions 10_000_000 \ 147 | --max_clicks 1_000_000 \ 148 | --behavior position \ 149 | --eta 0.75 150 | mv $@.tmp $@ 151 | 152 | $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog : $(BUILD)/baselines/istella.pth | $(BUILD)/clicklogs/ 153 | python -m experiments.simulate_clicks --input_data $(ISTELLA_DIR)/train.txt \ 154 | --ranker $(BUILD)/baselines/istella.pth \ 155 | --output_log $@.tmp \ 156 | --sessions 10_000_000 \ 157 | --max_clicks 1_000_000 \ 158 | --behavior position \ 159 | --eta 1.0 160 | mv $@.tmp $@ 161 | 162 | $(BUILD)/clicklogs/istella_1m_position_eta_1.25.clog : $(BUILD)/baselines/istella.pth | $(BUILD)/clicklogs/ 163 | python -m experiments.simulate_clicks --input_data $(ISTELLA_DIR)/train.txt \ 164 | --ranker $(BUILD)/baselines/istella.pth \ 165 | --output_log $@.tmp \ 166 | --sessions 10_000_000 \ 167 | --max_clicks 1_000_000 \ 168 | --behavior position \ 169 | --eta 1.25 170 | mv $@.tmp $@ 171 | 172 | $(BUILD)/clicklogs/istella_1m_position_eta_1.5.clog : $(BUILD)/baselines/istella.pth | $(BUILD)/clicklogs/ 173 | python -m experiments.simulate_clicks --input_data $(ISTELLA_DIR)/train.txt \ 174 | --ranker $(BUILD)/baselines/istella.pth \ 175 | --output_log $@.tmp \ 176 | --sessions 10_000_000 \ 177 | --max_clicks 1_000_000 \ 178 | --behavior position \ 179 | --eta 1.5 180 | mv $@.tmp $@ 181 | 182 | $(BUILD)/clicklogs/istella_1m_position_eta_2.0.clog : $(BUILD)/baselines/istella.pth | $(BUILD)/clicklogs/ 183 | python -m experiments.simulate_clicks --input_data $(ISTELLA_DIR)/train.txt \ 184 | --ranker $(BUILD)/baselines/istella.pth \ 185 | --output_log $@.tmp \ 186 | --sessions 10_000_000 \ 187 | --max_clicks 1_000_000 \ 188 | --behavior position \ 189 | --eta 2.0 190 | mv $@.tmp $@ 191 | 192 | $(BUILD)/clicklogs/istella_1m_nearrandom_eta_1.0.clog : $(BUILD)/baselines/istella.pth | $(BUILD)/clicklogs/ 193 | python -m experiments.simulate_clicks --input_data $(ISTELLA_DIR)/train.txt \ 194 | --ranker $(BUILD)/baselines/istella.pth \ 195 | --output_log $@.tmp \ 196 | --sessions 10_000_000 \ 197 | --max_clicks 1_000_000 \ 198 | --behavior nearrandom \ 199 | --eta 1.0 200 | mv $@.tmp $@ 201 | -------------------------------------------------------------------------------- /makescripts/directories.mk: -------------------------------------------------------------------------------- 1 | # Directories 2 | $(BUILD)/skylines/ : 3 | mkdir -p $(BUILD)/skylines/ 4 | 5 | $(BUILD)/baselines/ : 6 | mkdir -p $(BUILD)/baselines/ 7 | 8 | $(BUILD)/clicklogs/ : 9 | mkdir -p $(BUILD)/clicklogs/ 10 | 11 | $(BUILD)/results/optimizers/ : 12 | mkdir -p $(BUILD)/results/optimizers/ 13 | 14 | $(BUILD)/results/batch_sizes/ : 15 | mkdir -p $(BUILD)/results/batch_sizes/ 16 | 17 | $(BUILD)/results/etas/ : 18 | mkdir -p $(BUILD)/results/etas/ 19 | 20 | $(BUILD)/plots/ : 21 | mkdir -p $(BUILD)/plots/ 22 | 23 | $(BUILD)/tables/ : 24 | mkdir -p $(BUILD)/tables/ 25 | -------------------------------------------------------------------------------- /makescripts/istella_batch_sizes.mk: -------------------------------------------------------------------------------- 1 | # Results for batch sizes experiment under istella dataset. 2 | istella_batch_sizes_10_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/istella_10_none_seed_42$(i).json) 3 | istella_batch_sizes_10_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/istella_10_weight_seed_42$(i).json) 4 | istella_batch_sizes_10_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/istella_10_sample_seed_42$(i).json) 5 | istella_batch_sizes_20_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/istella_20_none_seed_42$(i).json) 6 | istella_batch_sizes_20_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/istella_20_weight_seed_42$(i).json) 7 | istella_batch_sizes_20_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/istella_20_sample_seed_42$(i).json) 8 | istella_batch_sizes_50_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/istella_50_none_seed_42$(i).json) 9 | istella_batch_sizes_50_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/istella_50_weight_seed_42$(i).json) 10 | istella_batch_sizes_50_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/istella_50_sample_seed_42$(i).json) 11 | istella_batch_sizes_repeat_5: istella_batch_sizes_10_repeat_5 istella_batch_sizes_20_repeat_5 istella_batch_sizes_50_repeat_5 12 | .PHONY: istella_batch_sizes_repeat_5 istella_batch_sizes_10_repeat_5 istella_batch_sizes_20_repeat_5 istella_batch_sizes_50_repeat_5 13 | 14 | # batch 10 15 | $(BUILD)/results/batch_sizes/istella_10_none_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 16 | python -m experiments.train \ 17 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 18 | --train_data $(ISTELLA_DIR)/train.txt \ 19 | --test_data $(ISTELLA_DIR)/test.txt \ 20 | --output $@.tmp \ 21 | --lr 1e-03 \ 22 | --optimizer sgd \ 23 | --ips_strategy none \ 24 | --batch_size 10 \ 25 | --log_every 1000 \ 26 | --eval_every 1000 \ 27 | --epochs 5 \ 28 | --seed $* $(TRAIN_ARGS) 29 | mv $@.tmp $@ 30 | 31 | $(BUILD)/results/batch_sizes/istella_10_weight_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 32 | python -m experiments.train \ 33 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 34 | --train_data $(ISTELLA_DIR)/train.txt \ 35 | --test_data $(ISTELLA_DIR)/test.txt \ 36 | --output $@.tmp \ 37 | --lr 3e-06 \ 38 | --optimizer sgd \ 39 | --ips_strategy weight \ 40 | --batch_size 10 \ 41 | --log_every 1000 \ 42 | --eval_every 1000 \ 43 | --epochs 5 \ 44 | --seed $* $(TRAIN_ARGS) 45 | mv $@.tmp $@ 46 | 47 | $(BUILD)/results/batch_sizes/istella_10_sample_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 48 | python -m experiments.train \ 49 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 50 | --train_data $(ISTELLA_DIR)/train.txt \ 51 | --test_data $(ISTELLA_DIR)/test.txt \ 52 | --output $@.tmp \ 53 | --lr 1e-05 \ 54 | --optimizer sgd \ 55 | --ips_strategy sample \ 56 | --batch_size 10 \ 57 | --log_every 1000 \ 58 | --eval_every 1000 \ 59 | --epochs 5 \ 60 | --seed $* $(TRAIN_ARGS) 61 | mv $@.tmp $@ 62 | 63 | # batch 20 64 | $(BUILD)/results/batch_sizes/istella_20_none_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 65 | python -m experiments.train \ 66 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 67 | --train_data $(ISTELLA_DIR)/train.txt \ 68 | --test_data $(ISTELLA_DIR)/test.txt \ 69 | --output $@.tmp \ 70 | --lr 1e-03 \ 71 | --optimizer sgd \ 72 | --ips_strategy none \ 73 | --batch_size 20 \ 74 | --log_every 500 \ 75 | --eval_every 500 \ 76 | --epochs 5 \ 77 | --seed $* $(TRAIN_ARGS) 78 | mv $@.tmp $@ 79 | 80 | $(BUILD)/results/batch_sizes/istella_20_weight_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 81 | python -m experiments.train \ 82 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 83 | --train_data $(ISTELLA_DIR)/train.txt \ 84 | --test_data $(ISTELLA_DIR)/test.txt \ 85 | --output $@.tmp \ 86 | --lr 1e-05 \ 87 | --optimizer sgd \ 88 | --ips_strategy weight \ 89 | --batch_size 20 \ 90 | --log_every 500 \ 91 | --eval_every 500 \ 92 | --epochs 5 \ 93 | --seed $* $(TRAIN_ARGS) 94 | mv $@.tmp $@ 95 | 96 | $(BUILD)/results/batch_sizes/istella_20_sample_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 97 | python -m experiments.train \ 98 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 99 | --train_data $(ISTELLA_DIR)/train.txt \ 100 | --test_data $(ISTELLA_DIR)/test.txt \ 101 | --output $@.tmp \ 102 | --lr 3e-05 \ 103 | --optimizer sgd \ 104 | --ips_strategy sample \ 105 | --batch_size 20 \ 106 | --log_every 500 \ 107 | --eval_every 500 \ 108 | --epochs 5 \ 109 | --seed $* $(TRAIN_ARGS) 110 | mv $@.tmp $@ 111 | 112 | # batch 50 113 | $(BUILD)/results/batch_sizes/istella_50_none_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 114 | python -m experiments.train \ 115 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 116 | --train_data $(ISTELLA_DIR)/train.txt \ 117 | --test_data $(ISTELLA_DIR)/test.txt \ 118 | --output $@.tmp \ 119 | --lr 3e-03 \ 120 | --optimizer sgd \ 121 | --ips_strategy none \ 122 | --batch_size 50 \ 123 | --log_every 200 \ 124 | --eval_every 200 \ 125 | --epochs 5 \ 126 | --seed $* $(TRAIN_ARGS) 127 | mv $@.tmp $@ 128 | 129 | $(BUILD)/results/batch_sizes/istella_50_weight_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 130 | python -m experiments.train \ 131 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 132 | --train_data $(ISTELLA_DIR)/train.txt \ 133 | --test_data $(ISTELLA_DIR)/test.txt \ 134 | --output $@.tmp \ 135 | --lr 3e-05 \ 136 | --optimizer sgd \ 137 | --ips_strategy weight \ 138 | --batch_size 50 \ 139 | --log_every 200 \ 140 | --eval_every 200 \ 141 | --epochs 5 \ 142 | --seed $* $(TRAIN_ARGS) 143 | mv $@.tmp $@ 144 | 145 | $(BUILD)/results/batch_sizes/istella_50_sample_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 146 | python -m experiments.train \ 147 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 148 | --train_data $(ISTELLA_DIR)/train.txt \ 149 | --test_data $(ISTELLA_DIR)/test.txt \ 150 | --output $@.tmp \ 151 | --lr 3e-05 \ 152 | --optimizer sgd \ 153 | --ips_strategy sample \ 154 | --batch_size 50 \ 155 | --log_every 200 \ 156 | --eval_every 200 \ 157 | --epochs 5 \ 158 | --seed $* $(TRAIN_ARGS) 159 | mv $@.tmp $@ 160 | -------------------------------------------------------------------------------- /makescripts/istella_etas.mk: -------------------------------------------------------------------------------- 1 | # Results for batch sizes experiment under istella dataset. 2 | istella_etas_0_5_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/istella_0.5_none_seed_42$(i).json) 3 | istella_etas_0_5_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/istella_0.5_weight_seed_42$(i).json) 4 | istella_etas_0_5_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/istella_0.5_sample_seed_42$(i).json) 5 | istella_etas_0_75_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/istella_0.75_none_seed_42$(i).json) 6 | istella_etas_0_75_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/istella_0.75_weight_seed_42$(i).json) 7 | istella_etas_0_75_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/istella_0.75_sample_seed_42$(i).json) 8 | istella_etas_1_0_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/istella_1.0_none_seed_42$(i).json) 9 | istella_etas_1_0_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/istella_1.0_weight_seed_42$(i).json) 10 | istella_etas_1_0_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/istella_1.0_sample_seed_42$(i).json) 11 | istella_etas_1_25_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/istella_1.25_none_seed_42$(i).json) 12 | istella_etas_1_25_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/istella_1.25_weight_seed_42$(i).json) 13 | istella_etas_1_25_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/istella_1.25_sample_seed_42$(i).json) 14 | istella_etas_1_5_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/istella_1.5_none_seed_42$(i).json) 15 | istella_etas_1_5_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/istella_1.5_weight_seed_42$(i).json) 16 | istella_etas_1_5_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/istella_1.5_sample_seed_42$(i).json) 17 | 18 | istella_etas_repeat_5: istella_etas_0_5_repeat_5 istella_etas_0_75_repeat_5 istella_etas_1_0_repeat_5 istella_etas_1_25_repeat_5 istella_etas_1_5_repeat_5 19 | .PHONY: istella_etas_repeat_5 istella_etas_0_5_repeat_5 istella_etas_0_75_repeat_5 istella_etas_1_0_repeat_5 istella_etas_1_25_repeat_5 istella_etas_1_5_repeat_5 20 | 21 | # Eta 1.0 22 | $(BUILD)/results/etas/istella_1.0_none_seed_%.json : $(BUILD)/results/optimizers/istella_sgd_none_seed_%.json | $(BUILD)/results/etas/ 23 | cp $(BUILD)/results/optimizers/istella_sgd_none_seed_$*.json $@ 24 | 25 | $(BUILD)/results/etas/istella_1.0_weight_seed_%.json : $(BUILD)/results/optimizers/istella_sgd_weight_seed_%.json | $(BUILD)/results/etas/ 26 | cp $(BUILD)/results/optimizers/istella_sgd_weight_seed_$*.json $@ 27 | 28 | $(BUILD)/results/etas/istella_1.0_sample_seed_%.json : $(BUILD)/results/optimizers/istella_sgd_sample_seed_%.json | $(BUILD)/results/etas/ 29 | cp $(BUILD)/results/optimizers/istella_sgd_sample_seed_$*.json $@ 30 | 31 | # Eta 0.5 32 | $(BUILD)/results/etas/istella_0.5_none_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_0.5.clog | $(BUILD)/results/etas/ 33 | python -m experiments.train \ 34 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_0.5.clog \ 35 | --train_data $(ISTELLA_DIR)/train.txt \ 36 | --test_data $(ISTELLA_DIR)/test.txt \ 37 | --output $@.tmp \ 38 | --lr 3e-05 \ 39 | --optimizer sgd \ 40 | --ips_strategy none \ 41 | --batch_size 1 \ 42 | --log_every 10000 \ 43 | --eval_every 10000 \ 44 | --epochs 5 \ 45 | --seed $* $(TRAIN_ARGS) 46 | mv $@.tmp $@ 47 | 48 | $(BUILD)/results/etas/istella_0.5_sample_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_0.5.clog | $(BUILD)/results/etas/ 49 | python -m experiments.train \ 50 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_0.5.clog \ 51 | --train_data $(ISTELLA_DIR)/train.txt \ 52 | --test_data $(ISTELLA_DIR)/test.txt \ 53 | --output $@.tmp \ 54 | --lr 3e-06 \ 55 | --optimizer sgd \ 56 | --ips_strategy sample \ 57 | --batch_size 1 \ 58 | --log_every 10000 \ 59 | --eval_every 10000 \ 60 | --epochs 5 \ 61 | --seed $* $(TRAIN_ARGS) 62 | mv $@.tmp $@ 63 | 64 | $(BUILD)/results/etas/istella_0.5_weight_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_0.5.clog | $(BUILD)/results/etas/ 65 | python -m experiments.train \ 66 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_0.5.clog \ 67 | --train_data $(ISTELLA_DIR)/train.txt \ 68 | --test_data $(ISTELLA_DIR)/test.txt \ 69 | --output $@.tmp \ 70 | --lr 3e-06 \ 71 | --optimizer sgd \ 72 | --ips_strategy weight \ 73 | --batch_size 1 \ 74 | --log_every 10000 \ 75 | --eval_every 10000 \ 76 | --epochs 5 \ 77 | --seed $* $(TRAIN_ARGS) 78 | mv $@.tmp $@ 79 | 80 | # Eta 0.75 81 | $(BUILD)/results/etas/istella_0.75_none_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_0.75.clog | $(BUILD)/results/etas/ 82 | python -m experiments.train \ 83 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_0.75.clog \ 84 | --train_data $(ISTELLA_DIR)/train.txt \ 85 | --test_data $(ISTELLA_DIR)/test.txt \ 86 | --output $@.tmp \ 87 | --lr 3e-05 \ 88 | --optimizer sgd \ 89 | --ips_strategy none \ 90 | --batch_size 1 \ 91 | --log_every 10000 \ 92 | --eval_every 10000 \ 93 | --epochs 5 \ 94 | --seed $* $(TRAIN_ARGS) 95 | mv $@.tmp $@ 96 | 97 | $(BUILD)/results/etas/istella_0.75_sample_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_0.75.clog | $(BUILD)/results/etas/ 98 | python -m experiments.train \ 99 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_0.75.clog \ 100 | --train_data $(ISTELLA_DIR)/train.txt \ 101 | --test_data $(ISTELLA_DIR)/test.txt \ 102 | --output $@.tmp \ 103 | --lr 3e-06 \ 104 | --optimizer sgd \ 105 | --ips_strategy sample \ 106 | --batch_size 1 \ 107 | --log_every 10000 \ 108 | --eval_every 10000 \ 109 | --epochs 5 \ 110 | --seed $* $(TRAIN_ARGS) 111 | mv $@.tmp $@ 112 | 113 | $(BUILD)/results/etas/istella_0.75_weight_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_0.75.clog | $(BUILD)/results/etas/ 114 | python -m experiments.train \ 115 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_0.75.clog \ 116 | --train_data $(ISTELLA_DIR)/train.txt \ 117 | --test_data $(ISTELLA_DIR)/test.txt \ 118 | --output $@.tmp \ 119 | --lr 1e-06 \ 120 | --optimizer sgd \ 121 | --ips_strategy weight \ 122 | --batch_size 1 \ 123 | --log_every 10000 \ 124 | --eval_every 10000 \ 125 | --epochs 5 \ 126 | --seed $* $(TRAIN_ARGS) 127 | mv $@.tmp $@ 128 | 129 | 130 | # Eta 1.25 131 | $(BUILD)/results/etas/istella_1.25_none_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.25.clog | $(BUILD)/results/etas/ 132 | python -m experiments.train \ 133 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.25.clog \ 134 | --train_data $(ISTELLA_DIR)/train.txt \ 135 | --test_data $(ISTELLA_DIR)/test.txt \ 136 | --output $@.tmp \ 137 | --lr 3e-04 \ 138 | --optimizer sgd \ 139 | --ips_strategy none \ 140 | --batch_size 1 \ 141 | --log_every 10000 \ 142 | --eval_every 10000 \ 143 | --epochs 5 \ 144 | --seed $* $(TRAIN_ARGS) 145 | mv $@.tmp $@ 146 | 147 | $(BUILD)/results/etas/istella_1.25_sample_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.25.clog | $(BUILD)/results/etas/ 148 | python -m experiments.train \ 149 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.25.clog \ 150 | --train_data $(ISTELLA_DIR)/train.txt \ 151 | --test_data $(ISTELLA_DIR)/test.txt \ 152 | --output $@.tmp \ 153 | --lr 1e-06 \ 154 | --optimizer sgd \ 155 | --ips_strategy sample \ 156 | --batch_size 1 \ 157 | --log_every 10000 \ 158 | --eval_every 10000 \ 159 | --epochs 5 \ 160 | --seed $* $(TRAIN_ARGS) 161 | mv $@.tmp $@ 162 | 163 | $(BUILD)/results/etas/istella_1.25_weight_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.25.clog | $(BUILD)/results/etas/ 164 | python -m experiments.train \ 165 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.25.clog \ 166 | --train_data $(ISTELLA_DIR)/train.txt \ 167 | --test_data $(ISTELLA_DIR)/test.txt \ 168 | --output $@.tmp \ 169 | --lr 3e-07 \ 170 | --optimizer sgd \ 171 | --ips_strategy weight \ 172 | --batch_size 1 \ 173 | --log_every 10000 \ 174 | --eval_every 10000 \ 175 | --epochs 5 \ 176 | --seed $* $(TRAIN_ARGS) 177 | mv $@.tmp $@ 178 | 179 | 180 | # Eta 1.5 181 | $(BUILD)/results/etas/istella_1.5_none_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.5.clog | $(BUILD)/results/etas/ 182 | python -m experiments.train \ 183 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.5.clog \ 184 | --train_data $(ISTELLA_DIR)/train.txt \ 185 | --test_data $(ISTELLA_DIR)/test.txt \ 186 | --output $@.tmp \ 187 | --lr 1e-03 \ 188 | --optimizer sgd \ 189 | --ips_strategy none \ 190 | --batch_size 1 \ 191 | --log_every 10000 \ 192 | --eval_every 10000 \ 193 | --epochs 5 \ 194 | --seed $* $(TRAIN_ARGS) 195 | mv $@.tmp $@ 196 | 197 | $(BUILD)/results/etas/istella_1.5_sample_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.5.clog | $(BUILD)/results/etas/ 198 | python -m experiments.train \ 199 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.5.clog \ 200 | --train_data $(ISTELLA_DIR)/train.txt \ 201 | --test_data $(ISTELLA_DIR)/test.txt \ 202 | --output $@.tmp \ 203 | --lr 3e-07 \ 204 | --optimizer sgd \ 205 | --ips_strategy sample \ 206 | --batch_size 1 \ 207 | --log_every 10000 \ 208 | --eval_every 10000 \ 209 | --epochs 5 \ 210 | --seed $* $(TRAIN_ARGS) 211 | mv $@.tmp $@ 212 | 213 | $(BUILD)/results/etas/istella_1.5_weight_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.5.clog | $(BUILD)/results/etas/ 214 | python -m experiments.train \ 215 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.5.clog \ 216 | --train_data $(ISTELLA_DIR)/train.txt \ 217 | --test_data $(ISTELLA_DIR)/test.txt \ 218 | --output $@.tmp \ 219 | --lr 1e-07 \ 220 | --optimizer sgd \ 221 | --ips_strategy weight \ 222 | --batch_size 1 \ 223 | --log_every 10000 \ 224 | --eval_every 10000 \ 225 | --epochs 5 \ 226 | --seed $* $(TRAIN_ARGS) 227 | mv $@.tmp $@ 228 | -------------------------------------------------------------------------------- /makescripts/istella_optimizers.mk: -------------------------------------------------------------------------------- 1 | # Results for optimizers experiment under istella dataset. 2 | istella_optimizers_sgd_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/istella_sgd_none_seed_42$(i).json) 3 | istella_optimizers_sgd_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/istella_sgd_weight_seed_42$(i).json) 4 | istella_optimizers_sgd_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/istella_sgd_sample_seed_42$(i).json) 5 | istella_optimizers_adam_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/istella_adam_none_seed_42$(i).json) 6 | istella_optimizers_adam_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/istella_adam_weight_seed_42$(i).json) 7 | istella_optimizers_adam_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/istella_adam_sample_seed_42$(i).json) 8 | istella_optimizers_adagrad_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/istella_adagrad_none_seed_42$(i).json) 9 | istella_optimizers_adagrad_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/istella_adagrad_weight_seed_42$(i).json) 10 | istella_optimizers_adagrad_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/istella_adagrad_sample_seed_42$(i).json) 11 | istella_optimizers_repeat_5 : istella_optimizers_sgd_repeat_5 istella_optimizers_adam_repeat_5 istella_optimizers_adagrad_repeat_5 12 | .PHONY: istella_optimizers_repeat_5 istella_optimizers_sgd_repeat_5 istella_optimizers_adam_repeat_5 istella_optimizers_adagrad_repeat_5 13 | 14 | # SGD 15 | $(BUILD)/results/optimizers/istella_sgd_none_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 16 | python -m experiments.train \ 17 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 18 | --train_data $(ISTELLA_DIR)/train.txt \ 19 | --test_data $(ISTELLA_DIR)/test.txt \ 20 | --output $@.tmp \ 21 | --lr 1e-04 \ 22 | --optimizer sgd \ 23 | --ips_strategy none \ 24 | --batch_size 1 \ 25 | --log_every 10_000 \ 26 | --eval_every 10_000 \ 27 | --epochs 5 \ 28 | --seed $* $(TRAIN_ARGS) 29 | mv $@.tmp $@ 30 | 31 | $(BUILD)/results/optimizers/istella_sgd_weight_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 32 | python -m experiments.train \ 33 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 34 | --train_data $(ISTELLA_DIR)/train.txt \ 35 | --test_data $(ISTELLA_DIR)/test.txt \ 36 | --output $@.tmp \ 37 | --lr 3e-07 \ 38 | --optimizer sgd \ 39 | --ips_strategy weight \ 40 | --batch_size 1 \ 41 | --log_every 10_000 \ 42 | --eval_every 10_000 \ 43 | --epochs 5 \ 44 | --seed $* $(TRAIN_ARGS) 45 | mv $@.tmp $@ 46 | 47 | $(BUILD)/results/optimizers/istella_sgd_sample_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 48 | python -m experiments.train \ 49 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 50 | --train_data $(ISTELLA_DIR)/train.txt \ 51 | --test_data $(ISTELLA_DIR)/test.txt \ 52 | --output $@.tmp \ 53 | --lr 1e-06 \ 54 | --optimizer sgd \ 55 | --ips_strategy sample \ 56 | --batch_size 1 \ 57 | --log_every 10_000 \ 58 | --eval_every 10_000 \ 59 | --epochs 5 \ 60 | --seed $* $(TRAIN_ARGS) 61 | mv $@.tmp $@ 62 | 63 | # Adam 64 | $(BUILD)/results/optimizers/istella_adam_none_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 65 | python -m experiments.train \ 66 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 67 | --train_data $(ISTELLA_DIR)/train.txt \ 68 | --test_data $(ISTELLA_DIR)/test.txt \ 69 | --output $@.tmp \ 70 | --lr 1e-03 \ 71 | --optimizer adam \ 72 | --ips_strategy none \ 73 | --batch_size 1 \ 74 | --log_every 10_000 \ 75 | --eval_every 10_000 \ 76 | --epochs 5 \ 77 | --seed $* $(TRAIN_ARGS) 78 | mv $@.tmp $@ 79 | 80 | $(BUILD)/results/optimizers/istella_adam_weight_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 81 | python -m experiments.train \ 82 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 83 | --train_data $(ISTELLA_DIR)/train.txt \ 84 | --test_data $(ISTELLA_DIR)/test.txt \ 85 | --output $@.tmp \ 86 | --lr 1e-04 \ 87 | --optimizer adam \ 88 | --ips_strategy weight \ 89 | --batch_size 1 \ 90 | --log_every 10_000 \ 91 | --eval_every 10_000 \ 92 | --epochs 5 \ 93 | --seed $* $(TRAIN_ARGS) 94 | mv $@.tmp $@ 95 | 96 | $(BUILD)/results/optimizers/istella_adam_sample_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 97 | python -m experiments.train \ 98 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 99 | --train_data $(ISTELLA_DIR)/train.txt \ 100 | --test_data $(ISTELLA_DIR)/test.txt \ 101 | --output $@.tmp \ 102 | --lr 3e-04 \ 103 | --optimizer adam \ 104 | --ips_strategy sample \ 105 | --batch_size 1 \ 106 | --log_every 10_000 \ 107 | --eval_every 10_000 \ 108 | --epochs 5 \ 109 | --seed $* $(TRAIN_ARGS) 110 | mv $@.tmp $@ 111 | 112 | # Adagrad 113 | $(BUILD)/results/optimizers/istella_adagrad_none_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 114 | python -m experiments.train \ 115 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 116 | --train_data $(ISTELLA_DIR)/train.txt \ 117 | --test_data $(ISTELLA_DIR)/test.txt \ 118 | --output $@.tmp \ 119 | --lr 3e-01 \ 120 | --optimizer adagrad \ 121 | --ips_strategy none \ 122 | --batch_size 1 \ 123 | --log_every 10_000 \ 124 | --eval_every 10_000 \ 125 | --epochs 5 \ 126 | --seed $* $(TRAIN_ARGS) 127 | mv $@.tmp $@ 128 | 129 | $(BUILD)/results/optimizers/istella_adagrad_weight_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 130 | python -m experiments.train \ 131 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 132 | --train_data $(ISTELLA_DIR)/train.txt \ 133 | --test_data $(ISTELLA_DIR)/test.txt \ 134 | --output $@.tmp \ 135 | --lr 1e-01 \ 136 | --optimizer adagrad \ 137 | --ips_strategy weight \ 138 | --batch_size 1 \ 139 | --log_every 10_000 \ 140 | --eval_every 10_000 \ 141 | --epochs 5 \ 142 | --seed $* $(TRAIN_ARGS) 143 | mv $@.tmp $@ 144 | 145 | $(BUILD)/results/optimizers/istella_adagrad_sample_seed_%.json : $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 146 | python -m experiments.train \ 147 | --click_log $(BUILD)/clicklogs/istella_1m_position_eta_1.0.clog \ 148 | --train_data $(ISTELLA_DIR)/train.txt \ 149 | --test_data $(ISTELLA_DIR)/test.txt \ 150 | --output $@.tmp \ 151 | --lr 1e-01 \ 152 | --optimizer adagrad \ 153 | --ips_strategy sample \ 154 | --batch_size 1 \ 155 | --log_every 10_000 \ 156 | --eval_every 10_000 \ 157 | --epochs 5 \ 158 | --seed $* $(TRAIN_ARGS) 159 | mv $@.tmp $@ 160 | -------------------------------------------------------------------------------- /makescripts/plots.mk: -------------------------------------------------------------------------------- 1 | # Phony target for easier dependencies 2 | plots: $(BUILD)/plots/batchsizes.pdf $(BUILD)/plots/etas.pdf $(BUILD)/plots/optimizers.pdf $(BUILD)/plots/toy-example.pdf 3 | .PHONY: plots 4 | 5 | # Scripts to generate plots in the paper 6 | $(BUILD)/plots/batchsizes.pdf : | yahoo_batch_sizes_repeat_5 istella_batch_sizes_repeat_5 $(BUILD)/plots/ 7 | python -m experiments.plots.batch_sizes \ 8 | --dataset test --json_files $(BUILD)/results/batch_sizes/* \ 9 | --out $@.tmp.pdf \ 10 | --format pdf \ 11 | --legend --height 3.3 --points 50 12 | mv $@.tmp.pdf $@ 13 | 14 | $(BUILD)/plots/batchsizes-dark.pdf : | yahoo_batch_sizes_repeat_5 istella_batch_sizes_repeat_5 $(BUILD)/plots/ 15 | python -m experiments.plots.batch_sizes \ 16 | --dataset test --json_files $(BUILD)/results/batch_sizes/* \ 17 | --out $@.tmp.pdf \ 18 | --format pdf \ 19 | --legend --height 3.3 --points 50 --darkstyle --datasets yahoo 20 | mv $@.tmp.pdf $@ 21 | 22 | $(BUILD)/plots/optimizers.pdf : | yahoo_optimizers_repeat_5 istella_optimizers_repeat_5 $(BUILD)/plots/ 23 | python -m experiments.plots.optimizers \ 24 | --dataset test --json_files $(BUILD)/results/optimizers/* \ 25 | --out $@.tmp.pdf \ 26 | --format pdf \ 27 | --legend --height 3.3 --points 50 28 | mv $@.tmp.pdf $@ 29 | 30 | $(BUILD)/plots/optimizers-dark.pdf : | yahoo_optimizers_repeat_5 istella_optimizers_repeat_5 $(BUILD)/plots/ 31 | python -m experiments.plots.optimizers \ 32 | --dataset test --json_files $(BUILD)/results/optimizers/* \ 33 | --out $@.tmp.pdf \ 34 | --format pdf \ 35 | --legend --height 3.3 --points 50 --darkstyle --datasets yahoo 36 | mv $@.tmp.pdf $@ 37 | 38 | $(BUILD)/plots/etas.pdf : | yahoo_etas_repeat_5 istella_etas_repeat_5 $(BUILD)/plots/ 39 | python -m experiments.plots.etas \ 40 | --dataset test --json_files $(BUILD)/results/etas/* \ 41 | --out $@.tmp.pdf \ 42 | --format pdf \ 43 | --legend --height 3.3 --points 50 44 | mv $@.tmp.pdf $@ 45 | 46 | $(BUILD)/plots/etas-dark.pdf : | yahoo_etas_repeat_5 istella_etas_repeat_5 $(BUILD)/plots/ 47 | python -m experiments.plots.etas \ 48 | --dataset test --json_files $(BUILD)/results/etas/* \ 49 | --out $@.tmp.pdf \ 50 | --format pdf \ 51 | --legend --height 3.3 --points 50 --darkstyle --datasets yahoo --etas "0.5" "1.0" "1.5" 52 | mv $@.tmp.pdf $@ 53 | 54 | $(BUILD)/plots/toy-example.pdf : | $(BUILD)/plots/ 55 | python -m experiments.plots.toy_sample --format pdf --out $@.tmp.pdf 56 | mv $@.tmp.pdf $@ 57 | -------------------------------------------------------------------------------- /makescripts/tables.mk: -------------------------------------------------------------------------------- 1 | # Phony target for easier dependencies 2 | tables: $(BUILD)/tables/batchsizes.tbl.tex $(BUILD)/tables/optimizers.tbl.tex $(BUILD)/tables/etas.tbl.tex 3 | .PHONY: tables 4 | 5 | # Scripts to generate tables in the paper 6 | $(BUILD)/tables/batchsizes.tbl.tex : $(BUILD)/skylines/yahoo.json $(BUILD)/skylines/istella.json | yahoo_batch_sizes_repeat_5 istella_batch_sizes_repeat_5 $(BUILD)/tables/ 7 | python -m experiments.tables.batch_sizes \ 8 | --json_files $(BUILD)/results/batch_sizes/*.json \ 9 | --skylines $(BUILD)/skylines/*.json \ 10 | --dataset test > $@.tmp 11 | mv $@.tmp $@ 12 | 13 | 14 | $(BUILD)/tables/optimizers.tbl.tex : $(BUILD)/skylines/yahoo.json $(BUILD)/skylines/istella.json | yahoo_optimizers_repeat_5 istella_optimizers_repeat_5 $(BUILD)/tables/ 15 | python -m experiments.tables.optimizers \ 16 | --json_files $(BUILD)/results/optimizers/*.json \ 17 | --skylines $(BUILD)/skylines/*.json \ 18 | --dataset test > $@.tmp 19 | mv $@.tmp $@ 20 | 21 | $(BUILD)/tables/etas.tbl.tex : $(BUILD)/skylines/yahoo.json $(BUILD)/skylines/istella.json | yahoo_etas_repeat_5 istella_etas_repeat_5 $(BUILD)/tables/ 22 | python -m experiments.tables.etas \ 23 | --json_files $(BUILD)/results/etas/*.json \ 24 | --skylines $(BUILD)/skylines/*.json \ 25 | --dataset test > $@.tmp 26 | mv $@.tmp $@ 27 | -------------------------------------------------------------------------------- /makescripts/yahoo_batch_sizes.mk: -------------------------------------------------------------------------------- 1 | # Results for batch sizes experiment under yahoo dataset. 2 | yahoo_batch_sizes_10_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/yahoo_10_none_seed_42$(i).json) 3 | yahoo_batch_sizes_10_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/yahoo_10_weight_seed_42$(i).json) 4 | yahoo_batch_sizes_10_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/yahoo_10_sample_seed_42$(i).json) 5 | yahoo_batch_sizes_20_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/yahoo_20_none_seed_42$(i).json) 6 | yahoo_batch_sizes_20_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/yahoo_20_weight_seed_42$(i).json) 7 | yahoo_batch_sizes_20_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/yahoo_20_sample_seed_42$(i).json) 8 | yahoo_batch_sizes_50_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/yahoo_50_none_seed_42$(i).json) 9 | yahoo_batch_sizes_50_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/yahoo_50_weight_seed_42$(i).json) 10 | yahoo_batch_sizes_50_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/batch_sizes/yahoo_50_sample_seed_42$(i).json) 11 | yahoo_batch_sizes_repeat_5 : yahoo_batch_sizes_10_repeat_5 yahoo_batch_sizes_20_repeat_5 yahoo_batch_sizes_50_repeat_5 12 | .PHONY: yahoo_batch_sizes_repeat_5 yahoo_batch_sizes_10_repeat_5 yahoo_batch_sizes_20_repeat_5 yahoo_batch_sizes_50_repeat_5 13 | 14 | # batch 10 15 | $(BUILD)/results/batch_sizes/yahoo_10_none_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 16 | python -m experiments.train \ 17 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 18 | --train_data $(YAHOO_DIR)/train.txt \ 19 | --test_data $(YAHOO_DIR)/test.txt \ 20 | --output $@.tmp \ 21 | --lr 3e+00 \ 22 | --optimizer sgd \ 23 | --ips_strategy none \ 24 | --batch_size 10 \ 25 | --log_every 1000 \ 26 | --eval_every 1000 \ 27 | --epochs 5 \ 28 | --seed $* $(TRAIN_ARGS) 29 | mv $@.tmp $@ 30 | 31 | $(BUILD)/results/batch_sizes/yahoo_10_weight_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 32 | python -m experiments.train \ 33 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 34 | --train_data $(YAHOO_DIR)/train.txt \ 35 | --test_data $(YAHOO_DIR)/test.txt \ 36 | --output $@.tmp \ 37 | --lr 1e-04 \ 38 | --optimizer sgd \ 39 | --ips_strategy weight \ 40 | --batch_size 10 \ 41 | --log_every 1000 \ 42 | --eval_every 1000 \ 43 | --epochs 5 \ 44 | --seed $* $(TRAIN_ARGS) 45 | mv $@.tmp $@ 46 | 47 | $(BUILD)/results/batch_sizes/yahoo_10_sample_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 48 | python -m experiments.train \ 49 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 50 | --train_data $(YAHOO_DIR)/train.txt \ 51 | --test_data $(YAHOO_DIR)/test.txt \ 52 | --output $@.tmp \ 53 | --lr 1e-03 \ 54 | --optimizer sgd \ 55 | --ips_strategy sample \ 56 | --batch_size 10 \ 57 | --log_every 1000 \ 58 | --eval_every 1000 \ 59 | --epochs 5 \ 60 | --seed $* $(TRAIN_ARGS) 61 | mv $@.tmp $@ 62 | 63 | # batch 20 64 | $(BUILD)/results/batch_sizes/yahoo_20_none_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 65 | python -m experiments.train \ 66 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 67 | --train_data $(YAHOO_DIR)/train.txt \ 68 | --test_data $(YAHOO_DIR)/test.txt \ 69 | --output $@.tmp \ 70 | --lr 3e+00 \ 71 | --optimizer sgd \ 72 | --ips_strategy none \ 73 | --batch_size 20 \ 74 | --log_every 500 \ 75 | --eval_every 500 \ 76 | --epochs 5 \ 77 | --seed $* $(TRAIN_ARGS) 78 | mv $@.tmp $@ 79 | 80 | $(BUILD)/results/batch_sizes/yahoo_20_weight_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 81 | python -m experiments.train \ 82 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 83 | --train_data $(YAHOO_DIR)/train.txt \ 84 | --test_data $(YAHOO_DIR)/test.txt \ 85 | --output $@.tmp \ 86 | --lr 3e-04 \ 87 | --optimizer sgd \ 88 | --ips_strategy weight \ 89 | --batch_size 20 \ 90 | --log_every 500 \ 91 | --eval_every 500 \ 92 | --epochs 5 \ 93 | --seed $* $(TRAIN_ARGS) 94 | mv $@.tmp $@ 95 | 96 | $(BUILD)/results/batch_sizes/yahoo_20_sample_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 97 | python -m experiments.train \ 98 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 99 | --train_data $(YAHOO_DIR)/train.txt \ 100 | --test_data $(YAHOO_DIR)/test.txt \ 101 | --output $@.tmp \ 102 | --lr 1e-03 \ 103 | --optimizer sgd \ 104 | --ips_strategy sample \ 105 | --batch_size 20 \ 106 | --log_every 500 \ 107 | --eval_every 500 \ 108 | --epochs 5 \ 109 | --seed $* $(TRAIN_ARGS) 110 | mv $@.tmp $@ 111 | 112 | # batch 50 113 | $(BUILD)/results/batch_sizes/yahoo_50_none_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 114 | python -m experiments.train \ 115 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 116 | --train_data $(YAHOO_DIR)/train.txt \ 117 | --test_data $(YAHOO_DIR)/test.txt \ 118 | --output $@.tmp \ 119 | --lr 1e-01 \ 120 | --optimizer sgd \ 121 | --ips_strategy none \ 122 | --batch_size 50 \ 123 | --log_every 200 \ 124 | --eval_every 200 \ 125 | --epochs 5 \ 126 | --seed $* $(TRAIN_ARGS) 127 | mv $@.tmp $@ 128 | 129 | $(BUILD)/results/batch_sizes/yahoo_50_weight_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 130 | python -m experiments.train \ 131 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 132 | --train_data $(YAHOO_DIR)/train.txt \ 133 | --test_data $(YAHOO_DIR)/test.txt \ 134 | --output $@.tmp \ 135 | --lr 3e-04 \ 136 | --optimizer sgd \ 137 | --ips_strategy weight \ 138 | --batch_size 50 \ 139 | --log_every 200 \ 140 | --eval_every 200 \ 141 | --epochs 5 \ 142 | --seed $* $(TRAIN_ARGS) 143 | mv $@.tmp $@ 144 | 145 | $(BUILD)/results/batch_sizes/yahoo_50_sample_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/batch_sizes/ 146 | python -m experiments.train \ 147 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 148 | --train_data $(YAHOO_DIR)/train.txt \ 149 | --test_data $(YAHOO_DIR)/test.txt \ 150 | --output $@.tmp \ 151 | --lr 1e-03 \ 152 | --optimizer sgd \ 153 | --ips_strategy sample \ 154 | --batch_size 50 \ 155 | --log_every 200 \ 156 | --eval_every 200 \ 157 | --epochs 5 \ 158 | --seed $* $(TRAIN_ARGS) 159 | mv $@.tmp $@ 160 | -------------------------------------------------------------------------------- /makescripts/yahoo_etas.mk: -------------------------------------------------------------------------------- 1 | # Results for batch sizes experiment under yahoo dataset. 2 | yahoo_etas_0_5_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/yahoo_0.5_none_seed_42$(i).json) 3 | yahoo_etas_0_5_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/yahoo_0.5_weight_seed_42$(i).json) 4 | yahoo_etas_0_5_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/yahoo_0.5_sample_seed_42$(i).json) 5 | yahoo_etas_0_75_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/yahoo_0.75_none_seed_42$(i).json) 6 | yahoo_etas_0_75_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/yahoo_0.75_weight_seed_42$(i).json) 7 | yahoo_etas_0_75_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/yahoo_0.75_sample_seed_42$(i).json) 8 | yahoo_etas_1_0_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/yahoo_1.0_none_seed_42$(i).json) 9 | yahoo_etas_1_0_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/yahoo_1.0_weight_seed_42$(i).json) 10 | yahoo_etas_1_0_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/yahoo_1.0_sample_seed_42$(i).json) 11 | yahoo_etas_1_25_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/yahoo_1.25_none_seed_42$(i).json) 12 | yahoo_etas_1_25_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/yahoo_1.25_weight_seed_42$(i).json) 13 | yahoo_etas_1_25_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/yahoo_1.25_sample_seed_42$(i).json) 14 | yahoo_etas_1_5_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/yahoo_1.5_none_seed_42$(i).json) 15 | yahoo_etas_1_5_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/yahoo_1.5_weight_seed_42$(i).json) 16 | yahoo_etas_1_5_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/etas/yahoo_1.5_sample_seed_42$(i).json) 17 | yahoo_etas_repeat_5: yahoo_etas_0_5_repeat_5 yahoo_etas_0_75_repeat_5 yahoo_etas_1_0_repeat_5 yahoo_etas_1_25_repeat_5 yahoo_etas_1_5_repeat_5 18 | .PHONY: yahoo_etas_repeat_5 yahoo_etas_0_5_repeat_5 yahoo_etas_0_75_repeat_5 yahoo_etas_1_0_repeat_5 yahoo_etas_1_25_repeat_5 yahoo_etas_1_5_repeat_5 19 | 20 | # Eta 1.0 21 | $(BUILD)/results/etas/yahoo_1.0_none_seed_%.json : $(BUILD)/results/optimizers/yahoo_sgd_none_seed_%.json | $(BUILD)/results/etas/ 22 | cp $(BUILD)/results/optimizers/yahoo_sgd_none_seed_$*.json $@ 23 | 24 | $(BUILD)/results/etas/yahoo_1.0_weight_seed_%.json : $(BUILD)/results/optimizers/yahoo_sgd_weight_seed_%.json | $(BUILD)/results/etas/ 25 | cp $(BUILD)/results/optimizers/yahoo_sgd_weight_seed_$*.json $@ 26 | 27 | $(BUILD)/results/etas/yahoo_1.0_sample_seed_%.json : $(BUILD)/results/optimizers/yahoo_sgd_sample_seed_%.json | $(BUILD)/results/etas/ 28 | cp $(BUILD)/results/optimizers/yahoo_sgd_sample_seed_$*.json $@ 29 | 30 | # Eta 0.5 31 | $(BUILD)/results/etas/yahoo_0.5_none_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_0.5.clog | $(BUILD)/results/etas/ 32 | python -m experiments.train \ 33 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_0.5.clog \ 34 | --train_data $(YAHOO_DIR)/train.txt \ 35 | --test_data $(YAHOO_DIR)/test.txt \ 36 | --output $@.tmp \ 37 | --lr 1e-02 \ 38 | --optimizer sgd \ 39 | --ips_strategy none \ 40 | --batch_size 1 \ 41 | --log_every 10000 \ 42 | --eval_every 10000 \ 43 | --epochs 5 \ 44 | --seed $* $(TRAIN_ARGS) 45 | mv $@.tmp $@ 46 | 47 | $(BUILD)/results/etas/yahoo_0.5_sample_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_0.5.clog | $(BUILD)/results/etas/ 48 | python -m experiments.train \ 49 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_0.5.clog \ 50 | --train_data $(YAHOO_DIR)/train.txt \ 51 | --test_data $(YAHOO_DIR)/test.txt \ 52 | --output $@.tmp \ 53 | --lr 1e-03 \ 54 | --optimizer sgd \ 55 | --ips_strategy sample \ 56 | --batch_size 1 \ 57 | --log_every 10000 \ 58 | --eval_every 10000 \ 59 | --epochs 5 \ 60 | --seed $* $(TRAIN_ARGS) 61 | mv $@.tmp $@ 62 | 63 | $(BUILD)/results/etas/yahoo_0.5_weight_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_0.5.clog | $(BUILD)/results/etas/ 64 | python -m experiments.train \ 65 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_0.5.clog \ 66 | --train_data $(YAHOO_DIR)/train.txt \ 67 | --test_data $(YAHOO_DIR)/test.txt \ 68 | --output $@.tmp \ 69 | --lr 1e-04 \ 70 | --optimizer sgd \ 71 | --ips_strategy weight \ 72 | --batch_size 1 \ 73 | --log_every 10000 \ 74 | --eval_every 10000 \ 75 | --epochs 5 \ 76 | --seed $* $(TRAIN_ARGS) 77 | mv $@.tmp $@ 78 | 79 | # Eta 0.75 80 | $(BUILD)/results/etas/yahoo_0.75_none_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_0.75.clog | $(BUILD)/results/etas/ 81 | python -m experiments.train \ 82 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_0.75.clog \ 83 | --train_data $(YAHOO_DIR)/train.txt \ 84 | --test_data $(YAHOO_DIR)/test.txt \ 85 | --output $@.tmp \ 86 | --lr 3e-01 \ 87 | --optimizer sgd \ 88 | --ips_strategy none \ 89 | --batch_size 1 \ 90 | --log_every 10000 \ 91 | --eval_every 10000 \ 92 | --epochs 5 \ 93 | --seed $* $(TRAIN_ARGS) 94 | mv $@.tmp $@ 95 | 96 | $(BUILD)/results/etas/yahoo_0.75_sample_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_0.75.clog | $(BUILD)/results/etas/ 97 | python -m experiments.train \ 98 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_0.75.clog \ 99 | --train_data $(YAHOO_DIR)/train.txt \ 100 | --test_data $(YAHOO_DIR)/test.txt \ 101 | --output $@.tmp \ 102 | --lr 1e-03 \ 103 | --optimizer sgd \ 104 | --ips_strategy sample \ 105 | --batch_size 1 \ 106 | --log_every 10000 \ 107 | --eval_every 10000 \ 108 | --epochs 5 \ 109 | --seed $* $(TRAIN_ARGS) 110 | mv $@.tmp $@ 111 | 112 | $(BUILD)/results/etas/yahoo_0.75_weight_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_0.75.clog | $(BUILD)/results/etas/ 113 | python -m experiments.train \ 114 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_0.75.clog \ 115 | --train_data $(YAHOO_DIR)/train.txt \ 116 | --test_data $(YAHOO_DIR)/test.txt \ 117 | --output $@.tmp \ 118 | --lr 3e-05 \ 119 | --optimizer sgd \ 120 | --ips_strategy weight \ 121 | --batch_size 1 \ 122 | --log_every 10000 \ 123 | --eval_every 10000 \ 124 | --epochs 5 \ 125 | --seed $* $(TRAIN_ARGS) 126 | mv $@.tmp $@ 127 | 128 | 129 | # Eta 1.25 130 | $(BUILD)/results/etas/yahoo_1.25_none_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.25.clog | $(BUILD)/results/etas/ 131 | python -m experiments.train \ 132 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.25.clog \ 133 | --train_data $(YAHOO_DIR)/train.txt \ 134 | --test_data $(YAHOO_DIR)/test.txt \ 135 | --output $@.tmp \ 136 | --lr 1e-02 \ 137 | --optimizer sgd \ 138 | --ips_strategy none \ 139 | --batch_size 1 \ 140 | --log_every 10000 \ 141 | --eval_every 10000 \ 142 | --epochs 5 \ 143 | --seed $* $(TRAIN_ARGS) 144 | mv $@.tmp $@ 145 | 146 | $(BUILD)/results/etas/yahoo_1.25_sample_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.25.clog | $(BUILD)/results/etas/ 147 | python -m experiments.train \ 148 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.25.clog \ 149 | --train_data $(YAHOO_DIR)/train.txt \ 150 | --test_data $(YAHOO_DIR)/test.txt \ 151 | --output $@.tmp \ 152 | --lr 1e-04 \ 153 | --optimizer sgd \ 154 | --ips_strategy sample \ 155 | --batch_size 1 \ 156 | --log_every 10000 \ 157 | --eval_every 10000 \ 158 | --epochs 5 \ 159 | --seed $* $(TRAIN_ARGS) 160 | mv $@.tmp $@ 161 | 162 | $(BUILD)/results/etas/yahoo_1.25_weight_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.25.clog | $(BUILD)/results/etas/ 163 | python -m experiments.train \ 164 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.25.clog \ 165 | --train_data $(YAHOO_DIR)/train.txt \ 166 | --test_data $(YAHOO_DIR)/test.txt \ 167 | --output $@.tmp \ 168 | --lr 3e-06 \ 169 | --optimizer sgd \ 170 | --ips_strategy weight \ 171 | --batch_size 1 \ 172 | --log_every 10000 \ 173 | --eval_every 10000 \ 174 | --epochs 5 \ 175 | --seed $* $(TRAIN_ARGS) 176 | mv $@.tmp $@ 177 | 178 | 179 | # Eta 1.5 180 | $(BUILD)/results/etas/yahoo_1.5_none_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.5.clog | $(BUILD)/results/etas/ 181 | python -m experiments.train \ 182 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.5.clog \ 183 | --train_data $(YAHOO_DIR)/train.txt \ 184 | --test_data $(YAHOO_DIR)/test.txt \ 185 | --output $@.tmp \ 186 | --lr 3e+00 \ 187 | --optimizer sgd \ 188 | --ips_strategy none \ 189 | --batch_size 1 \ 190 | --log_every 10000 \ 191 | --eval_every 10000 \ 192 | --epochs 5 \ 193 | --seed $* $(TRAIN_ARGS) 194 | mv $@.tmp $@ 195 | 196 | $(BUILD)/results/etas/yahoo_1.5_sample_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.5.clog | $(BUILD)/results/etas/ 197 | python -m experiments.train \ 198 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.5.clog \ 199 | --train_data $(YAHOO_DIR)/train.txt \ 200 | --test_data $(YAHOO_DIR)/test.txt \ 201 | --output $@.tmp \ 202 | --lr 3e-03 \ 203 | --optimizer sgd \ 204 | --ips_strategy sample \ 205 | --batch_size 1 \ 206 | --log_every 10000 \ 207 | --eval_every 10000 \ 208 | --epochs 5 \ 209 | --seed $* $(TRAIN_ARGS) 210 | mv $@.tmp $@ 211 | 212 | $(BUILD)/results/etas/yahoo_1.5_weight_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.5.clog | $(BUILD)/results/etas/ 213 | python -m experiments.train \ 214 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.5.clog \ 215 | --train_data $(YAHOO_DIR)/train.txt \ 216 | --test_data $(YAHOO_DIR)/test.txt \ 217 | --output $@.tmp \ 218 | --lr 1e-06 \ 219 | --optimizer sgd \ 220 | --ips_strategy weight \ 221 | --batch_size 1 \ 222 | --log_every 10000 \ 223 | --eval_every 10000 \ 224 | --epochs 5 \ 225 | --seed $* $(TRAIN_ARGS) 226 | mv $@.tmp $@ 227 | -------------------------------------------------------------------------------- /makescripts/yahoo_optimizers.mk: -------------------------------------------------------------------------------- 1 | # Results for optimizers experiment under yahoo dataset. 2 | yahoo_optimizers_sgd_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/yahoo_sgd_none_seed_42$(i).json) 3 | yahoo_optimizers_sgd_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/yahoo_sgd_weight_seed_42$(i).json) 4 | yahoo_optimizers_sgd_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/yahoo_sgd_sample_seed_42$(i).json) 5 | yahoo_optimizers_adam_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/yahoo_adam_none_seed_42$(i).json) 6 | yahoo_optimizers_adam_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/yahoo_adam_weight_seed_42$(i).json) 7 | yahoo_optimizers_adam_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/yahoo_adam_sample_seed_42$(i).json) 8 | yahoo_optimizers_adagrad_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/yahoo_adagrad_none_seed_42$(i).json) 9 | yahoo_optimizers_adagrad_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/yahoo_adagrad_weight_seed_42$(i).json) 10 | yahoo_optimizers_adagrad_repeat_5: $(foreach i,1 2 3 4 5,$(BUILD)/results/optimizers/yahoo_adagrad_sample_seed_42$(i).json) 11 | yahoo_optimizers_repeat_5 : yahoo_optimizers_sgd_repeat_5 yahoo_optimizers_adam_repeat_5 yahoo_optimizers_adagrad_repeat_5 12 | .PHONY: yahoo_optimizers_repeat_5 yahoo_optimizers_sgd_repeat_5 yahoo_optimizers_adam_repeat_5 yahoo_optimizers_adagrad_repeat_5 13 | 14 | # SGD 15 | $(BUILD)/results/optimizers/yahoo_sgd_none_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 16 | python -m experiments.train \ 17 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 18 | --train_data $(YAHOO_DIR)/train.txt \ 19 | --test_data $(YAHOO_DIR)/test.txt \ 20 | --output $@.tmp \ 21 | --lr 3e-01 \ 22 | --optimizer sgd \ 23 | --ips_strategy none \ 24 | --batch_size 1 \ 25 | --log_every 10_000 \ 26 | --eval_every 10_000 \ 27 | --epochs 5 \ 28 | --seed $* $(TRAIN_ARGS) 29 | mv $@.tmp $@ 30 | 31 | $(BUILD)/results/optimizers/yahoo_sgd_weight_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 32 | python -m experiments.train \ 33 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 34 | --train_data $(YAHOO_DIR)/train.txt \ 35 | --test_data $(YAHOO_DIR)/test.txt \ 36 | --output $@.tmp \ 37 | --lr 1e-05 \ 38 | --optimizer sgd \ 39 | --ips_strategy weight \ 40 | --batch_size 1 \ 41 | --log_every 10_000 \ 42 | --eval_every 10_000 \ 43 | --epochs 5 \ 44 | --seed $* $(TRAIN_ARGS) 45 | mv $@.tmp $@ 46 | 47 | $(BUILD)/results/optimizers/yahoo_sgd_sample_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 48 | python -m experiments.train \ 49 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 50 | --train_data $(YAHOO_DIR)/train.txt \ 51 | --test_data $(YAHOO_DIR)/test.txt \ 52 | --output $@.tmp \ 53 | --lr 1e-04 \ 54 | --optimizer sgd \ 55 | --ips_strategy sample \ 56 | --batch_size 1 \ 57 | --log_every 10_000 \ 58 | --eval_every 10_000 \ 59 | --epochs 5 \ 60 | --seed $* $(TRAIN_ARGS) 61 | mv $@.tmp $@ 62 | 63 | # Adam 64 | $(BUILD)/results/optimizers/yahoo_adam_none_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 65 | python -m experiments.train \ 66 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 67 | --train_data $(YAHOO_DIR)/train.txt \ 68 | --test_data $(YAHOO_DIR)/test.txt \ 69 | --output $@.tmp \ 70 | --lr 3e-02 \ 71 | --optimizer adam \ 72 | --ips_strategy none \ 73 | --batch_size 1 \ 74 | --log_every 10_000 \ 75 | --eval_every 10_000 \ 76 | --epochs 5 \ 77 | --seed $* $(TRAIN_ARGS) 78 | mv $@.tmp $@ 79 | 80 | $(BUILD)/results/optimizers/yahoo_adam_weight_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 81 | python -m experiments.train \ 82 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 83 | --train_data $(YAHOO_DIR)/train.txt \ 84 | --test_data $(YAHOO_DIR)/test.txt \ 85 | --output $@.tmp \ 86 | --lr 1e-04 \ 87 | --optimizer adam \ 88 | --ips_strategy weight \ 89 | --batch_size 1 \ 90 | --log_every 10_000 \ 91 | --eval_every 10_000 \ 92 | --epochs 5 \ 93 | --seed $* $(TRAIN_ARGS) 94 | mv $@.tmp $@ 95 | 96 | $(BUILD)/results/optimizers/yahoo_adam_sample_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 97 | python -m experiments.train \ 98 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 99 | --train_data $(YAHOO_DIR)/train.txt \ 100 | --test_data $(YAHOO_DIR)/test.txt \ 101 | --output $@.tmp \ 102 | --lr 1e-02 \ 103 | --optimizer adam \ 104 | --ips_strategy sample \ 105 | --batch_size 1 \ 106 | --log_every 10_000 \ 107 | --eval_every 10_000 \ 108 | --epochs 5 \ 109 | --seed $* $(TRAIN_ARGS) 110 | mv $@.tmp $@ 111 | 112 | # Adagrad 113 | $(BUILD)/results/optimizers/yahoo_adagrad_none_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 114 | python -m experiments.train \ 115 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 116 | --train_data $(YAHOO_DIR)/train.txt \ 117 | --test_data $(YAHOO_DIR)/test.txt \ 118 | --output $@.tmp \ 119 | --lr 3e+00 \ 120 | --optimizer adagrad \ 121 | --ips_strategy none \ 122 | --batch_size 1 \ 123 | --log_every 10_000 \ 124 | --eval_every 10_000 \ 125 | --epochs 5 \ 126 | --seed $* $(TRAIN_ARGS) 127 | mv $@.tmp $@ 128 | 129 | $(BUILD)/results/optimizers/yahoo_adagrad_weight_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 130 | python -m experiments.train \ 131 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 132 | --train_data $(YAHOO_DIR)/train.txt \ 133 | --test_data $(YAHOO_DIR)/test.txt \ 134 | --output $@.tmp \ 135 | --lr 3e-01 \ 136 | --optimizer adagrad \ 137 | --ips_strategy weight \ 138 | --batch_size 1 \ 139 | --log_every 10_000 \ 140 | --eval_every 10_000 \ 141 | --epochs 5 \ 142 | --seed $* $(TRAIN_ARGS) 143 | mv $@.tmp $@ 144 | 145 | $(BUILD)/results/optimizers/yahoo_adagrad_sample_seed_%.json : $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog | $(BUILD)/results/optimizers/ 146 | python -m experiments.train \ 147 | --click_log $(BUILD)/clicklogs/yahoo_1m_position_eta_1.0.clog \ 148 | --train_data $(YAHOO_DIR)/train.txt \ 149 | --test_data $(YAHOO_DIR)/test.txt \ 150 | --output $@.tmp \ 151 | --lr 1e+00 \ 152 | --optimizer adagrad \ 153 | --ips_strategy sample \ 154 | --batch_size 1 \ 155 | --log_every 10_000 \ 156 | --eval_every 10_000 \ 157 | --epochs 5 \ 158 | --seed $* $(TRAIN_ARGS) 159 | mv $@.tmp $@ 160 | -------------------------------------------------------------------------------- /requirements.in: -------------------------------------------------------------------------------- 1 | numpy 2 | torch 3 | git+ssh://git@github.com/rjagerman/pytorchltr.git@61f5cb87f8bb513bde1a6dec2c3142790228c840#egg=pytorchltr 4 | matplotlib 5 | pytorch-ignite 6 | torchcontrib 7 | tikzplotlib 8 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile requirements.in 6 | # 7 | cycler==0.10.0 # via matplotlib 8 | future==0.18.2 # via torch 9 | joblib==0.14.1 # via scikit-learn 10 | kiwisolver==1.2.0 # via matplotlib 11 | matplotlib==3.2.1 # via -r requirements.in, tikzplotlib 12 | numpy==1.18.3 # via -r requirements.in, matplotlib, pytorchltr, scikit-learn, scipy, tikzplotlib, torch 13 | pillow==7.1.1 # via tikzplotlib 14 | pyparsing==2.4.7 # via matplotlib 15 | python-dateutil==2.8.1 # via matplotlib 16 | pytorch-ignite==0.3.0 # via -r requirements.in 17 | git+ssh://git@github.com/rjagerman/pytorchltr.git@61f5cb87f8bb513bde1a6dec2c3142790228c840#egg=pytorchltr # via -r requirements.in 18 | scikit-learn==0.22.2.post1 # via pytorchltr 19 | scipy==1.4.1 # via pytorchltr, scikit-learn 20 | six==1.14.0 # via cycler, python-dateutil 21 | tikzplotlib==0.9.1 # via -r requirements.in 22 | torch==1.5.0 # via -r requirements.in, pytorch-ignite, pytorchltr 23 | torchcontrib==0.0.2 # via -r requirements.in 24 | --------------------------------------------------------------------------------