├── LICENSE ├── README.md ├── docs ├── data │ └── data.md ├── how_to_start │ ├── Configuration.md │ ├── Develop_A_New_Model.md │ └── Get_Started.md ├── img │ └── new_loss.png ├── index.md ├── ltr_adhoc │ ├── about.md │ └── lambda_framework.md ├── ltr_adversarial │ ├── about.md │ └── irgan.md ├── ltr_diversification │ ├── Direct_Metric_Optimization.md │ └── about.md ├── ltr_gbm │ ├── LambdaMART.md │ └── about.md └── metric │ └── metric.md ├── img ├── histogram_split.png └── new_loss.png ├── mkdocs.yml ├── pt_ranking.py ├── ptranking ├── __init__.py ├── base │ ├── __init__.py │ ├── adhoc_ranker.py │ ├── list_ranker.py │ ├── point_ranker.py │ ├── ranker.py │ └── utils.py ├── data │ ├── __init__.py │ └── data_utils.py ├── ltr_adhoc │ ├── __init__.py │ ├── eval │ │ ├── __init__.py │ │ ├── ltr.py │ │ └── parameter.py │ ├── listwise │ │ ├── __init__.py │ │ ├── approxNDCG.py │ │ ├── dasalc.py │ │ ├── lambdaloss.py │ │ ├── lambdarank.py │ │ ├── listmle.py │ │ ├── listnet.py │ │ ├── mdprank.py │ │ ├── rank_cosine.py │ │ ├── softrank.py │ │ ├── st_listnet.py │ │ └── wassrank │ │ │ ├── __init__.py │ │ │ ├── pytorch_wasserstein.py │ │ │ ├── wassRank.py │ │ │ └── wasserstein_cost_mat.py │ ├── pairwise │ │ ├── __init__.py │ │ └── ranknet.py │ ├── pointwise │ │ ├── __init__.py │ │ └── rank_mse.py │ └── util │ │ ├── __init__.py │ │ ├── bin_utils.py │ │ ├── gather_utils.py │ │ ├── lambda_utils.py │ │ ├── one_hot_utils.py │ │ └── sampling_utils.py ├── ltr_adversarial │ ├── __init__.py │ ├── base │ │ ├── __init__.py │ │ ├── ad_machine.py │ │ └── ad_player.py │ ├── eval │ │ ├── __init__.py │ │ ├── ad_parameter.py │ │ └── ltr_adversarial.py │ ├── listwise │ │ ├── __init__.py │ │ ├── irfgan_list.py │ │ ├── irgan_list.py │ │ ├── list_discriminator.py │ │ └── list_generator.py │ ├── pairwise │ │ ├── __init__.py │ │ ├── irfgan_pair.py │ │ ├── irgan_pair.py │ │ ├── pair_discriminator.py │ │ └── pair_generator.py │ ├── pointwise │ │ ├── __init__.py │ │ ├── irfgan_point.py │ │ ├── irgan_point.py │ │ ├── point_discriminator.py │ │ └── point_generator.py │ └── util │ │ ├── __init__.py │ │ ├── f_divergence.py │ │ ├── list_probability.py │ │ ├── list_sampling.py │ │ ├── pair_sampling.py │ │ ├── point_sampling.py │ │ └── semi_utils.py ├── ltr_diversification │ ├── __init__.py │ ├── base │ │ ├── __init__.py │ │ ├── div_list_ranker.py │ │ ├── div_mdn_ranker.py │ │ ├── div_point_ranker.py │ │ └── diversity_ranker.py │ ├── eval │ │ ├── __init__.py │ │ ├── div_parameter.py │ │ └── ltr_diversification.py │ ├── one_by_one │ │ └── __init__.py │ ├── score_and_sort │ │ ├── __init__.py │ │ ├── daletor.py │ │ └── div_prob_ranker.py │ └── util │ │ ├── __init__.py │ │ ├── div_data.py │ │ ├── div_lambda_utils.py │ │ ├── prob_utils.py │ │ └── sim_utils.py ├── ltr_global.py ├── ltr_tree │ ├── __init__.py │ ├── eval │ │ ├── __init__.py │ │ ├── ltr_tree.py │ │ └── tree_parameter.py │ ├── lambdamart │ │ ├── __init__.py │ │ ├── lightgbm_lambdaMART.py │ │ └── note │ └── util │ │ ├── __init__.py │ │ └── lightgbm_util.py ├── metric │ ├── __init__.py │ ├── adhoc │ │ ├── __init__.py │ │ └── adhoc_metric.py │ ├── metric_utils.py │ ├── smooth_metric │ │ ├── __init__.py │ │ └── metric_as_opt_objective.py │ └── srd │ │ ├── WT_Div_0912_Implicit_qrels.txt │ │ ├── __init__.py │ │ ├── diversity_metric.py │ │ ├── ndeval │ │ └── ndeval.c └── utils │ ├── __init__.py │ ├── args │ ├── __init__.py │ └── argsUtil.py │ ├── bigdata │ ├── BigPickle.py │ └── __init__.py │ ├── numpy │ ├── __init__.py │ └── np_extensions.py │ └── pytorch │ ├── __init__.py │ └── pt_extensions.py ├── setup.py ├── testing ├── __init__.py ├── data │ ├── __init__.py │ └── testing_data_utils.py ├── ltr_adhoc │ ├── __init__.py │ ├── json │ │ ├── ApproxNDCGParameter.json │ │ ├── Data_Eval_ScoringFunction.json │ │ ├── LambdaLossParameter.json │ │ ├── LambdaRankParameter.json │ │ ├── RankNetParameter.json │ │ ├── STListNetParameter.json │ │ └── WassRankParameter.json │ └── testing_ltr_adhoc.py ├── ltr_adversarial │ ├── __init__.py │ ├── json │ │ ├── Ad_Data_Eval_ScoringFunction.json │ │ ├── IRGAN_ListParameter.json │ │ ├── IRGAN_PairParameter.json │ │ └── IRGAN_PointParameter.json │ └── testing_ltr_adversarial.py ├── ltr_diversification │ ├── __init__.py │ ├── json │ │ ├── DALETORParameter.json │ │ ├── DivProbRankerParameter.json │ │ └── Div_Data_Eval_ScoringFunction.json │ └── testing_ltr_diversification.py ├── ltr_tree │ ├── __init__.py │ ├── json │ │ ├── LightGBMLambdaMARTParameter.json │ │ └── Tree_Data_Eval_ScoringFunction.json │ └── testing_ltr_tree.py └── metric │ ├── __init__.py │ └── testing_metric.py └── tutorial ├── __init__.py ├── ptranking_demo_dataset_statistics.ipynb ├── ptranking_demo_ltr.ipynb ├── ptranking_empirical_risk_minimization.ipynb ├── ptranking_gbm.ipynb ├── ptranking_ir_metric.ipynb └── ptranking_lambda_framework.ipynb /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 pt-ranking 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # What's New? 2 | 3 | - The recent representative methods (such as [MO4SRD](https://wildltr.github.io/ptranking/) and [DALETOR](https://dl.acm.org/doi/abs/10.1145/3442381.3449831)) for Search Result Diversification by directly optimizing the evaluation metric (e.g., alpha-nDCG) have been added. (02/22/2022) 4 | 5 | - Different types of neural scoring functions are supported now, namely **pointwise neural scoring function** (mainly consists of feedforward layers) and **listwise neural scoring function** (mainly builds upon multi-head self-attention Layer). (02/22/2022) 6 | 7 | 8 | # Introduction 9 | 10 | This open-source project, referred to as **PTRanking** (Learning-to-Rank in PyTorch) aims to provide scalable and extendable implementations of typical learning-to-rank methods based on PyTorch. On one hand, this project enables a uniform comparison over several benchmark datasets leading to an in-depth understanding of previous learning-to-rank methods. On the other hand, this project makes it easy to develop and incorporate newly proposed models, so as to expand the territory of techniques on learning-to-rank. 11 | 12 | **Key Features**: 13 | 14 | - A number of representative learning-to-rank models for addressing **Ad-hoc Ranking** and **Search Result Diversification**, including not only the traditional optimization framework via empirical risk minimization but also the adversarial optimization framework 15 | - Supports widely used benchmark datasets. Meanwhile, random masking of the ground-truth labels with a specified ratio is also supported 16 | - Supports different metrics, such as Precision, MAP, nDCG, nERR, alpha-nDCG and ERR-IA. 17 | - Highly configurable functionalities for fine-tuning hyper-parameters, e.g., grid-search over hyper-parameters of a specific model 18 | - Provides easy-to-use APIs for developing a new learning-to-rank model 19 | 20 | Please refer to the [documentation site](https://wildltr.github.io/ptranking/) for more details. 21 | -------------------------------------------------------------------------------- /docs/data/data.md: -------------------------------------------------------------------------------- 1 | # Supported Datasets and Formats 2 | 3 | ## Popular Benchmark Datasets 4 | 5 | -- **[LETOR4.0](https://www.microsoft.com/en-us/research/project/letor-learning-rank-information-retrieval/)** 6 | (MQ2007 \| MQ2008 \| MQ2007-semi \| MQ2008-semi \| MQ2007-list \| MQ2008-list ) 7 | 8 | -- **[MSLR-WEB](https://www.microsoft.com/en-us/research/project/mslr/)** (MSLR-WEB10K \| MSLR-WEB30K) 9 | 10 | -- **[Yahoo! LETOR](https://webscope.sandbox.yahoo.com/catalog.php?datatype=c)** (including Set1 \| Set2) 11 | 12 | -- **[Istella](http://quickrank.isti.cnr.it/istella-dataset/)** (Istella-S \| Istella \| Istella-X) 13 | 14 | These above datasets can be directly used once downloaded. **Please note that:** 15 | 16 | - Semi-supervised datasets (MQ2007-semi | MQ2008-semi) have the same format as that for supervised ranking setting. The only difference is that the semi-supervised datasets in this setting contain both judged and undged query-document pairs 17 | (**in training set but not in validation and testing set**)(The relevance label “-1” indicates the query-document pair is not judged) while the datasets for supervised ranking contain only judged query-document pair. 18 | 19 | - According to [Introducing LETOR 4.0 Datasets](https://arxiv.org/abs/1306.2597), queryLevelNorm version refers to that: conduct query level normalization in the way of using MIN. This data can be directly used for learning. 20 | They further provide 5-fold partitions of this version for cross fold validation. Thus there is no need to perform query_level_scale again for {MQ2007_super | MQ2008_super | MQ2007_semi | MQ2008_semi}. 21 | But for {MSLRWEB10K | MSLRWEB30K}, the query-level normalization is **not conducted yet**. 22 | 23 | - For Yahoo! LETOR, the query-level normalization is already done. 24 | 25 | - For Istella! LETOR, the query-level normalization is **not conducted yet**. We note that ISTELLA contains extremely large features, e.g., 1.79769313486e+308, we replace features of this kind with a constant 1000000. 26 | 27 | ## LibSVM formats 28 | 29 | PT-Ranking currently supports to ingest data with the LibSVM formats 30 | 31 | - LETOR datasets in LibSVM format 32 | 33 | \ qid\: : ... : 34 | 35 | For example: 36 | 37 | 4 qid:105 2:0.4 8:0.7 50:0.5 38 | 39 | 40 | 1 qid:105 5:0.5 30:0.7 32:0.4 48:0.53 41 | 42 | 0 qid:210 4:0.9 38:0.01 39:0.5 45:0.7 43 | 44 | 1 qid:210 1:0.2 8:0.9 31:0.93 40:0.6 45 | 46 | The above sample dataset includes two queries, the query "105" has 2 documents, the corresponding ground-truth labels are 4 and 1, respectively. 47 | 48 | - Converting LETOR datasets into LibSVM format with a corresponding **group** file 49 | 50 | This functionality is required when using the implementation of LambdaMART provided in [LightGBM](https://lightgbm.readthedocs.io/en/latest/). 51 | -------------------------------------------------------------------------------- /docs/how_to_start/Configuration.md: -------------------------------------------------------------------------------- 1 | ## Configuration Strategy 2 | 3 | An easy-to-use configuration is necessary for any ML library. PT-Ranking offers a self-contained strategy. 4 | In other words, we appeal to particularly designed class objects for setting. For example, **DataSetting** for data loading, **EvalSetting** for evaluation setting and **ModelParameter** for a model's parameter setting. Moreover, configuration with **json files** is also supported for DataSetting, EvalSetting and ModelParameter, which is the recommended way. 5 | 6 | Thanks to this strategy, on one hand, we can initialize the settings for data-loading, evaluation, and models in a simple way. In particular, the parameter setting of a model is self-contained, and easy to customize. On the other hand, given the specified setting, e.g., settings with json files, it is very easy to reproduce an experiment. 7 | 8 | ## Setting on Loading A Dataset 9 | 10 | When loading a dataset, the meta-information and preprocessing are specified with **[DataSetting](https://github.com/ptranking/ptranking.github.io/raw/master/ptranking/ltr_adhoc/eval/parameter.py)**. Taking the json file for initializing DataSetting for example, 11 | 12 | - "data_id":"MQ2008_Super", # the ID of an adopted dataset 13 | - "dir_data":"/Users/dryuhaitao/WorkBench/Corpus/LETOR4.0/MQ2008/", # the location of an adopted dataset 14 | 15 | - "min_docs":[10], # the minimum number of documents per query. Otherwise, the query instance is not used. 16 | - "min_rele":[1], # the minimum number of relevant documents. Otherwise, the query instance is not used. 17 | - "sample_rankings_per_q":[1], # the sample rankings per query 18 | 19 | - "binary_rele":[false], # whether convert multi-graded labels into binary labels 20 | - "unknown_as_zero":[false], # whether convert '-1' (i.e., unlabeled documents) as zero 21 | - "presort":[true] # whether sort documents based on ground-truth labels in advance 22 | 23 | ## Setting on Evaluation 24 | 25 | When testing a specific learning-to-rank method, the evaluation details are specified with **[EvalSetting](https://github.com/ptranking/ptranking.github.io/raw/master/ptranking/ltr_adhoc/eval/parameter.py)**. Taking the json file for initializing EvalSetting for example, 26 | 27 | - "dir_output":"/Users/dryuhaitao/WorkBench/CodeBench/Bench_Output/NeuralLTR/Listwise/", # output directory of results 28 | 29 | - "epochs":100, # the number of epoches for training 30 | 31 | - "do_validation":true, # perform validation or not 32 | 33 | - "vali_k":5, # the cutoff value for validation, e.g., nDCG@5 34 | - "cutoffs":[1, 3, 5, 10, 20, 50], # the cutoff values for evaluation 35 | 36 | - "loss_guided":false, # whether the selection of trained model is based on loss function or validation 37 | 38 | - "do_log":true, # logging the training outputs 39 | - "log_step":2, # the step-size of logging 40 | - "do_summary":false, 41 | 42 | - "mask_label":false, # mask ground-truth labels 43 | - "mask_type":["rand_mask_all"], 44 | - "mask_ratio":[0.2] 45 | 46 | ## Parameter Setting 47 | 48 | ### Parameters for Base Scoring Function 49 | For most learning-to-rank methods, PT-Ranking offers deep neural networks as the basis to construct a scoring function. Therefore, we use [ScoringFunctionParameter](https://github.com/ptranking/ptranking.github.io/raw/master/ptranking/ltr_adhoc/eval/parameter.py) to specify the details, such as the number of layers and activation function. Taking the json file for initializing ScoringFunctionParameter for example, 50 | 51 | - "BN":[true], # batch normalization 52 | - "RD":[false], # residual module 53 | - "layers":[5], # number of layers 54 | - "apply_tl_af":[true], # use activation function for the last layer 55 | - "hd_hn_tl_af":["R"] # type of activation function 56 | 57 | ### Parameters for Loss Function 58 | Besides the configuration of the scoring function, for some relatively complex learning-to-rank methods, we also need to specify some parameters for the loss function. In this case, it is required to develop the subclass ModelAParameter by inheriting **[ModelParameter](https://github.com/ptranking/ptranking.github.io/raw/master/ptranking/ltr_adhoc/eval/parameter.py)** and customizing the functions, such as to_para_string(), default_para_dict() and grid_search(). Please refer to [LambdaRankParameter](https://github.com/ptranking/ptranking.github.io/raw/master/ptranking/ltr_adhoc/listwise/lambdarank.py) as an example. 59 | 60 | ## Prepare Configuration Json Files 61 | 62 | When evaluating a method, two json files are commonly required: 63 | 64 | - **Data_Eval_ScoringFunction.json**, which specifies the detailed settings for data loading (i.e, DataSetting), evaluation (i.e, EvalSetting) and the parameters for base scoring function (i.e, ScoringFunctionParameter). Please refer to [Data_Eval_ScoringFunction](https://github.com/ptranking/ptranking.github.io/raw/master/ptranking/testing/ltr_adhoc/json/) as an example. 65 | 66 | - **XParameter**, which specifies the parameters for model **X**. Please refer to [LambdaRankParameter](https://github.com/ptranking/ptranking.github.io/raw/master/ptranking/ltr_adhoc/listwise/lambdarank.py) as an example. 67 | -------------------------------------------------------------------------------- /docs/how_to_start/Develop_A_New_Model.md: -------------------------------------------------------------------------------- 1 | ## Develop Your Own Learning-to-Rank Method 2 | 3 | PT-Ranking offers deep neural networks as the basis to construct a scoring function based on PyTorch and can thus fully leverage the advantages of PyTorch. 4 | NeuralRanker is a class that represents a general learning-to-rank model. 5 | A key component of NeuralRanker is the neural scoring function. The configurable hyper-parameters include activation function, number of layers, number of neurons per layer, etc. 6 | All specific learning-to-rank models inherit NeuralRanker and mainly differ in the way of computing the training loss. 7 | The following figure shows the main step in developing a new learning-to-rank model based on Empirical Risk Minimization, 8 | where batch_preds and batch_stds correspond to outputs of the scoring function and ground-truth lables, respectively. 9 | We can observe that the main work is to define the surrogate loss function. 10 | 11 | ![](https://github.com/ptranking/ptranking.github.io/raw/master/img/new_loss.png) 12 | 13 | When incorporating a newly developed model (say ModelA), it is commonly required to develop the subclass ModelAParameter by inheriting **[ModelParameter](https://github.com/ptranking/ptranking.github.io/raw/master/ptranking/ltr_adhoc/eval/parameter.py)** and customizing the functions, such as to_para_string(), default_para_dict() and grid_search(). Please refer to [Configuration](./Configuration.md) for detailed description on parameter setting and [LambdaRankParameter](https://github.com/ptranking/ptranking.github.io/raw/master/ptranking/ltr_adhoc/listwise/lambdarank.py) as an example. 14 | 15 | To fully leverage PT-Ranking, one needs to [be familiar with PyTorch](https://pytorch.org/tutorials/beginner/deep_learning_60min_blitz.html). 16 | 17 | For detailed introduction on learning-to-rank, please refer to the book: [Learning to Rank for Information Retrieval](https://link.springer.com/book/10.1007/978-3-642-14267-3). 18 | -------------------------------------------------------------------------------- /docs/how_to_start/Get_Started.md: -------------------------------------------------------------------------------- 1 | ## Requirements 2 | 3 | 1. Prepare a virtual environment with Python 3.* via `conda`, `venv` or others. 4 | 5 | 2. Install Pytorch following the [instructions](https://pytorch.org/get-started/locally/) 6 | 7 | 3. Install scikit-learn following the [instructions](https://scikit-learn.org/stable/install.html#installation-instructions) 8 | 9 | ## Command-line Usage 10 | 11 | 1. Download source code 12 | 13 | ``` 14 | git clone https://github.com/wildltr/ptranking 15 | ``` 16 | 2. Download [Supported Datasets](../data/data.md), such as **MQ2008** and unrar the .rar file. 17 | 18 | ``` 19 | wget "https://lyoz5a.ch.files.1drv.com/y4mM8g8v4d2mFfO5djKT-ELADpDDRcsVwXRSaZu-9rlOlgvW62Qeuc8hFe_wr6m5NZMnUSEfr6QpMP81ZIQIiwI4BnoHmIZT9Sraf53AmhhIfLi531DOKYZTy4MtDHbBC7dn_Z9DSKvLJZhERPIamAXCrONg7WrFPiG0sTpOXl3-YEYZ1scTslmNyg2a__3YalWRMyEIipY56sy97pb68Sdww" -O MQ2008.rar 20 | ``` 21 | 3. Prepare the required json files (Data_Eval_ScoringFunction.json and XParameter.json, please refer to [Configuration](./Configuration.md) for more information) for specifying evaluation details. 22 | 23 | 4. Run the following command script on your Terminal/Command Prompt: 24 | 25 | (1) Without using GPU 26 | ``` 27 | python pt_ranking.py -model ListMLE -dir_json /home/dl-box/WorkBench/Dropbox/CodeBench/GitPool/wildltr_ptranking/testing/ltr_adhoc/json/ 28 | ``` 29 | (2) Using GPU 30 | ``` 31 | python pt_ranking.py -cuda 0 -model ListMLE -dir_json /home/dl-box/WorkBench/Dropbox/CodeBench/GitPool/wildltr_ptranking/testing/ltr_adhoc/json/ 32 | ``` 33 | The meaning of each supported argument is: 34 | ``` 35 | optional arguments: 36 | -h, --help show this help message and exit 37 | -cuda CUDA specify the gpu id if needed, such as 0 or 1. 38 | -model MODEL specify the learning-to-rank method 39 | -debug quickly check the setting in a debug mode 40 | -dir_json DIR_JSON the path of json files specifying the evaluation 41 | details. 42 | ``` 43 | 44 | ## PyCharm Usage 45 | 46 | 1. Install [PyCharm](https://www.jetbrains.com/pycharm/) (either Professional version or Community version) 47 | 48 | 2. Download source code 49 | 50 | ``` 51 | git clone https://github.com/wildltr/ptranking 52 | ``` 53 | 3. Open the unzipped source code with PyCharm as a new project 54 | 55 | 4. Test the supported learning-to-rank models by selectively running the following files, where the setting arguments can be changed accordingly 56 | ``` 57 | testing/ltr_adhoc/testing_ltr_adhoc.py 58 | testing/ltr_adversarial/testing_ltr_adversarial.py 59 | testing/ltr_tree/testing_ltr_tree.py 60 | ``` 61 | 62 | ## Python-package Usage 63 | 64 | TBA 65 | 66 | Install ptranking: pip install ptranking 67 | 68 | ## Demo Scripts 69 | 70 | To get a taste of learning-to-rank models without writing any code, you could try the following script. You just need to specify the model name, the dataset id, as well as the directories for input and output. 71 | 72 | - [Jupyter Notebook example on RankNet & LambdaRank](https://github.com/ptranking/ptranking.github.io/raw/master/tutorial/) 73 | 74 | To get familiar with the process of data loading, you could try the following script, namely, get the statistics of a dataset. 75 | 76 | - [Jupyter Notebook example on getting dataset statistics](https://github.com/ptranking/ptranking.github.io/raw/master/tutorial/) 77 | -------------------------------------------------------------------------------- /docs/img/new_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/docs/img/new_loss.png -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Learning-to-Rank in PyTorch 2 | 3 | ## Introduction 4 | 5 | This open-source project, referred to as **PTRanking** (Learning-to-Rank in PyTorch) aims to provide scalable and extendable implementations of typical learning-to-rank methods based on PyTorch. On one hand, this project enables a uniform comparison over several benchmark datasets, leading to an in-depth understanding of previous learning-to-rank methods. On the other hand, this project makes it easy to develop and incorporate newly proposed models, so as to expand the territory of techniques on learning-to-rank. 6 | 7 | **Key Features**: 8 | 9 | - A number of representative learning-to-rank models for addressing **Ad-hoc Ranking** and **Search Result Diversification**, including not only the traditional optimization framework via empirical risk minimization but also the adversarial optimization framework 10 | - Supports widely used benchmark datasets. Meanwhile, random masking of the ground-truth labels with a specified ratio is also supported 11 | - Supports different metrics, such as Precision, MAP, nDCG, nERR, alpha-nDCG and ERR-IA. 12 | - Highly configurable functionalities for fine-tuning hyper-parameters, e.g., grid-search over hyper-parameters of a specific model 13 | - Provides easy-to-use APIs for developing a new learning-to-rank model 14 | 15 | ### Source Code 16 | 17 | Please refer to the Github Repository [PT-Ranking](https://github.com/wildltr/ptranking/) for detailed implementations. 18 | 19 | ## Implemented models 20 | 21 | - Typical Learning-to-Rank Methods for Ad-hoc Ranking 22 | 23 | | |Model| 24 | |:----|:----| 25 | | Pointwise | RankMSE | 26 | | Pairwise | RankNet | 27 | | Listwise | ListNet ・ ListMLE ・ RankCosine ・ LambdaRank ・ ApproxNDCG ・ WassRank ・ STListNet ・ LambdaLoss| 28 | 29 | - Learning-to-Rank Methods for Search Result Diversification 30 | 31 | | |Model| 32 | |:----|:----| 33 | | Score-and-sort strategy | MO4SRD ・ DALETOR| 34 | 35 | - Adversarial Learning-to-Rank Methods for Ad-hoc Ranking 36 | 37 | | |Model| 38 | |:----|:----| 39 | | Pointwise | IR_GAN_Point | 40 | | Pairwise | IR_GAN_Pair | 41 | | Listwise | IR_GAN_List | 42 | 43 | - Learning-to-rank Methods Based on Gradient Boosting Decision Trees (GBDT) (based on LightGBM) 44 | 45 | | |Model| 46 | |:----|:----| 47 | | Listwise | LightGBMLambdaMART | 48 | 49 | #### References 50 | 51 | - **RankNet**: Chris Burges, Tal Shaked, Erin Renshaw, Ari Lazier, Matt Deeds, Nicole Hamilton, and Greg Hullender. 2005. Learning to rank using gradient descent. In Proceedings of the 22nd ICML. 89–96. 52 | 53 | - **RankSVM**: Joachims, Thorsten. Optimizing Search Engines Using Clickthrough Data. Proceedings of the Eighth ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, 133–142, 2002. 54 | 55 | - **LambdaRank**: Christopher J.C. Burges, Robert Ragno, and Quoc Viet Le. 2006. Learning to Rank with Nonsmooth Cost Functions. In Proceedings of NIPS conference. 193–200. 56 | 57 | - **ListNet**: Zhe Cao, Tao Qin, Tie-Yan Liu, Ming-Feng Tsai, and Hang Li. 2007. Learning to Rank: From Pairwise Approach to Listwise Approach. In Proceedings of the 24th ICML. 129–136. 58 | 59 | - **ListMLE**: Fen Xia, Tie-Yan Liu, Jue Wang, Wensheng Zhang, and Hang Li. 2008. Listwise Approach to Learning to Rank: Theory and Algorithm. In Proceedings of the 25th ICML. 1192–1199. 60 | 61 | - **RankCosine**: Tao Qin, Xu-Dong Zhang, Ming-Feng Tsai, De-Sheng Wang, Tie-Yan Liu, and Hang Li. 2008. Query-level loss functions for information retrieval. Information Processing and Management 44, 2 (2008), 838–855. 62 | 63 | - **AppoxNDCG**: Tao Qin, Tie-Yan Liu, and Hang Li. 2010. A general approximation framework for direct optimization of information retrieval measures. Journal of Information Retrieval 13, 4 (2010), 375–397. 64 | 65 | - **LambdaMART**: Q. Wu, C.J.C. Burges, K. Svore and J. Gao. Adapting Boosting for Information Retrieval Measures. Journal of Information Retrieval, 2007. 66 | (We note that the implementation is provided by [LightGBM](https://lightgbm.readthedocs.io/en/latest/)) 67 | 68 | - **IRGAN**: Wang, Jun and Yu, Lantao and Zhang, Weinan and Gong, Yu and Xu, Yinghui and Wang, Benyou and Zhang, Peng and Zhang, Dell. IRGAN: A Minimax Game for Unifying Generative and Discriminative Information Retrieval Models. Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval, 515–524, 2017. (**Besides the pointwise and pairiwse adversarial learning-to-rank methods introduced in the paper, we also include the listwise version in PT-Ranking**) 69 | 70 | - **LambdaLoss** Xuanhui Wang, Cheng Li, Nadav Golbandi, Mike Bendersky and Marc Najork. The LambdaLoss Framework for Ranking Metric Optimization. Proceedings of The 27th ACM International Conference on Information and Knowledge Management (CIKM '18), 1313-1322, 2018. 71 | 72 | - **WassRank**: Hai-Tao Yu, Adam Jatowt, Hideo Joho, Joemon Jose, Xiao Yang and Long Chen. WassRank: Listwise Document Ranking Using Optimal Transport Theory. Proceedings of the 12th International Conference on Web Search and Data Mining (WSDM), 24-32, 2019. 73 | 74 | - Bruch, Sebastian and Han, Shuguang and Bendersky, Michael and Najork, Marc. A Stochastic Treatment of Learning to Rank Scoring Functions. Proceedings of the 13th International Conference on Web Search and Data Mining (WSDM), 61–69, 2020. 75 | 76 | - **MO4SRD**: Hai-Tao Yu. Optimize What You EvaluateWith: Search Result Diversification Based on Metric 77 | Optimization. The 36th AAAI Conference on Artificial Intelligence, 2022. 78 | 79 | - **DALETOR**: Le Yan, Zhen Qin, Rama Kumar Pasumarthi, Xuanhui Wang, Michael Bendersky. Diversification-Aware Learning to Rank 80 | using Distributed Representation. In Proceedings of the Web Conference 2021, 127–136. 81 | 82 | ## Test Setting 83 | 84 | PyTorch (>=1.3) 85 | 86 | Python (3) 87 | 88 | Ubuntu 16.04 LTS 89 | 90 | ## Call for Contribution 91 | 92 | We are adding more learning-to-rank models all the time. Please submit an issue if there is something you want to have implemented and included. Meanwhile, 93 | anyone who are interested in any kinds of contributions and/or collaborations are warmly welcomed. 94 | 95 | ## Citation 96 | 97 | If you use PTRanking in your research, please use the following BibTex entry. 98 | 99 | ``` 100 | @misc{yu2020ptranking, 101 | title={PT-Ranking: A Benchmarking Platform for Neural Learning-to-Rank}, 102 | author={Hai-Tao Yu}, 103 | year={2020}, 104 | eprint={2008.13368}, 105 | archivePrefix={arXiv}, 106 | primaryClass={cs.IR} 107 | } 108 | ``` 109 | 110 | ## Relevant Resources 111 | 112 | | Name | Language | Deep Learning | 113 | |---|---|---| 114 | | [PTRanking](https://ptranking.github.io/) | Python | [PyTorch](https://pytorch.org) | 115 | | [TF-Ranking](https://github.com/tensorflow/ranking) | Python | [TensorFlow](https://tensorflow.org) | 116 | | [MatchZoo](https://github.com/NTMC-Community/MatchZoo) | Python | [Keras](https://github.com/keras-team/keras) / [PyTorch](https://pytorch.org) | 117 | | [Shoelace](https://github.com/rjagerman/shoelace) | Python | [Chainer](https://chainer.org) | 118 | | [LEROT](https://bitbucket.org/ilps/lerot) | Python | x | 119 | | [Rank Lib](http://www.lemurproject.org/ranklib.php) | Java | x | 120 | | [Propensity SVM^Rank](http://www.cs.cornell.edu/people/tj/svm_light/svm_proprank.html) | C | x | 121 | | [QuickRank](https://github.com/hpclab/quickrank) | C++ | x | 122 | -------------------------------------------------------------------------------- /docs/ltr_adhoc/about.md: -------------------------------------------------------------------------------- 1 | 2 | ## About LTR_Adhoc 3 | By **LTR_Adhoc**, we refer to the traditional learning-to-rank methods based on the Empirical Risk Minimization Framework, which is detailed in [ptranking_empirical_risk_minimization.ipynb](https://github.com/ptranking/ptranking.github.io/raw/master/tutorial/). 4 | 5 | Major learning-to-rank approaches can be classified into three categories: **pointwise**, **pairwise**, and **listwise**. The key distinctions are the underlying hypotheses, loss functions, the input and output spaces. 6 | 7 | The typical pointwise approaches include regression-based [1], classification-based [2], and ordinal regression-based algorithms [3, 4]. The loss functions of these algorithms is defined on the basis of each individual document. 8 | 9 | The pairwise approaches care about the relative order between two documents. The goal of learning is to maximize the number of correctly ordered document pairs. The assumption is that the optimal ranking of documents can be achieved if all the document pairs are correctly ordered. Towards this end, many representative methods have been proposed [5,6,7,8,9]. 10 | 11 | The listwise approaches take all the documents associated with the same query in the training data as the input. In particular, there are two types of loss functions when performing listwise learning. For the first type, the loss function is related to a specific evaluation metric (e.g., nDCG and ERR). Due to the non-differentiability and non-decomposability of the commonly used metrics, the methods of this type either try to optimize the upper bounds as surrogate objective functions [10, 11, 12] or approximate the target metric using some smooth 12 | functions [13, 14, 15]. However, there are still some open issues regarding the first type methods. On one hand, some adopted surrogate functions or approximated metrics are not convex, which makes it hard to optimize. On the other hand, the relationship between the surrogate function and the adopted metric has not been sufficiently investigated, which makes it unclear whether optimizing the surrogate functions can indeed optimize the target metric. For the second type, the loss function is not explicitly related to a specific evaluation metric. The loss function reflects the discrepancy between the predicted ranking and the ground-truth ranking. Example algorithms include []. Although no particular evaluation metrics are directly involved and optimized here, it is possible that the learned ranking function can achieve good performance in terms of evaluation metrics. 13 | 14 | ## References 15 | 16 | - [1] David Cossock and Tong Zhang. 2006. Subset Ranking Using Regression. In 17 | Proceedings of the 19th Annual Conference on Learning Theory. 605–619. 18 | - [2] Ramesh Nallapati. 2004. Discriminative Models for Information Retrieval. In 19 | Proceedings of the 27th SIGIR. 64–71. 20 | - [3] Wei Chu and Zoubin Ghahramani. 2005. Gaussian Processes for Ordinal Regression. 21 | Journal of Machine Learning Research 6 (2005), 1019–1041. 22 | - [4] Wei Chu and S. Sathiya Keerthi. 2005. New Approaches to Support Vector Ordinal 23 | Regression. In Proceedings of the 22nd ICML. 145–152. 24 | - [5] Chris Burges, Tal Shaked, Erin Renshaw, Ari Lazier, Matt Deeds, Nicole Hamilton, 25 | and Greg Hullender. 2005. Learning to rank using gradient descent. In Proceedings 26 | of the 22nd ICML. 89–96. 27 | - [6] Yoav Freund, Raj Iyer, Robert E. Schapire, and Yoram Singer. 2003. An Efficient 28 | Boosting Algorithm for Combining Preferences. Journal of Machine Learning 29 | Research 4 (2003), 933–969. 30 | - [7] Thorsten Joachims. 2002. Optimizing search engines using clickthrough data. In 31 | Proceedings of the 8th KDD. 133–142. 32 | - [8] Libin Shen and Aravind K. Joshi. 2005. Ranking and Reranking with Perceptron. 33 | Machine Learning 60, 1-3 (2005), 73–96. 34 | - [9] Fajie Yuan, Guibing Guo, Joemon Jose, Long Chen, Hai-Tao Yu, andWeinan Zhang. 35 | 2016. LambdaFM: Learning Optimal Ranking with Factorization Machines Using 36 | Lambda Surrogates. In Proceedings of the 25th CIKM. 227–236. 37 | - [10] Olivier Chapelle, Quoc Le, and Alex Smola. 2007. Large margin optimization of 38 | ranking measures. In NIPS workshop on Machine Learning for Web Search. 39 | - [11] Jun Xu and Hang Li. 2007. AdaRank: a boosting algorithm for information 40 | retrieval. In Proceedings of the 30th SIGIR. 391–398. 41 | - [12] Yisong Yue, Thomas Finley, Filip Radlinski, and Thorsten Joachims. 2007. A 42 | Support Vector Method for Optimizing Average Precision. In Proceedings of the 43 | 30th SIGIR. 271–278. 44 | - [13] John Guiver and Edward Snelson. 2008. Learning to Rank with SoftRank and 45 | Gaussian Processes. In Proceedings of the 31st SIGIR. 259–266. 46 | - [14] Tao Qin, Tie-Yan Liu, and Hang Li. 2010. A general approximation framework 47 | for direct optimization of information retrieval measures. Journal of Information 48 | Retrieval 13, 4 (2010), 375–397. 49 | - [15] Michael Taylor, John Guiver, Stephen Robertson, and Tom Minka. 2008. SoftRank: 50 | Optimizing Non-smooth Rank Metrics. In Proceedings of the 1st WSDM. 77–86. 51 | - [16] Christopher J.C. Burges, Robert Ragno, and Quoc Viet Le. 2006. Learning to Rank 52 | with Nonsmooth Cost Functions. In Proceedings of NIPS conference. 193–200. 53 | - [17] Zhe Cao, Tao Qin, Tie-Yan Liu, Ming-Feng Tsai, and Hang Li. 2007. Learning to 54 | Rank: From Pairwise Approach to Listwise Approach. In Proceedings of the 24th 55 | ICML. 129–136. 56 | - [18] Fen Xia, Tie-Yan Liu, Jue Wang, Wensheng Zhang, and Hang Li. 2008. Listwise 57 | Approach to Learning to Rank: Theory and Algorithm. In Proceedings of the 25th 58 | ICML. 1192–1199. 59 | -------------------------------------------------------------------------------- /docs/ltr_adhoc/lambda_framework.md: -------------------------------------------------------------------------------- 1 | 2 | ## From RankNet to LambdaRank to LambdaMART: A Revisit 3 | 4 | Please refer to [ptranking_lambda_framework.ipynb](https://github.com/ptranking/ptranking.github.io/raw/master/tutorial/) for detailed description. 5 | -------------------------------------------------------------------------------- /docs/ltr_adversarial/about.md: -------------------------------------------------------------------------------- 1 | 2 | ## About LTR_Adversarial 3 | By **LTR_Adversarial**, we refer to the adversarial learning-to-rank methods inspired by the generative adversarial network 4 | (GAN) and its variants. 5 | -------------------------------------------------------------------------------- /docs/ltr_adversarial/irgan.md: -------------------------------------------------------------------------------- 1 | ## IRGAN 2 | 3 | Please refer to the following paper for detailed description. 4 | 5 | - **IRGAN**: Wang, Jun and Yu, Lantao and Zhang, Weinan and Gong, Yu and Xu, Yinghui and Wang, Benyou and Zhang, Peng and Zhang, Dell. IRGAN: A Minimax Game for Unifying Generative and Discriminative Information Retrieval Models. Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval, 515–524, 2017. 6 | -------------------------------------------------------------------------------- /docs/ltr_diversification/Direct_Metric_Optimization.md: -------------------------------------------------------------------------------- 1 | Please refer to the following papers for detailed description. 2 | 3 | - Hai-Tao Yu. Optimize What You EvaluateWith: Search Result Diversification Based on Metric Optimization. The 36th AAAI Conference on Artificial Intelligence, 2022. 4 | - Le Yan, Zhen Qin, Rama Kumar Pasumarthi, Xuanhui Wang, Michael Bendersky. Diversification-Aware Learning to Rank using Distributed Representation. In Proceedings of the Web Conference 2021, 127–136. 5 | -------------------------------------------------------------------------------- /docs/ltr_diversification/about.md: -------------------------------------------------------------------------------- 1 | ## About LTR_Diversification 2 | By **LTR_Diversification**, we refer to the learning-to-rank methods for search result diversification. 3 | -------------------------------------------------------------------------------- /docs/ltr_gbm/LambdaMART.md: -------------------------------------------------------------------------------- 1 | 2 | ## From RankNet to LambdaRank to LambdaMART: A Revisit 3 | 4 | Please refer to [ptranking_gbm.ipynb](https://github.com/ptranking/ptranking.github.io/raw/master/tutorial/) for detailed description. 5 | -------------------------------------------------------------------------------- /docs/ltr_gbm/about.md: -------------------------------------------------------------------------------- 1 | 2 | ## About LTR_GBM 3 | By **LTR_GBM**, we refer to the learning-to-rank methods based on the framework of Gradient Boosting Machine (GBM). 4 | -------------------------------------------------------------------------------- /docs/metric/metric.md: -------------------------------------------------------------------------------- 1 | 2 | In PT-Ranking, the widely used IR metrics, such as **precision**, **average precision**, **nDCG** and **ERR**, are included. 3 | 4 | ## Definition 5 | 6 | For the definition of each metric, please refer to [ptranking_ir_metric.ipynb](https://github.com/ptranking/ptranking.github.io/raw/master/tutorial/) for a brief description. 7 | 8 | ## Implementation 9 | 10 | For the implementation based on PyTorch, please refer to [adhoc_metric.py](https://github.com/ptranking/ptranking.github.io/raw/master/ptranking/metric/) and [testing_metric.py](https://github.com/ptranking/ptranking.github.io/raw/master/testing/metric/) for a reference. 11 | -------------------------------------------------------------------------------- /img/histogram_split.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/img/histogram_split.png -------------------------------------------------------------------------------- /img/new_loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/img/new_loss.png -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: PT-Ranking 2 | nav: 3 | - Home: index.md 4 | - How-to-Start: 5 | - how_to_start/Get_Started.md 6 | - how_to_start/Configuration.md 7 | - how_to_start/Develop_A_New_Model.md 8 | - Data: data/data.md 9 | - Metric: metric/metric.md 10 | - LTR_Adhoc: 11 | - ltr_adhoc/about.md 12 | - ltr_adhoc/Lambda_Framework.md 13 | - LTR_Diversification: 14 | - ltr_diversification/about.md 15 | - ltr_diversification/Direct_Metric_Optimization.md 16 | - LTR_Adversarial: 17 | - ltr_adversarial/about.md 18 | - ltr_adversarial/IRGAN.md 19 | - LTR_GBM: 20 | - ltr_gbm/about.md 21 | - ltr_gbm/LambdaMART.md 22 | 23 | theme: 24 | name: 'material' 25 | palette: 26 | primary: 'red' 27 | accent: 'red' 28 | markdown_extensions: 29 | - codehilite 30 | - toc: 31 | permalink: true 32 | - attr_list 33 | repo_name: 'PT-Ranking' 34 | repo_url: 'https://github.com/wildltr/ptranking/' 35 | -------------------------------------------------------------------------------- /pt_ranking.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | 6 | """ 7 | 8 | import sys 9 | from ptranking.utils.args.argsUtil import ArgsUtil 10 | 11 | from ptranking.ltr_adhoc.eval.ltr import LTREvaluator, LTR_ADHOC_MODEL 12 | from ptranking.ltr_tree.eval.ltr_tree import TreeLTREvaluator, LTR_TREE_MODEL 13 | from ptranking.ltr_adversarial.eval.ltr_adversarial import AdLTREvaluator, LTR_ADVERSARIAL_MODEL 14 | 15 | 16 | """ 17 | The command line usage: 18 | 19 | (1) Without using GPU 20 | python pt_ranking.py -model ListMLE -dir_json /home/dl-box/WorkBench/Dropbox/CodeBench/GitPool/wildltr_ptranking/testing/ltr_adhoc/json/ 21 | 22 | (2) Using GPU 23 | python pt_ranking.py -cuda 0 -model ListMLE -dir_json /home/dl-box/WorkBench/Dropbox/CodeBench/GitPool/wildltr_ptranking/testing/ltr_adhoc/json/ 24 | 25 | """ 26 | 27 | if __name__ == '__main__': 28 | 29 | """ 30 | >>> Learning-to-Rank Models <<< 31 | (1) Optimization based on Empirical Risk Minimization 32 | ----------------------------------------------------------------------------------------- 33 | | Pointwise | RankMSE | 34 | ----------------------------------------------------------------------------------------- 35 | | Pairwise | RankNet | 36 | ----------------------------------------------------------------------------------------- 37 | | Listwise | LambdaRank % ListNet % ListMLE % RankCosine % ApproxNDCG % WassRank | 38 | | | STListNet % LambdaLoss | 39 | ----------------------------------------------------------------------------------------- 40 | 41 | (2) Adversarial Optimization 42 | ----------------------------------------------------------------------------------------- 43 | | Pointwise | IRGAN_Point | 44 | ----------------------------------------------------------------------------------------- 45 | | Pairwise | IRGAN_Pair | 46 | ----------------------------------------------------------------------------------------- 47 | | Listwise | IRGAN_List | 48 | ----------------------------------------------------------------------------------------- 49 | 50 | (3) Tree-based Model (provided by LightGBM) 51 | ----------------------------------------------------------------------------------------- 52 | | LightGBMLambdaMART | 53 | ----------------------------------------------------------------------------------------- 54 | 55 | 56 | >>> Supported Datasets <<< 57 | ----------------------------------------------------------------------------------------- 58 | | LETTOR | MQ2007_Super % MQ2008_Super % MQ2007_Semi % MQ2008_Semi | 59 | ----------------------------------------------------------------------------------------- 60 | | MSLRWEB | MSLRWEB10K % MSLRWEB30K | 61 | ----------------------------------------------------------------------------------------- 62 | | Yahoo_LTR | Set1 % Set2 | 63 | ----------------------------------------------------------------------------------------- 64 | | ISTELLA_LTR | Istella_S | Istella | Istella_X | 65 | ----------------------------------------------------------------------------------------- 66 | 67 | """ 68 | 69 | args_obj = ArgsUtil(given_root='./') 70 | l2r_args = args_obj.get_l2r_args() 71 | 72 | if l2r_args.model in LTR_ADHOC_MODEL: 73 | evaluator = LTREvaluator(cuda=l2r_args.cuda) 74 | 75 | elif l2r_args.model in LTR_ADVERSARIAL_MODEL: 76 | evaluator = AdLTREvaluator(cuda=l2r_args.cuda) 77 | 78 | elif l2r_args.model in LTR_TREE_MODEL: 79 | evaluator = TreeLTREvaluator() 80 | 81 | else: 82 | args_obj.args_parser.print_help() 83 | sys.exit() 84 | 85 | print('Started evaluation with pt_ranking !') 86 | evaluator.run(model_id=l2r_args.model, dir_json=l2r_args.dir_json, debug=l2r_args.debug, config_with_json=True) 87 | print('Finished evaluation with pt_ranking !') 88 | 89 | -------------------------------------------------------------------------------- /ptranking/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /ptranking/base/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /ptranking/base/adhoc_ranker.py: -------------------------------------------------------------------------------- 1 | 2 | from ptranking.data.data_utils import LABEL_TYPE 3 | from ptranking.base.list_ranker import ListNeuralRanker 4 | from ptranking.base.point_ranker import PointNeuralRanker 5 | 6 | 7 | class AdhocNeuralRanker(PointNeuralRanker, ListNeuralRanker): 8 | ''' 9 | A combination of PointNeuralRanker & PENeuralRanker 10 | ''' 11 | def __init__(self, id='AdhocNeuralRanker', sf_para_dict=None, weight_decay=1e-3, gpu=False, device=None): 12 | self.id = id 13 | self.gpu, self.device = gpu, device 14 | 15 | self.sf_para_dict = sf_para_dict 16 | self.sf_id = sf_para_dict['sf_id'] 17 | assert self.sf_id in ['pointsf', 'listsf'] 18 | 19 | self.opt, self.lr = sf_para_dict['opt'], sf_para_dict['lr'] 20 | self.weight_decay = weight_decay 21 | 22 | self.stop_check_freq = 10 23 | 24 | if 'listsf' == self.sf_id: 25 | self.encoder_type = self.sf_para_dict[self.sf_para_dict['sf_id']]['encoder_type'] 26 | 27 | def init(self): 28 | if 'pointsf' == self.sf_id: 29 | PointNeuralRanker.init(self) 30 | elif 'listsf' == self.sf_id: 31 | ListNeuralRanker.init(self) 32 | 33 | def get_parameters(self): 34 | if 'pointsf' == self.sf_id: 35 | return PointNeuralRanker.get_parameters(self) 36 | elif 'listsf' == self.sf_id: 37 | return ListNeuralRanker.get_parameters(self) 38 | 39 | def forward(self, batch_q_doc_vectors): 40 | if 'pointsf' == self.sf_id: 41 | return PointNeuralRanker.forward(self, batch_q_doc_vectors) 42 | elif 'listsf' == self.sf_id: 43 | return ListNeuralRanker.forward(self, batch_q_doc_vectors) 44 | 45 | def eval_mode(self): 46 | if 'pointsf' == self.sf_id: 47 | PointNeuralRanker.eval_mode(self) 48 | elif 'listsf' == self.sf_id: 49 | ListNeuralRanker.eval_mode(self) 50 | 51 | def train_mode(self): 52 | if 'pointsf' == self.sf_id: 53 | PointNeuralRanker.train_mode(self) 54 | elif 'listsf' == self.sf_id: 55 | ListNeuralRanker.train_mode(self) 56 | 57 | # depreated due to moving to Evaluator 58 | def _validation(self, vali_data=None, vali_metric=None, k=5, presort=False, max_label=None, label_type=LABEL_TYPE.MultiLabel, device='cpu'): 59 | if 'nDCG' == vali_metric: 60 | return self.ndcg_at_k(test_data=vali_data, k=k, label_type=label_type, presort=presort, device=device) 61 | elif 'nERR' == vali_metric: 62 | return self.nerr_at_k(test_data=vali_data, k=k, label_type=label_type, 63 | max_label=max_label, presort=presort, device=device) 64 | elif 'AP' == vali_metric: 65 | return self.ap_at_k(test_data=vali_data, k=k, presort=presort, device=device) 66 | elif 'P' == vali_metric: 67 | return self.p_at_k(test_data=vali_data, k=k, device=device) 68 | else: 69 | raise NotImplementedError 70 | 71 | def save(self, dir, name): 72 | if 'pointsf' == self.sf_id: 73 | PointNeuralRanker.save(self, dir=dir, name=name) 74 | elif 'listsf' == self.sf_id: 75 | ListNeuralRanker.save(self, dir=dir, name=name) 76 | 77 | def load(self, file_model, device): 78 | if 'pointsf' == self.sf_id: 79 | PointNeuralRanker.load(self, file_model=file_model, device=device) 80 | elif 'listsf' == self.sf_id: 81 | ListNeuralRanker.load(self, file_model=file_model, device=device) 82 | 83 | def get_tl_af(self): 84 | if 'pointsf' == self.sf_id: 85 | PointNeuralRanker.get_tl_af(self) 86 | elif 'listsf' == self.sf_id: 87 | self.sf_para_dict[self.sf_para_dict['sf_id']]['AF'] 88 | -------------------------------------------------------------------------------- /ptranking/base/point_ranker.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | 4 | import torch 5 | 6 | from ptranking.base.ranker import NeuralRanker 7 | from ptranking.base.utils import get_stacked_FFNet 8 | 9 | class PointNeuralRanker(NeuralRanker): 10 | ''' 11 | A one-size-fits-all neural ranker. 12 | Given the documents associated with the same query, this ranker scores each document independently. 13 | ''' 14 | def __init__(self, id='PointNeuralRanker', sf_para_dict=None, weight_decay=1e-3, gpu=False, device=None): 15 | super(PointNeuralRanker, self).__init__(id=id, sf_para_dict=sf_para_dict, weight_decay=weight_decay, gpu=gpu, device=device) 16 | 17 | def init(self): 18 | self.point_sf = self.config_point_neural_scoring_function() 19 | self.config_optimizer() 20 | 21 | def config_point_neural_scoring_function(self): 22 | point_sf = self.ini_pointsf(**self.sf_para_dict[self.sf_para_dict['sf_id']]) 23 | if self.gpu: point_sf = point_sf.to(self.device) 24 | 25 | return point_sf 26 | 27 | def get_parameters(self): 28 | return self.point_sf.parameters() 29 | 30 | def ini_pointsf(self, num_features=None, h_dim=100, out_dim=1, num_layers=3, AF='R', TL_AF='S', apply_tl_af=False, 31 | BN=True, bn_type=None, bn_affine=False, dropout=0.1): 32 | ''' 33 | Initialization of a feed-forward neural network 34 | ''' 35 | ff_dims = [num_features] 36 | for i in range(num_layers): 37 | ff_dims.append(h_dim) 38 | ff_dims.append(out_dim) 39 | 40 | point_sf = get_stacked_FFNet(ff_dims=ff_dims, AF=AF, TL_AF=TL_AF, apply_tl_af=apply_tl_af, dropout=dropout, 41 | BN=BN, bn_type=bn_type, bn_affine=bn_affine, device=self.device) 42 | return point_sf 43 | 44 | 45 | def forward(self, batch_q_doc_vectors): 46 | ''' 47 | Forward pass through the scoring function, where each document is scored independently. 48 | @param batch_q_doc_vectors: [batch_size, num_docs, num_features], the latter two dimensions {num_docs, num_features} denote feature vectors associated with the same query. 49 | @return: 50 | ''' 51 | batch_size, num_docs, num_features = batch_q_doc_vectors.size() 52 | 53 | _batch_preds = self.point_sf(batch_q_doc_vectors) 54 | batch_preds = _batch_preds.view(-1, num_docs) # [batch_size x num_docs, 1] -> [batch_size, num_docs] 55 | return batch_preds 56 | 57 | def eval_mode(self): 58 | self.point_sf.eval() 59 | 60 | def train_mode(self): 61 | self.point_sf.train(mode=True) 62 | 63 | def save(self, dir, name): 64 | if not os.path.exists(dir): 65 | os.makedirs(dir) 66 | 67 | torch.save(self.point_sf.state_dict(), dir + name) 68 | 69 | def load(self, file_model, **kwargs): 70 | device = kwargs['device'] 71 | self.point_sf.load_state_dict(torch.load(file_model, map_location=device)) 72 | 73 | def get_tl_af(self): 74 | return self.sf_para_dict[self.sf_para_dict['sf_id']]['TL_AF'] 75 | -------------------------------------------------------------------------------- /ptranking/data/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/eval/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/listwise/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/listwise/approxNDCG.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | Tao Qin, Tie-Yan Liu, and Hang Li. 2010. 6 | A general approximation framework for direct optimization of information retrieval measures. 7 | Journal of Information Retrieval 13, 4 (2010), 375–397. 8 | """ 9 | 10 | import torch 11 | 12 | from ptranking.data.data_utils import LABEL_TYPE 13 | from ptranking.base.adhoc_ranker import AdhocNeuralRanker 14 | from ptranking.ltr_adhoc.eval.parameter import ModelParameter 15 | from ptranking.metric.adhoc.adhoc_metric import torch_dcg_at_k 16 | from ptranking.base.utils import robust_sigmoid 17 | 18 | 19 | def get_approx_ranks(input, alpha=10, device=None): 20 | ''' get approximated rank positions: Equation-11 in the paper''' 21 | batch_pred_diffs = torch.unsqueeze(input, dim=2) - torch.unsqueeze(input, dim=1) # computing pairwise differences, i.e., Sij or Sxy 22 | 23 | batch_indicators = robust_sigmoid(torch.transpose(batch_pred_diffs, dim0=1, dim1=2), alpha, device) # using {-1.0*} may lead to a poor performance when compared with the above way; 24 | 25 | batch_hat_pis = torch.sum(batch_indicators, dim=2) + 0.5 # get approximated rank positions, i.e., hat_pi(x) 26 | 27 | return batch_hat_pis 28 | 29 | 30 | def approxNDCG(batch_preds=None, batch_stds=None, alpha=10, label_type=None, device=None): 31 | batch_hat_pis = get_approx_ranks(batch_preds, alpha=alpha, device=device) 32 | 33 | ''' since the input standard labels are sorted in advance, thus directly used ''' 34 | # sorted_labels, _ = torch.sort(batch_stds, dim=1, descending=True) # for optimal ltr_adhoc based on standard labels 35 | batch_idcgs = torch_dcg_at_k(batch_sorted_labels=batch_stds, cutoff=None, label_type=label_type, device=device) # ideal dcg given standard labels 36 | 37 | batch_gains = torch.pow(2.0, batch_stds) - 1.0 38 | 39 | batch_dcg = torch.sum(torch.div(batch_gains, torch.log2(batch_hat_pis + 1)), dim=1) 40 | batch_approx_nDCG = torch.div(batch_dcg, batch_idcgs) 41 | 42 | return batch_approx_nDCG 43 | 44 | 45 | def approxNDCG_loss(batch_preds=None, batch_ideal_rankings=None, alpha=10, label_type=None, device=None): 46 | batch_hat_pis = get_approx_ranks(batch_preds, alpha=alpha, device=device) 47 | 48 | # ideal dcg given optimally ordered labels 49 | batch_idcgs = torch_dcg_at_k(batch_rankings=batch_ideal_rankings, cutoff=None, label_type=label_type, device=device) 50 | 51 | if LABEL_TYPE.MultiLabel == label_type: 52 | batch_gains = torch.pow(2.0, batch_ideal_rankings) - 1.0 53 | elif LABEL_TYPE.Permutation == label_type: 54 | batch_gains = batch_ideal_rankings 55 | else: 56 | raise NotImplementedError 57 | 58 | batch_dcg = torch.sum(torch.div(batch_gains, torch.log2(batch_hat_pis + 1)), dim=1) 59 | batch_approx_nDCG = torch.div(batch_dcg, batch_idcgs) 60 | 61 | batch_loss = -torch.sum(batch_approx_nDCG) 62 | return batch_loss 63 | 64 | 65 | 66 | 67 | class ApproxNDCG(AdhocNeuralRanker): 68 | ''' 69 | Tao Qin, Tie-Yan Liu, and Hang Li. 2010. 70 | A general approximation framework for direct optimization of information retrieval measures. 71 | Journal of Information Retrieval 13, 4 (2010), 375–397. 72 | ''' 73 | 74 | def __init__(self, sf_para_dict=None, model_para_dict=None, gpu=False, device=None): 75 | super(ApproxNDCG, self).__init__(id='ApproxNDCG', sf_para_dict=sf_para_dict, gpu=gpu, device=device) 76 | self.alpha = model_para_dict['alpha'] 77 | 78 | def uniform_eval_setting(self, **kwargs): 79 | eval_dict = kwargs['eval_dict'] 80 | if eval_dict["do_validation"] and not eval_dict['vali_metric']=='nDCG': 81 | eval_dict['vali_metric'] = "nDCG" 82 | 83 | def custom_loss_function(self, batch_preds, batch_std_labels, **kwargs): 84 | ''' 85 | @param batch_preds: [batch, ranking_size] each row represents the relevance predictions for documents associated with the same query 86 | @param batch_std_labels: [batch, ranking_size] each row represents the standard relevance grades for documents associated with the same query 87 | @param kwargs: 88 | @return: 89 | ''' 90 | label_type = kwargs['label_type'] 91 | assert label_type == LABEL_TYPE.MultiLabel 92 | 93 | if 'presort' in kwargs and kwargs['presort']: 94 | target_batch_preds, batch_ideal_rankings = batch_preds, batch_std_labels 95 | else: 96 | batch_ideal_rankings, batch_ideal_desc_inds = torch.sort(batch_std_labels, dim=1, descending=True) 97 | target_batch_preds = torch.gather(batch_preds, dim=1, index=batch_ideal_desc_inds) 98 | 99 | ''' 100 | Given the ideal rankings, the optimization objective is to maximize the approximated nDCG based on differentiable rank positions 101 | ''' 102 | batch_loss = approxNDCG_loss(batch_preds=target_batch_preds, batch_ideal_rankings=batch_ideal_rankings, 103 | alpha=self.alpha, label_type=label_type, device=self.device) 104 | 105 | self.optimizer.zero_grad() 106 | batch_loss.backward() 107 | self.optimizer.step() 108 | 109 | return batch_loss 110 | 111 | #------- 112 | def get_apxndcg_paras_str(model_para_dict, log=False): 113 | s1 = ':' if log else '_' 114 | apxNDCG_paras_str = s1.join(['Alpha', str(model_para_dict['alpha'])]) 115 | 116 | return apxNDCG_paras_str 117 | 118 | ###### Parameter of ApproxNDCG ###### 119 | 120 | class ApproxNDCGParameter(ModelParameter): 121 | ''' Parameter class for ApproxNDCG ''' 122 | def __init__(self, debug=False, para_json=None): 123 | super(ApproxNDCGParameter, self).__init__(model_id='ApproxNDCG', para_json=para_json) 124 | self.debug = debug 125 | 126 | def default_para_dict(self): 127 | """ 128 | Default parameter setting for ApproxNDCG 129 | :return: 130 | """ 131 | self.apxNDCG_para_dict = dict(model_id=self.model_id, alpha=10.) 132 | return self.apxNDCG_para_dict 133 | 134 | def to_para_string(self, log=False, given_para_dict=None): 135 | """ 136 | String identifier of parameters 137 | :param log: 138 | :param given_para_dict: a given dict, which is used for maximum setting w.r.t. grid-search 139 | :return: 140 | """ 141 | # using specified para-dict or inner para-dict 142 | apxNDCG_para_dict = given_para_dict if given_para_dict is not None else self.apxNDCG_para_dict 143 | 144 | s1 = ':' if log else '_' 145 | apxNDCG_paras_str = s1.join(['Alpha', str(apxNDCG_para_dict['alpha'])]) 146 | return apxNDCG_paras_str 147 | 148 | def grid_search(self): 149 | """ 150 | Iterator of parameter settings for ApproxNDCG 151 | """ 152 | if self.use_json: 153 | choice_alpha = self.json_dict['alpha'] 154 | else: 155 | choice_alpha = [10.0] if self.debug else [10.0] # 1.0, 10.0, 50.0, 100.0 156 | 157 | for alpha in choice_alpha: 158 | self.apxNDCG_para_dict = dict(model_id=self.model_id, alpha=alpha) 159 | yield self.apxNDCG_para_dict 160 | -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/listwise/dasalc.py: -------------------------------------------------------------------------------- 1 | 2 | import torch 3 | import torch.nn.functional as F 4 | 5 | from ptranking.base.list_ranker import ListNeuralRanker 6 | 7 | class DASALC(ListNeuralRanker): 8 | ''' 9 | Zhen Qin, Le Yan, Honglei Zhuang, Yi Tay, Rama Kumar Pasumarthi, Xuanhui Wang, Mike Bendersky, and Marc Najork. 10 | Are Neural Rankers still Outperformed by Gradient Boosted Decision Trees?. In Proceedings of ICLR, 2021. 11 | ''' 12 | def __init__(self, sf_para_dict=None, gpu=False, device=None): 13 | super(DASALC, self).__init__(id='DASALC', sf_para_dict=sf_para_dict, gpu=gpu, device=device) 14 | assert 'listsf' == sf_para_dict['sf_id'] 15 | 16 | def custom_loss_function(self, batch_preds, batch_std_labels, **kwargs): 17 | ''' 18 | The Top-1 approximated ListNet loss, which reduces to a softmax and simple cross entropy. 19 | @param batch_preds: [batch, ranking_size] each row represents the relevance predictions for documents associated with the same query 20 | @param batch_std_labels: [batch, ranking_size] each row represents the standard relevance grades for documents associated with the same query 21 | @param kwargs: 22 | @return: 23 | ''' 24 | #print('batch_preds', batch_preds.size()) 25 | #print('batch_stds', batch_stds.size()) 26 | 27 | 28 | # todo-as-note: log(softmax(x)), doing these two operations separately is slower, and numerically unstable. 29 | # c.f. https://pytorch.org/docs/stable/_modules/torch/nn/functional.html 30 | batch_loss = torch.sum(-torch.sum(F.softmax(batch_std_labels, dim=1) * F.log_softmax(batch_preds, dim=1), dim=1)) 31 | 32 | self.optimizer.zero_grad() 33 | batch_loss.backward() 34 | self.optimizer.step() 35 | 36 | return batch_loss 37 | -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/listwise/lambdarank.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | Christopher J.C. Burges, Robert Ragno, and Quoc Viet Le. 2006. 6 | Learning to Rank with Nonsmooth Cost Functions. In Proceedings of NIPS conference. 193–200. 7 | """ 8 | 9 | import torch 10 | import torch.nn.functional as F 11 | 12 | from ptranking.data.data_utils import LABEL_TYPE 13 | from ptranking.metric.metric_utils import get_delta_ndcg 14 | from ptranking.base.adhoc_ranker import AdhocNeuralRanker 15 | from ptranking.ltr_adhoc.eval.parameter import ModelParameter 16 | from ptranking.ltr_adhoc.util.lambda_utils import get_pairwise_comp_probs 17 | 18 | class LambdaRank(AdhocNeuralRanker): 19 | ''' 20 | Christopher J.C. Burges, Robert Ragno, and Quoc Viet Le. 2006. 21 | Learning to Rank with Nonsmooth Cost Functions. In Proceedings of NIPS conference. 193–200. 22 | ''' 23 | def __init__(self, sf_para_dict=None, model_para_dict=None, gpu=False, device=None): 24 | super(LambdaRank, self).__init__(id='LambdaRank', sf_para_dict=sf_para_dict, gpu=gpu, device=device) 25 | self.sigma = model_para_dict['sigma'] 26 | 27 | def custom_loss_function(self, batch_preds, batch_std_labels, **kwargs): 28 | ''' 29 | @param batch_preds: [batch, ranking_size] each row represents the relevance predictions for documents associated with the same query 30 | @param batch_std_labels: [batch, ranking_size] each row represents the standard relevance grades for documents associated with the same query 31 | @param kwargs: 32 | @return: 33 | ''' 34 | assert 'label_type' in kwargs and LABEL_TYPE.MultiLabel == kwargs['label_type'] 35 | label_type = kwargs['label_type'] 36 | assert 'presort' in kwargs and kwargs['presort'] is True # aiming for direct usage of ideal ranking 37 | 38 | # sort documents according to the predicted relevance 39 | batch_descending_preds, batch_pred_desc_inds = torch.sort(batch_preds, dim=1, descending=True) 40 | # reorder batch_stds correspondingly so as to make it consistent. 41 | # BTW, batch_stds[batch_preds_sorted_inds] only works with 1-D tensor 42 | batch_predict_rankings = torch.gather(batch_std_labels, dim=1, index=batch_pred_desc_inds) 43 | 44 | batch_p_ij, batch_std_p_ij = get_pairwise_comp_probs(batch_preds=batch_descending_preds, 45 | batch_std_labels=batch_predict_rankings, 46 | sigma=self.sigma) 47 | 48 | batch_delta_ndcg = get_delta_ndcg(batch_ideal_rankings=batch_std_labels, 49 | batch_predict_rankings=batch_predict_rankings, 50 | label_type=label_type, device=self.device) 51 | 52 | _batch_loss = F.binary_cross_entropy(input=torch.triu(batch_p_ij, diagonal=1), 53 | target=torch.triu(batch_std_p_ij, diagonal=1), 54 | weight=torch.triu(batch_delta_ndcg, diagonal=1), reduction='none') 55 | 56 | batch_loss = torch.sum(torch.sum(_batch_loss, dim=(2, 1))) 57 | 58 | self.optimizer.zero_grad() 59 | batch_loss.backward() 60 | self.optimizer.step() 61 | 62 | return batch_loss 63 | 64 | 65 | ###### Parameter of LambdaRank ###### 66 | 67 | class LambdaRankParameter(ModelParameter): 68 | ''' Parameter class for LambdaRank ''' 69 | def __init__(self, debug=False, para_json=None): 70 | super(LambdaRankParameter, self).__init__(model_id='LambdaRank', para_json=para_json) 71 | self.debug = debug 72 | 73 | def default_para_dict(self): 74 | """ 75 | Default parameter setting for LambdaRank 76 | :return: 77 | """ 78 | self.lambda_para_dict = dict(model_id=self.model_id, sigma=1.0) 79 | return self.lambda_para_dict 80 | 81 | def to_para_string(self, log=False, given_para_dict=None): 82 | """ 83 | String identifier of parameters 84 | :param log: 85 | :param given_para_dict: a given dict, which is used for maximum setting w.r.t. grid-search 86 | :return: 87 | """ 88 | # using specified para-dict or inner para-dict 89 | lambda_para_dict = given_para_dict if given_para_dict is not None else self.lambda_para_dict 90 | 91 | s1, s2 = (':', '\n') if log else ('_', '_') 92 | lambdarank_para_str = s1.join(['Sigma', '{:,g}'.format(lambda_para_dict['sigma'])]) 93 | return lambdarank_para_str 94 | 95 | def grid_search(self): 96 | """ 97 | Iterator of parameter settings for LambdaRank 98 | """ 99 | if self.use_json: 100 | choice_sigma = self.json_dict['sigma'] 101 | else: 102 | choice_sigma = [5.0, 1.0] if self.debug else [1.0] # 1.0, 10.0, 50.0, 100.0 103 | 104 | for sigma in choice_sigma: 105 | self.lambda_para_dict = dict(model_id=self.model_id, sigma=sigma) 106 | yield self.lambda_para_dict 107 | -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/listwise/listmle.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | ListMLE: Fen Xia, Tie-Yan Liu, Jue Wang, Wensheng Zhang, and Hang Li. 2008. Listwise Approach to Learning to Rank: Theory and Algorithm. 6 | In Proceedings of the 25th ICML. 1192–1199. 7 | """ 8 | 9 | import torch 10 | 11 | from ptranking.base.adhoc_ranker import AdhocNeuralRanker 12 | from ptranking.ltr_adhoc.util.sampling_utils import arg_shuffle_ties 13 | 14 | class LogCumsumExp(torch.autograd.Function): 15 | ''' 16 | The PyTorch OP corresponding to the operation: log{ |sum_k^m{ exp{pred_k} } } 17 | ''' 18 | @staticmethod 19 | def forward(ctx, input): 20 | ''' 21 | In the forward pass we receive a context object and a Tensor containing the input; 22 | we must return a Tensor containing the output, and we can use the context object to cache objects for use in the backward pass. 23 | Specifically, ctx is a context object that can be used to stash information for backward computation. 24 | You can cache arbitrary objects for use in the backward pass using the ctx.save_for_backward method. 25 | :param ctx: 26 | :param input: i.e., batch_preds of [batch, ranking_size], each row represents the relevance predictions for documents within a ltr_adhoc 27 | :return: [batch, ranking_size], each row represents the log_cumsum_exp value 28 | ''' 29 | 30 | m, _ = torch.max(input, dim=1, keepdim=True) #a transformation aiming for higher stability when computing softmax() with exp() 31 | y = input - m 32 | y = torch.exp(y) 33 | y_backward_cumsum = torch.flip(torch.cumsum(torch.flip(y, dims=[1]), dim=1), dims=[1]) #row-wise cumulative sum, from tail to head 34 | fd_output = torch.log(y_backward_cumsum) + m # corresponding to the '-m' operation 35 | 36 | ctx.save_for_backward(input, fd_output) 37 | 38 | return fd_output 39 | 40 | 41 | @staticmethod 42 | def backward(ctx, grad_output): 43 | ''' 44 | In the backward pass we receive the context object and 45 | a Tensor containing the gradient of the loss with respect to the output produced during the forward pass (i.e., forward's output). 46 | We can retrieve cached data from the context object, and 47 | must compute and return the gradient of the loss with respect to the input to the forward function. 48 | Namely, grad_output is the gradient of the loss w.r.t. forward's output. Here we first compute the gradient (denoted as grad_out_wrt_in) of forward's output w.r.t. forward's input. 49 | Based on the chain rule, grad_output * grad_out_wrt_in would be the desired output, i.e., the gradient of the loss w.r.t. forward's input 50 | :param ctx: 51 | :param grad_output: 52 | :return: 53 | ''' 54 | 55 | input, fd_output = ctx.saved_tensors 56 | #chain rule 57 | bk_output = grad_output * (torch.exp(input) * torch.cumsum(torch.exp(-fd_output), dim=1)) 58 | 59 | return bk_output 60 | 61 | 62 | apply_LogCumsumExp = LogCumsumExp.apply 63 | 64 | 65 | class ListMLE(AdhocNeuralRanker): 66 | ''' 67 | ListMLE: Fen Xia, Tie-Yan Liu, Jue Wang, Wensheng Zhang, and Hang Li. 2008. Listwise Approach to Learning to Rank: Theory and Algorithm. 68 | In Proceedings of the 25th ICML. 1192–1199. 69 | ''' 70 | def __init__(self, sf_para_dict=None, model_para_dict=None, gpu=False, device=None): 71 | super(ListMLE, self).__init__(id='ListMLE', sf_para_dict=sf_para_dict, gpu=gpu, device=device) 72 | 73 | def custom_loss_function(self, batch_preds, batch_std_labels, **kwargs): 74 | ''' 75 | @param batch_preds: [batch, ranking_size] each row represents the relevance predictions for documents associated with the same query 76 | @param batch_std_labels: [batch, ranking_size] each row represents the standard relevance grades for documents associated with the same query 77 | @param kwargs: 78 | @return: 79 | ''' 80 | # shuffle per epoch rather than using the same order for a query 81 | batch_shuffle_ties_inds = arg_shuffle_ties(batch_rankings=batch_std_labels, descending=True, device=self.device) 82 | batch_preds_shuffled_ties = torch.gather(batch_preds, dim=1, index=batch_shuffle_ties_inds) 83 | 84 | # 1 using self-defined op since torch.flip() is later added 85 | ''' 86 | batch_logcumsumexps = apply_LogCumsumExp(target_batch_preds) 87 | batch_loss = torch.sum(batch_logcumsumexps - target_batch_preds) 88 | ''' 89 | 90 | # 2 since torch.flip() is available now, the loss can also be directly computed without defining a new op 91 | #''' 92 | m, _ = torch.max(batch_preds_shuffled_ties, dim=1, keepdim=True) # a transformation aiming for higher stability when computing softmax() with exp() 93 | y = batch_preds_shuffled_ties - m 94 | y = torch.exp(y) 95 | y_backward_cumsum = torch.flip(torch.cumsum(torch.flip(y, dims=[1]), dim=1), dims=[1]) # row-wise cumulative sum, from tail to head 96 | batch_logcumsumexps = torch.log(y_backward_cumsum) + m # corresponding to the '-m' operation 97 | batch_loss = torch.sum(torch.sum((batch_logcumsumexps - batch_preds_shuffled_ties), dim=1)) 98 | #''' 99 | 100 | self.optimizer.zero_grad() 101 | batch_loss.backward() 102 | self.optimizer.step() 103 | 104 | return batch_loss 105 | -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/listwise/listnet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | Zhe Cao, Tao Qin, Tie-Yan Liu, Ming-Feng Tsai, and Hang Li. 2007. 6 | Learning to Rank: From Pairwise Approach to Listwise Approach. In Proceedings of the 24th ICML. 129–136. 7 | """ 8 | 9 | import torch 10 | import torch.nn.functional as F 11 | 12 | from ptranking.base.adhoc_ranker import AdhocNeuralRanker 13 | 14 | class ListNet(AdhocNeuralRanker): 15 | ''' 16 | Zhe Cao, Tao Qin, Tie-Yan Liu, Ming-Feng Tsai, and Hang Li. 2007. 17 | Learning to Rank: From Pairwise Approach to Listwise Approach. In Proceedings of the 24th ICML. 129–136. 18 | ''' 19 | def __init__(self, sf_para_dict=None, gpu=False, device=None): 20 | super(ListNet, self).__init__(id='ListNet', sf_para_dict=sf_para_dict, gpu=gpu, device=device) 21 | 22 | def custom_loss_function(self, batch_preds, batch_std_labels, **kwargs): 23 | ''' 24 | The Top-1 approximated ListNet loss, which reduces to a softmax and simple cross entropy. 25 | @param batch_preds: [batch, ranking_size] each row represents the relevance predictions for documents associated with the same query 26 | @param batch_std_labels: [batch, ranking_size] each row represents the standard relevance grades for documents associated with the same query 27 | @param kwargs: 28 | @return: 29 | ''' 30 | ''' 31 | #- deprecated way -# 32 | batch_top1_pros_pred = F.softmax(batch_preds, dim=1) 33 | batch_top1_pros_std = F.softmax(batch_stds, dim=1) 34 | batch_loss = torch.sum(-torch.sum(batch_top1_pros_std * torch.log(batch_top1_pros_pred), dim=1)) 35 | ''' 36 | 37 | # todo-as-note: log(softmax(x)), doing these two operations separately is slower, and numerically unstable. 38 | # c.f. https://pytorch.org/docs/stable/_modules/torch/nn/functional.html 39 | batch_loss = torch.sum(-torch.sum(F.softmax(batch_std_labels, dim=1) * F.log_softmax(batch_preds, dim=1), dim=1)) 40 | 41 | self.optimizer.zero_grad() 42 | batch_loss.backward() 43 | self.optimizer.step() 44 | 45 | return batch_loss 46 | -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/listwise/mdprank.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import torch 5 | 6 | from itertools import product 7 | 8 | from ptranking.base.adhoc_ranker import AdhocNeuralRanker 9 | from ptranking.metric.adhoc.adhoc_metric import torch_ndcg_at_k 10 | from ptranking.ltr_adhoc.eval.parameter import ModelParameter 11 | from ptranking.ltr_adhoc.util.sampling_utils import sample_ranking_PL, sample_ranking_PL_gumbel_softmax 12 | 13 | class MDPRank(AdhocNeuralRanker): 14 | def __init__(self, sf_para_dict=None, model_para_dict=None, gpu=False, device=None): 15 | super(MDPRank, self).__init__(id='MDPRank', sf_para_dict=sf_para_dict, gpu=gpu, device=device) 16 | self.gamma = model_para_dict['gamma'] 17 | self.top_k = model_para_dict['top_k'] 18 | self.temperature = model_para_dict['temperature'] 19 | self.distribution = model_para_dict['distribution'] # 'PL', 'STPL' 20 | 21 | # checking the quality of sampled rankings 22 | self.pg_checking = False 23 | 24 | def custom_loss_function(self, batch_preds, batch_std_labels, **kwargs): 25 | ''' 26 | @param batch_preds: [batch, ranking_size] each row represents the relevance predictions for documents associated with the same query 27 | @param batch_std_labels: [batch, ranking_size] each row represents the standard relevance grades for documents associated with the same query 28 | @param kwargs: 29 | @return: 30 | ''' 31 | # aiming for meaningful batch-normalization, please set {train_rough_batch_size, validation_rough_batch_size, test_rough_batch_size = 1, 1, 1} 32 | assert 1 == batch_preds.size(0) 33 | 34 | assert 'presort' in kwargs and kwargs['presort'] is True # aiming for direct usage of ideal ranking 35 | 36 | if 'PL' == self.distribution: 37 | batch_sample_inds, batch_action_preds = sample_ranking_PL(batch_preds=batch_preds, only_indices=False, temperature=self.temperature) 38 | 39 | elif 'STPL' == self.distribution: 40 | batch_sample_inds, batch_action_preds = sample_ranking_PL_gumbel_softmax( 41 | batch_preds=batch_preds, only_indices=False, temperature=self.temperature, device=self.device) 42 | else: 43 | raise NotImplementedError 44 | 45 | top_k = batch_std_labels.size(1) if self.top_k is None else self.top_k 46 | batch_action_stds = torch.gather(batch_std_labels, dim=1, index=batch_sample_inds) 47 | 48 | if self.pg_checking: 49 | sample_metric_values = torch_ndcg_at_k(batch_predict_rankings=batch_action_stds, 50 | batch_ideal_rankings=batch_std_labels, k=5, device=self.device) 51 | 52 | # TODO alternative metrics, such as AP and NERR 53 | batch_gains = torch.pow(2.0, batch_action_stds) - 1.0 54 | batch_ranks = torch.arange(top_k, dtype=torch.float, device=self.device).view(1, -1) 55 | batch_discounts = torch.log2(2.0 + batch_ranks) 56 | batch_rewards = batch_gains[:, 0:top_k] / batch_discounts 57 | # the long-term return of the sampled episode starting from t 58 | """ this is also the key difference, equivalently, weighting is different """ 59 | batch_G_t = torch.flip(torch.cumsum(torch.flip(batch_rewards, dims=[1]), dim=1), dims=[1]) 60 | 61 | if self.gamma != 1.0: 62 | return_discounts = torch.cumprod(torch.ones(top_k).view(1, -1) * self.gamma, dim=1) 63 | batch_G_t = batch_G_t * return_discounts 64 | 65 | m, _ = torch.max(batch_action_preds, dim=1, keepdim=True) # a transformation aiming for higher stability when computing softmax() with exp() 66 | y = batch_action_preds - m 67 | y = torch.exp(y) 68 | y_cumsum_t2h = torch.flip(torch.cumsum(torch.flip(y, dims=[1]), dim=1), dims=[1]) # row-wise cumulative sum, from tail to head 69 | batch_logcumsumexps = torch.log(y_cumsum_t2h) + m # corresponding to the '-m' operation 70 | batch_neg_log_probs = batch_logcumsumexps[:, 0:top_k] - batch_action_preds[:, 0:top_k] 71 | batch_loss = torch.sum(torch.sum(batch_neg_log_probs * batch_G_t[:, 0:top_k], dim=1)) 72 | 73 | self.optimizer.zero_grad() 74 | batch_loss.backward() 75 | self.optimizer.step() 76 | 77 | if self.pg_checking: 78 | return sample_metric_values 79 | else: 80 | return batch_loss 81 | 82 | ###### Parameter of FastMDPRank ###### 83 | 84 | class MDPRankParameter(ModelParameter): 85 | ''' Parameter class for FastMDPRank ''' 86 | 87 | def __init__(self, debug=False, para_json=None): 88 | super(MDPRankParameter, self).__init__(model_id='MDPRank', para_json=para_json) 89 | self.debug = debug 90 | 91 | def default_para_dict(self): 92 | """ 93 | Default parameter setting for FastMDPRank 94 | """ 95 | self.MDPRank_para_dict = dict(model_id=self.model_id, temperature=1.0, gamma=1.0, top_k=10, 96 | distribution='PL') 97 | return self.MDPRank_para_dict 98 | 99 | def to_para_string(self, log=False, given_para_dict=None): 100 | """ 101 | String identifier of parameters 102 | :param log: 103 | :param given_para_dict: a given dict, which is used for maximum setting w.r.t. grid-search 104 | """ 105 | # using specified para-dict or inner para-dict 106 | MDPRank_para_dict = given_para_dict if given_para_dict is not None else self.MDPRank_para_dict 107 | 108 | s1 = ':' if log else '_' 109 | top_k, distribution, gamma, temperature=MDPRank_para_dict['top_k'], MDPRank_para_dict['distribution'],\ 110 | MDPRank_para_dict['gamma'], MDPRank_para_dict['temperature'] 111 | fastMDPRank_para_str = s1.join([str(top_k), distribution, 'G', '{:,g}'.format(gamma), 112 | 'T', '{:,g}'.format(temperature)]) 113 | return fastMDPRank_para_str 114 | 115 | def grid_search(self): 116 | """ 117 | Iterator of parameter settings for FastMDPRank 118 | """ 119 | if self.use_json: 120 | #choice_metric = json_dict['metric'] 121 | choice_topk = self.json_dict['top_k'] 122 | choice_distribution = self.json_dict['distribution'] 123 | choice_temperature = self.json_dict['temperature'] 124 | choice_gamma = self.json_dict['gamma'] 125 | else: 126 | #choice_metric = ['NERR', 'nDCG', 'AP'] # 'nDCG', 'AP', 'NERR' 127 | choice_topk = [10] if self.debug else [10] 128 | choice_distribution = ['PL'] 129 | choice_temperature = [.1] if self.debug else [1.0] # 1.0, 10.0 130 | choice_gamma = [1.0] 131 | 132 | for top_k, distribution, temperature, gamma in product(choice_topk, choice_distribution, choice_temperature, choice_gamma): 133 | self.MDPRank_para_dict = dict(model_id=self.model_id, top_k=top_k, gamma=gamma, 134 | distribution=distribution, temperature=temperature) 135 | yield self.MDPRank_para_dict 136 | -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/listwise/rank_cosine.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | Tao Qin, Xu-Dong Zhang, Ming-Feng Tsai, De-Sheng Wang, Tie-Yan Liu, and Hang Li. 2008. 7 | Query-level loss functions for information retrieval. Information Processing and Management 44, 2 (2008), 838–855. 8 | """ 9 | 10 | import torch 11 | import torch.nn as nn 12 | 13 | from ptranking.base.adhoc_ranker import AdhocNeuralRanker 14 | 15 | cos = nn.CosineSimilarity(dim=1) 16 | 17 | class RankCosine(AdhocNeuralRanker): 18 | ''' 19 | Tao Qin, Xu-Dong Zhang, Ming-Feng Tsai, De-Sheng Wang, Tie-Yan Liu, and Hang Li. 2008. 20 | Query-level loss functions for information retrieval. Information Processing and Management 44, 2 (2008), 838–855. 21 | ''' 22 | def __init__(self, sf_para_dict=None, gpu=False, device=None): 23 | super(RankCosine, self).__init__(id='RankCosine', sf_para_dict=sf_para_dict, gpu=gpu, device=device) 24 | 25 | def custom_loss_function(self, batch_preds, batch_std_labels, **kwargs): 26 | ''' 27 | @param batch_preds: [batch, ranking_size] each row represents the relevance predictions for documents associated with the same query 28 | @param batch_std_labels: [batch, ranking_size] each row represents the standard relevance grades for documents associated with the same query 29 | @param kwargs: 30 | @return: 31 | ''' 32 | batch_loss = torch.sum((1.0 - cos(batch_preds, batch_std_labels)) / 0.5) 33 | 34 | self.optimizer.zero_grad() 35 | batch_loss.backward() 36 | self.optimizer.step() 37 | 38 | return batch_loss 39 | -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/listwise/softrank.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | { 6 | author = {Taylor, Michael and Guiver, John and Robertson, Stephen and Minka, Tom}, 7 | title = {SoftRank: Optimizing Non-Smooth Rank Metrics}, 8 | year = {2008}, 9 | } 10 | """ 11 | 12 | from itertools import product 13 | 14 | import torch 15 | 16 | from ptranking.data.data_utils import LABEL_TYPE 17 | from ptranking.metric.metric_utils import torch_dcg_at_k 18 | from ptranking.base.adhoc_ranker import AdhocNeuralRanker 19 | from ptranking.ltr_adhoc.eval.parameter import ModelParameter 20 | 21 | class SoftRank(AdhocNeuralRanker): 22 | ''' 23 | author = {Taylor, Michael and Guiver, John and Robertson, Stephen and Minka, Tom}, 24 | title = {SoftRank: Optimizing Non-Smooth Rank Metrics}, 25 | ''' 26 | def __init__(self, sf_para_dict=None, model_para_dict=None, gpu=False, device=None): 27 | super(SoftRank, self).__init__(id='SoftRank', sf_para_dict=sf_para_dict, gpu=gpu, device=device) 28 | delta = model_para_dict['delta'] 29 | self.delta = torch.tensor([delta], device=self.device) 30 | self.top_k = model_para_dict['top_k'] 31 | self.metric = model_para_dict['metric'] 32 | 33 | def custom_loss_function(self, batch_preds, batch_std_labels, **kwargs): 34 | ''' 35 | @param batch_preds: [batch, ranking_size] each row represents the mean predictions for documents associated with the same query 36 | @param batch_std_labels: [batch, ranking_size] each row represents the standard relevance grades for documents associated with the same query 37 | @param kwargs: 38 | @return: 39 | ''' 40 | assert 'presort' in kwargs and kwargs['presort'] is True # aiming for direct usage of ideal ranking 41 | assert 'nDCG' == self.metric # TODO support more metrics 42 | assert LABEL_TYPE.MultiLabel == kwargs['label_type'] # other types are not considered yet 43 | label_type = kwargs['label_type'] 44 | 45 | batch_mus = batch_preds 46 | 47 | ''' expected ranks ''' 48 | # f_ij, i.e., mean difference 49 | batch_pairsub_mus = torch.unsqueeze(batch_mus, dim=2) - torch.unsqueeze(batch_mus, dim=1) 50 | # variance w.r.t. s_i - s_j, which is equal to sigma^2_i + sigma^2_j 51 | pairsub_vars = 2 * self.delta**2 52 | # \Phi(0)$ 53 | batch_Phi0 = 0.5 * torch.erfc(batch_pairsub_mus / torch.sqrt(2 * pairsub_vars)) 54 | # remove diagonal entries 55 | batch_Phi0_subdiag = torch.triu(batch_Phi0, diagonal=1) + torch.tril(batch_Phi0, diagonal=-1) 56 | batch_expt_ranks = torch.sum(batch_Phi0_subdiag, dim=2) + 1.0 57 | 58 | batch_gains = torch.pow(2.0, batch_std_labels) - 1.0 59 | batch_dists = 1.0 / torch.log2(batch_expt_ranks + 1.0) # discount co-efficients 60 | batch_idcgs = torch_dcg_at_k(batch_rankings=batch_std_labels, label_type=label_type, device=self.device) 61 | 62 | #TODO check the effect of removing batch_idcgs 63 | if self.top_k is None: 64 | batch_dcgs = batch_dists * batch_gains 65 | batch_expt_nDCG = torch.sum(batch_dcgs/batch_idcgs, dim=1) 66 | batch_loss = - torch.sum(batch_expt_nDCG) 67 | else: 68 | k = min(self.top_k, batch_std_labels.size(1)) 69 | batch_dcgs = batch_dists[:, 0:k] * batch_gains[:, 0:k] 70 | batch_expt_nDCG_k = torch.sum(batch_dcgs/batch_idcgs, dim=1) 71 | batch_loss = - torch.sum(batch_expt_nDCG_k) 72 | 73 | self.optimizer.zero_grad() 74 | batch_loss.backward() 75 | self.optimizer.step() 76 | 77 | return batch_loss 78 | 79 | 80 | ###### Parameter of SoftRank ###### 81 | 82 | class SoftRankParameter(ModelParameter): 83 | ''' Parameter class for SoftRank ''' 84 | def __init__(self, debug=False, para_json=None): 85 | super(SoftRankParameter, self).__init__(model_id='SoftRank', para_json=para_json) 86 | self.debug = debug 87 | 88 | def default_para_dict(self): 89 | """ 90 | Default parameter setting for SoftRank 91 | :return: 92 | """ 93 | self.soft_para_dict = dict(model_id=self.model_id, delta=2.0, metric='nDCG', top_k=None) 94 | return self.soft_para_dict 95 | 96 | def to_para_string(self, log=False, given_para_dict=None): 97 | """ 98 | String identifier of parameters 99 | :param log: 100 | :param given_para_dict: a given dict, which is used for maximum setting w.r.t. grid-search 101 | :return: 102 | """ 103 | # using specified para-dict or inner para-dict 104 | soft_para_dict = given_para_dict if given_para_dict is not None else self.soft_para_dict 105 | 106 | s1, s2 = (':', '\n') if log else ('_', '_') 107 | 108 | metric, delta, top_k = soft_para_dict['metric'], soft_para_dict['delta'], soft_para_dict['top_k'] 109 | if top_k is not None: 110 | softrank_para_str = s1.join([metric, str(top_k), 'Delta', '{:,g}'.format(delta)]) 111 | else: 112 | softrank_para_str = s1.join([metric, 'Delta', '{:,g}'.format(delta)]) 113 | 114 | return softrank_para_str 115 | 116 | def grid_search(self): 117 | """ 118 | Iterator of parameter settings for SoftRank 119 | """ 120 | if self.use_json: 121 | choice_topk = self.json_dict['top_k'] 122 | choice_delta = self.json_dict['delta'] 123 | choice_metric = self.json_dict['metric'] 124 | else: 125 | choice_delta = [5.0, 1.0] if self.debug else [1.0] # 1.0, 10.0, 50.0, 100.0 126 | choice_metric = ['nDCG'] # 'nDCG' 127 | choice_topk = [None] if self.debug else [None] 128 | 129 | for delta, top_k, metric in product(choice_delta, choice_topk, choice_metric): 130 | self.soft_para_dict = dict(model_id=self.model_id, delta=delta, top_k=top_k, metric=metric) 131 | yield self.soft_para_dict 132 | -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/listwise/st_listnet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | author = {Bruch, Sebastian and Han, Shuguang and Bendersky, Michael and Najork, Marc}, 6 | title = {A Stochastic Treatment of Learning to Rank Scoring Functions}, 7 | year = {2020}, 8 | booktitle = {Proceedings of the 13th International Conference on Web Search and Data Mining}, 9 | pages = {61–69} 10 | """ 11 | 12 | import torch 13 | import torch.nn.functional as F 14 | 15 | from ptranking.base.adhoc_ranker import AdhocNeuralRanker 16 | from ptranking.ltr_adhoc.eval.parameter import ModelParameter 17 | 18 | 19 | EPS = 1e-20 20 | 21 | class STListNet(AdhocNeuralRanker): 22 | ''' 23 | author = {Bruch, Sebastian and Han, Shuguang and Bendersky, Michael and Najork, Marc}, 24 | title = {A Stochastic Treatment of Learning to Rank Scoring Functions}, 25 | year = {2020}, 26 | booktitle = {Proceedings of the 13th International Conference on Web Search and Data Mining}, 27 | pages = {61–69} 28 | ''' 29 | def __init__(self, sf_para_dict=None, model_para_dict=None, gpu=False, device=None): 30 | super(STListNet, self).__init__(id='STListNet', sf_para_dict=sf_para_dict, gpu=gpu, device=device) 31 | self.temperature = model_para_dict['temperature'] 32 | 33 | def custom_loss_function(self, batch_preds, batch_std_labels, **kwargs): 34 | ''' 35 | The Top-1 approximated ListNet loss, which reduces to a softmax and simple cross entropy. 36 | @param batch_preds: [batch, ranking_size] each row represents the relevance predictions for documents associated with the same query 37 | @param batch_std_labels: [batch, ranking_size] each row represents the standard relevance grades for documents associated with the same query 38 | @param kwargs: 39 | @return: 40 | ''' 41 | unif = torch.rand(batch_preds.size(), device=self.device) # [batch_size, ranking_size] 42 | 43 | gumbel = -torch.log(-torch.log(unif + EPS) + EPS) # Sample from gumbel distribution 44 | 45 | batch_preds = (batch_preds + gumbel) / self.temperature 46 | 47 | # todo-as-note: log(softmax(x)), doing these two operations separately is slower, and numerically unstable. 48 | # c.f. https://pytorch.org/docs/stable/_modules/torch/nn/functional.html 49 | batch_loss = torch.sum(-torch.sum(F.softmax(batch_std_labels, dim=1) * F.log_softmax(batch_preds, dim=1), dim=1)) 50 | 51 | self.optimizer.zero_grad() 52 | batch_loss.backward() 53 | self.optimizer.step() 54 | 55 | return batch_loss 56 | 57 | ###### Parameter of STListNet ###### 58 | 59 | class STListNetParameter(ModelParameter): 60 | ''' Parameter class for STListNet ''' 61 | def __init__(self, debug=False, para_json=None): 62 | super(STListNetParameter, self).__init__(model_id='STListNet', para_json=para_json) 63 | self.debug = debug 64 | 65 | def default_para_dict(self): 66 | """ 67 | Default parameter setting for STListNet 68 | :return: 69 | """ 70 | self.stlistnet_para_dict = dict(model_id=self.model_id, temperature=1.0) 71 | return self.stlistnet_para_dict 72 | 73 | def to_para_string(self, log=False, given_para_dict=None): 74 | """ 75 | String identifier of parameters 76 | :param log: 77 | :param given_para_dict: a given dict, which is used for maximum setting w.r.t. grid-search 78 | :return: 79 | """ 80 | # using specified para-dict or inner para-dict 81 | stlistnet_para_dict = given_para_dict if given_para_dict is not None else self.stlistnet_para_dict 82 | 83 | s1 = ':' if log else '_' 84 | stlistnet_para_str = s1.join(['Tem', str(stlistnet_para_dict['temperature'])]) 85 | return stlistnet_para_str 86 | 87 | 88 | def grid_search(self): 89 | """ 90 | Iterator of parameter settings for STListNet 91 | :param debug: 92 | :return: 93 | """ 94 | if self.use_json: 95 | choice_temperature = self.json_dict['temperature'] 96 | else: 97 | choice_temperature = [1.0] if self.debug else [1.0] # 1.0, 10.0, 50.0, 100.0 98 | 99 | for temperature in choice_temperature: 100 | self.stlistnet_para_dict = dict(model_id=self.model_id, temperature=temperature) 101 | yield self.stlistnet_para_dict 102 | -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/listwise/wassrank/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/pairwise/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/pairwise/ranknet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | Chris Burges, Tal Shaked, Erin Renshaw, Ari Lazier, Matt Deeds, Nicole Hamilton, and Greg Hullender. 2005. 6 | Learning to rank using gradient descent. In Proceedings of the 22nd ICML. 89–96. 7 | """ 8 | 9 | import torch 10 | import torch.nn.functional as F 11 | 12 | from ptranking.base.adhoc_ranker import AdhocNeuralRanker 13 | from ptranking.ltr_adhoc.eval.parameter import ModelParameter 14 | from ptranking.ltr_adhoc.util.lambda_utils import get_pairwise_comp_probs 15 | 16 | class RankNet(AdhocNeuralRanker): 17 | ''' 18 | Chris Burges, Tal Shaked, Erin Renshaw, Ari Lazier, Matt Deeds, Nicole Hamilton, and Greg Hullender. 2005. 19 | Learning to rank using gradient descent. In Proceedings of the 22nd ICML. 89–96. 20 | ''' 21 | def __init__(self, sf_para_dict=None, model_para_dict=None, gpu=False, device=None): 22 | super(RankNet, self).__init__(id='RankNet', sf_para_dict=sf_para_dict, gpu=gpu, device=device) 23 | self.sigma = model_para_dict['sigma'] 24 | 25 | def custom_loss_function(self, batch_preds, batch_std_labels, **kwargs): 26 | ''' 27 | @param batch_preds: [batch, ranking_size] each row represents the relevance predictions for documents associated with the same query 28 | @param batch_std_labels: [batch, ranking_size] each row represents the standard relevance grades for documents associated with the same query 29 | @param kwargs: 30 | @return: 31 | ''' 32 | batch_p_ij, batch_std_p_ij = get_pairwise_comp_probs(batch_preds=batch_preds, batch_std_labels=batch_std_labels, 33 | sigma=self.sigma) 34 | _batch_loss = F.binary_cross_entropy(input=torch.triu(batch_p_ij, diagonal=1), 35 | target=torch.triu(batch_std_p_ij, diagonal=1), reduction='none') 36 | batch_loss = torch.sum(torch.sum(_batch_loss, dim=(2, 1))) 37 | 38 | self.optimizer.zero_grad() 39 | batch_loss.backward() 40 | self.optimizer.step() 41 | 42 | return batch_loss 43 | 44 | ###### Parameter of RankNet ###### 45 | 46 | class RankNetParameter(ModelParameter): 47 | ''' Parameter class for RankNet ''' 48 | def __init__(self, debug=False, para_json=None): 49 | super(RankNetParameter, self).__init__(model_id='RankNet', para_json=para_json) 50 | self.debug = debug 51 | 52 | def default_para_dict(self): 53 | """ 54 | Default parameter setting for RankNet 55 | """ 56 | self.ranknet_para_dict = dict(model_id=self.model_id, sigma=1.0) 57 | return self.ranknet_para_dict 58 | 59 | def to_para_string(self, log=False, given_para_dict=None): 60 | """ 61 | String identifier of parameters 62 | :param log: 63 | :param given_para_dict: a given dict, which is used for maximum setting w.r.t. grid-search 64 | :return: 65 | """ 66 | # using specified para-dict or inner para-dict 67 | ranknet_para_dict = given_para_dict if given_para_dict is not None else self.ranknet_para_dict 68 | 69 | s1, s2 = (':', '\n') if log else ('_', '_') 70 | ranknet_para_str = s1.join(['Sigma', '{:,g}'.format(ranknet_para_dict['sigma'])]) 71 | return ranknet_para_str 72 | 73 | def grid_search(self): 74 | """ 75 | Iterator of parameter settings for RankNet 76 | """ 77 | if self.use_json: 78 | choice_sigma = self.json_dict['sigma'] 79 | else: 80 | choice_sigma = [5.0, 1.0] if self.debug else [1.0] # 1.0, 10.0, 50.0, 100.0 81 | 82 | for sigma in choice_sigma: 83 | self.ranknet_para_dict = dict(model_id=self.model_id, sigma=sigma) 84 | yield self.ranknet_para_dict 85 | -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/pointwise/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | 6 | """ -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/pointwise/rank_mse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | Viewing the prediction of relevance as a conventional regression problem. 6 | """ 7 | 8 | import torch 9 | import torch.nn.functional as F 10 | 11 | from ptranking.base.adhoc_ranker import AdhocNeuralRanker 12 | 13 | def rankMSE_loss_function(relevance_preds=None, std_labels=None): 14 | ''' 15 | Ranking loss based on mean square error TODO adjust output scale w.r.t. output layer activation function 16 | @param batch_preds: [batch, ranking_size] each row represents the relevance predictions for documents associated with the same query 17 | @param batch_std_labels: [batch, ranking_size] each row represents the standard relevance grades for documents associated with the same query 18 | @return: 19 | ''' 20 | _batch_loss = F.mse_loss(relevance_preds, std_labels, reduction='none') 21 | batch_loss = torch.mean(torch.sum(_batch_loss, dim=1)) 22 | return batch_loss 23 | 24 | class RankMSE(AdhocNeuralRanker): 25 | def __init__(self, sf_para_dict=None, gpu=False, device=None): 26 | super(RankMSE, self).__init__(id='RankMSE', sf_para_dict=sf_para_dict, gpu=gpu, device=device) 27 | #self.TL_AF = self.get_tl_af() 28 | 29 | def custom_loss_function(self, batch_preds, batch_std_labels, **kwargs): 30 | ''' 31 | :param batch_preds: [batch, ranking_size] each row represents the relevance predictions for documents within a ltr_adhoc 32 | :param batch_std_labels: [batch, ranking_size] each row represents the standard relevance grades for documents within a ltr_adhoc 33 | :return: 34 | ''' 35 | batch_loss = rankMSE_loss_function(batch_preds, batch_std_labels) 36 | 37 | self.optimizer.zero_grad() 38 | batch_loss.backward() 39 | self.optimizer.step() 40 | 41 | return batch_loss 42 | -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/util/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/util/bin_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ 8 | 9 | import torch 10 | 11 | def batch_count(batch_std_labels=None, max_rele_grade=None, descending=False, gpu=False): 12 | """ 13 | Todo now an api is already provided by pytorch 14 | :param batch_std_labels: 15 | :param max_rele_grade: 16 | :param descending: 17 | :param gpu: 18 | :return: 19 | """ 20 | rele_grades = torch.arange(max_rele_grade+1).type(torch.cuda.FloatTensor) if gpu else torch.arange(max_rele_grade+1).type(torch.FloatTensor) 21 | if descending: rele_grades, _ = torch.sort(rele_grades, descending=True) 22 | 23 | batch_cnts = torch.stack([(batch_std_labels == g).sum(dim=1) for g in rele_grades]) 24 | batch_cnts = torch.t(batch_cnts) 25 | return batch_cnts 26 | -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/util/gather_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | 6 | """ 7 | 8 | import torch 9 | 10 | import numpy as np 11 | 12 | 13 | ####### 14 | # For Pair Extraction 15 | ####### 16 | 17 | PAIR_TYPE = ['All', 'NoTies', 'No00', '00', 'Inversion'] 18 | 19 | def torch_batch_triu(batch_mats=None, k=0, pair_type='All', batch_std_labels=None, gpu=False, device=None): 20 | ''' 21 | Get unique document pairs being consistent with the specified pair_type. This function is used to avoid duplicate computation. 22 | 23 | All: pairs including both pairs of documents across different relevance levels and 24 | pairs of documents having the same relevance level. 25 | NoTies: the pairs consisting of two documents of the same relevance level are removed 26 | No00: the pairs consisting of two non-relevant documents are removed 27 | 28 | :param batch_mats: [batch, m, m] 29 | :param k: the offset w.r.t. the diagonal line: k=0 means including the diagonal line, k=1 means upper triangular part without the diagonal line 30 | :return: 31 | ''' 32 | assert batch_mats.size(1) == batch_mats.size(2) 33 | assert pair_type in PAIR_TYPE 34 | 35 | m = batch_mats.size(1) # the number of documents 36 | 37 | if pair_type == 'All': 38 | row_inds, col_inds = np.triu_indices(m, k=k) 39 | 40 | elif pair_type == 'No00': 41 | assert batch_std_labels.size(0) == 1 42 | 43 | row_inds, col_inds = np.triu_indices(m, k=k) 44 | std_labels = torch.squeeze(batch_std_labels, 0) 45 | labels = std_labels.cpu().numpy() if gpu else std_labels.data.numpy() 46 | 47 | pairs = [e for e in zip(row_inds, col_inds) if not (0 == labels[e[0]] and 0 == labels[e[1]])] # remove pairs of 00 comparisons 48 | row_inds = [e[0] for e in pairs] 49 | col_inds = [e[1] for e in pairs] 50 | 51 | elif pair_type == 'NoTies': 52 | assert batch_std_labels.size(0) == 1 53 | std_labels = torch.squeeze(batch_std_labels, 0) 54 | 55 | row_inds, col_inds = np.triu_indices(m, k=k) 56 | 57 | labels = std_labels.cpu().numpy() if gpu else std_labels.data.numpy() 58 | pairs = [e for e in zip(row_inds, col_inds) if labels[e[0]]!=labels[e[1]]] # remove pairs of documents of the same level 59 | row_inds = [e[0] for e in pairs] 60 | col_inds = [e[1] for e in pairs] 61 | 62 | #tor_row_inds = torch.LongTensor(row_inds).to(device) if gpu else torch.LongTensor(row_inds) 63 | #tor_col_inds = torch.LongTensor(col_inds).to(device) if gpu else torch.LongTensor(col_inds) 64 | tor_row_inds = torch.LongTensor(row_inds, device) 65 | tor_col_inds = torch.LongTensor(col_inds, device) 66 | batch_triu = batch_mats[:, tor_row_inds, tor_col_inds] 67 | 68 | return batch_triu # shape: [batch_size, number of pairs] 69 | 70 | 71 | def torch_triu_indice(k=0, pair_type='All', batch_label=None, gpu=False, device=None): 72 | ''' 73 | Get unique document pairs being consistent with the specified pair_type. This function is used to avoid duplicate computation. 74 | 75 | All: pairs including both pairs of documents across different relevance levels and 76 | pairs of documents having the same relevance level. 77 | NoTies: the pairs consisting of two documents of the same relevance level are removed 78 | No00: the pairs consisting of two non-relevant documents are removed 79 | Inversion: the pairs that are inverted order, i.e., the 1st doc is less relevant than the 2nd doc 80 | 81 | :param batch_mats: [batch, m, m] 82 | :param k: the offset w.r.t. the diagonal line: k=0 means including the diagonal line, k=1 means upper triangular part without the diagonal line 83 | :return: 84 | ''' 85 | assert pair_type in PAIR_TYPE 86 | 87 | m = batch_label.size(1) # the number of documents 88 | if pair_type == 'All': 89 | row_inds, col_inds = np.triu_indices(m, k=k) 90 | 91 | elif pair_type == 'No00': 92 | assert batch_label.size(0) == 1 93 | 94 | row_inds, col_inds = np.triu_indices(m, k=k) 95 | std_labels = torch.squeeze(batch_label, 0) 96 | labels = std_labels.cpu().numpy() if gpu else std_labels.data.numpy() 97 | 98 | pairs = [e for e in zip(row_inds, col_inds) if not (0==labels[e[0]] and 0==labels[e[1]])] # remove pairs of 00 comparisons 99 | row_inds = [e[0] for e in pairs] 100 | col_inds = [e[1] for e in pairs] 101 | 102 | elif pair_type == '00': # the pairs consisting of two non-relevant documents 103 | assert batch_label.size(0) == 1 104 | 105 | row_inds, col_inds = np.triu_indices(m, k=k) 106 | std_labels = torch.squeeze(batch_label, 0) 107 | labels = std_labels.cpu().numpy() if gpu else std_labels.data.numpy() 108 | 109 | pairs = [e for e in zip(row_inds, col_inds) if (0 == labels[e[0]] and 0 == labels[e[1]])] # remove pairs of 00 comparisons 110 | row_inds = [e[0] for e in pairs] 111 | col_inds = [e[1] for e in pairs] 112 | 113 | elif pair_type == 'NoTies': 114 | assert batch_label.size(0) == 1 115 | std_labels = torch.squeeze(batch_label, 0) 116 | 117 | row_inds, col_inds = np.triu_indices(m, k=k) 118 | 119 | labels = std_labels.cpu().numpy() if gpu else std_labels.data.numpy() 120 | pairs = [e for e in zip(row_inds, col_inds) if labels[e[0]]!=labels[e[1]]] # remove pairs of documents of the same level 121 | row_inds = [e[0] for e in pairs] 122 | col_inds = [e[1] for e in pairs] 123 | 124 | elif pair_type == 'Inversion': 125 | assert batch_label.size(0) == 1 126 | std_labels = torch.squeeze(batch_label, 0) 127 | 128 | row_inds, col_inds = np.triu_indices(m, k=k) 129 | 130 | labels = std_labels.cpu().numpy() if gpu else std_labels.data.numpy() 131 | pairs = [e for e in zip(row_inds, col_inds) if labels[e[0]] < labels[e[1]]] # remove pairs of documents of the same level 132 | row_inds = [e[0] for e in pairs] 133 | col_inds = [e[1] for e in pairs] 134 | 135 | else: 136 | raise NotImplementedError 137 | 138 | #tor_row_inds = torch.LongTensor(row_inds).to(device) if gpu else torch.LongTensor(row_inds) 139 | #tor_col_inds = torch.LongTensor(col_inds).to(device) if gpu else torch.LongTensor(col_inds) 140 | tor_row_inds = torch.LongTensor(row_inds, device) 141 | tor_col_inds = torch.LongTensor(col_inds, device) 142 | #batch_triu = batch_mats[:, tor_row_inds, tor_col_inds] 143 | 144 | return tor_row_inds, tor_col_inds # shape: [number of pairs] 145 | 146 | -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/util/lambda_utils.py: -------------------------------------------------------------------------------- 1 | 2 | import torch 3 | 4 | 5 | def get_pairwise_comp_probs(batch_preds, batch_std_labels, sigma=None): 6 | ''' 7 | Get the predicted and standard probabilities p_ij which denotes d_i beats d_j 8 | @param batch_preds: 9 | @param batch_std_labels: 10 | @param sigma: 11 | @return: 12 | ''' 13 | # computing pairwise differences w.r.t. predictions, i.e., s_i - s_j 14 | batch_s_ij = torch.unsqueeze(batch_preds, dim=2) - torch.unsqueeze(batch_preds, dim=1) 15 | batch_p_ij = torch.sigmoid(sigma * batch_s_ij) 16 | 17 | # computing pairwise differences w.r.t. standard labels, i.e., S_{ij} 18 | batch_std_diffs = torch.unsqueeze(batch_std_labels, dim=2) - torch.unsqueeze(batch_std_labels, dim=1) 19 | # ensuring S_{ij} \in {-1, 0, 1} 20 | batch_Sij = torch.clamp(batch_std_diffs, min=-1.0, max=1.0) 21 | batch_std_p_ij = 0.5 * (1.0 + batch_Sij) 22 | 23 | return batch_p_ij, batch_std_p_ij -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/util/one_hot_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | 6 | """ 7 | 8 | import torch 9 | 10 | def get_one_hot_reprs(batch_stds, gpu=False): 11 | """ Get one-hot representation of batch ground-truth labels """ 12 | batch_size = batch_stds.size(0) 13 | hist_size = batch_stds.size(1) 14 | int_batch_stds = batch_stds.type(torch.cuda.LongTensor) if gpu else batch_stds.type(torch.LongTensor) 15 | 16 | hot_batch_stds = torch.cuda.FloatTensor(batch_size, hist_size, 3) if gpu else torch.FloatTensor(batch_size, hist_size, 3) 17 | hot_batch_stds.zero_() 18 | hot_batch_stds.scatter_(2, torch.unsqueeze(int_batch_stds, 2), 1) 19 | 20 | return hot_batch_stds 21 | -------------------------------------------------------------------------------- /ptranking/ltr_adhoc/util/sampling_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import torch 5 | 6 | EPS = 1e-20 7 | 8 | ####### 9 | # Module for listwise sampling 10 | # We note that: each time we only sample one ranking per query since multiple times of sampling just corresonds to more epoches 11 | ####### 12 | 13 | def arg_shuffle_ties(batch_rankings, descending=True, device=None): 14 | '''Shuffle ties, and return the corresponding indice ''' 15 | batch_size, ranking_size = batch_rankings.size() 16 | if batch_size > 1: 17 | list_rperms = [] 18 | for _ in range(batch_size): 19 | list_rperms.append(torch.randperm(ranking_size, device=device)) 20 | batch_rperms = torch.stack(list_rperms, dim=0) 21 | else: 22 | batch_rperms = torch.randperm(ranking_size, device=device).view(1, -1) 23 | 24 | batch_shuffled_rankings = torch.gather(batch_rankings, dim=1, index=batch_rperms) 25 | batch_desc_inds = torch.argsort(batch_shuffled_rankings, descending=descending) 26 | batch_shuffle_ties_inds = torch.gather(batch_rperms, dim=1, index=batch_desc_inds) 27 | 28 | return batch_shuffle_ties_inds 29 | 30 | 31 | def sample_ranking_PL(batch_preds, only_indices=True, temperature=1.0): 32 | ''' 33 | Sample one ranking per query based on Plackett-Luce model 34 | @param batch_preds: [batch_size, ranking_size] each row denotes the relevance predictions for documents associated with the same query 35 | @param only_indices: only return the indices or not 36 | ''' 37 | if torch.isnan(batch_preds).any(): # checking is needed for later PL model 38 | print('batch_preds', batch_preds) 39 | print('Including NaN error.') 40 | 41 | if 1.0 != temperature: 42 | target_batch_preds = torch.div(batch_preds, temperature) 43 | else: 44 | target_batch_preds = batch_preds 45 | 46 | batch_m, _ = torch.max(target_batch_preds, dim=1, keepdim=True) # a transformation aiming for higher stability when computing softmax() with exp() 47 | m_target_batch_preds = target_batch_preds - batch_m 48 | batch_exps = torch.exp(m_target_batch_preds) 49 | batch_sample_inds = torch.multinomial(batch_exps, replacement=False, num_samples=batch_preds.size(1)) 50 | 51 | if only_indices: 52 | return batch_sample_inds 53 | else: 54 | # sort batch_preds according to the sample order 55 | # w.r.t. top-k, we need the remaining part, but we don't consider the orders among the remaining parts 56 | batch_preds_in_sample_order = torch.gather(batch_preds, dim=1, index=batch_sample_inds) 57 | return batch_sample_inds, batch_preds_in_sample_order 58 | 59 | 60 | def sample_ranking_PL_gumbel_softmax(batch_preds, only_indices=True, temperature=1.0, device=None): 61 | ''' 62 | Sample a ranking based stochastic Plackett-Luce model, where gumble noise is added 63 | @param batch_preds: [batch_size, ranking_size] each row denotes the relevance predictions for documents associated with the same query 64 | @param only_indices: only return the indices or not 65 | ''' 66 | unif = torch.rand(batch_preds.size(), device=device) # [batch_size, ranking_size] 67 | 68 | gumbel = -torch.log(-torch.log(unif + EPS) + EPS) # Sample from gumbel distribution 69 | 70 | if only_indices: 71 | batch_logits = batch_preds + gumbel 72 | _, batch_sample_inds = torch.sort(batch_logits, dim=1, descending=True) 73 | return batch_sample_inds 74 | else: 75 | if 1.0 == temperature: 76 | batch_logits = batch_preds + gumbel 77 | else: 78 | batch_logits = (batch_preds + gumbel) / temperature 79 | 80 | batch_logits_in_sample_order, batch_sample_inds = torch.sort(batch_logits, dim=1, descending=True) 81 | return batch_sample_inds, batch_logits_in_sample_order 82 | 83 | ###### 84 | # 85 | ###### 86 | 87 | def unique_count(std_labels, descending=True): 88 | asc_std_labels, _ = torch.sort(std_labels) 89 | uni_elements, inds = torch.unique(asc_std_labels, sorted=True, return_inverse=True) 90 | asc_uni_cnts = torch.stack([(asc_std_labels == e).sum() for e in uni_elements]) 91 | 92 | if descending: 93 | des_uni_cnts = torch.flip(asc_uni_cnts, dims=[0]) 94 | return des_uni_cnts 95 | else: 96 | return asc_uni_cnts 97 | 98 | 99 | def batch_global_unique_count(batch_std_labels, max_rele_lavel, descending=True, gpu=False): 100 | ''' ''' 101 | batch_asc_std_labels, _ = torch.sort(batch_std_labels, dim=1) 102 | # default ascending order 103 | global_uni_elements = torch.arange(max_rele_lavel+1).type(torch.cuda.FloatTensor) if gpu else torch.arange(max_rele_lavel+1).type(torch.FloatTensor) 104 | 105 | asc_uni_cnts = torch.cat([(batch_asc_std_labels == e).sum(dim=1, keepdim=True) for e in global_uni_elements], dim=1) # row-wise count per element 106 | 107 | if descending: 108 | des_uni_cnts = torch.flip(asc_uni_cnts, dims=[1]) 109 | return des_uni_cnts 110 | else: 111 | return asc_uni_cnts 112 | 113 | def uniform_rand_per_label(uni_cnts, device='cpu'): 114 | """ can be compatible with batch """ 115 | num_unis = uni_cnts.size(0) # number of unique elements 116 | inner_rand_inds = (torch.rand(num_unis) * uni_cnts.type(torch.FloatTensor)).type(torch.LongTensor) # random index w.r.t each interval 117 | begs = torch.cumsum(torch.cat([torch.tensor([0.], dtype=torch.long, device=device), uni_cnts[0:num_unis - 1]]), dim=0) # begin positions of each interval within the same vector 118 | # print('begin positions', begs) 119 | rand_inds_per_label = begs + inner_rand_inds 120 | # print('random index', rand_inds_per_label) # random index tensor([ 0, 1, 3, 6, 10]) ([0, 2, 3, 5, 8]) 121 | 122 | return rand_inds_per_label 123 | 124 | 125 | def sample_per_label(batch_rankings, batch_stds): 126 | assert 1 == batch_stds.size(0) 127 | 128 | std_labels = torch.squeeze(batch_stds) 129 | des_uni_cnts = unique_count(std_labels) 130 | rand_inds_per_label = uniform_rand_per_label(des_uni_cnts) 131 | 132 | sp_batch_rankings = batch_rankings[:, rand_inds_per_label, :] 133 | sp_batch_stds = batch_stds[:, rand_inds_per_label] 134 | 135 | return sp_batch_rankings, sp_batch_stds 136 | 137 | 138 | if __name__ == '__main__': 139 | #1 140 | #cuda = 'cuda:0' 141 | cuda = 'cpu' 142 | target_batch_stds = torch.randn(size=(3, 5), device=cuda) 143 | 144 | batch_shuffle_ties_inds = arg_shuffle_ties(target_batch_stds=target_batch_stds, descending=True, device=cuda) 145 | print('batch_shuffle_ties_inds', batch_shuffle_ties_inds) 146 | 147 | ''' 148 | std_labels = tensor([3, 3, 2, 1, 1, 1, 0, 0, 0, 0]) 149 | des_uni_cnts = unique_count(std_labels) 150 | print('des_uni_cnts', des_uni_cnts) 151 | 152 | batch_global_des_uni_cnts = batch_global_unique_count(std_labels.view(2, -1), max_rele_lavel=4) 153 | print(std_labels.view(2, -1)) 154 | print('batch_global_des_uni_cnts', batch_global_des_uni_cnts) 155 | 156 | rand_inds_per_label = uniform_rand_per_label(des_uni_cnts) 157 | print('random elements', std_labels[rand_inds_per_label]) 158 | ''' 159 | -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/base/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/base/ad_machine.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | class AdversarialMachine(): 6 | ''' 7 | An abstract adversarial learning-to-rank framework 8 | ''' 9 | 10 | def __init__(self, eval_dict=None, data_dict=None, gpu=False, device=None): 11 | #todo double-check the necessity of these two arguments 12 | self.eval_dict = eval_dict 13 | self.data_dict = data_dict 14 | self.gpu, self.device = gpu, device 15 | 16 | def pre_check(self): 17 | pass 18 | 19 | def burn_in(self, train_data=None): 20 | pass 21 | 22 | def mini_max_train(self, train_data=None, generator=None, discriminator=None, d_epoches=1, g_epoches=1, global_buffer=None): 23 | pass 24 | 25 | def fill_global_buffer(self): 26 | ''' 27 | Buffer some global information for higher efficiency. 28 | We note that global information may differ w.r.t. the model 29 | ''' 30 | pass 31 | 32 | def generate_data(self, train_data=None, generator=None, **kwargs): 33 | pass 34 | 35 | def train_generator(self, train_data=None, generated_data=None, generator=None, discriminator=None, **kwargs): 36 | pass 37 | 38 | def train_discriminator(self, train_data=None, generated_data=None, discriminator=None, **kwargs): 39 | pass 40 | 41 | def reset_generator(self): 42 | pass 43 | 44 | def reset_discriminator(self): 45 | pass 46 | 47 | def reset_generator_discriminator(self): 48 | self.reset_generator() 49 | self.reset_discriminator() 50 | 51 | def get_generator(self): 52 | return None 53 | 54 | def get_discriminator(self): 55 | return None 56 | -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/base/ad_player.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from ptranking.base.adhoc_ranker import AdhocNeuralRanker 5 | 6 | class AdversarialPlayer(AdhocNeuralRanker): 7 | ''' 8 | An adversarial player, which is used as a component of AdversarialMachine 9 | ''' 10 | def __init__(self, id=None, sf_para_dict=None, weight_decay=1e-3, gpu=False, device=None): 11 | super(AdversarialPlayer, self).__init__(id=id, sf_para_dict=sf_para_dict, 12 | weight_decay=weight_decay, gpu=gpu, device=device) 13 | -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/eval/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | 6 | """ -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/listwise/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/listwise/list_discriminator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | 6 | """ 7 | 8 | from ptranking.ltr_adversarial.base.ad_player import AdversarialPlayer 9 | 10 | class List_Discriminator(AdversarialPlayer): 11 | ''' 12 | A listwise discriminator 13 | ''' 14 | def __init__(self, sf_para_dict=None, gpu=False, device=None): 15 | super(List_Discriminator, self).__init__(sf_para_dict=sf_para_dict, gpu=gpu, device=device) 16 | -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/listwise/list_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | 6 | """ 7 | 8 | from ptranking.ltr_adversarial.base.ad_player import AdversarialPlayer 9 | 10 | class List_Generator(AdversarialPlayer): 11 | ''' 12 | A listwise generator 13 | ''' 14 | def __init__(self, sf_para_dict=None, gpu=False, device=None): 15 | super(List_Generator, self).__init__(sf_para_dict=sf_para_dict, gpu=gpu, device=device) 16 | -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/pairwise/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | 6 | """ -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/pairwise/pair_discriminator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ 8 | 9 | from ptranking.ltr_adversarial.base.ad_player import AdversarialPlayer 10 | 11 | class Pair_Discriminator(AdversarialPlayer): 12 | ''' 13 | A pairwise discriminator 14 | ''' 15 | def __init__(self, sf_para_dict=None, gpu=False, device=None): 16 | super(Pair_Discriminator, self).__init__(sf_para_dict=sf_para_dict, gpu=gpu, device=device) 17 | -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/pairwise/pair_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ 8 | 9 | from ptranking.ltr_adversarial.base.ad_player import AdversarialPlayer 10 | 11 | class Pair_Generator(AdversarialPlayer): 12 | ''' 13 | A pairwise generator 14 | ''' 15 | def __init__(self, sf_para_dict=None, gpu=False, device=None): 16 | super(Pair_Generator, self).__init__(sf_para_dict=sf_para_dict, gpu=gpu, device=device) 17 | -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/pointwise/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | 6 | """ -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/pointwise/point_discriminator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from ptranking.ltr_adversarial.base.ad_player import AdversarialPlayer 5 | 6 | class Point_Discriminator(AdversarialPlayer): 7 | ''' 8 | A pointwise discriminator 9 | ''' 10 | def __init__(self, sf_para_dict=None, gpu=False, device=None): 11 | super(Point_Discriminator, self).__init__(sf_para_dict=sf_para_dict, gpu=gpu, device=device) 12 | -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/pointwise/point_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from ptranking.ltr_adversarial.base.ad_player import AdversarialPlayer 5 | 6 | class Point_Generator(AdversarialPlayer): 7 | ''' 8 | A pointwise generator 9 | ''' 10 | def __init__(self, sf_para_dict=None, gpu=False, device=None): 11 | super(Point_Generator, self).__init__(sf_para_dict=sf_para_dict, gpu=gpu, device=device) 12 | -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/util/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | 6 | """ -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/util/f_divergence.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import math 5 | 6 | import torch 7 | 8 | 9 | def get_f_divergence_functions(f_div_str=None): 10 | ''' 11 | the activation function is chosen as a monotone increasing function 12 | ''' 13 | if 'TVar' == f_div_str: # Total variation 14 | def activation_f(v): 15 | return 0.5 * torch.tanh(v) 16 | 17 | def conjugate_f(t): 18 | return t 19 | 20 | elif 'KL' == f_div_str: # Kullback-Leibler 21 | def activation_f(v): 22 | return v 23 | 24 | def conjugate_f(t): 25 | return torch.exp(t-1) 26 | 27 | elif 'RKL' == f_div_str: # Reverse KL 28 | def activation_f(v): 29 | return -torch.exp(-v) 30 | 31 | def conjugate_f(t): 32 | return -1.0 - torch.log(-t) 33 | 34 | elif 'PC' == f_div_str: # Pearson chi-square 35 | def activation_f(v): 36 | return v 37 | 38 | def conjugate_f(t): 39 | return 0.25 * torch.pow(t, exponent=2.0) + t 40 | 41 | elif 'NC' == f_div_str: # Neyman chi-square 42 | def activation_f(v): 43 | return 1.0 - torch.exp(-v) 44 | 45 | def conjugate_f(t): 46 | return 2.0 - 2.0 * torch.sqrt(1.0-t) 47 | 48 | elif 'SH' == f_div_str: # Squared Hellinger 49 | def activation_f(v): 50 | return 1.0 - torch.exp(-v) 51 | 52 | def conjugate_f(t): 53 | return t/(1.0-t) 54 | 55 | elif 'JS' == f_div_str: # Jensen-Shannon 56 | def activation_f(v): 57 | return torch.log(torch.tensor(2.0)) - torch.log(1.0 + torch.exp(-v)) 58 | 59 | def conjugate_f(t): 60 | return -torch.log(2.0 - torch.exp(t)) 61 | 62 | elif 'JSW' == f_div_str: # Jensen-Shannon-weighted 63 | def activation_f(v): 64 | return -math.pi*torch.log(math.pi) - torch.log(1.0+torch.exp(-v)) 65 | 66 | def conjugate_f(t): 67 | return (1.0-math.pi)*torch.log((1.0-math.pi)/(1.0-math.pi*torch.exp(t/math.pi))) 68 | 69 | elif 'GAN' == f_div_str: # GAN 70 | def activation_f(v): 71 | return -torch.log(1.0 + torch.exp(-v)) 72 | 73 | def conjugate_f(t): 74 | return -torch.log(1.0 - torch.exp(t)) 75 | 76 | return activation_f, conjugate_f 77 | -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/util/list_probability.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | 6 | """ 7 | 8 | import torch 9 | 10 | from ptranking.ltr_adhoc.listwise.listmle import apply_LogCumsumExp 11 | 12 | 13 | """ 14 | Plackett_Luce 15 | """ 16 | 17 | def ranking_prob_Plackett_Luce(batch_preds): 18 | batch_log_prob = log_ranking_prob_Plackett_Luce(batch_preds) 19 | batch_prob = torch.exp(batch_log_prob) 20 | 21 | return batch_prob 22 | 23 | 24 | def log_ranking_prob_Plackett_Luce(batch_preds): 25 | assert 2 == len(batch_preds.size()) 26 | 27 | batch_logcumsumexps = apply_LogCumsumExp(batch_preds) 28 | batch_log_prob = torch.sum(batch_preds - batch_logcumsumexps, dim=1) 29 | 30 | return batch_log_prob 31 | 32 | 33 | """ 34 | Bradley_Terry 35 | """ 36 | 37 | def ranking_prob_Bradley_Terry(batch_preds): 38 | batch_log_ranking_prob = log_ranking_prob_Bradley_Terry(batch_preds) 39 | batch_BT_ranking_prob = torch.exp(batch_log_ranking_prob) 40 | 41 | return batch_BT_ranking_prob 42 | 43 | 44 | def log_ranking_prob_Bradley_Terry(batch_preds): 45 | ''' 46 | :param batch_preds: [batch_size, list_size] 47 | :return: 48 | ''' 49 | assert 2 == len(batch_preds.size()) 50 | 51 | max_v = torch.max(batch_preds) 52 | new_batch_preds = torch.exp(batch_preds - max_v) 53 | 54 | batch_numerators = torch.unsqueeze(new_batch_preds, dim=2).repeat(1, 1, batch_preds.size(1)) 55 | 56 | batch_denominaotrs = torch.unsqueeze(new_batch_preds, dim=2) + torch.unsqueeze(new_batch_preds, dim=1) 57 | 58 | batch_BT_probs = batch_numerators / batch_denominaotrs 59 | 60 | batch_log_ranking_prob = torch.sum(torch.sum(torch.triu(torch.log(batch_BT_probs), diagonal=1), dim=2), dim=1) 61 | 62 | return batch_log_ranking_prob 63 | -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/util/list_sampling.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ 8 | 9 | import torch 10 | from torch.nn import functional as F 11 | 12 | 13 | EPS = 1e-20 14 | 15 | 16 | def gumbel_softmax(logits, samples_per_query, temperature=1.0, cuda=False, cuda_device=None): 17 | ''' 18 | 19 | :param logits: [1, ranking_size] 20 | :param num_samples_per_query: number of stochastic rankings to generate 21 | :param temperature: 22 | :return: 23 | ''' 24 | assert 1 == logits.size(0) and 2 == len(logits.size()) 25 | 26 | unif = torch.rand(samples_per_query, logits.size(1)) # [num_samples_per_query, ranking_size] 27 | if cuda: unif = unif.to(cuda_device) 28 | 29 | gumbel = -torch.log(-torch.log(unif + EPS) + EPS) # Sample from gumbel distribution 30 | 31 | logit = (logits + gumbel) / temperature 32 | 33 | y = F.softmax(logit, dim=1) 34 | 35 | # i.e., #return F.softmax(logit, dim=1) 36 | return y 37 | 38 | def sample_ranking_PL_gumbel_softmax(batch_preds, num_sample_ranking=1, only_indices=True, temperature=1.0, gpu=False, device=None): 39 | ''' 40 | Sample a ranking based stochastic Plackett-Luce model, where gumble noise is added 41 | @param batch_preds: [1, ranking_size] vector of relevance predictions for documents associated with the same query 42 | @param num_sample_ranking: number of rankings to sample 43 | @param only_indices: only return the indices or not 44 | @return: 45 | ''' 46 | if num_sample_ranking > 1: 47 | target_batch_preds = batch_preds.expand(num_sample_ranking, -1) 48 | else: 49 | target_batch_preds = batch_preds 50 | 51 | unif = torch.rand(target_batch_preds.size()) # [num_samples_per_query, ranking_size] 52 | if gpu: unif = unif.to(device) 53 | 54 | gumbel = -torch.log(-torch.log(unif + EPS) + EPS) # Sample from gumbel distribution 55 | 56 | if only_indices: 57 | batch_logits = target_batch_preds + gumbel 58 | _, batch_indices = torch.sort(batch_logits, dim=1, descending=True) 59 | return batch_indices 60 | else: 61 | if 1.0 == temperature: 62 | batch_logits = target_batch_preds + gumbel 63 | else: 64 | batch_logits = (target_batch_preds + gumbel) / temperature 65 | 66 | batch_logits_sorted, batch_indices = torch.sort(batch_logits, dim=1, descending=True) 67 | return batch_indices, batch_logits_sorted 68 | 69 | 70 | def arg_shuffle_ties(target_batch_stds, descending=True, gpu=False, device=None): 71 | ''' Shuffle ties, and return the corresponding indice ''' 72 | batch_size, ranking_size = target_batch_stds.size() 73 | if batch_size > 1: 74 | list_rperms = [] 75 | for _ in range(batch_size): 76 | list_rperms.append(torch.randperm(ranking_size)) 77 | batch_rperms = torch.stack(list_rperms, dim=0) 78 | else: 79 | batch_rperms = torch.randperm(ranking_size).view(1, -1) 80 | 81 | if gpu: batch_rperms = batch_rperms.to(device) 82 | 83 | shuffled_target_batch_stds = torch.gather(target_batch_stds, dim=1, index=batch_rperms) 84 | batch_sorted_inds = torch.argsort(shuffled_target_batch_stds, descending=descending) 85 | batch_shuffle_ties_inds = torch.gather(batch_rperms, dim=1, index=batch_sorted_inds) 86 | 87 | return batch_shuffle_ties_inds 88 | -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/util/point_sampling.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | 6 | """ 7 | 8 | import torch 9 | 10 | def generate_true_docs(qid, truth_label, num_samples, dict_true_inds=None): 11 | ''' 12 | :param qid: 13 | :param truth_label: it is fine if including '-1' 14 | :param num_samples: 15 | :param dict_true_inds: 16 | :return: 17 | ''' 18 | if dict_true_inds is not None and qid in dict_true_inds: 19 | true_inds, size_unique = dict_true_inds[qid] 20 | 21 | if num_samples is None or num_samples>size_unique: 22 | num_samples = int(0.5*size_unique) 23 | if num_samples < 1: 24 | num_samples = 1 25 | 26 | rand_inds = torch.multinomial(torch.ones(size_unique), num_samples, replacement=False) 27 | return true_inds[rand_inds] 28 | else: 29 | # [z, n] If input has n dimensions, then the resulting indices tensor out is of size (z×n), where z is the total number of non-zero elements in the input tensor. 30 | true_inds = torch.nonzero(torch.gt(truth_label, 0), as_tuple=False) 31 | 32 | size_unique = true_inds.size(0) 33 | true_inds = true_inds[:, 0] 34 | if dict_true_inds is not None: #buffer 35 | dict_true_inds[qid] = (true_inds, size_unique) 36 | 37 | if num_samples is None or num_samples>size_unique: 38 | num_samples = int(0.5*size_unique) 39 | if num_samples < 1: 40 | num_samples = 1 41 | 42 | rand_inds = torch.multinomial(torch.ones(size_unique), num_samples, replacement=False) 43 | return true_inds[rand_inds] -------------------------------------------------------------------------------- /ptranking/ltr_adversarial/util/semi_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /ptranking/ltr_diversification/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/ptranking/ltr_diversification/__init__.py -------------------------------------------------------------------------------- /ptranking/ltr_diversification/base/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/ptranking/ltr_diversification/base/__init__.py -------------------------------------------------------------------------------- /ptranking/ltr_diversification/base/div_list_ranker.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import copy 4 | 5 | import torch 6 | 7 | from ptranking.base.utils import get_stacked_FFNet 8 | from ptranking.base.list_ranker import MultiheadAttention, PositionwiseFeedForward, Encoder, EncoderLayer, ListNeuralRanker 9 | 10 | dc = copy.deepcopy 11 | 12 | class DivListNeuralRanker(ListNeuralRanker): 13 | ''' 14 | A univariate scoring function for diversified ranking, where listwise information is integrated. 15 | ''' 16 | def __init__(self, id='DivListNeuralRanker', sf_para_dict=None, weight_decay=1e-3, gpu=False, device=None): 17 | super(DivListNeuralRanker, self).__init__(id=id, sf_para_dict=sf_para_dict, weight_decay=weight_decay, gpu=gpu, device=device) 18 | self.encoder_type = self.sf_para_dict[self.sf_para_dict['sf_id']]['encoder_type'] 19 | 20 | def ini_listsf(self, num_features=None, n_heads=2, encoder_layers=2, dropout=0.1, encoder_type=None, 21 | ff_dims=[256, 128, 64], out_dim=1, AF='R', TL_AF='GE', apply_tl_af=False, 22 | BN=True, bn_type=None, bn_affine=False): 23 | ''' 24 | Initialization the univariate scoring function for diversified ranking. 25 | ''' 26 | # the input size according to the used dataset 27 | encoder_num_features, fc_num_features = num_features * 3, num_features * 6 28 | 29 | ''' Component-1: stacked multi-head self-attention (MHSA) blocks ''' 30 | mhsa = MultiheadAttention(hid_dim=encoder_num_features, n_heads=n_heads, dropout=dropout, device=self.device) 31 | 32 | if 'AllRank' == encoder_type: 33 | fc = PositionwiseFeedForward(encoder_num_features, hid_dim=encoder_num_features, dropout=dropout) 34 | encoder = Encoder(layer=EncoderLayer(hid_dim=encoder_num_features, mhsa=dc(mhsa), encoder_type=encoder_type, 35 | fc=fc, dropout=dropout), 36 | num_layers=encoder_layers, encoder_type=encoder_type) 37 | 38 | elif 'DASALC' == encoder_type: # we note that feature normalization strategy is different from AllRank 39 | encoder = Encoder(layer=EncoderLayer(hid_dim=encoder_num_features, mhsa=dc(mhsa), encoder_type=encoder_type), 40 | num_layers=encoder_layers, encoder_type=encoder_type) 41 | 42 | elif 'AttnDIN' == encoder_type: 43 | encoder = Encoder(layer=EncoderLayer(hid_dim=encoder_num_features, mhsa=dc(mhsa), encoder_type=encoder_type), 44 | num_layers=encoder_layers, encoder_type=encoder_type) 45 | else: 46 | raise NotImplementedError 47 | 48 | ''' Component-2: univariate scoring function ''' 49 | uni_ff_dims = [fc_num_features] 50 | uni_ff_dims.extend(ff_dims) 51 | uni_ff_dims.append(out_dim) 52 | uni_sf = get_stacked_FFNet(ff_dims=uni_ff_dims, AF=AF, TL_AF=TL_AF, apply_tl_af=apply_tl_af, 53 | BN=BN, bn_type=bn_type, bn_affine=bn_affine, device=self.device) 54 | if self.gpu: 55 | encoder = encoder.to(self.device) 56 | uni_sf = uni_sf.to(self.device) 57 | 58 | list_sf = {'encoder': encoder, 'uni_sf': uni_sf} 59 | return list_sf 60 | 61 | def div_forward(self, q_repr, doc_reprs): 62 | latent_cross_reprs = q_repr * doc_reprs 63 | # TODO is it OK if using expand as boradcasting? 64 | cat_1st_reprs = torch.cat((q_repr.expand(doc_reprs.size(0), -1), doc_reprs, latent_cross_reprs), 1) 65 | 66 | if 'AllRank' == self.encoder_type: 67 | #batch_FC_mappings = self.list_sf['head_ffnns'](cat_reprs) 68 | batch_encoder_mappings = self.list_sf['encoder'](torch.unsqueeze(cat_1st_reprs, dim=0)) 69 | 70 | elif 'DASALC' == self.encoder_type: 71 | #batch_FC_mappings = self.list_sf['head_ffnns'](cat_reprs) 72 | batch_encoder_mappings = self.list_sf['encoder'](torch.unsqueeze(cat_1st_reprs, dim=0)) 73 | 74 | elif 'AttnDIN' == self.encoder_type: 75 | #batch_FC_mappings = self.list_sf['head_ffnns'](cat_reprs) # -> the same shape as the output of encoder 76 | batch_encoder_mappings = self.list_sf['encoder'](torch.unsqueeze(cat_1st_reprs, dim=0)) # the batch dimension is required 77 | else: 78 | raise NotImplementedError 79 | 80 | encoder_mappings = torch.squeeze(batch_encoder_mappings, dim=0) 81 | cat_2nd_reprs = torch.cat((q_repr.expand(doc_reprs.size(0), -1), doc_reprs, latent_cross_reprs, encoder_mappings), dim=1) 82 | batch_preds = self.list_sf['uni_sf'](cat_2nd_reprs) 83 | batch_preds = batch_preds.view(1, -1) # [num_docs, 1] -> [batch, ranking_size] 84 | 85 | return batch_preds 86 | 87 | def get_parameters(self): 88 | all_parameters = list(self.list_sf['encoder'].parameters()) +\ 89 | list(self.list_sf['uni_sf'].parameters()) 90 | return all_parameters 91 | 92 | def eval_mode(self): 93 | self.list_sf['encoder'].eval() 94 | self.list_sf['uni_sf'].eval() 95 | 96 | def train_mode(self): 97 | self.list_sf['encoder'].train(mode=True) 98 | self.list_sf['uni_sf'].train(mode=True) 99 | 100 | def save(self, dir, name): 101 | if not os.path.exists(dir): 102 | os.makedirs(dir) 103 | 104 | torch.save({"encoder": self.list_sf['encoder'].state_dict(), 105 | "uni_sf": self.list_sf['uni_sf'].state_dict()}, dir + name) 106 | 107 | def load(self, file_model, **kwargs): 108 | checkpoint = torch.load(file_model) 109 | self.list_sf['encoder'].load_state_dict(checkpoint["encoder"]) 110 | self.list_sf['uni_sf'].load_state_dict(checkpoint["uni_sf"]) 111 | -------------------------------------------------------------------------------- /ptranking/ltr_diversification/base/div_point_ranker.py: -------------------------------------------------------------------------------- 1 | 2 | import torch 3 | 4 | from ptranking.base.point_ranker import PointNeuralRanker 5 | 6 | class DivPointNeuralRanker(PointNeuralRanker): 7 | ''' 8 | A one-size-fits-all neural ranker. 9 | Given the documents associated with the same query, this ranker scores each document independently. 10 | ''' 11 | def __init__(self, id='DivPointNeuralRanker', sf_para_dict=None, weight_decay=1e-3, gpu=False, device=None): 12 | super(DivPointNeuralRanker, self).__init__(id=id, sf_para_dict=sf_para_dict, weight_decay=weight_decay, gpu=gpu, device=device) 13 | 14 | def div_forward(self, q_repr, doc_reprs): 15 | num_docs, num_features = doc_reprs.size() 16 | 17 | latent_cross_reprs = q_repr * doc_reprs 18 | #TODO is it OK if using expand as boradcasting? 19 | cat_reprs = torch.cat((q_repr.expand(doc_reprs.size(0), -1), latent_cross_reprs, doc_reprs), 1) 20 | 21 | _pred = self.point_sf(cat_reprs) 22 | batch_pred = _pred.view(-1, num_docs) # -> [batch_size, num_docs], where batch_size=1 since only one query 23 | 24 | return batch_pred 25 | -------------------------------------------------------------------------------- /ptranking/ltr_diversification/base/diversity_ranker.py: -------------------------------------------------------------------------------- 1 | 2 | from ptranking.ltr_diversification.base.div_list_ranker import DivListNeuralRanker 3 | from ptranking.ltr_diversification.base.div_point_ranker import DivPointNeuralRanker 4 | 5 | class DiversityNeuralRanker(DivPointNeuralRanker, DivListNeuralRanker): 6 | ''' 7 | A combination of PointNeuralRanker & PENeuralRanker 8 | ''' 9 | def __init__(self, id='DiversityNeuralRanker', sf_para_dict=None, weight_decay=1e-3, gpu=False, device=None): 10 | self.id = id 11 | self.gpu, self.device = gpu, device 12 | 13 | self.sf_para_dict = sf_para_dict 14 | self.sf_id = sf_para_dict['sf_id'] 15 | assert self.sf_id in ['pointsf', 'listsf'] 16 | 17 | self.opt, self.lr = sf_para_dict['opt'], sf_para_dict['lr'] 18 | self.weight_decay = weight_decay 19 | 20 | self.stop_check_freq = 10 21 | 22 | if 'pointsf' == self.sf_id: # corresponding to the concatenation operation, i.e., q_repr + doc_repr + latent_cross 23 | self.sf_para_dict[self.sf_id]['num_features'] *= 3 24 | elif 'listsf' == self.sf_id: 25 | self.encoder_type = self.sf_para_dict[self.sf_para_dict['sf_id']]['encoder_type'] 26 | 27 | 28 | def init(self): 29 | if self.sf_id.startswith('pointsf'): 30 | DivPointNeuralRanker.init(self) 31 | elif self.sf_id.startswith('listsf'): 32 | DivListNeuralRanker.init(self) 33 | 34 | def get_parameters(self): 35 | if self.sf_id.startswith('pointsf'): 36 | return DivPointNeuralRanker.get_parameters(self) 37 | elif self.sf_id.startswith('listsf'): 38 | return DivListNeuralRanker.get_parameters(self) 39 | 40 | def div_forward(self, q_repr, doc_reprs): 41 | if self.sf_id.startswith('pointsf'): 42 | return DivPointNeuralRanker.div_forward(self, q_repr, doc_reprs) 43 | elif self.sf_id.startswith('listsf'): 44 | return DivListNeuralRanker.div_forward(self, q_repr, doc_reprs) 45 | 46 | def eval_mode(self): 47 | if self.sf_id.startswith('pointsf'): 48 | DivPointNeuralRanker.eval_mode(self) 49 | elif self.sf_id.startswith('listsf'): 50 | DivListNeuralRanker.eval_mode(self) 51 | 52 | def div_validation(self, vali_data=None, vali_metric=None, k=5, max_label=None, device='cpu'): 53 | if 'aNDCG' == vali_metric: 54 | return self.alpha_ndcg_at_k(test_data=vali_data, k=k, device=device) 55 | elif 'nERR-IA' == vali_metric: # nERR-IA is better choice than ERR-IA with no normalization 56 | return self.nerr_ia_at_k(test_data=vali_data, k=k, max_label=max_label, device=device) 57 | else: 58 | raise NotImplementedError 59 | 60 | def train_mode(self): 61 | if self.sf_id.startswith('pointsf'): 62 | DivPointNeuralRanker.train_mode(self) 63 | elif self.sf_id.startswith('listsf'): 64 | DivListNeuralRanker.train_mode(self) 65 | 66 | def save(self, dir, name): 67 | if self.sf_id.startswith('pointsf'): 68 | DivPointNeuralRanker.save(self, dir=dir, name=name) 69 | elif self.sf_id.startswith('listsf'): 70 | DivListNeuralRanker.save(self, dir=dir, name=name) 71 | 72 | def load(self, file_model, **kwargs): 73 | if self.sf_id.startswith('pointsf'): 74 | DivPointNeuralRanker.load(self, file_model=file_model, **kwargs) 75 | elif self.sf_id.startswith('listsf'): 76 | DivListNeuralRanker.load(self, file_model=file_model, **kwargs) 77 | 78 | def get_tl_af(self): 79 | if self.sf_id.startswith('pointsf'): 80 | DivPointNeuralRanker.get_tl_af(self) 81 | elif self.sf_id.startswith('listsf'): 82 | self.sf_para_dict[self.sf_para_dict['sf_id']]['AF'] 83 | -------------------------------------------------------------------------------- /ptranking/ltr_diversification/eval/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/ptranking/ltr_diversification/eval/__init__.py -------------------------------------------------------------------------------- /ptranking/ltr_diversification/one_by_one/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/ptranking/ltr_diversification/one_by_one/__init__.py -------------------------------------------------------------------------------- /ptranking/ltr_diversification/score_and_sort/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/ptranking/ltr_diversification/score_and_sort/__init__.py -------------------------------------------------------------------------------- /ptranking/ltr_diversification/score_and_sort/daletor.py: -------------------------------------------------------------------------------- 1 | 2 | import torch 3 | from itertools import product 4 | 5 | from ptranking.base.utils import robust_sigmoid 6 | from ptranking.ltr_adhoc.eval.parameter import ModelParameter 7 | from ptranking.ltr_diversification.base.diversity_ranker import DiversityNeuralRanker 8 | 9 | def get_approx_ranks(batch_preds, rt=None, device=None, q_doc_rele_mat=None): 10 | ''' get approximated rank positions: Equation-7 in the paper''' 11 | batch_pred_diffs = torch.unsqueeze(batch_preds, dim=2) - torch.unsqueeze(batch_preds, dim=1) # computing pairwise differences, i.e., Sij or Sxy 12 | 13 | batch_indicators = robust_sigmoid(torch.transpose(batch_pred_diffs, dim0=1, dim1=2), rt, device) # using {-1.0*} may lead to a poor performance when compared with the above way; 14 | 15 | batch_hat_pis = torch.sum(batch_indicators, dim=2) + 0.5 # get approximated rank positions, i.e., hat_pi(x) 16 | 17 | _q_doc_rele_mat = torch.unsqueeze(q_doc_rele_mat, dim=1) 18 | batch_q_doc_rele_mat = _q_doc_rele_mat.expand(-1, q_doc_rele_mat.size(1), -1) # duplicate w.r.t. each subtopic -> [num_subtopics, ranking_size, ranking_size] 19 | prior_cover_cnts = torch.sum(batch_indicators * batch_q_doc_rele_mat, dim=2) - q_doc_rele_mat/2.0 # [num_subtopics, num_docs] 20 | 21 | return batch_hat_pis, prior_cover_cnts 22 | 23 | def alphaDCG_as_a_loss(batch_preds=None, q_doc_rele_mat=None, rt=10, device=None, alpha=0.5, top_k=10): 24 | """ 25 | There are two ways to formulate the loss: (1) using the ideal order; (2) using the predicted order (TBA) 26 | """ 27 | batch_hat_pis, prior_cover_cnts = get_approx_ranks(batch_preds, rt=rt, device=device, q_doc_rele_mat=q_doc_rele_mat) 28 | 29 | batch_per_subtopic_gains = q_doc_rele_mat * torch.pow((1.0-alpha), prior_cover_cnts) / torch.log2(1.0 + batch_hat_pis) 30 | batch_global_gains = torch.sum(batch_per_subtopic_gains, dim=1) 31 | 32 | if top_k is None: 33 | alpha_DCG = torch.sum(batch_global_gains) 34 | else: 35 | alpha_DCG = torch.sum(batch_global_gains[0:top_k]) 36 | 37 | batch_loss = -alpha_DCG 38 | return batch_loss 39 | 40 | 41 | class DALETOR(DiversityNeuralRanker): 42 | """Description 43 | Le Yan Zhen Qin Rama Kumar Pasumarthi Xuanhui Wang Mike Bendersky. 44 | Diversification-Aware Learning to Rank using Distributed Representation. 45 | The Web Conference 2021 (WWW) 46 | """ 47 | 48 | def __init__(self, sf_para_dict=None, model_para_dict=None, gpu=False, device=None): 49 | super(DALETOR, self).__init__(id='DALETOR', sf_para_dict=sf_para_dict, gpu=gpu, device=device) 50 | self.rt = model_para_dict['rt'] 51 | self.top_k = model_para_dict['top_k'] 52 | 53 | def div_custom_loss_function(self, batch_preds, q_doc_rele_mat, **kwargs): 54 | ''' 55 | :param batch_preds: [batch, ranking_size] each row represents the relevance predictions for documents within a ltr_adhoc 56 | :param batch_stds: [batch, ranking_size] each row represents the standard relevance grades for documents within a ltr_adhoc 57 | :return: 58 | ''' 59 | assert 'presort' in kwargs and kwargs['presort'] is True # aiming for directly optimising alpha-nDCG over top-k documents 60 | 61 | batch_loss = alphaDCG_as_a_loss(batch_preds=batch_preds, q_doc_rele_mat=q_doc_rele_mat, 62 | rt=self.rt, top_k=self.top_k, device=self.device) 63 | 64 | self.optimizer.zero_grad() 65 | batch_loss.backward() 66 | self.optimizer.step() 67 | 68 | return batch_loss 69 | 70 | 71 | ###### Parameter of DALETOR ###### 72 | 73 | class DALETORParameter(ModelParameter): 74 | ''' Parameter class for DALETOR ''' 75 | 76 | def __init__(self, debug=False, para_json=None): 77 | super(DALETORParameter, self).__init__(model_id='DALETOR', para_json=para_json) 78 | self.debug = debug 79 | 80 | def default_para_dict(self): 81 | """ 82 | Default parameter setting for DALETOR. Here rt (reversed T) corresponds to 1/T in paper. 83 | :return: 84 | """ 85 | if self.use_json: 86 | top_k = self.json_dict['top_k'][0] 87 | rt = self.json_dict['rt'][0] # corresponds to 1/T in paper 88 | self.DALETOR_para_dict = dict(model_id=self.model_id, rt=rt, top_k=top_k) 89 | else: 90 | self.DALETOR_para_dict = dict(model_id=self.model_id, rt=10., top_k=10) 91 | 92 | return self.DALETOR_para_dict 93 | 94 | def to_para_string(self, log=False, given_para_dict=None): 95 | """ 96 | String identifier of parameters 97 | :param log: 98 | :param given_para_dict: a given dict, which is used for maximum setting w.r.t. grid-search 99 | :return: 100 | """ 101 | # using specified para-dict or inner para-dict 102 | DALETOR_para_dict = given_para_dict if given_para_dict is not None else self.DALETOR_para_dict 103 | 104 | rt, top_k = DALETOR_para_dict['rt'], DALETOR_para_dict['top_k'] 105 | 106 | s1 = ':' if log else '_' 107 | if top_k is None: 108 | DALETOR_paras_str = s1.join(['rt', str(rt), 'topk', 'Full']) 109 | else: 110 | DALETOR_paras_str = s1.join(['rt', str(rt), 'topk', str(top_k)]) 111 | return DALETOR_paras_str 112 | 113 | def grid_search(self): 114 | """ 115 | Iterator of parameter settings for ApproxNDCG 116 | """ 117 | if self.use_json: 118 | choice_rt = self.json_dict['rt'] # corresponds to 1/T in paper 119 | choice_topk = self.json_dict['top_k'] # the cutoff value of optimising objective alpha-nDCG@k 120 | else: 121 | choice_rt = [10.0] if self.debug else [10.0] # 1.0, 10.0, 50.0, 100.0 122 | choice_topk = [10] if self.debug else [10] 123 | 124 | for rt, top_k in product(choice_rt, choice_topk): 125 | self.DALETOR_para_dict = dict(model_id=self.model_id, rt=rt, top_k=top_k) 126 | yield self.DALETOR_para_dict -------------------------------------------------------------------------------- /ptranking/ltr_diversification/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/ptranking/ltr_diversification/util/__init__.py -------------------------------------------------------------------------------- /ptranking/ltr_diversification/util/div_lambda_utils.py: -------------------------------------------------------------------------------- 1 | 2 | import torch 3 | 4 | 5 | def get_pairwise_comp_probs(batch_preds, std_q_doc_rele_mat, sigma=None): 6 | ''' 7 | Get the predicted and standard probabilities p_ij which denotes d_i beats d_j, the subtopic labels are aggregated. 8 | @param batch_preds: 9 | @param batch_std_labels: 10 | @param sigma: 11 | @return: 12 | ''' 13 | # standard pairwise differences per-subtopic, i.e., S_{ij} 14 | subtopic_std_diffs = torch.unsqueeze(std_q_doc_rele_mat, dim=2) - torch.unsqueeze(std_q_doc_rele_mat, dim=1) 15 | # ensuring S_{ij} \in {-1, 0, 1} 16 | subtopic_std_Sij = torch.clamp(subtopic_std_diffs, min=-1.0, max=1.0) 17 | subtopic_std_p_ij = 0.5 * (1.0 + subtopic_std_Sij) 18 | batch_std_p_ij = torch.mean(subtopic_std_p_ij, dim=0, keepdim=True) 19 | 20 | # computing pairwise differences, i.e., s_i - s_j 21 | batch_s_ij = torch.unsqueeze(batch_preds, dim=2) - torch.unsqueeze(batch_preds, dim=1) 22 | batch_p_ij = torch.sigmoid(sigma * batch_s_ij) 23 | 24 | return batch_p_ij, batch_std_p_ij 25 | 26 | def get_prob_pairwise_comp_probs(batch_pairsub_mus, batch_pairsub_vars, q_doc_rele_mat): 27 | ''' 28 | The difference of two normal random variables is another normal random variable. 29 | pairsub_mu & pairsub_var denote the corresponding mean & variance of the difference of two normal random variables 30 | p_ij denotes the probability that d_i beats d_j 31 | @param batch_pairsub_mus: 32 | @param batch_pairsub_vars: 33 | @param batch_std_labels: 34 | @return: 35 | ''' 36 | subtopic_std_diffs = torch.unsqueeze(q_doc_rele_mat, dim=2) - torch.unsqueeze(q_doc_rele_mat, dim=1) 37 | subtopic_std_Sij = torch.clamp(subtopic_std_diffs, min=-1.0, max=1.0) # ensuring S_{ij} \in {-1, 0, 1} 38 | subtopic_std_p_ij = 0.5 * (1.0 + subtopic_std_Sij) 39 | batch_std_p_ij = torch.mean(subtopic_std_p_ij, dim=0, keepdim=True) 40 | 41 | batch_p_ij = 1.0 - 0.5 * torch.erfc(batch_pairsub_mus / torch.sqrt(2 * batch_pairsub_vars)) 42 | 43 | return batch_p_ij, batch_std_p_ij -------------------------------------------------------------------------------- /ptranking/ltr_diversification/util/sim_utils.py: -------------------------------------------------------------------------------- 1 | 2 | import torch 3 | 4 | def batch_cosine_similarity(x1, x2=None, eps=1e-8): 5 | ''' 6 | :param x1: [batch_size, num_docs, num_features] 7 | :param x2: the same shape or None 8 | :param eps: 9 | :return: 10 | ''' 11 | x2 = x1 if x2 is None else x2 12 | w1 = x1.norm(p=2, dim=2, keepdim=True) 13 | #print('w1', w1.size(), '\n', w1) 14 | w2 = w1 if x2 is x1 else x2.norm(p=2, dim=2, keepdim=True) 15 | batch_numerator = torch.bmm(x1, x2.permute(0, 2, 1)) 16 | batch_denominator = torch.bmm(w1, w2.permute(0, 2, 1)).clamp(min=eps) 17 | return batch_numerator/batch_denominator -------------------------------------------------------------------------------- /ptranking/ltr_global.py: -------------------------------------------------------------------------------- 1 | 2 | """Description 3 | Global Variable 4 | """ 5 | 6 | """ Seed """ 7 | ltr_seed = 137 8 | 9 | """ A Tiny Value """ 10 | epsilon = 1e-8 11 | -------------------------------------------------------------------------------- /ptranking/ltr_tree/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | 6 | """ -------------------------------------------------------------------------------- /ptranking/ltr_tree/eval/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by Hai-Tao Yu | 2020/06/02 | https://y-research.github.io 5 | 6 | """Description 7 | 8 | """ -------------------------------------------------------------------------------- /ptranking/ltr_tree/eval/tree_parameter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import json 5 | from itertools import product 6 | 7 | from ptranking.ltr_adhoc.eval.parameter import DataSetting, EvalSetting 8 | from ptranking.data.data_utils import get_scaler_setting, MSLETOR_SEMI, get_data_meta 9 | 10 | 11 | class TreeDataSetting(DataSetting): 12 | """ 13 | Class object for data settings w.r.t. data loading and pre-process w.r.t. tree-based method 14 | """ 15 | def __init__(self, debug=False, data_id=None, dir_data=None, tree_data_json=None): 16 | super(TreeDataSetting, self).__init__(debug=debug, data_id=data_id, dir_data=dir_data, data_json=tree_data_json) 17 | 18 | def default_setting(self): 19 | """ 20 | A default setting for data loading when running lambdaMART 21 | """ 22 | scaler_id = None 23 | unknown_as_zero = True if self.data_id in MSLETOR_SEMI else False # since lambdaMART is a supervised method 24 | binary_rele = False # using the original values 25 | train_presort, validation_presort, test_presort = False, False, False 26 | train_rough_batch_size, validation_rough_batch_size, test_rough_batch_size = 1, 1, 1 27 | 28 | scale_data, scaler_id, scaler_level = get_scaler_setting(data_id=self.data_id, scaler_id=scaler_id) 29 | 30 | # more data settings that are rarely changed 31 | self.data_dict = dict(data_id=self.data_id, dir_data=self.dir_data, min_docs=10, min_rele=1, 32 | unknown_as_zero=unknown_as_zero, binary_rele=binary_rele, train_presort=train_presort, 33 | validation_presort=validation_presort, test_presort=test_presort, train_rough_batch_size=train_rough_batch_size, 34 | validation_rough_batch_size=validation_rough_batch_size, test_rough_batch_size=test_rough_batch_size, 35 | scale_data=scale_data, scaler_id=scaler_id, scaler_level=scaler_level) 36 | 37 | data_meta = get_data_meta(data_id=self.data_id) # add meta-information 38 | self.data_dict.update(data_meta) 39 | 40 | return self.data_dict 41 | 42 | 43 | class TreeEvalSetting(EvalSetting): 44 | """ 45 | Class object for evaluation settings w.r.t. tree-based methods 46 | """ 47 | def __init__(self, debug=False, dir_output=None, tree_eval_json=None): 48 | super(TreeEvalSetting, self).__init__(debug=debug, dir_output=dir_output, eval_json=tree_eval_json) 49 | 50 | def to_eval_setting_string(self, log=False): 51 | """ 52 | String identifier of eval-setting 53 | :param log: 54 | :return: 55 | """ 56 | eval_dict = self.eval_dict 57 | s1, s2 = (':', '\n') if log else ('_', '_') 58 | 59 | early_stop_or_boost_round, do_validation = eval_dict['early_stop_or_boost_round'], eval_dict['do_validation'] 60 | if do_validation: 61 | eval_string = s1.join(['EarlyStop', str(early_stop_or_boost_round)]) 62 | else: 63 | eval_string = s1.join(['BoostRound', str(early_stop_or_boost_round)]) 64 | 65 | return eval_string 66 | 67 | def default_setting(self): 68 | """ 69 | A default setting for evaluation 70 | """ 71 | do_validation = True if self.debug else True 72 | do_log = False if self.debug else True 73 | early_stop_or_boost_round = 10 if self.debug else 200 74 | 75 | # more evaluation settings that are rarely changed 76 | self.eval_dict = dict(debug=self.debug, grid_search=False, dir_output=self.dir_output, do_log=do_log, 77 | cutoffs=[1, 3, 5, 10, 20, 50], do_validation=do_validation, 78 | mask_label=False, early_stop_or_boost_round=early_stop_or_boost_round) 79 | 80 | return self.eval_dict 81 | 82 | def grid_search(self): 83 | """ 84 | Iterator of settings for evaluation 85 | """ 86 | if self.use_json: 87 | dir_output = self.json_dict['dir_output'] 88 | early_stop_or_boost_round = 20 if self.debug else self.json_dict['early_stop_or_boost_round'] 89 | do_validation = self.json_dict['do_validation'] 90 | cutoffs = self.json_dict['cutoffs'] 91 | do_log = self.json_dict['do_log'] 92 | mask_label = self.json_dict['mask']['mask_label'] 93 | choice_mask_type = self.json_dict['mask']['mask_type'] 94 | choice_mask_ratio = self.json_dict['mask']['mask_ratio'] 95 | 96 | base_dict = dict(debug=False, grid_search=True, dir_output=dir_output) 97 | else: 98 | base_dict = dict(debug=self.debug, grid_search=True, dir_output=self.dir_output) 99 | early_stop_or_boost_round = 20 if self.debug else 100 100 | do_validation = False if self.debug else True # True, False 101 | cutoffs = [1, 3, 5, 10, 20, 50] 102 | do_log = False if self.debug else True 103 | 104 | mask_label = False if self.debug else False 105 | choice_mask_type = ['rand_mask_all'] 106 | choice_mask_ratio = [0.2] 107 | 108 | self.eval_dict = dict(early_stop_or_boost_round=early_stop_or_boost_round, do_validation=do_validation, 109 | cutoffs=cutoffs, do_log=do_log, mask_label=mask_label) 110 | self.eval_dict.update(base_dict) 111 | 112 | if mask_label: 113 | for mask_type, mask_ratio in product(choice_mask_type, choice_mask_ratio): 114 | mask_dict = dict(mask_type=mask_type, mask_ratio=mask_ratio) 115 | self.eval_dict.update(mask_dict) 116 | yield self.eval_dict 117 | else: 118 | yield self.eval_dict 119 | -------------------------------------------------------------------------------- /ptranking/ltr_tree/lambdamart/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | 6 | """ -------------------------------------------------------------------------------- /ptranking/ltr_tree/lambdamart/note: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ## LightGBM ## 5 | 6 | (1) 7 | 'min_sum_hessian_in_leaf': 100 8 | 9 | A too big value may lead to the following case: 10 | 11 | ''' 12 | [10] valid_0's ndcg@1: 0.113095 valid_0's ndcg@2: 0.161686 valid_0's ndcg@3: 0.202154 valid_0's ndcg@4: 0.214604 valid_0's ndcg@5: 0.23572 13 | [20] valid_0's ndcg@1: 0.113095 valid_0's ndcg@2: 0.161686 valid_0's ndcg@3: 0.202154 valid_0's ndcg@4: 0.214604 valid_0's ndcg@5: 0.23572 14 | [30] valid_0's ndcg@1: 0.113095 valid_0's ndcg@2: 0.161686 valid_0's ndcg@3: 0.202154 valid_0's ndcg@4: 0.214604 valid_0's ndcg@5: 0.23572 15 | [40] valid_0's ndcg@1: 0.113095 valid_0's ndcg@2: 0.161686 valid_0's ndcg@3: 0.202154 valid_0's ndcg@4: 0.214604 valid_0's ndcg@5: 0.23572 16 | [50] valid_0's ndcg@1: 0.113095 valid_0's ndcg@2: 0.161686 valid_0's ndcg@3: 0.202154 valid_0's ndcg@4: 0.214604 valid_0's ndcg@5: 0.23572 17 | [60] valid_0's ndcg@1: 0.113095 valid_0's ndcg@2: 0.161686 valid_0's ndcg@3: 0.202154 valid_0's ndcg@4: 0.214604 valid_0's ndcg@5: 0.23572 18 | [70] valid_0's ndcg@1: 0.113095 valid_0's ndcg@2: 0.161686 valid_0's ndcg@3: 0.202154 valid_0's ndcg@4: 0.214604 valid_0's ndcg@5: 0.23572 19 | [80] valid_0's ndcg@1: 0.113095 valid_0's ndcg@2: 0.161686 valid_0's ndcg@3: 0.202154 valid_0's ndcg@4: 0.214604 valid_0's ndcg@5: 0.23572 20 | ''' 21 | -------------------------------------------------------------------------------- /ptranking/ltr_tree/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/ptranking/ltr_tree/util/__init__.py -------------------------------------------------------------------------------- /ptranking/metric/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /ptranking/metric/adhoc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/ptranking/metric/adhoc/__init__.py -------------------------------------------------------------------------------- /ptranking/metric/metric_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ 8 | import re 9 | 10 | import torch 11 | 12 | from ptranking.data.data_utils import LABEL_TYPE 13 | from ptranking.metric.adhoc.adhoc_metric import torch_dcg_at_k 14 | 15 | ####### 16 | # For Delta Metrics 17 | ####### 18 | 19 | def get_delta_ndcg(batch_ideal_rankings, batch_predict_rankings, label_type=LABEL_TYPE.MultiLabel, device='cpu'): 20 | ''' 21 | Delta-nDCG w.r.t. pairwise swapping of the currently predicted ltr_adhoc 22 | :param batch_ideal_rankings: the standard labels sorted in a descending order 23 | :param batch_predicted_rankings: the standard labels sorted based on the corresponding predictions 24 | :return: 25 | ''' 26 | # ideal discount cumulative gains 27 | batch_idcgs = torch_dcg_at_k(batch_rankings=batch_ideal_rankings, label_type=label_type, device=device) 28 | 29 | if LABEL_TYPE.MultiLabel == label_type: 30 | batch_gains = torch.pow(2.0, batch_predict_rankings) - 1.0 31 | elif LABEL_TYPE.Permutation == label_type: 32 | batch_gains = batch_predict_rankings 33 | else: 34 | raise NotImplementedError 35 | 36 | batch_n_gains = batch_gains / batch_idcgs # normalised gains 37 | batch_ng_diffs = torch.unsqueeze(batch_n_gains, dim=2) - torch.unsqueeze(batch_n_gains, dim=1) 38 | 39 | batch_std_ranks = torch.arange(batch_predict_rankings.size(1), dtype=torch.float, device=device) 40 | batch_dists = 1.0 / torch.log2(batch_std_ranks + 2.0) # discount co-efficients 41 | batch_dists = torch.unsqueeze(batch_dists, dim=0) 42 | batch_dists_diffs = torch.unsqueeze(batch_dists, dim=2) - torch.unsqueeze(batch_dists, dim=1) 43 | batch_delta_ndcg = torch.abs(batch_ng_diffs) * torch.abs(batch_dists_diffs) # absolute changes w.r.t. pairwise swapping 44 | 45 | return batch_delta_ndcg 46 | 47 | 48 | def metric_results_to_string(list_scores=None, list_cutoffs=None, split_str=', ', metric='nDCG'): 49 | """ 50 | Convert metric results to a string representation 51 | :param list_scores: 52 | :param list_cutoffs: 53 | :param split_str: 54 | :return: 55 | """ 56 | list_str = [] 57 | for i in range(len(list_scores)): 58 | list_str.append(metric + '@{}:{:.4f}'.format(list_cutoffs[i], list_scores[i])) 59 | return split_str.join(list_str) 60 | 61 | def tryint(s): 62 | try: 63 | return int(s) 64 | except: 65 | return s 66 | 67 | def alphanum_key(s): 68 | """ 69 | Turn a string into a list of string and number chunks. 70 | "z23a" -> ["z", 23, "a"] 71 | """ 72 | return [ tryint(c) for c in re.split('([0-9]+)', s) ] 73 | 74 | def sort_nicely(l): 75 | """ Sort the given list in the way that humans expect.""" 76 | l.sort(key=alphanum_key, reverse=True) 77 | 78 | def test_sort(): 79 | tmp_list = ['net_params_epoch_2.pkl', 'net_params_epoch_34.pkl', 'net_params_epoch_8.pkl'] 80 | print(sort_nicely(tmp_list)) 81 | print(tmp_list) 82 | 83 | 84 | def get_opt_model(list_model_names): 85 | sort_nicely(list_model_names) 86 | return list_model_names[0] 87 | -------------------------------------------------------------------------------- /ptranking/metric/smooth_metric/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/ptranking/metric/smooth_metric/__init__.py -------------------------------------------------------------------------------- /ptranking/metric/srd/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/ptranking/metric/srd/__init__.py -------------------------------------------------------------------------------- /ptranking/metric/srd/ndeval: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/ptranking/metric/srd/ndeval -------------------------------------------------------------------------------- /ptranking/utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Description 5 | 6 | """ -------------------------------------------------------------------------------- /ptranking/utils/args/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /ptranking/utils/args/argsUtil.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | Class for parsing command-line input 7 | """ 8 | 9 | import argparse 10 | 11 | class ArgsUtil(object): 12 | def __init__(self, given_root=None): 13 | self.args_parser = argparse.ArgumentParser('Run pt_ranking.') 14 | self.ini_l2r_args(given_root=given_root) 15 | 16 | def ini_l2r_args(self, given_root=None): 17 | self.given_root = given_root 18 | 19 | ''' gpu ''' 20 | self.args_parser.add_argument('-cuda', type=int, help='specify the gpu id if needed, such as 0 or 1.', default=None) 21 | 22 | ''' model ''' 23 | self.args_parser.add_argument('-model', type=str, help='specify the learning-to-rank method') 24 | 25 | ''' debug ''' 26 | self.args_parser.add_argument("-debug", action="store_true", help="quickly check the setting in a debug mode") 27 | 28 | ''' path of json files specifying the evaluation details ''' 29 | self.args_parser.add_argument('-dir_json', type=str, help='the path of json files specifying the evaluation details.') 30 | 31 | def update_if_required(self, args): 32 | return args 33 | 34 | def get_l2r_args(self): 35 | l2r_args = self.args_parser.parse_args() 36 | l2r_args = self.update_if_required(l2r_args) 37 | return l2r_args 38 | 39 | -------------------------------------------------------------------------------- /ptranking/utils/bigdata/BigPickle.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | 4 | ## due to the restriction of 4GB ## 5 | max_bytes = 2**31 - 1 6 | 7 | def pickle_save(target, file): 8 | bytes_out = pickle.dumps(target, protocol=4) 9 | with open(file, 'wb') as f_out: 10 | for idx in range(0, len(bytes_out), max_bytes): 11 | f_out.write(bytes_out[idx:idx + max_bytes]) 12 | 13 | 14 | def pickle_load(file): 15 | file_size = os.path.getsize(file) 16 | with open(file, 'rb') as f_in: 17 | bytes_in = bytearray(0) 18 | for _ in range(0, file_size, max_bytes): 19 | bytes_in += f_in.read(max_bytes) 20 | data = pickle.loads(bytes_in) 21 | return data -------------------------------------------------------------------------------- /ptranking/utils/bigdata/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /ptranking/utils/numpy/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /ptranking/utils/numpy/np_extensions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ 8 | 9 | import numpy as np 10 | 11 | 12 | def np_shuffle_ties(vec, descending=True): 13 | ''' 14 | namely, randomly permuate ties 15 | :param vec: 16 | :param descending: the sorting order w.r.t. the input vec 17 | :return: 18 | ''' 19 | if len(vec.shape) > 1: 20 | raise NotImplementedError 21 | else: 22 | length = vec.shape[0] 23 | perm = np.random.permutation(length) 24 | shuffled_vec = sorted(vec[perm], reverse=descending) 25 | return shuffled_vec 26 | 27 | def np_arg_shuffle_ties(vec, descending=True): 28 | ''' the same as np_shuffle_ties, but return the corresponding indice ''' 29 | if len(vec.shape) > 1: 30 | raise NotImplementedError 31 | else: 32 | length = vec.shape[0] 33 | perm = np.random.permutation(length) 34 | if descending: 35 | sorted_shuffled_vec_inds = np.argsort(-vec[perm]) 36 | else: 37 | sorted_shuffled_vec_inds = np.argsort(vec[perm]) 38 | 39 | shuffle_ties_inds = perm[sorted_shuffled_vec_inds] 40 | return shuffle_ties_inds 41 | 42 | def test_np_shuffle_ties(): 43 | np_arr = np.asarray([0.8, 0.8, 0.7, 0.7, 0.5, 0.5]) 44 | 45 | print(np_shuffle_ties(vec=np_arr, descending=True)) 46 | inds = np_arg_shuffle_ties(np_arr) 47 | print(inds) 48 | print(np_arr[inds]) 49 | 50 | def np_softmax(xs): 51 | ys = xs - np.max(xs) 52 | exps = np.exp(ys) 53 | return exps/exps.sum(axis=0) 54 | 55 | def np_plackett_luce_sampling(items, probs, softmaxed=False): 56 | ''' 57 | sample a ltr_adhoc based on the Plackett-Luce model 58 | :param vec: a vector of values, the higher, the more possible the corresponding entry will be sampled 59 | :return: the indice of the corresponding ltr_adhoc 60 | ''' 61 | if softmaxed: 62 | ranking = np.random.choice(items, size=len(probs), p=probs, replace=False) 63 | else: 64 | probs = np_softmax(probs) 65 | ranking = np.random.choice(items, size=len(probs), p=probs, replace=False) 66 | 67 | return ranking 68 | 69 | def test_pl_sampling(): 70 | pass 71 | 72 | if __name__ == '__main__': 73 | #1 74 | test_np_shuffle_ties() -------------------------------------------------------------------------------- /ptranking/utils/pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | from os import path 4 | 5 | this_directory = path.abspath(path.dirname(__file__)) 6 | with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: 7 | long_description = f.read() 8 | 9 | install_requires = [ 10 | 'numpy', 11 | #'scikit-learn', 12 | 'tqdm', 13 | #'torch >= 1.6.0', # todo torch-gpu 14 | #'torchvision', 15 | ] 16 | 17 | extras_requires = None 18 | 19 | setuptools.setup( 20 | name="ptranking", 21 | version="0.0.3", 22 | author="II-Research", 23 | author_email="yuhaitao@slis.tsukuba.ac.jp", 24 | description="A library of scalable and extendable implementations of typical learning-to-rank methods based on PyTorch.", 25 | license="MIT License", 26 | keywords=['Learning-to-rank', 'PyTorch'], 27 | url="https://github.com/wildltr/ptranking", 28 | packages=setuptools.find_namespace_packages(include=["ptranking", "ptranking.*"]), 29 | long_description=long_description, 30 | long_description_content_type='text/markdown', 31 | classifiers=[ 32 | "Development Status :: 3 - Alpha", 33 | "Programming Language :: Python :: 3", 34 | "License :: OSI Approved :: MIT License", 35 | "Operating System :: OS Independent", 36 | ], 37 | install_requires=install_requires, 38 | extras_require=extras_requires 39 | ) 40 | 41 | #todo package_data 42 | -------------------------------------------------------------------------------- /testing/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by Hai-Tao Yu | 22/08/2020 | https://ii-research.github.io 5 | 6 | """Description 7 | 8 | """ 9 | -------------------------------------------------------------------------------- /testing/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/testing/data/__init__.py -------------------------------------------------------------------------------- /testing/ltr_adhoc/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by Hai-Tao Yu | 22/08/2020 | https://ii-research.github.io 5 | 6 | """Description 7 | 8 | """ 9 | -------------------------------------------------------------------------------- /testing/ltr_adhoc/json/ApproxNDCGParameter.json: -------------------------------------------------------------------------------- 1 | { 2 | "alpha":[10.0] 3 | } 4 | -------------------------------------------------------------------------------- /testing/ltr_adhoc/json/Data_Eval_ScoringFunction.json: -------------------------------------------------------------------------------- 1 | { 2 | "DataSetting": { 3 | "data_id":"MQ2008_Super", 4 | "dir_data":"/Users/iimac/Workbench/Corpus/L2R/LETOR4.0/MQ2008/", 5 | "min_docs":[10], 6 | "min_rele":[1], 7 | "scaler_id":"StandardScaler", 8 | 9 | "binary_rele":[false], 10 | "unknown_as_zero":[false], 11 | "tr_batch_size":[100] 12 | }, 13 | 14 | "EvalSetting": { 15 | "dir_output":"/Users/iimac/Workbench/CodeBench/Output/NeuralLTR/", 16 | "epochs":2, 17 | 18 | "do_validation":true, 19 | 20 | "vali_k":5, 21 | "cutoffs":[1, 3, 5, 10, 20, 50], 22 | 23 | "loss_guided":false, 24 | 25 | "do_log":true, 26 | "log_step":1, 27 | "do_summary":false, 28 | 29 | "mask":{ 30 | "mask_label":false, 31 | "mask_type":["rand_mask_all"], 32 | "mask_ratio":[0.2] 33 | } 34 | }, 35 | 36 | "SFParameter": { 37 | "sf_id":"pointsf", 38 | "opt":["Adam"], 39 | "lr":[0.001], 40 | "pointsf":{ 41 | "BN":[true], 42 | "bn_type":["BN"], 43 | "bn_affine":[true], 44 | "layers":[5], 45 | "AF":["GE"], 46 | "TL_AF":["GE"], 47 | "apply_tl_af":[true] 48 | }, 49 | "listsf":{ 50 | "BN":[true], 51 | "bn_type":["BN2"], 52 | "bn_affine":[false], 53 | "RD":[false], 54 | "apply_tl_af":[false], 55 | "AF":["R"], 56 | "ff_dims":[128, 256, 512], 57 | "encoder_type":["DASALC"], 58 | "encoder_layers":[3], 59 | "n_heads":[2] 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /testing/ltr_adhoc/json/LambdaLossParameter.json: -------------------------------------------------------------------------------- 1 | { 2 | "k":[5], 3 | "mu":[5.0], 4 | "sigma":[1.0], 5 | "loss_type":["NDCG_Loss2"] 6 | } 7 | -------------------------------------------------------------------------------- /testing/ltr_adhoc/json/LambdaRankParameter.json: -------------------------------------------------------------------------------- 1 | { 2 | "sigma":[1.0] 3 | } 4 | -------------------------------------------------------------------------------- /testing/ltr_adhoc/json/RankNetParameter.json: -------------------------------------------------------------------------------- 1 | { 2 | "sigma":[1.0] 3 | } 4 | -------------------------------------------------------------------------------- /testing/ltr_adhoc/json/STListNetParameter.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/testing/ltr_adhoc/json/STListNetParameter.json -------------------------------------------------------------------------------- /testing/ltr_adhoc/json/WassRankParameter.json: -------------------------------------------------------------------------------- 1 | { 2 | "mode":["WassLossSta"], 3 | "itr":[10], 4 | "lam":[0.1], 5 | "cost_type":["eg"], 6 | "non_rele_gap":[10], 7 | "var_penalty":[2.71828], 8 | "group_base":[4], 9 | "smooth":["ST"], 10 | "norm":["BothST"] 11 | } 12 | -------------------------------------------------------------------------------- /testing/ltr_adhoc/testing_ltr_adhoc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | A simple script for testing either in-built methods or newly added methods 7 | """ 8 | 9 | import torch 10 | import numpy as np 11 | 12 | from ptranking.ltr_global import ltr_seed 13 | from ptranking.ltr_adhoc.eval.ltr import LTREvaluator 14 | 15 | np.random.seed(seed=ltr_seed) 16 | torch.manual_seed(seed=ltr_seed) 17 | 18 | if __name__ == '__main__': 19 | """ 20 | >>> Learning-to-Rank Models <<< 21 | (1) Optimization based on Empirical Risk Minimization 22 | ----------------------------------------------------------------------------------------- 23 | | Pointwise | RankMSE | 24 | ----------------------------------------------------------------------------------------- 25 | | Pairwise | RankNet | 26 | ----------------------------------------------------------------------------------------- 27 | | Listwise | LambdaRank % ListNet % ListMLE % RankCosine % ApproxNDCG % WassRank | 28 | | | STListNet % LambdaLoss | 29 | ----------------------------------------------------------------------------------------- 30 | 31 | 32 | >>> Supported Datasets <<< 33 | ----------------------------------------------------------------------------------------- 34 | | LETTOR | MQ2007_Super % MQ2008_Super % MQ2007_Semi % MQ2008_Semi | 35 | ----------------------------------------------------------------------------------------- 36 | | MSLRWEB | MSLRWEB10K % MSLRWEB30K | 37 | ----------------------------------------------------------------------------------------- 38 | | Yahoo_LTR | Set1 % Set2 | 39 | ----------------------------------------------------------------------------------------- 40 | | ISTELLA_LTR | Istella_S % Istella % Istella_X | 41 | ----------------------------------------------------------------------------------------- 42 | 43 | """ 44 | 45 | cuda = None # the gpu id, e.g., 0 or 1, otherwise, set it as None indicating to use cpu 46 | 47 | debug = True # in a debug mode, we just check whether the model can operate 48 | 49 | config_with_json = False # specify configuration with json files or not 50 | 51 | reproduce = False # given pre-trained models, reproduce experiments 52 | 53 | models_to_run = [ 54 | #'RankMSE', 55 | 'RankNet', 56 | #'LambdaRank', 57 | #'ListNet', 58 | #'ListMLE', 59 | #'RankCosine', 60 | #'ApproxNDCG', 61 | #'WassRank', 62 | #'STListNet', 63 | #'LambdaLoss' 64 | ] 65 | 66 | evaluator = LTREvaluator(cuda=cuda) 67 | 68 | if config_with_json: # specify configuration with json files 69 | # the directory of json files 70 | #dir_json = '/Users/dryuhaitao/WorkBench/Dropbox/CodeBench/GitPool/wildltr_ptranking/testing/ltr_adhoc/json/' 71 | dir_json = '/Users/solar/WorkBench/Dropbox/CodeBench/GitPool/wildltr_ptranking/testing/ltr_adhoc/json/' 72 | #dir_json = '/home/dl-box/WorkBench/Dropbox/CodeBench/GitPool/wildltr_ptranking/testing/ltr_adhoc/json/' 73 | 74 | # reco - linear 75 | #dir_json = '/Users/dryuhaitao/WorkBench/Experiments/RECO/Linear/' 76 | # reco - non-linear 77 | #dir_json = '/Users/dryuhaitao/WorkBench/Experiments/RECO/Nonlinear/' 78 | 79 | # auto 1 layer with activation yahoo 80 | #dir_json = '/home/dl-box/WorkBench/ExperimentBench/Auto/ERM/yahoo/' 81 | 82 | # auto 1 layer without activation yahoo 83 | #dir_json = '/home/dl-box/WorkBench/ExperimentBench/Auto/ERM/yahoo/' 84 | 85 | # auto 1 layer with activation ms30k 86 | #dir_json = '/home/dl-box/WorkBench/ExperimentBench/Auto/ERM/ms30k/' 87 | 88 | # auto 1 layer without activation yahoo 89 | #dir_json = '/home/dl-box/WorkBench/ExperimentBench/Auto/ERM/ms30k/' 90 | 91 | #dir_json = '/home/dl-box/WorkBench/ExperimentBench/Auto/ERM/mq2008/' 92 | 93 | #dir_json = '/Users/iimac/Workbench/ExperimentBench/PGRanking/ms30k/' 94 | 95 | for model_id in models_to_run: 96 | evaluator.run(debug=debug, model_id=model_id, config_with_json=config_with_json, dir_json=dir_json) 97 | 98 | else: # specify configuration manually 99 | ''' pointsf | listsf, namely the type of neural scoring function ''' 100 | sf_id = 'pointsf' 101 | 102 | ''' Selected dataset ''' 103 | #data_id = 'Set1' 104 | #data_id = 'MSLRWEB30K' 105 | data_id = 'MQ2008_Super' 106 | 107 | ''' By grid_search, we can explore the effects of different hyper-parameters of a model ''' 108 | grid_search = False 109 | 110 | ''' Location of the adopted data ''' 111 | #dir_data = '/Users/dryuhaitao/WorkBench/Corpus/' + 'LETOR4.0/MQ2008/' 112 | #dir_data = '/home/dl-box/WorkBench/Datasets/L2R/LETOR4.0/MQ2008/' 113 | #dir_data = '/Users/solar/WorkBench/Datasets/L2R/LETOR4.0/MQ2008/' 114 | dir_data = '/Users/iimac/Workbench/Corpus/L2R/LETOR4.0/MQ2008/' 115 | 116 | #data_id = 'Istella_X' 117 | #dir_data = '/home/dl-box/WorkBench/Datasets/L2R/ISTELLA_L2R/Istella_X/' 118 | 119 | #data_id = 'Istella' 120 | #dir_data = '/home/dl-box/WorkBench/Datasets/L2R/ISTELLA_L2R/Istella/' 121 | 122 | #data_id = 'Istella_S' 123 | #dir_data = '/home/dl-box/WorkBench/Datasets/L2R/ISTELLA_L2R/Istella_S/' 124 | 125 | ''' Output directory ''' 126 | #dir_output = '/Users/dryuhaitao/WorkBench/CodeBench/Bench_Output/NeuralLTR/Listwise/' 127 | #dir_output = '/home/dl-box/WorkBench/CodeBench/PyCharmProject/Project_output/Out_L2R/Listwise/' 128 | #dir_output = '/Users/solar/WorkBench/CodeBench/PyCharmProject/Project_output/Out_L2R/' 129 | dir_output = '/Users/iimac/Workbench/CodeBench/Output/NeuralLTR/' 130 | 131 | for model_id in models_to_run: 132 | evaluator.run(debug=debug, model_id=model_id, sf_id=sf_id, grid_search=grid_search, 133 | data_id=data_id, dir_data=dir_data, dir_output=dir_output, reproduce=reproduce) 134 | -------------------------------------------------------------------------------- /testing/ltr_adversarial/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by Hai-Tao Yu | 22/08/2020 | https://ii-research.github.io 5 | 6 | """Description 7 | 8 | """ 9 | -------------------------------------------------------------------------------- /testing/ltr_adversarial/json/Ad_Data_Eval_ScoringFunction.json: -------------------------------------------------------------------------------- 1 | { 2 | "AdDataSetting": { 3 | "data_id":"MQ2008_Super", 4 | "dir_data":"/Users/iimac/Workbench/Corpus/L2R/LETOR4.0/MQ2008/", 5 | "min_docs":[10], 6 | "min_rele":[1], 7 | "scaler_id":"StandardScaler", 8 | 9 | "binary_rele":[false], 10 | "unknown_as_zero":[false] 11 | }, 12 | 13 | "AdEvalSetting": { 14 | "dir_output":"/Users/iimac/Workbench/CodeBench/Output/NeuralLTR/", 15 | 16 | "epochs":2, 17 | 18 | "do_validation":true, 19 | 20 | "vali_k":5, 21 | "cutoffs":[1, 3, 5, 10, 20, 50], 22 | 23 | "loss_guided":false, 24 | 25 | "do_log":true, 26 | "log_step":1, 27 | "do_summary":false, 28 | 29 | "mask":{ 30 | "mask_label":false, 31 | "mask_type":["rand_mask_all"], 32 | "mask_ratio":[0.2] 33 | } 34 | }, 35 | 36 | "SFParameter": { 37 | "sf_id":"pointsf", 38 | "opt":["Adam"], 39 | "lr":[0.0001], 40 | "pointsf":{ 41 | "BN":[false], 42 | "bn_type":["BN"], 43 | "bn_affine":[true], 44 | "layers":[5], 45 | "AF":["GE"], 46 | "TL_AF":["GE"], 47 | "apply_tl_af":[true] 48 | }, 49 | "listsf":{ 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /testing/ltr_adversarial/json/IRGAN_ListParameter.json: -------------------------------------------------------------------------------- 1 | { 2 | "d_g_epoch":["1-1"], 3 | "temperature":[0.5], 4 | "samples_per_query":[5], 5 | "ad_training_order":["DG"], 6 | "top_k":[5], 7 | "PL_D":[true], 8 | "repTrick":[false], 9 | "dropLog":[true] 10 | } 11 | -------------------------------------------------------------------------------- /testing/ltr_adversarial/json/IRGAN_PairParameter.json: -------------------------------------------------------------------------------- 1 | { 2 | "d_g_epoch":["1-1"], 3 | "temperature":[0.5], 4 | "samples_per_query":[5], 5 | "ad_training_order":["DG"], 6 | "losstype_d":["svm"] 7 | } 8 | -------------------------------------------------------------------------------- /testing/ltr_adversarial/json/IRGAN_PointParameter.json: -------------------------------------------------------------------------------- 1 | { 2 | "d_g_epoch":["1-1"], 3 | "temperature":[0.5], 4 | "samples_per_query":[5], 5 | "ad_training_order":["DG"] 6 | } 7 | -------------------------------------------------------------------------------- /testing/ltr_adversarial/testing_ltr_adversarial.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ 8 | 9 | import torch 10 | 11 | import numpy as np 12 | 13 | from ptranking.ltr_global import ltr_seed 14 | from ptranking.ltr_adversarial.eval.ltr_adversarial import AdLTREvaluator 15 | 16 | np.random.seed(seed=ltr_seed) 17 | torch.manual_seed(seed=ltr_seed) 18 | 19 | 20 | if __name__ == '__main__': 21 | 22 | """ 23 | >>> Learning-to-Rank Models <<< 24 | 25 | (2) Adversarial Optimization 26 | ----------------------------------------------------------------------------------------- 27 | | Pointwise | IRGAN_Point % IRFGAN_Point | 28 | ----------------------------------------------------------------------------------------- 29 | | Pairwise | IRGAN_Pair % IRFGAN_Pair | 30 | ----------------------------------------------------------------------------------------- 31 | | Listwise | IRGAN_List % IRFGAN_List | 32 | ----------------------------------------------------------------------------------------- 33 | 34 | 35 | >>> Supported Datasets <<< 36 | ----------------------------------------------------------------------------------------- 37 | | LETTOR | MQ2007_Super % MQ2008_Super % MQ2007_Semi % MQ2008_Semi | 38 | ----------------------------------------------------------------------------------------- 39 | | MSLRWEB | MSLRWEB10K % MSLRWEB30K | 40 | ----------------------------------------------------------------------------------------- 41 | | Yahoo_LTR | Set1 % Set2 | 42 | ----------------------------------------------------------------------------------------- 43 | | ISTELLA_LTR | Istella_S | Istella | Istella_X | 44 | ----------------------------------------------------------------------------------------- 45 | | IRGAN_MQ2008_Semi | 46 | ----------------------------------------------------------------------------------------- 47 | 48 | """ 49 | 50 | cuda = None # the gpu id, e.g., 0 or 1, otherwise, set it as None indicating to use cpu 51 | 52 | debug = True # in a debug mode, we just check whether the model can operate 53 | 54 | config_with_json = False # specify configuration with json files or not 55 | 56 | models_to_run = [ 57 | # 'IRGAN_Point', 58 | 'IRGAN_Pair', 59 | # 'IRGAN_List', 60 | 61 | # 'IRFGAN_Point', 62 | # 'IRFGAN_Pair', 63 | # 'IRFGAN_List' 64 | ] 65 | 66 | evaluator = AdLTREvaluator(cuda=cuda) 67 | 68 | if config_with_json: # specify configuration with json files 69 | # the directory of json files 70 | dir_json = '/Users/dryuhaitao/WorkBench/Dropbox/CodeBench/GitPool/wildltr_ptranking/testing/ltr_adversarial/json/' 71 | #dir_json = '/home/dl-box/WorkBench/Dropbox/CodeBench/GitPool/wildltr_ptranking/testing/ltr_adversarial/json/' 72 | for model_id in models_to_run: 73 | evaluator.run(debug=debug, model_id=model_id, config_with_json=config_with_json, dir_json=dir_json) 74 | 75 | else: # specify configuration manually 76 | data_id = 'MQ2008_Super' 77 | 78 | ''' pointsf | listsf, namely the type of neural scoring function ''' 79 | sf_id = 'pointsf' 80 | 81 | ''' Location of the adopted data ''' 82 | #dir_data = '/home/dl-box/WorkBench/Datasets/L2R/LETOR4.0/MQ2008/' 83 | #dir_data = '/Users/solar/WorkBench/Datasets/L2R/LETOR4.0/MQ2008/' 84 | dir_data = '/Users/iimac/Workbench/Corpus/L2R/LETOR4.0/MQ2007/' 85 | 86 | ''' Output directory ''' 87 | #dir_output = '/Users/dryuhaitao/WorkBench/CodeBench/Bench_Output/NeuralLTR/ALTR/' 88 | #dir_output = '/home/dl-box/WorkBench/CodeBench/PyCharmProject/Project_output/Out_L2R/Listwise/' 89 | #dir_output = '/Users/solar/WorkBench/CodeBench/PyCharmProject/Project_output/Out_L2R/' 90 | dir_output = '/Users/iimac/Workbench/CodeBench/Output/NeuralLTR/' 91 | 92 | grid_search = False # with grid_search, we can explore the effects of different hyper-parameters of a model 93 | 94 | for model_id in models_to_run: 95 | evaluator.run(debug=debug, model_id=model_id, sf_id=sf_id, 96 | data_id=data_id, dir_data=dir_data, dir_output=dir_output, grid_search=grid_search) 97 | -------------------------------------------------------------------------------- /testing/ltr_diversification/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/testing/ltr_diversification/__init__.py -------------------------------------------------------------------------------- /testing/ltr_diversification/json/DALETORParameter.json: -------------------------------------------------------------------------------- 1 | { 2 | "top_k":[10], 3 | "rt":[10.0] 4 | } 5 | -------------------------------------------------------------------------------- /testing/ltr_diversification/json/DivProbRankerParameter.json: -------------------------------------------------------------------------------- 1 | { 2 | "cluster":[false], 3 | "K":[1, 3, 5, 10, 30, 100], 4 | "opt_id":["LambdaPairCLS"], 5 | "top_k":[10], 6 | "sort_id":["ExpRele"], 7 | "limit_delta":[true, false] 8 | } 9 | -------------------------------------------------------------------------------- /testing/ltr_diversification/json/Div_Data_Eval_ScoringFunction.json: -------------------------------------------------------------------------------- 1 | { 2 | "DivDataSetting": { 3 | "data_id":"WT_Div_0912_Implicit", 4 | "dir_data":"/Users/iimac/Workbench/Corpus/L2R/TREC_WebTrack_Div_2009_2012_Implicit/" 5 | }, 6 | 7 | "DivEvalSetting": { 8 | "dir_output":"/Users/iimac/Workbench/CodeBench/Output/DivLTR/", 9 | "epochs":100, 10 | 11 | "do_validation":true, 12 | 13 | "vali_k":5, 14 | "cutoffs":[1, 3, 5, 10, 20, 50], 15 | 16 | "loss_guided":false, 17 | 18 | "do_log":true, 19 | "log_step":1, 20 | "do_summary":false 21 | }, 22 | 23 | "DivSFParameter": { 24 | "opt":["Adam"], 25 | "lr":[0.01], 26 | "sf_id":"listsf", 27 | "pointsf":{ 28 | "BN":[true], 29 | "bn_type":["BN"], 30 | "bn_affine":[true], 31 | "RD":[false], 32 | "layers":[5], 33 | "apply_tl_af":[true], 34 | "hd_hn_tl_af":["GE"] 35 | }, 36 | "listsf":{ 37 | "encoder_type":["DASALC"], 38 | "encoder_layers":[3], 39 | "n_heads":[2], 40 | "BN":[true], 41 | "bn_type":["BN"], 42 | "bn_affine":[true], 43 | "RD":[false], 44 | "ff_dims":[128, 256, 512], 45 | "apply_tl_af":[false], 46 | "AF":["R"] 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /testing/ltr_diversification/testing_ltr_diversification.py: -------------------------------------------------------------------------------- 1 | """Description 2 | A simple script for testing either in-built methods or newly added methods 3 | """ 4 | 5 | import torch 6 | import numpy as np 7 | 8 | from ptranking.ltr_global import ltr_seed 9 | from ptranking.ltr_diversification.eval.ltr_diversification import DivLTREvaluator 10 | 11 | np.random.seed(seed=ltr_seed) 12 | torch.manual_seed(seed=ltr_seed) 13 | 14 | if __name__ == '__main__': 15 | """ 16 | >>> Models <<< 17 | (1) Learning-to-Rank & Search Result Diversification 18 | 19 | >>> Supported Datasets <<< 20 | ----------------------------------------------------------------------------------------- 21 | | WebTrack_Div_2009_2012 | 22 | ----------------------------------------------------------------------------------------- 23 | 24 | """ 25 | 26 | cuda = None # the gpu id, e.g., 0 or 1, otherwise, set it as None indicating to use cpu 27 | 28 | debug = True # in a debug mode, we just check whether the model can operate 29 | 30 | config_with_json = False # specify configuration with json files or not 31 | 32 | reproduce = False 33 | 34 | models_to_run = [ 35 | #'DALETOR', 36 | 'DivProbRanker', 37 | ] 38 | 39 | evaluator = DivLTREvaluator(cuda=cuda) 40 | 41 | if config_with_json: # specify configuration with json files 42 | # the directory of json files 43 | #dir_json = '/Users/iimac/II-Research Dropbox/Hai-Tao Yu/CodeBench/GitPool/drl_ptranking/testing/ltr_diversification/json/' 44 | 45 | #dir_json = '/home/user/T2_Workbench/ExperimentBench/SRD_AAAI/DivLTR/GE_S/Opt_aNDCG/' 46 | #dir_json = '/home/user/T2_Workbench/ExperimentBench/SRD_AAAI/DivLTR/GE_S/Opt_nERRIA/' 47 | 48 | # DivProbRanker - SuperSoft - Full 49 | # dir_json = '/home/user/T2_Workbench/ExperimentBench/SRD_AAAI/DivLTR/GE_Full/Opt_aNDCG/' 50 | #dir_json = '/home/user/T2_Workbench/ExperimentBench/SRD_AAAI/DivLTR/R_Full/Opt_aNDCG/' 51 | 52 | # reproduce 53 | #dir_json = '/home/user/T2_Workbench/ExperimentBench/SRD_AAAI/DivLTR/R_reproduce/aNDCG/' 54 | #dir_json = '/home/user/T2_Workbench/ExperimentBench/SRD_AAAI/DivLTR/R_reproduce/nERRIA/' 55 | #dir_json = '/home/user/T2_Workbench/ExperimentBench/SRD_AAAI/DivLTR/DivSoftRank_reproduce/aNDCG/' 56 | #dir_json = '/home/user/T2_Workbench/ExperimentBench/SRD_AAAI/DivLTR/DivSoftRank_reproduce/nERRIA/' 57 | 58 | #dir_json = '/home/user/T2_Workbench/ExperimentBench/SRD_AAAI/DivLTR/DALETOR_reproduce/' 59 | #dir_json = '/T2Root/dl-box/T2_WorkBench/ExperimentBench/TwinRank_WWW/Div_reproduce/aNDCG/' # nERR-IA 60 | dir_json = '/T2Root/dl-box/T2_WorkBench/ExperimentBench/TwinRank_WWW/Div_reproduce/nERR-IA/' # 61 | 62 | 63 | for model_id in models_to_run: 64 | evaluator.run(debug=debug, model_id=model_id, config_with_json=config_with_json, dir_json=dir_json, 65 | reproduce=reproduce) 66 | 67 | else: # specify configuration manually 68 | sf_id = 'listsf' # pointsf | listsf | listsf_co, namely the type of neural scoring function 69 | 70 | ''' Selected dataset ''' 71 | data_id = 'WT_Div_0912_Implicit' 72 | 73 | ''' By grid_search, we can explore the effects of different hyper-parameters of a model ''' 74 | grid_search = False 75 | 76 | ''' Location of the adopted data ''' 77 | dir_data = '/Users/iimac/Workbench/Corpus/L2R/TREC_WebTrack_Div_2009_2012_Implicit/' 78 | #dir_data = '/home/user/T2_Workbench/Corpus/L2R/TREC_WebTrack_Div_2009_2012_Implicit/' 79 | 80 | ''' Output directory ''' 81 | #dir_output = '/home/user/T2_Workbench/Project_output/Out_L2R/DivLTR/' 82 | dir_output = '/Users/iimac/Workbench/CodeBench/Output/DivLTR/' 83 | 84 | for model_id in models_to_run: 85 | evaluator.run(debug=debug, model_id=model_id, sf_id=sf_id, grid_search=grid_search, 86 | data_id=data_id, dir_data=dir_data, dir_output=dir_output) 87 | -------------------------------------------------------------------------------- /testing/ltr_tree/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by Hai-Tao Yu | 22/08/2020 | https://ii-research.github.io 5 | 6 | """Description 7 | 8 | """ 9 | -------------------------------------------------------------------------------- /testing/ltr_tree/json/LightGBMLambdaMARTParameter.json: -------------------------------------------------------------------------------- 1 | { 2 | "BT":["gbdt"], 3 | "metric":["ndcg"], 4 | "leaves":[400], 5 | "trees":[1000], 6 | "MiData":[50], 7 | "MSH":[200], 8 | "LR":[0.05] 9 | } 10 | -------------------------------------------------------------------------------- /testing/ltr_tree/json/Tree_Data_Eval_ScoringFunction.json: -------------------------------------------------------------------------------- 1 | { 2 | "DataSetting": { 3 | "data_id":"MQ2008_Super", 4 | "dir_data":"/Users/dryuhaitao/WorkBench/Corpus/LETOR4.0/MQ2008/", 5 | 6 | "min_docs":[10], 7 | "min_rele":[1], 8 | "train_batch_size":[1], 9 | 10 | "binary_rele":[false], 11 | "unknown_as_zero":[false], 12 | "train_presort":[false] 13 | }, 14 | 15 | "EvalSetting": { 16 | "dir_output":"/Users/dryuhaitao/WorkBench/CodeBench/Bench_Output/NeuralLTR/Listwise/", 17 | 18 | "epochs":100, 19 | 20 | "do_validation":true, 21 | 22 | "cutoffs":[1, 3, 5, 10, 20, 50], 23 | 24 | "do_log":false, 25 | 26 | "mask":{ 27 | "mask_label":false, 28 | "mask_type":["rand_mask_all"], 29 | "mask_ratio":[0.2] 30 | } 31 | }, 32 | 33 | "SFParameter": {} 34 | } 35 | -------------------------------------------------------------------------------- /testing/ltr_tree/testing_ltr_tree.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ 8 | 9 | import numpy as np 10 | 11 | from ptranking.ltr_global import ltr_seed 12 | from ptranking.ltr_tree.eval.ltr_tree import TreeLTREvaluator 13 | 14 | np.random.seed(seed=ltr_seed) 15 | 16 | 17 | if __name__ == '__main__': 18 | 19 | """ 20 | >>> Tree-based Learning-to-Rank Models <<< 21 | 22 | (3) Tree-based Model 23 | ----------------------------------------------------------------------------------------- 24 | | LightGBMLambdaMART | 25 | ----------------------------------------------------------------------------------------- 26 | 27 | >>> Supported Datasets <<< 28 | ----------------------------------------------------------------------------------------- 29 | | LETTOR | MQ2007_Super % MQ2008_Super % MQ2007_Semi % MQ2008_Semi | 30 | ----------------------------------------------------------------------------------------- 31 | | MSLRWEB | MSLRWEB10K % MSLRWEB30K | 32 | ----------------------------------------------------------------------------------------- 33 | | Yahoo_LTR | Set1 % Set2 | 34 | ----------------------------------------------------------------------------------------- 35 | | ISTELLA_LTR | Istella_S | Istella | Istella_X | 36 | ----------------------------------------------------------------------------------------- 37 | 38 | """ 39 | 40 | debug = True # in a debug mode, we just check whether the model can operate 41 | 42 | config_with_json = False # specify configuration with json files or not 43 | 44 | evaluator = TreeLTREvaluator() 45 | 46 | if config_with_json: # specify configuration with json files 47 | # the directory of json files 48 | #dir_json = '/Users/dryuhaitao/WorkBench/Dropbox/CodeBench/GitPool/wildltr_ptranking/testing/ltr_tree/json/' 49 | 50 | dir_json = '/home/dl-box/WorkBench/ExperimentBench/Cmp_LTR_OptFrames/LambdaMART/mq2008/' 51 | 52 | evaluator.run(debug=debug, model_id='LightGBMLambdaMART', config_with_json=config_with_json, dir_json=dir_json) 53 | 54 | else: 55 | data_id = 'MQ2008_Super' 56 | #data_id = 'MSLRWEB30K' 57 | #data_id = 'MQ2008_List' 58 | 59 | #dir_data = '/home/dl-box/WorkBench/Datasets/L2R/LETOR4.0/MQ2008/' 60 | #dir_data = '/Users/solar/WorkBench/Datasets/L2R/LETOR4.0/MQ2008/' 61 | #dir_data = '/Users/dryuhaitao/WorkBench/Corpus/' + 'LETOR4.0/MQ2008/' 62 | #dir_data = '/home/dl-box/WorkBench/Datasets/L2R/MSLR-WEB30K/' 63 | #dir_data = '/Users/dryuhaitao/WorkBench/Corpus/' + 'LETOR4.0/MQ2008-list/' 64 | dir_data = '/Users/iimac/Workbench/Corpus/L2R/LETOR4.0/MQ2008/' 65 | 66 | #data_id = 'Istella_S' 67 | #dir_data = '/home/dl-box/WorkBench/Datasets/L2R/ISTELLA_L2R/' 68 | 69 | ''' output directory ''' 70 | #dir_output = '/Users/dryuhaitao/WorkBench/CodeBench/Bench_Output/NeuralLTR/Listwise/' 71 | #dir_output = '/home/dl-box/WorkBench/CodeBench/PyCharmProject/Project_output/Out_L2R/Cmp_LTR_OptFrames/' 72 | #dir_output = '/Users/solar/WorkBench/CodeBench/PyCharmProject/Project_output/Out_L2R/' 73 | dir_output = '/Users/iimac/Workbench/CodeBench/Output/NeuralLTR/' 74 | 75 | grid_search = False # with grid_search, we can explore the effects of different hyper-parameters of a model 76 | 77 | evaluator.run(debug=debug, model_id='LightGBMLambdaMART', 78 | data_id=data_id, dir_data=dir_data, dir_output=dir_output, grid_search=grid_search) 79 | -------------------------------------------------------------------------------- /testing/metric/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wildltr/ptranking/f1d366cd4355fb9f38fb97cb89b224b182d9ca87/testing/metric/__init__.py -------------------------------------------------------------------------------- /testing/metric/testing_metric.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """Description 6 | 7 | """ 8 | 9 | import torch 10 | 11 | from scipy import stats 12 | 13 | from ptranking.metric.adhoc.adhoc_metric import torch_ap_at_k, torch_ndcg_at_k, torch_nerr_at_k, \ 14 | torch_ap_at_ks, torch_ndcg_at_ks, torch_kendall_tau, torch_nerr_at_ks 15 | 16 | 17 | def test_ap(): 18 | ''' todo-as-note: the denominator should be carefully checked when using AP@k ''' 19 | # here we assume that there five relevant documents, but the system just retrieves three of them 20 | sys_sorted_labels = torch.Tensor([1.0, 0.0, 1.0, 0.0, 1.0]) 21 | std_sorted_labels = torch.Tensor([1.0, 1.0, 1.0, 1.0, 1.0]) 22 | ap_at_ks = torch_ap_at_ks(sys_sorted_labels.view(1, -1), std_sorted_labels.view(1, -1), ks=[1, 3, 5]) 23 | print(ap_at_ks.size(), ap_at_ks) # tensor([1.0000, 0.5556, 0.4533]) 24 | ap_at_k = torch_ap_at_k(sys_sorted_labels.view(1, -1), std_sorted_labels.view(1, -1), k=3) 25 | print(ap_at_k.size(), ap_at_k) # tensor([1.0000, 0.5556, 0.4533]) 26 | 27 | sys_sorted_labels = torch.Tensor([1.0, 0.0, 1.0, 0.0, 1.0]) 28 | std_sorted_labels = torch.Tensor([1.0, 1.0, 1.0, 0.0, 0.0]) 29 | ap_at_ks = torch_ap_at_ks(sys_sorted_labels.view(1, -1), std_sorted_labels.view(1, -1), ks=[1, 3, 5]) 30 | print(ap_at_ks) # tensor([1.0000, 0.5556, 0.7556]) 31 | 32 | # here we assume that there four relevant documents, the system just retrieves four of them 33 | sys_sorted_labels = torch.Tensor([1.0, 1.0, 0.0, 1.0, 0.0, 0.0, 1.0]) 34 | std_sorted_labels = torch.Tensor([1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0]) 35 | ap_at_ks = torch_ap_at_ks(sys_sorted_labels.view(1, -1), std_sorted_labels.view(1, -1), ks=[1, 2, 3, 5, 7]) 36 | print(ap_at_ks) # tensor([1.0000, 1.0000, 0.6667, 0.6875, 0.8304]) 37 | ap_at_k = torch_ap_at_k(sys_sorted_labels.view(1, -1), std_sorted_labels.view(1, -1), k=5) 38 | print(ap_at_k) # tensor([1.0000, 1.0000, 0.6667, 0.6875, 0.8304]) 39 | print() 40 | 41 | 42 | def test_ndcg(): 43 | sys_sorted_labels = torch.Tensor([1.0, 1.0, 0.0, 1.0, 0.0, 0.0, 1.0]) 44 | std_sorted_labels = torch.Tensor([1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0]) 45 | ndcg_at_ks = torch_ndcg_at_ks(sys_sorted_labels.view(1, -1), std_sorted_labels.view(1, -1), ks=[1, 2, 3, 4, 5, 6, 7]) 46 | print(ndcg_at_ks.size(), ndcg_at_ks) # tensor([1.0000, 1.0000, 0.7654, 0.8048, 0.8048, 0.8048, 0.9349]) 47 | ndcg_at_k = torch_ndcg_at_k(sys_sorted_labels.view(1, -1), std_sorted_labels.view(1, -1), k=4) 48 | print(ndcg_at_k.size(), ndcg_at_k) # tensor([1.0000, 1.0000, 0.7654, 0.8048, 0.8048, 0.8048, 0.9349]) 49 | print() 50 | 51 | 52 | 53 | def test_nerr(): 54 | sys_sorted_labels = torch.Tensor([3.0, 2.0, 4.0]) 55 | std_sorted_labels = torch.Tensor([4.0, 3.0, 2.0]) 56 | # convert to batch mode 57 | batch_nerr_at_ks = torch_nerr_at_ks(sys_sorted_labels.view(1, -1), std_sorted_labels.view(1, -1), ks=[1, 2, 3]) 58 | print(batch_nerr_at_ks.size(), batch_nerr_at_ks) # tensor([0.4667, 0.5154, 0.6640]) 59 | batch_nerr_at_k = torch_nerr_at_k(sys_sorted_labels.view(1, -1), std_sorted_labels.view(1, -1), k=2) 60 | print(batch_nerr_at_k.size(), batch_nerr_at_k) # tensor([0.4667, 0.5154, 0.6640]) 61 | print() 62 | 63 | 64 | def test_kendall_tau(): 65 | reference = torch.Tensor([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]) 66 | sys_1 = torch.Tensor([2.0, 1.0, 5.0, 3.0, 4.0, 6.0, 7.0, 9.0, 8.0, 10.0]) 67 | sys_2 = torch.Tensor([10.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 1.0]) 68 | 69 | tau_1 = torch_kendall_tau(sys_1, natural_ascending_as_reference=True) 70 | print('tau_1', tau_1) 71 | 72 | tau_2 = torch_kendall_tau(sys_2, natural_ascending_as_reference=True) 73 | print('tau_2', tau_2) 74 | 75 | tau, p = stats.kendalltau(reference.data.data.numpy(), sys_1) 76 | print('scipy-1', tau, p) 77 | 78 | tau, p = stats.kendalltau(reference.data.numpy(), sys_2) 79 | print('scipy-2', tau, p) 80 | 81 | print() 82 | print('-----------------------') 83 | 84 | 85 | res_reference, _ = torch.sort(reference, dim=0, descending=True) 86 | 87 | tau_1 = torch_kendall_tau(sys_1, natural_ascending_as_reference=False) 88 | print('tau_1', tau_1) 89 | 90 | tau_2 = torch_kendall_tau(sys_2, natural_ascending_as_reference=False) 91 | print('tau_2', tau_2) 92 | 93 | tau, p = stats.kendalltau(res_reference.data.numpy(), sys_1) 94 | print('scipy-1', tau, p) 95 | 96 | tau, p = stats.kendalltau(res_reference.data.numpy(), sys_2) 97 | print('scipy-2', tau, p) 98 | 99 | 100 | if __name__ == '__main__': 101 | 102 | #1 103 | test_ap() 104 | 105 | #2 106 | test_nerr() 107 | 108 | #3 109 | test_ndcg() 110 | 111 | 112 | 113 | 114 | 115 | -------------------------------------------------------------------------------- /tutorial/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by Hai-Tao Yu | 22/08/2020 | https://ii-research.github.io 5 | 6 | """Description 7 | 8 | """ 9 | -------------------------------------------------------------------------------- /tutorial/ptranking_demo_dataset_statistics.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "pycharm": { 7 | "name": "#%% md\n" 8 | } 9 | }, 10 | "source": [ 11 | "## Check the statistics of a specified dataset with default setting" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "metadata": { 18 | "pycharm": { 19 | "name": "#%%\n" 20 | } 21 | }, 22 | "outputs": [ 23 | { 24 | "ename": "TypeError", 25 | "evalue": "__init__() got an unexpected keyword argument 'train'", 26 | "output_type": "error", 27 | "traceback": [ 28 | "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", 29 | "\u001B[0;31mTypeError\u001B[0m Traceback (most recent call last)", 30 | "\u001B[0;32m\u001B[0m in \u001B[0;36m\u001B[0;34m\u001B[0m\n\u001B[1;32m 14\u001B[0m \u001B[0mbuffer\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;32mFalse\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 15\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 16\u001B[0;31m \u001B[0mcheck_dataset_statistics\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdata_id\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mdata_id\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdir_data\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mdir_data\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mbuffer\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mbuffer\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 17\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 18\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", 31 | "\u001B[0;32m~/WorkBench/Dropbox/CodeBench/GitPool/wildltr_ptranking/testing/data/testing_data_utils.py\u001B[0m in \u001B[0;36mcheck_dataset_statistics\u001B[0;34m(data_id, dir_data, buffer)\u001B[0m\n\u001B[1;32m 126\u001B[0m \u001B[0mmin_doc\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mmax_doc\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0msum_rele\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mget_min_max_docs\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtrain_dataset\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mtrain_dataset\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mvali_dataset\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mNone\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtest_dataset\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mtest_dataset\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 127\u001B[0m \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 128\u001B[0;31m \u001B[0mtrain_dataset\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mLTRDataset\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtrain\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mfile\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mfile_train\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdata_id\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mdata_id\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mshuffle\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mFalse\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mbuffer\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mbuffer\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 129\u001B[0m \u001B[0mvali_dataset\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mLTRDataset\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtrain\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mFalse\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mfile\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mfile_vali\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdata_id\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mdata_id\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mshuffle\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mFalse\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mbuffer\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mbuffer\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 130\u001B[0m \u001B[0mtest_dataset\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mLTRDataset\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mtrain\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mFalse\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mfile\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mfile_test\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mdata_id\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mdata_id\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mshuffle\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mFalse\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mbuffer\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mbuffer\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", 32 | "\u001B[0;31mTypeError\u001B[0m: __init__() got an unexpected keyword argument 'train'" 33 | ] 34 | } 35 | ], 36 | "source": [ 37 | "from testing.data.testing_data_utils import check_dataset_statistics\n", 38 | "\n", 39 | "\"\"\" An example script \"\"\"\n", 40 | "\n", 41 | "''' The dataset identifier '''\n", 42 | "data_id = 'MQ2007_Super'\n", 43 | "\n", 44 | "\n", 45 | "''' The directory of the above dataset '''\n", 46 | "dir_data = '/Users/dryuhaitao/WorkBench/Corpus/LETOR4.0/MQ2007/'\n", 47 | "#dir_data = '/home/dl-box/WorkBench/Datasets/L2R/LETOR4.0/MQ2007/'\n", 48 | "\n", 49 | "''' Whether buffer the loaded dataset as numpy.tensor & PyTorch.tensor '''\n", 50 | "buffer = False\n", 51 | "\n", 52 | "check_dataset_statistics(data_id=data_id, dir_data=dir_data, buffer=buffer)\n", 53 | "\n" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": null, 59 | "metadata": {}, 60 | "outputs": [], 61 | "source": [] 62 | } 63 | ], 64 | "metadata": { 65 | "kernelspec": { 66 | "name": "pycharm-443bb574", 67 | "language": "python", 68 | "display_name": "PyCharm (ptl2r)" 69 | }, 70 | "language_info": { 71 | "codemirror_mode": { 72 | "name": "ipython", 73 | "version": 3 74 | }, 75 | "file_extension": ".py", 76 | "mimetype": "text/x-python", 77 | "name": "python", 78 | "nbconvert_exporter": "python", 79 | "pygments_lexer": "ipython3", 80 | "version": "3.7.6" 81 | } 82 | }, 83 | "nbformat": 4, 84 | "nbformat_minor": 1 85 | } -------------------------------------------------------------------------------- /tutorial/ptranking_empirical_risk_minimization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "source": [ 6 | "## The Empirical Risk Minimization Framework\n", 7 | "\n", 8 | "Let $\\mathcal{Q}$ and $\\mathcal{D}$ be the query space and the document space, respectively, we use $\\Phi:\\mathcal{Q}\\times\\mathcal{D}\\rightarrow\\mathcal{Z}=\\mathbb{R}^{d}$ to denote the mapping function for generating a feature vector for a query-document pair, where $\\mathcal{Z}$ represents the $d$-dimensional feature space. We use $\\mathcal{R}=\\mathbb{R}_{+}$ to denote the space of the ground-truth relevance scores each document receives. Thus for each query, we have a list of document feature vectors $X=(X_{1},...,X_{m})\\in\\mathcal{X}=\\mathcal{Z}^{m}$ and a corresponding list $Y=(Y_{1},...,Y_{m})\\in\\mathcal{Y}=\\mathcal{R}^{m}$ of ground-truth relevance scores. In practice, we get independently and identically distributed (i.i.d) samples $\\mathcal{S}=\\{(X_{i},Y_{i})\\}_{i=1}^{n}$\n", 9 | "from an unknown joint distribution $P(\\cdot,\\cdot)$ over $\\mathcal{X}\\times\\mathcal{Y}$. We use $f_{\\theta}:X\\rightarrow\\mathbb{R}_{+}^{m}$ parameterized by $\\theta\\in\\Theta$ to denote the real-valued ranking function, which assigns each document a score. The scores of the documents associated with the same query are used to rank the documents. We measure the loss of ranking documents for a query using $f_{\\theta}$ with the loss function $\\ell(f_{\\theta}(X),Y)$. The goal is to learn the optimal ranking function over a hypothesis space $\\mathcal{F}$ of ranking functions that can \\emph{minimize the expected risk} as defined below:\n", 10 | "\n", 11 | "\\begin{equation}\n", 12 | "\t\\min_{f_{\\theta}\\in\\mathcal{F}}\\Re(f_{\\theta})=\\min_{f_{\\theta}\\in\\mathcal{F}}\\int_{\\mathcal{X}\\times\\mathcal{Y}}\\ell(f_{\\theta}(X),Y)dP(X,Y)\n", 13 | "\\end{equation}\n", 14 | "\n", 15 | "Typically, $\\Re(f_{\\theta})$ is intractable to optimize directly and the joint distribution is unknown, we appeal to the \\emph{empirical risk minimization} to approximate the expected risk, which is defined as follows:\n", 16 | "\n", 17 | "\\begin{equation}\n", 18 | "\t\\min_{f_{\\theta}\\in\\mathcal{F}}\\hat{\\Re}(f_{\\theta};\\mathcal{S})=\\min_{f_{\\theta}\\in\\mathcal{F}}\\frac{1}{n}\\sum_{i=1}^{n}\\ell(f_{\\theta}(X),Y)\n", 19 | "\\end{equation}\n", 20 | "\n", 21 | "Given the above optimization framework, one can design various ranking methods by deploying different loss functions to learn the parameters $\\theta$ based on the training documents. In the testing phase, the predicted ranking can be obtained efficiently by sorting the testing documents in descending order of their individual relevance scores $f_{\\theta}(X_{i})$.\n" 22 | ], 23 | "metadata": { 24 | "collapsed": false, 25 | "pycharm": { 26 | "name": "#%% md\n" 27 | } 28 | } 29 | } 30 | ], 31 | "metadata": { 32 | "kernelspec": { 33 | "display_name": "Python 3", 34 | "language": "python", 35 | "name": "python3" 36 | }, 37 | "language_info": { 38 | "codemirror_mode": { 39 | "name": "ipython", 40 | "version": 2 41 | }, 42 | "file_extension": ".py", 43 | "mimetype": "text/x-python", 44 | "name": "python", 45 | "nbconvert_exporter": "python", 46 | "pygments_lexer": "ipython2", 47 | "version": "2.7.6" 48 | } 49 | }, 50 | "nbformat": 4, 51 | "nbformat_minor": 0 52 | } -------------------------------------------------------------------------------- /tutorial/ptranking_ir_metric.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "source": [ 6 | "## Review on IR Metrics\n", 7 | "\n", 8 | "Given the predictions $\\mathbf{y}=f(\\mathbf{x})$ and the corresponding ground-truth\n", 9 | "labels $\\mathbf{y}^{*}$, we use $Y^{+}=\\{i|y_{i}^{*}>0\\}$ and $Y^{-}=\\{j|y_{j}^{*}=0\\}$ to represent the sets of relevant documents and non-relevant documents, respectively. We use $\\mathbf{b}^{*}=\\mathbb{I}\\{\\mathbf{y}^{*}>0\\}$ with $b_{j}^{*}=\\mathbb{I}\\{y_{j}^{*}>0\\}$\n", 10 | "to represent the binarized ground-truth, and the cumulative sum on\n", 11 | "$\\mathbf{b}^{*}$ is given as $B_{k}^{*}=\\sum_{j=1}^{k}b_{j}^{*}$. The scoring function $f$ induces\n", 12 | "a ranking $\\bar{\\mathbf{y}}$.\n", 13 | "The corresponding ground-truth labels are $\\mathbf{y}^{**}$. Furthermore, we denote the binarized ground-truth labels as $\\mathbf{b}^{**}=\\mathbb{I}\\{\\mathbf{y}^{**}>0\\}$\n", 14 | "with $b_{j}^{**}=\\mathbb{I}\\{y_{j}^{**}>0\\}$, and the cumulative\n", 15 | "sum on $\\mathbf{b}^{**}$ is given as $B_{k}^{**}=\\sum_{j=1}^{k}b_{j}^{**}$.\n", 16 | "\n", 17 | "To evaluate the effectiveness of a scoring function, a number of IR\n", 18 | "metrics have been proposed to emphasize the items that are ranked\n", 19 | "at higher positions. In general, the IR metrics are computed\n", 20 | "based on the list of ground-truth labels $\\bar{\\mathbf{y}}$ induced by $f$. For example, the binary-relevance IR metrics measure the performance of\n", 21 | "a specific ranking model based on $\\mathbf{b}^{**}$, such as precision and AP. The graded-relevance IR metrics measure the performance of a specific ranking model based on $\\mathbf{y}^{**}$,\n", 22 | "such as nDCG and ERR.\n", 23 | "\n", 24 | "## Precision\n", 25 | "\n", 26 | "Precision@k measures the proportion\n", 27 | "of relevant documents retrieved at a given truncation position, which is defined as:\n", 28 | "\\begin{equation}\n", 29 | "Pre@k=\\frac{1}{k}\\sum_{j=1}^{k}b_{j}^{**}\n", 30 | "\\end{equation}\n", 31 | "\n", 32 | "where $k$ denotes the truncation position.\n", 33 | "\n", 34 | "## Average Precision\n", 35 | "\n", 36 | "Different from Precision@k that does not take into account the position\n", 37 | "at which a document is ranked, Average Precision (AP) is a **rank-sensitive**\n", 38 | "metric, which builds upon Precision as follows:\n", 39 | "\\begin{equation}\n", 40 | "AP=\\frac{1}{|Y^{+}|}\\sum_{j}b_{j}^{**}\\times Pre@j\n", 41 | "\\end{equation}\n", 42 | "\n", 43 | "Then Mean Average Precision (MAP) is defined\n", 44 | "as the mean of AP scores over a set of queries.\n", 45 | "\n", 46 | "## Normalized Discounted Cumulative Gain (nDCG)\n", 47 | "\n", 48 | "Normalized Discounted Cumulative Gain (nDCG) is a **graded-relevance\n", 49 | "rank-sensitive** metric. The discounted cumulative\n", 50 | "gain (DCG) of a ranked list is given as $DCG@k=\\sum_{j=1}^{k}\\frac{2^{y_{j}^{**}}-1}{\\log_{2}(j+1)}$,\n", 51 | "where $G_{j}=2^{y_{j}^{**}}-1$ is usually referred to as the gain\n", 52 | "value of the $j$-th document.\n", 53 | "We denote the maximum DCG value attained by the ideal ranking as $DCG^{*}$,\n", 54 | "then normalizing DCG with $DCG^{*}$ gives nDCG as follows:\n", 55 | "\n", 56 | "\\begin{equation}\n", 57 | "nDCG@k=\\frac{DCG@k}{DCG^{*}@k}\n", 58 | "\\end{equation}\n", 59 | "\n", 60 | "## Expected Reciprocal Rank (ERR)\n", 61 | "\n", 62 | "Expected Reciprocal Rank (ERR) is another popular graded-relevance rank-sensitive metric. Let $Pr(j)$ be the relevance probability of the document at rank\n", 63 | "$j$. In accordance with the gain function for nDCG, the relevance\n", 64 | "probability is commonly calculated as $Pr(j)=\\frac{2^{y_{j}^{**}}-1}{2^{\\max(\\mathbf{y}^{**})}}$.\n", 65 | "ERR interprets the relevance probability as the probability that the\n", 66 | "user is satisfied with the document at a rank position. Thus the probability\n", 67 | "that the user is dissatisfied with the documents at ranks from $1$\n", 68 | "to $k$ is given as $Disp(1,k)=\\prod_{i=1}^{k}(1-Pr(i))$. ERR is\n", 69 | "then defined as\n", 70 | "\n", 71 | "\\begin{equation}\n", 72 | "ERR@k=\\sum_{j=1}^{k}\\frac{Disp(1,j-1)\\cdot Pr(j)}{j}\n", 73 | "\\end{equation}\n", 74 | "\n", 75 | "\n", 76 | "Let $ERR^{*}$ the maximum ERR value attained by the ideal ranking, we have the normalized ERR as follows:\n", 77 | "\n", 78 | "\\begin{equation}\n", 79 | "nERR@k=\\frac{ERR@k}{ERR^{*}@k}\n", 80 | "\\end{equation}" 81 | ], 82 | "metadata": { 83 | "collapsed": false, 84 | "pycharm": { 85 | "name": "#%% md\n" 86 | } 87 | } 88 | } 89 | ], 90 | "metadata": { 91 | "kernelspec": { 92 | "display_name": "Python 3", 93 | "language": "python", 94 | "name": "python3" 95 | }, 96 | "language_info": { 97 | "codemirror_mode": { 98 | "name": "ipython", 99 | "version": 2 100 | }, 101 | "file_extension": ".py", 102 | "mimetype": "text/x-python", 103 | "name": "python", 104 | "nbconvert_exporter": "python", 105 | "pygments_lexer": "ipython2", 106 | "version": "2.7.6" 107 | } 108 | }, 109 | "nbformat": 4, 110 | "nbformat_minor": 0 111 | } --------------------------------------------------------------------------------