├── Rankers ├── __init__.py ├── models │ ├── __init__.py │ ├── RankNet.py │ ├── AdaRank.py │ ├── LambdaMart2.py │ ├── LambdaMart.py │ ├── ListNet.py │ ├── DirectRanker.py │ └── LambdaRank.py └── helpers.py ├── RankLib-2.10.jar ├── table1 ├── run_jobs.py ├── create_runscripts.py ├── gridsearch.py └── get_results.py ├── data ├── MSLR-WEB10K │ └── README.md ├── MQ2008 │ └── README.md ├── MQ2007 │ └── README.md ├── get_data.sh └── delete_zeros.py ├── rustlib ├── Cargo.toml ├── pyproject.toml ├── src │ └── lib.rs └── Cargo.lock ├── table2 ├── run_ranknet_eval.sh └── ranknet_eval.py ├── fig4 ├── gen_fig4c.py ├── gen_fig4a.py ├── gen_fig4b.py └── gen_fig2d.py ├── requirements.txt ├── tableB1 ├── timeana.py └── run_time_ana.sh ├── fig5 ├── plot_fig5.py ├── training_size_mslr.py ├── training_size.py ├── training_label.py └── training_label_mslr.py ├── test_all.sh ├── fig6 └── plot_sorted_list.py └── README.md /Rankers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Rankers/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /RankLib-2.10.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kramerlab/direct-ranker/HEAD/RankLib-2.10.jar -------------------------------------------------------------------------------- /table1/run_jobs.py: -------------------------------------------------------------------------------- 1 | from multiprocessing import Pool, cpu_count 2 | import os 3 | 4 | 5 | def run_script(f): 6 | os.system("source run_scripts/{}".format(f)) 7 | 8 | with Pool(cpu_count()) as p: 9 | p.map(run_script, [f for f in os.listdir("run_scripts")]) 10 | -------------------------------------------------------------------------------- /data/MSLR-WEB10K/README.md: -------------------------------------------------------------------------------- 1 | https://www.microsoft.com/en-us/research/project/mslr/ 2 | https://8kmjpq.dm.files.1drv.com/y4mhS-AvDnGSEBQTsG8VZrDSVbJLzlvgwHSBPDSAK9E8kfl_aHGcMhKy9FUSR7VSYk9J3vWkhcMwhlUHpHP_nR2YjxwZLORpn3k5ihHyvS8gtDLk7XSaNJ6b2jxBmc57_qa8lzgTo8fmbUhdaVYmb27fshRzYSufOgs3AkNw7urLhLJdVTKlyfT6zn3vSIM1a9V8oBbiGwN2DpQopirV8jH8w 3 | -------------------------------------------------------------------------------- /data/MQ2008/README.md: -------------------------------------------------------------------------------- 1 | https://www.microsoft.com/en-us/research/project/letor-learning-rank-information-retrieval/letor-4-0/ 2 | https://lyoz5a.ch.files.1drv.com/y4mM8g8v4d2mFfO5djKT-ELADpDDRcsVwXRSaZu-9rlOlgvW62Qeuc8hFe_wr6m5NZMnUSEfr6QpMP81ZIQIiwI4BnoHmIZT9Sraf53AmhhIfLi531DOKYZTy4MtDHbBC7dn_Z9DSKvLJZhERPIamAXCrONg7WrFPiG0sTpOXl3-YEYZ1scTslmNyg2a__3YalWRMyEIipY56sy97pb68Sdww -------------------------------------------------------------------------------- /data/MQ2007/README.md: -------------------------------------------------------------------------------- 1 | https://www.microsoft.com/en-us/research/project/letor-learning-rank-information-retrieval/letor-4-0/ 2 | https://lyoz5a-ch3301.files.1drv.com/y4mWKvFV-J52ArYcpuGty_7ArjYHb_gXJysvFMS0Ww2Uy9YAhUGdiaA4zyImaQIsa57gf3ugoLVea9D6LgWPsVgqwwNTWyc29TsgpXR6fgdS8ATy-svdAFbFRylhISnNVMgJpVvHC3pHmAeLfkYigMJkED9EZSnhPTgqEJh1JVtsdsKro5Rj8O8XwDSWdSj_7DRpygGQCWsbTJki0J1XskSHA -------------------------------------------------------------------------------- /rustlib/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rustlib" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | [lib] 8 | name = "rustlib" 9 | crate-type = ["cdylib"] 10 | 11 | [dependencies] 12 | pyo3 = "0.18.1" 13 | itertools = "0.10.5" 14 | rayon = "1.6.1" 15 | rand = "0.8.5" 16 | -------------------------------------------------------------------------------- /rustlib/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["maturin>=0.14,<0.15"] 3 | build-backend = "maturin" 4 | 5 | [project] 6 | name = "rustlib" 7 | requires-python = ">=3.7" 8 | classifiers = [ 9 | "Programming Language :: Rust", 10 | "Programming Language :: Python :: Implementation :: CPython", 11 | "Programming Language :: Python :: Implementation :: PyPy", 12 | ] 13 | 14 | 15 | [tool.maturin] 16 | features = ["pyo3/extension-module"] 17 | -------------------------------------------------------------------------------- /table2/run_ranknet_eval.sh: -------------------------------------------------------------------------------- 1 | mkdir .log 2 | ln -s ../Rankers . 3 | 4 | python ranknet_eval.py --path ../data/MSLR-WEB10K/ --activation sigmoid --epoch 10 > .log/log_sig_sgd & 5 | python ranknet_eval.py --path ../data/MSLR-WEB10K/ --activation tanh --epoch 10 > .log/log_tanh_sgd & 6 | python ranknet_eval.py --path ../data/MSLR-WEB10K/ --activation hard_sigmoid --epoch 10 > .log/log_hard_sgd & 7 | python ranknet_eval.py --path ../data/MSLR-WEB10K/ --activation linear --epoch 10 > .log/log_linear_sgd & 8 | 9 | python ranknet_eval.py --path ../data/MSLR-WEB10K/ --activation sigmoid --optimizer Adam --epoch 10 > .log/log_sig_adam & 10 | python ranknet_eval.py --path ../data/MSLR-WEB10K/ --activation tanh --optimizer Adam --epoch 10 > .log/log_tanh_adam & 11 | python ranknet_eval.py --path ../data/MSLR-WEB10K/ --activation hard_sigmoid --optimizer Adam --epoch 10 > .log/log_hard_adam & 12 | python ranknet_eval.py --path ../data/MSLR-WEB10K/ --activation linear --optimizer Adam --epoch 10 > .log/log_linear_adam & 13 | -------------------------------------------------------------------------------- /data/get_data.sh: -------------------------------------------------------------------------------- 1 | # get MQ2007 2 | wget --output-document MQ2007.rar https://lyoz5a-ch3301.files.1drv.com/y4mWKvFV-J52ArYcpuGty_7ArjYHb_gXJysvFMS0Ww2Uy9YAhUGdiaA4zyImaQIsa57gf3ugoLVea9D6LgWPsVgqwwNTWyc29TsgpXR6fgdS8ATy-svdAFbFRylhISnNVMgJpVvHC3pHmAeLfkYigMJkED9EZSnhPTgqEJh1JVtsdsKro5Rj8O8XwDSWdSj_7DRpygGQCWsbTJki0J1XskSHA 3 | unrar x MQ2007.rar 4 | rm MQ2007.rar 5 | 6 | # get MQ2008 7 | wget --output-document MQ2008.rar https://lyoz5a.ch.files.1drv.com/y4mM8g8v4d2mFfO5djKT-ELADpDDRcsVwXRSaZu-9rlOlgvW62Qeuc8hFe_wr6m5NZMnUSEfr6QpMP81ZIQIiwI4BnoHmIZT9Sraf53AmhhIfLi531DOKYZTy4MtDHbBC7dn_Z9DSKvLJZhERPIamAXCrONg7WrFPiG0sTpOXl3-YEYZ1scTslmNyg2a__3YalWRMyEIipY56sy97pb68Sdww 8 | unrar x MQ2008.rar 9 | rm MQ2008.rar 10 | 11 | # remove zeros from MQ2007/2008 12 | python delete_zeros.py 13 | 14 | # get MSLR-WEB10K 15 | wget --output-document MSLR-WEB10K.zip https://8kmjpq.dm.files.1drv.com/y4mhS-AvDnGSEBQTsG8VZrDSVbJLzlvgwHSBPDSAK9E8kfl_aHGcMhKy9FUSR7VSYk9J3vWkhcMwhlUHpHP_nR2YjxwZLORpn3k5ihHyvS8gtDLk7XSaNJ6b2jxBmc57_qa8lzgTo8fmbUhdaVYmb27fshRzYSufOgs3AkNw7urLhLJdVTKlyfT6zn3vSIM1a9V8oBbiGwN2DpQopirV8jH8w 16 | unzip MSLR-WEB10K.zip -d MSLR-WEB10K 17 | rm MSLR-WEB10K.zip 18 | -------------------------------------------------------------------------------- /fig4/gen_fig4c.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from Rankers.helpers import gen_set, train_model 4 | 5 | import matplotlib.pyplot as plt 6 | import numpy as np 7 | 8 | import argparse 9 | 10 | 11 | if __name__ == '__main__': 12 | 13 | parser = argparse.ArgumentParser(description='Get some run values.') 14 | parser.add_argument('--test', type=int, default=0, help='Run in test mode') 15 | 16 | args = parser.parse_args() 17 | 18 | os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 19 | 20 | test = False 21 | if args.test == 1: 22 | test = True 23 | 24 | ndcg_mean = [] 25 | ndcg_err = [] 26 | for sigma in np.arange(0, 1.1, 0.1): 27 | gen_set(n_feats=70, n_relevance=5, sigma_label=sigma, path="fig2c", test=test) 28 | mean, err = train_model(num_features=70, path="fig2c", test=test) 29 | ndcg_mean.append(mean) 30 | ndcg_err.append(err) 31 | 32 | plt.errorbar(np.arange(0, 1.1, 0.1), ndcg_mean, yerr=ndcg_err, alpha=0.3) 33 | plt.scatter(np.arange(0, 1.1, 0.1), ndcg_mean) 34 | plt.xlabel(r'$\sigma$') 35 | plt.ylabel(r'nDCG@20') 36 | plt.savefig("fig2c.pdf") 37 | plt.close() 38 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==1.3.0 2 | astunparse==1.6.3 3 | cachetools==5.2.0 4 | certifi==2022.12.7 5 | charset-normalizer==2.1.1 6 | contourpy==1.0.6 7 | cycler==0.11.0 8 | flatbuffers==22.12.6 9 | fonttools==4.38.0 10 | gast==0.4.0 11 | google-auth==2.15.0 12 | google-auth-oauthlib==0.4.6 13 | google-pasta==0.2.0 14 | grpcio==1.51.1 15 | h5py==3.7.0 16 | idna==3.4 17 | joblib==1.2.0 18 | keras==2.11.0 19 | kiwisolver==1.4.4 20 | libclang==14.0.6 21 | Markdown==3.4.1 22 | MarkupSafe==2.1.1 23 | matplotlib==3.6.2 24 | numpy==1.24.0 25 | oauthlib==3.2.2 26 | opt-einsum==3.3.0 27 | packaging==22.0 28 | pandas==1.5.2 29 | patsy==0.5.3 30 | Pillow==9.3.0 31 | protobuf==3.19.6 32 | pyasn1==0.4.8 33 | pyasn1-modules==0.2.8 34 | pyparsing==3.0.9 35 | python-dateutil==2.8.2 36 | pytz==2022.7 37 | requests==2.28.1 38 | requests-oauthlib==1.3.1 39 | rsa==4.9 40 | scikit-learn==1.2.0 41 | scikit-posthocs==0.7.0 42 | scipy==1.9.3 43 | seaborn==0.12.1 44 | six==1.16.0 45 | statsmodels==0.13.5 46 | tensorboard==2.11.0 47 | tensorboard-data-server==0.6.1 48 | tensorboard-plugin-wit==1.8.1 49 | tensorflow==2.11.0 50 | tensorflow-estimator==2.11.0 51 | tensorflow-io-gcs-filesystem==0.29.0 52 | termcolor==2.1.1 53 | threadpoolctl==3.1.0 54 | tqdm==4.64.1 55 | typing_extensions==4.4.0 56 | urllib3==1.26.13 57 | Werkzeug==2.2.2 58 | wrapt==1.14.1 59 | -------------------------------------------------------------------------------- /data/delete_zeros.py: -------------------------------------------------------------------------------- 1 | for directory in ["MQ2008", "MQ2007"]: 2 | for fold in ["Fold"+str(i) for i in range(1,6)]: 3 | for f in ["train.txt","test.txt","vali.txt"]: 4 | path = directory+"/"+fold+"/"+f 5 | queries = [] 6 | qid = "" 7 | for line in open(path): 8 | s = line.split() 9 | # Check if new query, and if so, append a new list of documents 10 | if s[1] != qid: 11 | queries.append([]) 12 | qid = s[1] 13 | # Append document to current query 14 | queries[-1].append(s) 15 | of = open(path,"w") 16 | for query in queries: 17 | # Check if there are non-zero documents in query 18 | non_zero = False 19 | for doc in query: 20 | if doc[0]!="0": 21 | non_zero = True 22 | break 23 | # If there are, write query into file 24 | if non_zero: 25 | for doc in query: 26 | of.write(doc[0]) 27 | for v in doc[1:]: 28 | of.write(" "+v) 29 | of.write("\n") 30 | 31 | 32 | -------------------------------------------------------------------------------- /fig4/gen_fig4a.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from Rankers.helpers import gen_set, train_model 4 | 5 | import matplotlib.pyplot as plt 6 | import numpy as np 7 | 8 | import argparse 9 | 10 | 11 | if __name__ == '__main__': 12 | 13 | parser = argparse.ArgumentParser(description='Get some run values.') 14 | parser.add_argument('--test', type=int, default=0, help='Run in test mode') 15 | 16 | args = parser.parse_args() 17 | 18 | os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 19 | 20 | test = False 21 | if args.test == 1: 22 | test = True 23 | 24 | ndcg_mean_70 = [] 25 | ndcg_err_70 = [] 26 | ndcg_mean_40 = [] 27 | ndcg_err_40 = [] 28 | for classes in np.arange(2, 20, 1): 29 | gen_set(n_feats=40, n_relevance=classes, path="fig2a", test=test) 30 | mean, err = train_model(num_features=40, path="fig2a", test=test) 31 | ndcg_mean_40.append(mean) 32 | ndcg_err_40.append(err) 33 | 34 | gen_set(n_feats=70, n_relevance=classes, path="fig2a", test=test) 35 | mean, err = train_model(num_features=70, path="fig2a", test=test) 36 | ndcg_mean_70.append(mean) 37 | ndcg_err_70.append(err) 38 | 39 | plt.errorbar(np.arange(2, 20, 1), ndcg_mean_40, yerr=ndcg_err_40, alpha=0.3) 40 | plt.scatter(np.arange(2, 20, 1), ndcg_mean_40, label="# Features 40") 41 | plt.errorbar(np.arange(2, 20, 1), ndcg_mean_70, yerr=ndcg_err_70, alpha=0.3) 42 | plt.scatter(np.arange(2, 20, 1), ndcg_mean_70, label="# Features 70") 43 | plt.xlabel(r'#Classes') 44 | plt.ylabel(r'nDCG@20') 45 | plt.savefig("fig2a.pdf") 46 | plt.close() 47 | -------------------------------------------------------------------------------- /fig4/gen_fig4b.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from Rankers.helpers import gen_set, train_model 4 | 5 | import matplotlib.pyplot as plt 6 | import numpy as np 7 | 8 | import argparse 9 | 10 | 11 | if __name__ == '__main__': 12 | 13 | parser = argparse.ArgumentParser(description='Get some run values.') 14 | parser.add_argument('--test', type=int, default=0, help='Run in test mode') 15 | 16 | args = parser.parse_args() 17 | 18 | os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 19 | 20 | test = False 21 | if args.test == 1: 22 | test = True 23 | 24 | ndcg_mean_5 = [] 25 | ndcg_err_5 = [] 26 | ndcg_mean_10 = [] 27 | ndcg_err_10 = [] 28 | for features in np.arange(2, 100, 5): 29 | gen_set(n_feats=features, n_relevance=5, path="fig2b", test=test) 30 | mean, err = train_model(num_features=int(features), path="fig2b", test=test) 31 | ndcg_mean_5.append(mean) 32 | ndcg_err_5.append(err) 33 | 34 | gen_set(n_feats=features, n_relevance=10, path="fig2b", test=test) 35 | mean, err = train_model(num_features=int(features), path="fig2b", test=test) 36 | ndcg_mean_10.append(mean) 37 | ndcg_err_10.append(err) 38 | 39 | plt.errorbar(np.arange(2, 100, 5), ndcg_mean_5, yerr=ndcg_err_5, alpha=0.3) 40 | plt.scatter(np.arange(2, 100, 5), ndcg_mean_5, label="# Classes 5") 41 | plt.errorbar(np.arange(2, 100, 5), ndcg_mean_10, yerr=ndcg_err_10, alpha=0.3) 42 | plt.scatter(np.arange(2, 100, 5), ndcg_mean_10, label="# Classes 10") 43 | plt.xlabel(r'#Features') 44 | plt.ylabel(r'nDCG@20') 45 | plt.savefig("fig2b.pdf") 46 | plt.close() 47 | -------------------------------------------------------------------------------- /fig4/gen_fig2d.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from Rankers.helpers import gen_set, train_model 4 | 5 | import matplotlib.pyplot as plt 6 | import numpy as np 7 | 8 | import argparse 9 | 10 | 11 | if __name__ == '__main__': 12 | 13 | parser = argparse.ArgumentParser(description='Get some run values.') 14 | parser.add_argument('--test', type=int, default=0, help='Run in test mode') 15 | 16 | args = parser.parse_args() 17 | 18 | os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 19 | 20 | test = False 21 | if args.test == 1: 22 | test = True 23 | 24 | ndcg_mean_70 = [] 25 | ndcg_err_70 = [] 26 | ndcg_mean_40 = [] 27 | ndcg_err_40 = [] 28 | n_trains = [int(i) for i in 10 ** np.arange(0, 7, 0.5) - 1 + 100] 29 | for n_train in n_trains: 30 | gen_set(n_train=n_train, n_feats=40, n_relevance=5, path="fig2a", test=test) 31 | mean, err = train_model(num_features=40, path="fig2a", test=test) 32 | ndcg_mean_40.append(mean) 33 | ndcg_err_40.append(err) 34 | 35 | gen_set(n_train=n_train, n_feats=70, n_relevance=5, path="fig2a", test=test) 36 | mean, err = train_model(num_features=70, path="fig2a", test=test) 37 | ndcg_mean_70.append(mean) 38 | ndcg_err_70.append(err) 39 | 40 | plt.errorbar(n_trains, ndcg_mean_40, yerr=ndcg_err_40, alpha=0.3) 41 | plt.scatter(n_trains, ndcg_mean_40, label="# Features 40") 42 | plt.errorbar(n_trains, ndcg_mean_70, yerr=ndcg_err_70, alpha=0.3) 43 | plt.scatter(n_trains, ndcg_mean_70, label="# Features 70") 44 | plt.xlabel(r'#Documents in training') 45 | plt.semilogx() 46 | plt.ylabel(r'nDCG@20') 47 | plt.savefig("fig2d.pdf") 48 | plt.close() 49 | -------------------------------------------------------------------------------- /Rankers/models/RankNet.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from Rankers.models.DirectRanker import DirectRanker 4 | 5 | def _ranknet_cost(y_actual, y_predicted): 6 | # mean(log(1+exp((1+pred)/2)) - (1+pred)/2) 7 | return tf.reduce_mean(tf.math.log(1+tf.math.exp((1+y_predicted)))-(1+y_predicted)) 8 | 9 | class RankNet(DirectRanker): 10 | """ 11 | TODO 12 | """ 13 | 14 | def __init__(self, 15 | # DirectRanker HPs 16 | hidden_layers_dr=[256, 128, 64, 20], 17 | feature_activation_dr='relu', 18 | ranking_activation_dr='linear', 19 | feature_bias_dr=True, 20 | kernel_initializer_dr=tf.random_normal_initializer, 21 | kernel_regularizer_dr=0.0, 22 | drop_out=0, 23 | # Common HPs 24 | scale_factor_train_sample=1, 25 | batch_size=200, 26 | loss=_ranknet_cost, 27 | learning_rate=0.0005, 28 | learning_rate_decay_rate=0, 29 | learning_rate_decay_steps=0, 30 | optimizer=tf.keras.optimizers.SGD, 31 | epoch=10, 32 | steps_per_epoch=None, 33 | # other variables 34 | verbose=0, 35 | validation_size=0.0, 36 | num_features=0, 37 | name="RankNet", 38 | dtype=tf.float32, 39 | print_summary=False, 40 | ): 41 | super().__init__( 42 | # DirectRanker HPs 43 | hidden_layers_dr=hidden_layers_dr, 44 | feature_activation_dr=feature_activation_dr, 45 | ranking_activation_dr=ranking_activation_dr, 46 | feature_bias_dr=feature_bias_dr, 47 | kernel_initializer_dr=kernel_initializer_dr, 48 | kernel_regularizer_dr=kernel_regularizer_dr, 49 | drop_out=drop_out, 50 | # Common HPs 51 | scale_factor_train_sample=scale_factor_train_sample, 52 | batch_size=batch_size, 53 | learning_rate=learning_rate, 54 | learning_rate_decay_rate=learning_rate_decay_rate, 55 | learning_rate_decay_steps=learning_rate_decay_steps, 56 | optimizer=optimizer, 57 | epoch=epoch, 58 | loss=loss, 59 | steps_per_epoch=steps_per_epoch, 60 | # other variables 61 | verbose=verbose, 62 | validation_size=validation_size, 63 | num_features=num_features, 64 | name=name, 65 | dtype=dtype, 66 | print_summary=print_summary, 67 | ) 68 | 69 | -------------------------------------------------------------------------------- /tableB1/timeana.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import argparse 4 | 5 | import numpy as np 6 | 7 | from Rankers.helpers import readData, readDataV1 8 | from Rankers.models.DirectRanker import DirectRanker 9 | from Rankers.models.RankNet import RankNet 10 | 11 | 12 | if __name__ == '__main__': 13 | 14 | parser = argparse.ArgumentParser(description='Get some run values.') 15 | parser.add_argument('--model', type=str, default='DirectRanker', help='Get model name') 16 | parser.add_argument('--cpu', type=int, default=1, help='Use CPU or GPU') 17 | parser.add_argument('--path', type=str, default="", help='Use CPU or GPU') 18 | 19 | args = parser.parse_args() 20 | 21 | if args.cpu == 1: 22 | os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 23 | 24 | if not os.path.exists("output"): 25 | os.makedirs("output") 26 | os.makedirs("output/data") 27 | 28 | list_of_train = [f"{args.path}/Fold1/train.txt", f"{args.path}/Fold2/train.txt", 29 | f"{args.path}/Fold3/train.txt", f"{args.path}/Fold4/train.txt", 30 | f"{args.path}/Fold5/train.txt"] 31 | 32 | for idx, train in enumerate(list_of_train): 33 | 34 | # Load model 35 | print("Load Model") 36 | if args.model == "DirectRankerV1": 37 | from Rankers.models.DirectRankerV1 import DirectRankerV1 38 | ranker = DirectRankerV1( 39 | # For debugging 40 | # max_steps=100, 41 | print_step=0, 42 | start_batch_size=3, 43 | end_batch_size=5, 44 | start_qids=10, 45 | end_qids=100, 46 | weight_regularization=0.0001, 47 | feature_bias=True, 48 | hidden_layers=[50] 49 | ) 50 | 51 | if args.model == "DirectRanker": 52 | ranker = DirectRanker( 53 | epoch=10, 54 | num_features=136 55 | ) 56 | 57 | if args.model == "RankNet": 58 | ranker = RankNet( 59 | epoch=10, 60 | num_features=136 61 | ) 62 | 63 | print("Reading data") 64 | start = time.time() 65 | if os.path.isfile(f"output/data/MSLR-WEB10K_x_train_fold{idx+1}.npy"): 66 | x_train = np.load(f"output/data/MSLR-WEB10K_x_train_fold{idx+1}.npy") 67 | y_train = np.load(f"output/data/MSLR-WEB10K_y_train_fold{idx+1}.npy") 68 | else: 69 | if args.model == "DirectRankerV1": 70 | x_train, y_train, _ = readDataV1( 71 | data_path=train, 72 | binary=True, 73 | at=10, 74 | number_features=136, 75 | synth_data=True, 76 | cut_zeros=True 77 | ) 78 | else: 79 | x_train, y_train, _ = readData( 80 | path=train, 81 | binary=True, 82 | at=10, 83 | number_features=136 84 | ) 85 | 86 | np.save(f"output/data/MSLR-WEB10K_x_train_fold{idx+1}", x_train) 87 | np.save(f"output/data/MSLR-WEB10K_y_train_fold{idx+1}", y_train) 88 | 89 | if args.model == "DirectRankerV1": 90 | ranker.fit(x_train, y_train, ranking=False) 91 | else: 92 | ranker.fit(x_train, y_train) 93 | end = time.time() 94 | 95 | print(end - start) 96 | 97 | with open(f"output/{args.model}_time_fold{idx+1}.txt", "w") as f: 98 | f.write(str(end-start)) 99 | -------------------------------------------------------------------------------- /Rankers/models/AdaRank.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import time 3 | import rustlib 4 | 5 | from sklearn.base import BaseEstimator 6 | from sklearn.model_selection import train_test_split 7 | 8 | 9 | class AdaRank(BaseEstimator): 10 | """AdaRank algorithm""" 11 | 12 | def __init__(self, T=500, estop=10, verbose=False): 13 | self.T = T 14 | self.estop = estop 15 | self.verbose = verbose 16 | self.best_score = 0 17 | self.best_round = 0 18 | 19 | def fit(self, x, y): 20 | """Fit a model to the data""" 21 | 22 | qid = np.array(x[:,0]) 23 | X = np.array(x[:,1:]) 24 | 25 | if self.estop > 0: 26 | X, X_valid, y, y_valid, qid, qid_valid = train_test_split(X, y, qid, test_size=0.33, shuffle=True) 27 | 28 | # init values 29 | docs_per_qid = [X[qid==qi] for qi in np.unique(qid)] 30 | y_per_qid = [y[qid==qi] for qi in np.unique(qid)] 31 | self.pred_train = [np.zeros(len(X[qid==qi])) for qi in np.unique(qid)] 32 | idx_per_qid = [qi for qi in range(len(np.unique(qid)))] 33 | 34 | # initial values 35 | n_queries = np.unique(qid).shape[0] 36 | p = [np.ones(n_queries, dtype=np.float64) / n_queries] 37 | self.h = [] 38 | self.alpha = [] 39 | 40 | if self.verbose: 41 | print("Get WeakRankers") 42 | weak_rankers = rustlib.WeakRanker(docs_per_qid, y_per_qid) 43 | # trainings loop 44 | for t in range(self.T): 45 | if self.verbose: 46 | print(f"Round {t}, #queries {n_queries}") 47 | 48 | start = time.time() 49 | current_h, current_alpha = weak_rankers.get_h_alpha(p[t]) 50 | self.h.append(current_h) 51 | self.alpha.append(current_alpha) 52 | end = time.time() 53 | if self.verbose: 54 | print("Get h and alpha time: ", end - start) 55 | 56 | # update p 57 | numerator = [] 58 | denominator = 0 59 | cur_score_list = [] 60 | start = time.time() 61 | for x_docs, y_docs, idx_docs in zip(docs_per_qid, y_per_qid, idx_per_qid): 62 | cur_score = rustlib.ndcg10(self.predict_proba_training(x_docs, t, idx_docs), y_docs) 63 | cur_score_list.append(cur_score) 64 | numerator.append(np.exp(-cur_score)) 65 | denominator += np.exp(-cur_score) 66 | p.append(np.array(numerator)/denominator) 67 | end = time.time() 68 | if self.verbose: 69 | print("Get p time: ", end - start) 70 | 71 | if self.estop > 0: 72 | cur_score_list = [] 73 | for qid in qid_valid: 74 | pred = self.predict_proba(X_valid[qid_valid==qid], t) 75 | cur_score_list.append(rustlib.ndcg10(pred, y_valid[qid_valid==qid])) 76 | cur_score = np.mean(cur_score_list) 77 | if self.best_score < cur_score: 78 | self.best_score = cur_score 79 | self.best_round = t 80 | if (t - self.best_round) > self.estop: 81 | self.T = self.best_round 82 | break 83 | 84 | if self.verbose: 85 | print(f"step:{t} alpha: {self.alpha[t]} score: {np.mean(cur_score_list)} best val_score: {self.best_score}") 86 | 87 | def predict_proba_training(self, X, t, qid): 88 | for xidx, xi in enumerate(X): 89 | self.pred_train[qid][xidx] += self.alpha[t] * xi[self.h[t]] 90 | return np.array(self.pred_train[qid]) 91 | 92 | def predict_proba(self, X, t=None): 93 | """Make predictions""" 94 | if t == None: # here we are not in the training 95 | t = self.T 96 | X = X[:,1:] 97 | else: 98 | t = t + 1 99 | pred = np.zeros(X.shape[0]) 100 | for ti in range(t): 101 | for xidx, xi in enumerate(X): 102 | pred[xidx] += self.alpha[ti] * xi[self.h[ti]] 103 | return np.array(pred) 104 | -------------------------------------------------------------------------------- /fig5/plot_fig5.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import pandas as pd 3 | import numpy as np 4 | import os 5 | import json 6 | 7 | 8 | def plot(path = 'synth_data', label=False): 9 | results_path = path + "/results" 10 | n_test = 10000 11 | folds = [0, 1, 2, 3, 4] 12 | models = ["ListNet", "DirectRanker", "RankNet", "LambdaRank"] 13 | msize = 5 14 | colors_points = [ 15 | (57 / 255, 106 / 255, 177 / 255), # blue 16 | (218 / 255, 124 / 255, 48 / 255), # orange 17 | (62 / 255, 150 / 255, 81 / 255), # green 18 | (204 / 255, 37 / 255, 41 / 255), # red 19 | (83 / 255, 81 / 255, 84 / 255), # black 20 | (107 / 255, 76 / 255, 154 / 255), # purple 21 | (146 / 255, 36 / 255, 40 / 255), # wine 22 | (148 / 255, 139 / 255, 61 / 255) # gold 23 | ] 24 | colors_bar = [ 25 | (114 / 255, 147 / 255, 203 / 255), # blue 26 | (225 / 255, 151 / 255, 76 / 255), # orange 27 | (132 / 255, 186 / 255, 91 / 255), # green 28 | (211 / 255, 94 / 255, 96 / 255), # red 29 | (128 / 255, 133 / 255, 133 / 255), # black 30 | (144 / 255, 103 / 255, 167 / 255), # purple 31 | (171 / 255, 104 / 255, 87 / 255), # wine 32 | (204 / 255, 194 / 255, 16 / 255) # gold 33 | ] 34 | 35 | train_list = np.logspace(2, 6, num=10) 36 | if label: 37 | train_list = np.concatenate([np.logspace(1, 2, num=3)[:2], train_list]) 38 | 39 | for n_train in train_list: 40 | if path == 'synth_data' and label == False: 41 | cur_path = results_path + "/results_{}_{}.json".format(int(n_train), n_test) 42 | elif label and path == 'synth_data': 43 | cur_path = results_path + "/results_label_{}_{}.json".format(int(n_train), n_test) 44 | elif label: 45 | cur_path = results_path + "/results_label_{}.json".format(int(n_train)) 46 | else: 47 | cur_path = results_path + "/results_{}.json".format(int(n_train)) 48 | 49 | if not os.path.isfile(cur_path): 50 | continue 51 | if label: 52 | if n_train > 50000: continue 53 | if path == 'mslr_data': 54 | n_train = n_train/112871 55 | else: 56 | n_train = n_train/50000 57 | 58 | with open(cur_path) as json_file: 59 | data = json.load(json_file) 60 | for model in models: 61 | try: 62 | nDCG = data["ndcg_{}".format(model)] 63 | except: 64 | print("No model {}".format(model)) 65 | continue 66 | if model == "ListNet": 67 | plt.scatter(n_train, nDCG[0], c=colors_points[0], marker='s', label=model, s=msize) 68 | if model == "DirectRanker": 69 | plt.scatter(n_train, nDCG[0], c=colors_points[1], marker='x', label=r"RankNet$^*$", s=msize) 70 | if model == "RankNet": 71 | plt.scatter(n_train, nDCG[0], c=colors_points[2], marker='>', label=model, s=msize) 72 | if model == "LambdaRank": 73 | plt.scatter(n_train, nDCG[0], c=colors_points[3], marker='>', label=model, s=msize) 74 | plt.errorbar(n_train, nDCG[0], yerr=nDCG[1], c=colors_bar[4], alpha=0.5, linestyle="None", zorder=0, label='_nolegend_') 75 | handles, labels = plt.gca().get_legend_handles_labels() 76 | by_label = dict(zip(labels, handles)) 77 | plt.legend(by_label.values(), by_label.keys(), loc="upper left") 78 | plt.xscale("log") 79 | if label: 80 | plt.xlabel("ratio \# relevant and \# unrelevant", fontsize=7) 81 | else: 82 | plt.xlabel("\# instances in training", fontsize=7) 83 | if path == 'synth_data': 84 | plt.ylabel("nDCG@20", fontsize=7) 85 | else: 86 | plt.ylabel("nDCG@10", fontsize=7) 87 | if label: 88 | plt.savefig('training_label_{}.pdf'.format(path)) 89 | plt.savefig('training_label_{}.png'.format(path)) 90 | else: 91 | plt.savefig('training_size_{}.pdf'.format(path)) 92 | plt.savefig('training_size_{}.png'.format(path)) 93 | plt.close() 94 | 95 | 96 | plot('synth_data') 97 | plot('synth_data', True) 98 | plot('mslr_data') 99 | plot('mslr_data', True) 100 | -------------------------------------------------------------------------------- /rustlib/src/lib.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | use itertools::Itertools; 3 | use rayon::prelude::*; 4 | use rand::random; 5 | 6 | fn ndcg(prediction: &Vec, y: &Vec, at: usize) -> f64 { 7 | let minat = std::cmp::min(at, y.len()); 8 | let iter_sorted_y = y.iter() 9 | .sorted_by(|a, b| b.partial_cmp(a).unwrap()) // sorting in reverse 10 | .take(minat); 11 | let iter_sorted_prediction = prediction.iter() 12 | .zip(y.iter()) 13 | .map(|(x,y)| (x, random::(), y)) 14 | .sorted_by(|a,b| b.partial_cmp(a).unwrap()) // sorting in reverse 15 | .take(minat); 16 | let (dcg, idcg) = iter_sorted_prediction.zip(iter_sorted_y) 17 | .enumerate() 18 | .fold((0.0, 0.0),|(_dcg,_idcg), (i, ((_, _, y_pred), y_ref))| { 19 | let denominator = (i as f64+2.0).log2(); 20 | let newdcg = _dcg + (2u64.pow(*y_pred) as f64-1.0)/denominator; 21 | let newidcg = _idcg + (2u64.pow(*y_ref) as f64-1.0)/denominator; 22 | (newdcg, newidcg) 23 | }); 24 | if idcg == 0.0 { 25 | return 0.0; 26 | } 27 | dcg/idcg 28 | } 29 | 30 | fn transpose(v: &Vec>) -> Vec> where T: Clone { 31 | assert!(!v.is_empty()); 32 | (0..v[0].len()) 33 | .map(|i| v.iter().map(|inner| inner[i].clone()).collect::>()) 34 | .collect() 35 | } 36 | 37 | #[pyclass] 38 | struct WeakRanker { 39 | docs_per_qid: Vec>>, 40 | y_per_qid: Vec>, 41 | ndcgs: Vec> 42 | } 43 | 44 | #[pymethods] 45 | impl WeakRanker { 46 | #[new] 47 | fn new(docs_per_qid: Vec>>, y_per_qid: Vec>) -> Self { 48 | let mut ndcgs = vec![vec![]]; 49 | (&docs_per_qid, &y_per_qid).into_par_iter() 50 | .map(|(xds, yd)| { 51 | transpose(xds).iter() 52 | .map(|feature| { 53 | ndcg(&feature, &yd, 10) 54 | }) 55 | .collect() 56 | }) 57 | .collect_into_vec(&mut ndcgs); 58 | WeakRanker { docs_per_qid, y_per_qid, ndcgs } 59 | } 60 | 61 | /// Computes alpha for the weak rankers 62 | fn get_alpha(&self, p_docs: Vec, h: usize) -> f64 { 63 | let (num, denom) = (&self.docs_per_qid, &self.y_per_qid, p_docs).into_par_iter() 64 | .map(|(x_docs_transpose, y_docs, p)| { 65 | let x_doc = x_docs_transpose.iter().map(|x| x[h]).collect(); 66 | let cur_score = ndcg(&x_doc, &y_docs,10); 67 | (p * (1.0 + cur_score), p * (1.0 - cur_score)) 68 | }) 69 | .reduce(|| (0.0, 0.0), |a,b| (a.0+b.0, a.1+b.1)); 70 | 0.5 * (num/denom).ln() 71 | } 72 | 73 | fn get_h_alpha(&self, p_docs: Vec) -> (usize, f64) { 74 | let mut feature_importance_vec = vec![vec![]]; 75 | self.ndcgs.par_iter() 76 | .zip(&p_docs) 77 | .map(|(feature, p)| feature.iter().map(|x| x*p).collect()) 78 | .collect_into_vec(&mut feature_importance_vec); 79 | let mut feature_importance = vec![0.0; feature_importance_vec[0].len()]; 80 | feature_importance_vec.iter() 81 | .for_each(|v| 82 | v.iter() 83 | .enumerate() 84 | .for_each(|(i, x)| feature_importance[i] += x)); let mut h = 0; for i in 1..feature_importance.len() { if feature_importance[i] > feature_importance[h] { 85 | h = i; 86 | } 87 | } 88 | (h, self.get_alpha(p_docs, h)) 89 | } 90 | } 91 | 92 | #[pyfunction] 93 | fn ndcg10(prediction: Vec, y: Vec) -> PyResult { 94 | Ok(ndcg(&prediction, &y, 10)) 95 | } 96 | 97 | #[test] 98 | fn test_ndcg() { 99 | let predictions = vec![0.3,0.2,0.5,0.4,0.1,0.6]; 100 | let ys = vec![1,0,0,1,2,2]; 101 | assert!((ndcg(&predictions, &ys, 10) - 0.858475).abs() < 0.00001) 102 | } 103 | 104 | #[test] 105 | fn transpose_test() { 106 | let v = vec![vec![1,2,3],vec![4,5,6],vec![7,8,9]]; 107 | let vt = vec![vec![1,4,7],vec![2,5,8],vec![3,6,9]]; 108 | assert_eq!(transpose(v),vt); 109 | } 110 | 111 | /// A Python module implemented in Rust. 112 | #[pymodule] 113 | fn rustlib(_py: Python, m: &PyModule) -> PyResult<()> { 114 | m.add_function(wrap_pyfunction!(ndcg10, m)?)?; 115 | m.add_class::()?; 116 | Ok(()) 117 | } 118 | -------------------------------------------------------------------------------- /test_all.sh: -------------------------------------------------------------------------------- 1 | echo "Start to test all scripts" 2 | echo "=========================" 3 | 4 | cd data 5 | if [ -d "MSLR-TEST" ]; then 6 | echo "Test dir is already created" 7 | else 8 | echo "Create Test dir ..." 9 | mkdir MSLR-TEST 10 | mkdir MSLR-TEST/Fold1 11 | mkdir MSLR-TEST/Fold2 12 | mkdir MSLR-TEST/Fold3 13 | mkdir MSLR-TEST/Fold4 14 | mkdir MSLR-TEST/Fold5 15 | head -n 1000 MSLR-WEB10K/Fold1/train.txt > MSLR-TEST/Fold1/train.txt 16 | head -n 1000 MSLR-WEB10K/Fold1/test.txt > MSLR-TEST/Fold1/test.txt 17 | head -n 1000 MSLR-WEB10K/Fold2/train.txt > MSLR-TEST/Fold2/train.txt 18 | head -n 1000 MSLR-WEB10K/Fold2/test.txt > MSLR-TEST/Fold2/test.txt 19 | head -n 1000 MSLR-WEB10K/Fold3/train.txt > MSLR-TEST/Fold3/train.txt 20 | head -n 1000 MSLR-WEB10K/Fold3/test.txt > MSLR-TEST/Fold3/test.txt 21 | head -n 1000 MSLR-WEB10K/Fold4/train.txt > MSLR-TEST/Fold4/train.txt 22 | head -n 1000 MSLR-WEB10K/Fold4/test.txt > MSLR-TEST/Fold4/test.txt 23 | head -n 1000 MSLR-WEB10K/Fold5/train.txt > MSLR-TEST/Fold5/train.txt 24 | head -n 1000 MSLR-WEB10K/Fold5/test.txt > MSLR-TEST/Fold5/test.txt 25 | fi 26 | cd - 27 | 28 | echo "Start to test Fig. 2 ..." 29 | sleep 1 30 | cd fig2 31 | ln -s ../Rankers . 32 | python gen_fig2a.py --test 1 33 | python gen_fig2b.py --test 1 34 | python gen_fig2c.py --test 1 35 | python gen_fig2d.py --test 1 36 | echo "Test Done" 37 | echo "Clean folder ..." 38 | sleep 1 39 | rm Rankers log 40 | cd - 41 | 42 | echo "Start to test Fig. 3 ..." 43 | sleep 1 44 | cd fig3 45 | ln -s ../Rankers . 46 | python training_label.py --test 1 47 | python training_label_mslr.py --test 1 --path ../data/MSLR-TEST/ 48 | python training_size.py --test 1 49 | python training_size_mslr.py --test 1 --path ../data/MSLR-TEST/ 50 | python plot_fig3.py 51 | echo "Test Done" 52 | echo "Clean folder ..." 53 | sleep 1 54 | rm -r mslr_data synth_data Rankers 55 | cd - 56 | 57 | echo "Start to test Fig. 4 ..." 58 | sleep 1 59 | cd fig4 60 | ln -s ../Rankers . 61 | python plot_sorted_list.py --test 1 --data ../data/MSLR-TEST/ 62 | echo "Test Done" 63 | echo "Clean folder ..." 64 | sleep 1 65 | rm Rankers 66 | cd - 67 | 68 | echo "Start to test Table 1 ..." 69 | sleep 1 70 | cd table1 71 | ln -s ../Rankers . 72 | rm -r run_scripts 73 | python create_runscripts.py --data ../data/MSLR-TEST --jarpath .. --resultsdir results_ranklib --datalabel MQ2007 --test 1 --model adarank,lambda 74 | python create_runscripts.py --data ../data/MSLR-TEST --jarpath .. --resultsdir results_ranklib --datalabel MQ2008 --test 1 --model adarank,lambda 75 | python create_runscripts.py --data ../data/MSLR-TEST --jarpath .. --resultsdir results_ranklib --datalabel MSLR-WEB10K --test 1 --model adarank,lambda 76 | python gridsearch.py --path ../data/MSLR-TEST --model DirectRankerV1 --jobs 5 --data test 77 | python gridsearch.py --path ../data/MSLR-TEST --model DirectRanker --jobs 5 --data test 78 | python gridsearch.py --path ../data/MSLR-TEST --model RankeNet --jobs 5 --data test 79 | python gridsearch.py --path ../data/MSLR-TEST --model ListNet --jobs 5 --data test 80 | python get_results.python 81 | echo "Test Done" 82 | echo "Clean folder ..." 83 | sleep 1 84 | rm -r run_scripts Rankers 85 | cd - 86 | 87 | echo "Start to test Table 2 ..." 88 | sleep 1 89 | cd table2 90 | ln -s ../Rankers . 91 | mkdir .log 92 | python ranknet_eval.py --path ../data/MSLR-TEST/ --activation sigmoid --epoch 1 --data test > .log/log_sig_sgd 93 | python ranknet_eval.py --path ../data/MSLR-TEST/ --activation tanh --epoch 1 --data test > .log/log_tanh_sgd 94 | python ranknet_eval.py --path ../data/MSLR-TEST/ --activation hard_sigmoid --epoch 1 --data test > .log/log_hard_sgd 95 | python ranknet_eval.py --path ../data/MSLR-TEST/ --activation sigmoid --optimizer Adam --data test --epoch 1 > .log/log_sig_adam 96 | python ranknet_eval.py --path ../data/MSLR-TEST/ --activation tanh --optimizer Adam --data test --epoch 1 > .log/log_tanh_adam 97 | python ranknet_eval.py --path ../data/MSLR-TEST/ --activation hard_sigmoid --optimizer Adam --data test --epoch 1 > .log/log_hard_adam 98 | python ranknet_eval.py --plot 1 99 | echo "Test Done" 100 | echo "Clean folder ..." 101 | sleep 1 102 | rm -r Rankers .log 103 | cd - 104 | 105 | echo "Start to test Table 4 ..." 106 | sleep 1 107 | cd table4 108 | ln -s ../Rankers . 109 | python timeana.py --model DirectRankerV1 --path ../data/MSLR-TEST 110 | python timeana.py --model RankNet --path ../data/MSLR-TEST 111 | echo "Test Done" 112 | echo "Clean folder ..." 113 | sleep 1 114 | rm -r output Rankers 115 | cd - 116 | -------------------------------------------------------------------------------- /fig6/plot_sorted_list.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | 4 | from Rankers.helpers import readData, gen_set, comparator, cmp_to_key 5 | from Rankers.models.DirectRanker import DirectRanker 6 | 7 | import matplotlib.pyplot as plt 8 | import matplotlib.cm as cm 9 | 10 | from collections import defaultdict 11 | from functools import partial 12 | 13 | import numpy as np 14 | from sklearn.utils import shuffle 15 | 16 | 17 | def plot(rankerValues, real_y, tuple_y, name, numberClass): 18 | 19 | colors = shuffle(cm.rainbow(np.linspace(0, 1, len(np.unique((real_y)))))) 20 | dict_of_idx = defaultdict(list) 21 | dict_of_y_values = defaultdict(list) 22 | 23 | for idx, y in enumerate(tuple_y): 24 | dict_of_y_values[y[0]].append(rankerValues[idx]) 25 | dict_of_y_values[y[1]].append(rankerValues[idx]) 26 | dict_of_idx[y[0]].append(idx) 27 | dict_of_idx[y[1]].append(idx + 1) 28 | 29 | for k in sorted(dict_of_y_values.keys()): 30 | plt.scatter(dict_of_idx[k], dict_of_y_values[k], color=colors[int(k)]) # , edgecolors='black') 31 | plt.legend(np.unique(real_y), loc=1, title="Class labels") 32 | 33 | plt.xlabel(r'$n$') 34 | plt.ylabel(r'$r(d_n,d_{n+1})$') 35 | plt.savefig(name + ".pdf") 36 | plt.close() 37 | 38 | 39 | def plot_sorted_list(estimator, X, y): 40 | X = np.array(X) 41 | y = np.array([[yi] for yi in y]) 42 | 43 | data = np.concatenate((X, y), axis=1) 44 | compar = partial(comparator, estimator=estimator) 45 | data = np.array(sorted(data, key=cmp_to_key(compar), reverse=True)) 46 | 47 | plot_values = [] 48 | real_y = [] 49 | tuple_y = [] 50 | for idx, _ in enumerate(data): 51 | if idx == len(data) - 1: 52 | break 53 | res = estimator.model.predict( 54 | [np.array([data[idx][:-1]]), np.array([data[idx+1][:-1]])], 55 | verbose=0 56 | ) 57 | plot_values.append(res[0][0]) 58 | tuple_y.append((data[idx][-1], data[idx + 1][-1])) 59 | real_y.append(int(data[idx][-1])) 60 | real_y.append(int(data[idx + 1][-1])) 61 | 62 | return plot_values, real_y, tuple_y 63 | 64 | 65 | def plot_own_data(path, n_relevance=3, test=False): 66 | gen_set(path=path, n_feats=136, n_relevance=n_relevance, test=test) 67 | 68 | x_train, y_train, _ = readData(path='output/train', binary=False, at=0) 69 | x_test, y_test, _ = readData(path='output/test', binary=False, at=0) 70 | 71 | # Load directRanker 72 | if test: 73 | dr = DirectRanker(epoch=1, num_features=x_train.shape[1]) 74 | else: 75 | dr = DirectRanker(hidden_layers_dr=[50, 5], num_features=x_train.shape[1], verbose=2) 76 | 77 | dr.fit(x_train, y_train) 78 | 79 | plot_values, real_y, tuple_y = plot_sorted_list(dr, x_test, y_test) 80 | plot(plot_values, real_y, tuple_y, "own_data_class_" + str(n_relevance), n_relevance) 81 | 82 | 83 | def plot_mslr(path, test=False, multi=False): 84 | 85 | if multi: 86 | x_train, y_train, _ = readData(path=f'{path}/Fold1/train.txt', binary=False, at=0) 87 | x_test, y_test, _ = readData(path=f'{path}/Fold1/test.txt', binary=False, at=0) 88 | else: 89 | x_train, y_train, _ = readData(path=f'{path}/Fold1/train.txt', binary=True, at=0) 90 | x_test, y_test, _ = readData(path=f'{path}/Fold1/test.txt', binary=False, at=0) 91 | 92 | # Load directRanker 93 | if test: 94 | dr = DirectRanker(epoch=1, num_features=x_train.shape[1]) 95 | else: 96 | dr = DirectRanker(hidden_layers_dr=[50, 5], num_features=x_train.shape[1], verbose=2) 97 | 98 | dr.fit(x_train, y_train) 99 | 100 | plot_values, real_y, tuple_y = plot_sorted_list(dr, x_test, y_test) 101 | if multi: 102 | plot(plot_values, real_y, tuple_y, "mslr_multi_" + str(multi), 5) 103 | else: 104 | plot(plot_values, real_y, tuple_y, "mslr_multi_" + str(multi), 2) 105 | 106 | 107 | if __name__ == '__main__': 108 | 109 | parser = argparse.ArgumentParser(description='Get some run values.') 110 | parser.add_argument('--test', type=int, default=0, help='Run in test mode') 111 | parser.add_argument('--data', type=str, default="", help='path to data') 112 | 113 | args = parser.parse_args() 114 | 115 | os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 116 | 117 | if not os.path.exists("output"): 118 | os.makedirs("output") 119 | 120 | test = False 121 | if args.test == 1: 122 | test = True 123 | 124 | print("Run own generated data test") 125 | plot_own_data("output", n_relevance=3, test=test) 126 | 127 | print("Run MSLR multi label") 128 | plot_mslr(args.data, test=test, multi=True) 129 | 130 | print("Run MSLR binary label") 131 | plot_mslr(args.data, test=test, multi=False) 132 | -------------------------------------------------------------------------------- /table2/ranknet_eval.py: -------------------------------------------------------------------------------- 1 | import os 2 | from Rankers.models.RankNet import RankNet 3 | from Rankers.helpers import nDCG_cls, readData, colors_points, colors_bar 4 | 5 | import numpy as np 6 | import pandas as pd 7 | import tensorflow as tf 8 | 9 | import argparse 10 | 11 | 12 | def plot(args): 13 | sig_adam_ranknet = np.load(f"{args.out_path}/sigmoid_MSLR10K_Adam_list_of_ndcg.npy") 14 | hsig_adam_ranknet = np.load(f"{args.out_path}/hard_sigmoid_MSLR10K_Adam_list_of_ndcg.npy") 15 | tanh_adam_ranknet = np.load(f"{args.out_path}/tanh_MSLR10K_Adam_list_of_ndcg.npy") 16 | linear_adam_ranknet = np.load(f"{args.out_path}/linear_MSLR10K_Adam_list_of_ndcg.npy") 17 | sig_sgd_ranknet = np.load(f"{args.out_path}/sigmoid_MSLR10K_SGD_list_of_ndcg.npy") 18 | hsig_sgd_ranknet = np.load(f"{args.out_path}/hard_sigmoid_MSLR10K_SGD_list_of_ndcg.npy") 19 | tanh_sgd_ranknet = np.load(f"{args.out_path}/tanh_MSLR10K_SGD_list_of_ndcg.npy") 20 | linear_sgd_ranknet = np.load(f"{args.out_path}/linear_MSLR10K_SGD_list_of_ndcg.npy") 21 | 22 | d = { 23 | 'Optimizer / Activation': ["Gradient descent", "Adam"], 24 | 'Sigmoid': [f"{round(np.mean(sig_sgd_ranknet),3)}({int(round(np.std(sig_sgd_ranknet),3)*1000)})", 25 | f"{round(np.mean(sig_adam_ranknet),3)}({int(round(np.std(sig_adam_ranknet),3)*1000)})"], 26 | 'Hard_Sigmoid': [f"{round(np.mean(hsig_sgd_ranknet),3)}({int(round(np.std(hsig_sgd_ranknet),3)*1000)})", 27 | f"{round(np.mean(hsig_adam_ranknet),3)}({int(round(np.std(hsig_adam_ranknet),3)*1000)})"], 28 | 'Tanh': [f"{round(np.mean(tanh_sgd_ranknet),3)}({int(round(np.std(tanh_sgd_ranknet),3)*1000)})", 29 | f"{round(np.mean(tanh_adam_ranknet),3)}({int(round(np.std(tanh_adam_ranknet),3)*1000)})"], 30 | 'Linear': [f"{round(np.mean(linear_sgd_ranknet),3)}({int(round(np.std(linear_sgd_ranknet),3)*1000)})", 31 | f"{round(np.mean(linear_adam_ranknet),3)}({int(round(np.std(linear_adam_ranknet),3)*1000)})"], 32 | } 33 | 34 | print(pd.DataFrame(data=d).to_latex(index=False)) 35 | 36 | 37 | if __name__ == '__main__': 38 | 39 | parser = argparse.ArgumentParser(description='Get some run values.') 40 | parser.add_argument('--activation', type=str, default='sigmoid', help='Get ranking activation') 41 | parser.add_argument('--optimizer', type=str, default='SGD', help='Get optimizer name') 42 | parser.add_argument('--epoch', type=int, default=100, help='Get number of epoch') 43 | parser.add_argument('--data', type=str, default='MSLR10K', help='Get data name') 44 | parser.add_argument('--path', type=str, default='', help='Get path to data') 45 | parser.add_argument('--out_path', type=str, default='ranknet_eval', help='Get path to data') 46 | parser.add_argument('--cpu', type=int, default=1, help='Use CPU or GPU') 47 | parser.add_argument('--plot', type=int, default=0, help='Plot the resutls') 48 | 49 | args = parser.parse_args() 50 | 51 | if args.plot == 1: 52 | plot(args) 53 | exit() 54 | 55 | if args.cpu == 1: 56 | os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 57 | 58 | if not os.path.exists(args.out_path): 59 | os.makedirs(args.out_path) 60 | os.makedirs(args.out_path + "/data") 61 | 62 | if args.optimizer == "SGD": 63 | optimizer = tf.keras.optimizers.SGD 64 | if args.optimizer == "Adam": 65 | optimizer = tf.keras.optimizers.Adam 66 | 67 | list_of_train = [f"{args.path}/Fold1/train.txt", f"{args.path}/Fold2/train.txt", 68 | f"{args.path}/Fold3/train.txt", f"{args.path}/Fold4/train.txt", 69 | f"{args.path}/Fold5/train.txt"] 70 | list_of_test = [f"{args.path}/Fold1/test.txt", f"{args.path}/Fold2/test.txt", 71 | f"{args.path}/Fold3/test.txt", f"{args.path}/Fold4/test.txt", 72 | f"{args.path}/Fold5/test.txt"] 73 | 74 | parameters = { 75 | 'hidden_layers_dr': [32, 20, 5], 76 | 'epoch': args.epoch, 77 | 'feature_activation_dr': 'sigmoid', 78 | 'verbose': 2, 79 | 'optimizer': optimizer, 80 | 'ranking_activation_dr': args.activation, 81 | } 82 | 83 | list_of_ndcg = [] 84 | for foldidx in range(len(list_of_train)): 85 | print("FOLD ", foldidx) 86 | 87 | if args.data == "MSLR10K" or args.data == "test": 88 | number_features = 136 89 | 90 | # read data 91 | print("Reading data") 92 | if os.path.isfile(f"{args.out_path}/data/{args.data}_x_train_fold{foldidx+1}.npy"): 93 | x_train = np.load(f"{args.out_path}/data/{args.data}_x_train_fold{foldidx+1}.npy") 94 | y_train = np.load(f"{args.out_path}/data/{args.data}_y_train_fold{foldidx+1}.npy") 95 | 96 | x_test = np.load(f"{args.out_path}/data/{args.data}_x_test_fold{foldidx+1}.npy") 97 | y_test = np.load(f"{args.out_path}/data/{args.data}_y_test_fold{foldidx+1}.npy") 98 | q_test = np.load(f"{args.out_path}/data/{args.data}_q_test_fold{foldidx+1}.npy") 99 | else: 100 | x_train, y_train, _ = readData( 101 | path=list_of_train[foldidx], 102 | binary=True, 103 | at=10, 104 | number_features=number_features 105 | ) 106 | x_test, y_test, q_test = readData( 107 | path=list_of_test[foldidx], 108 | binary=False, 109 | at=10, 110 | number_features=number_features 111 | ) 112 | 113 | np.save(f"{args.out_path}/data/{args.data}_x_train_fold{foldidx+1}", x_train) 114 | np.save(f"{args.out_path}/data/{args.data}_y_train_fold{foldidx+1}", y_train) 115 | 116 | np.save(f"{args.out_path}/data/{args.data}_x_test_fold{foldidx+1}", x_test) 117 | np.save(f"{args.out_path}/data/{args.data}_y_test_fold{foldidx+1}", y_test) 118 | np.save(f"{args.out_path}/data/{args.data}_q_test_fold{foldidx+1}", q_test) 119 | 120 | print("Training") 121 | ranker = RankNet(**parameters, num_features=x_train.shape[1]) 122 | 123 | ranker.fit(x_train, y_train) 124 | 125 | print("Eval") 126 | nDCG_l = [] 127 | pred = ranker.predict_proba(x_test) 128 | for n in np.unique(q_test): 129 | nDCG_l.append(nDCG_cls(pred[q_test==n], x_test[q_test==n], y_test[q_test==n], at=10, prediction=True)) 130 | 131 | print(f"Test on Fold {foldidx}: NDCG@10 = {np.mean(nDCG_l)}") 132 | 133 | list_of_ndcg.append(np.mean(nDCG_l)) 134 | 135 | np.save(f"{args.out_path}/{args.activation}_{args.data}_{args.optimizer}_list_of_ndcg", list_of_ndcg) 136 | -------------------------------------------------------------------------------- /Rankers/models/LambdaMart2.py: -------------------------------------------------------------------------------- 1 | # code adapted from https://github.com/discobot/LambdaMart/blob/master/mart.py 2 | 3 | import math 4 | import numpy as np 5 | import math 6 | from optparse import OptionParser 7 | from sklearn.tree import DecisionTreeRegressor 8 | from sklearn.base import BaseEstimator 9 | from collections import defaultdict 10 | from copy import deepcopy 11 | from multiprocessing import Pool 12 | from itertools import chain 13 | import time 14 | 15 | class Ensemble: 16 | def __init__(self, rate): 17 | self.trees = [] 18 | self.rate = rate 19 | 20 | def __len__(self): 21 | return len(self.trees) 22 | 23 | def add(self, tree): 24 | self.trees.append(tree) 25 | 26 | def eval_one(self, object): 27 | return self.eval([object])[0] 28 | 29 | def eval(self, objects): 30 | results = np.zeros(len(objects)) 31 | for tree in self.trees: 32 | results += tree.predict(objects) * self.rate 33 | return results 34 | 35 | def remove(self, number): 36 | self.trees = self.trees[:-number] 37 | 38 | 39 | def groupby(score, query): 40 | result = [] 41 | this_query = None 42 | for s, q in zip(score, query): 43 | if q != this_query: 44 | result.append([]) 45 | this_query = q 46 | result[-1].append(s) 47 | result = map(np.array, result) 48 | return result 49 | 50 | 51 | def point_dcg(arg): 52 | i, label = arg 53 | return (2 ** label - 1) / math.log(i + 2, 2) 54 | 55 | 56 | def dcg(scores): 57 | return sum(map(point_dcg, enumerate(scores))) 58 | 59 | 60 | def ndcg(page, k=10): 61 | model_top = page[:k] 62 | 63 | true_top = np.array([]) 64 | if len(page) > 10: 65 | true_top = np.partition(page, -10)[-k:] 66 | true_top.sort() 67 | else: 68 | true_top = np.sort(page) 69 | true_top = true_top[::-1] 70 | 71 | 72 | max_dcg = dcg(true_top) 73 | model_dcg = dcg(model_top) 74 | 75 | if max_dcg == 0: 76 | return 1 77 | 78 | return model_dcg / max_dcg 79 | 80 | 81 | def score(prediction, true_score, query, k=10): 82 | true_pages = groupby(true_score, query) 83 | model_pages = groupby(prediction, query) 84 | 85 | total_ndcg = [] 86 | 87 | for true_page, model_page in zip(true_pages, model_pages): 88 | page = true_page[np.argsort(model_page)[::-1]] 89 | total_ndcg.append(ndcg(page, k)) 90 | 91 | return sum(total_ndcg) / len(total_ndcg) 92 | 93 | 94 | def query_lambdas(page, k=10): 95 | true_page, model_page = page 96 | worst_order = np.argsort(true_page) 97 | 98 | true_page = true_page[worst_order] 99 | model_page = model_page[worst_order] 100 | 101 | 102 | model_order = np.argsort(model_page) 103 | 104 | idcg = dcg(np.sort(true_page)[-10:][::-1]) 105 | 106 | size = len(true_page) 107 | position_score = np.zeros((size, size)) 108 | 109 | for i in range(size): 110 | for j in range(size): 111 | position_score[model_order[i], model_order[j]] = \ 112 | point_dcg((model_order[j], true_page[model_order[i]])) 113 | 114 | lambdas = np.zeros(size) 115 | 116 | for i in range(size): 117 | for j in range(size): 118 | if true_page[i] > true_page[j]: 119 | 120 | delta_dcg = position_score[i][j] - position_score[i][i] 121 | delta_dcg += position_score[j][i] - position_score[j][j] 122 | 123 | delta_ndcg = abs(delta_dcg / idcg) 124 | 125 | rho = 1 / (1 + math.exp(model_page[i] - model_page[j])) 126 | 127 | lam = rho * delta_ndcg 128 | 129 | lambdas[j] -= lam 130 | lambdas[i] += lam 131 | return lambdas 132 | 133 | 134 | def compute_lambdas(prediction, true_score, query, k=10): 135 | true_pages = groupby(true_score, query) 136 | model_pages = groupby(prediction, query) 137 | 138 | pool = Pool() 139 | lambdas = pool.map(query_lambdas, zip(true_pages, model_pages)) 140 | return list(chain(*lambdas)) 141 | 142 | 143 | def mart_responces(prediction, true_score): 144 | return true_score - prediction 145 | 146 | 147 | class LambdaMart(BaseEstimator): 148 | 149 | def __init__(self, number_of_trees=5, learning_rate=0.1, max_depth=50, min_samples_split=2): 150 | """ 151 | This is the constructor for the LambdaMART object. 152 | Parameters 153 | ---------- 154 | number_of_trees : int (default: 5) 155 | Number of trees LambdaMART goes through 156 | learning_rate : float (default: 0.1) 157 | Rate at which we update our prediction with each tree 158 | """ 159 | 160 | self.number_of_trees = number_of_trees 161 | self.learning_rate = learning_rate 162 | self.max_depth = max_depth 163 | self.min_samples_split = min_samples_split 164 | self.trees = [] 165 | 166 | def fit(self, x, y): 167 | """ 168 | Fits the model on the training data. 169 | """ 170 | 171 | q = x[:,0] 172 | x = x[:,1:] 173 | 174 | scores = y 175 | queries = q 176 | features = x 177 | 178 | ensemble = Ensemble(self.learning_rate) 179 | 180 | print("Training starts...") 181 | model_output = np.zeros(len(features)) 182 | 183 | for i in range(self.number_of_trees): 184 | print(" Iteration: " + str(i + 1)) 185 | 186 | # Compute psedo responces (lambdas) 187 | # witch act as training label for document 188 | start = time.time() 189 | print(" --generating labels") 190 | lambdas = compute_lambdas(model_output, scores, queries, k=10) 191 | print(" --done", str(time.time() - start) + " sec") 192 | 193 | # create tree and append it to the model 194 | print(" --fitting tree") 195 | start = time.time() 196 | tree = DecisionTreeRegressor(max_depth=self.max_depth) 197 | tree.fit(features, lambdas) 198 | 199 | print(" --done", str(time.time() - start) + " sec") 200 | print(" --adding tree to ensemble") 201 | ensemble.add(tree) 202 | 203 | # update model score 204 | print(" --generating step prediction") 205 | prediction = tree.predict(features) 206 | 207 | print(" --updating full model output") 208 | model_output += self.learning_rate * prediction 209 | 210 | # train_score 211 | start = time.time() 212 | print(" --scoring on train") 213 | train_score = score(model_output, scores, queries, 10) 214 | print(" --iteration train score " + str(train_score) + ", took " + str(time.time() - start) + "sec to calculate") 215 | 216 | # finishing up 217 | print("Finished sucessfully.") 218 | print("------------------------------------------------") 219 | self.ensemble = ensemble 220 | 221 | def predict_proba(self, data): 222 | """ 223 | Predicts the scores for the test dataset. 224 | Parameters 225 | ---------- 226 | data : Numpy array of documents of a single query 227 | 228 | Returns 229 | ------- 230 | predicted_scores : Numpy array of scores 231 | This contains an array or the predicted scores for the documents. 232 | """ 233 | return self.ensemble.eval(data[:,1:]) 234 | -------------------------------------------------------------------------------- /table1/create_runscripts.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os.path 3 | import optparse 4 | 5 | 6 | parameter_adarank = { 7 | "Name" : "AdaRank", 8 | "rankerNumber" : 3, 9 | "-round" : [500, 1000, 2000], 10 | "-tolerance" : [0.01, 0.001, 0.002, 0.005], 11 | "-max" : [2, 5, 10] 12 | } 13 | 14 | parameter_lambda = { 15 | "Name" : "LambdaMart", 16 | "rankerNumber" : 6, 17 | "-tree" : [1000, 2000], 18 | "-leaf" : [5, 10, 15], 19 | "-shrinkage" : [0.1, 0.01] 20 | } 21 | 22 | parameter_ranknet = { 23 | "Name" : "RankNet", 24 | "rankerNumber" : 1, 25 | "-epoch" : [50, 100, 200, 300], 26 | "-layer" : [1, 5, 10], 27 | "-node" : [10, 20, 100] 28 | } 29 | 30 | parameter_listnet = { 31 | "Name" : "ListNet", 32 | "rankerNumber" : 7, 33 | "-epoch" : [1000, 1500, 2000], 34 | "-lr" : [0.0001, "0.00001"] 35 | } 36 | 37 | dics = {"lambda": parameter_lambda, 38 | "adarank": parameter_adarank, 39 | "ranknet": parameter_ranknet, 40 | "listnet": parameter_listnet 41 | } 42 | 43 | parameter_adarank_test = { 44 | "Name" : "AdaRank", 45 | "rankerNumber" : 3, 46 | "-round" : [1], 47 | } 48 | 49 | parameter_lambda_test = { 50 | "Name" : "LambdaMart", 51 | "rankerNumber" : 6, 52 | "-tree" : [10], 53 | } 54 | 55 | parameter_ranknet_test = { 56 | "Name" : "RankNet", 57 | "rankerNumber" : 1, 58 | "-epoch" : [1], 59 | } 60 | 61 | parameter_listnet_test = { 62 | "Name" : "ListNet", 63 | "rankerNumber" : 7, 64 | "-epoch" : [1], 65 | } 66 | 67 | dics_test = {"lambda": parameter_lambda_test, 68 | "adarank": parameter_adarank_test, 69 | "ranknet": parameter_ranknet_test, 70 | "listnet": parameter_listnet_test 71 | } 72 | 73 | sbatch_pre = """#!/bin/bash 74 | #SBATCH --ntasks 4 75 | #SBATCH --qos bbdefault 76 | #SBATCH --mail-type ALL 77 | #SBATCH --mem 45G # 45GB 78 | #SBATCH --error=slurm-%j.err 79 | #SBATCH --time=72:00:00 80 | set -e 81 | module purge; module load bluebear # this line is required 82 | module load Java/1.8.0_92 83 | echo "Java now ..." 84 | """ 85 | 86 | 87 | def options(): 88 | parser = optparse.OptionParser() 89 | parser.add_option("-d", "--data", dest="data_dir", 90 | help="where to find the data DIRECTORY") 91 | parser.add_option("-m", "--model", dest="model_names", 92 | help="Model names (lambda, ranknet, adarank, listnet) in , sep") 93 | parser.add_option("-l", "--datalabel", dest="data_label", 94 | help="tag the experiments by label") 95 | parser.add_option("-j", "--jarpath", dest="jar_path", 96 | help="where to look for the jar file") 97 | parser.add_option("-r", "--resultsdir", dest="results_dir", 98 | help="directory to store results") 99 | parser.add_option("-c", "--cluster", dest="cluster", 100 | help="run on the cluster") 101 | parser.add_option("-t", "--test", dest="test", 102 | help="run in test mode") 103 | parser.add_option("-b", "--ttest", dest="ttest", 104 | help="run 15 fold test mode") 105 | 106 | (options, args) = parser.parse_args() 107 | return (options, args) 108 | 109 | 110 | def main(data_dir,data_label,jar_path,results_dir,cluster,test,ttest,model_names): 111 | simpleJobCounter = 0 112 | if not os.path.exists('run_scripts'): 113 | os.makedirs('run_scripts') 114 | run_dics = [] 115 | for model in model_names.split(","): 116 | if test == "1": 117 | run_dics.append(dics_test[model]) 118 | if test != "1": 119 | run_dics.append(dics[model]) 120 | if ttest == "1": folds = ["Fold1", "Fold2", "Fold3", "Fold4", "Fold5", "Fold6", "Fold7", "Fold8", "Fold9", "Fold10", "Fold11", "Fold12", "Fold13", "Fold14", "Fold15"] 121 | if ttest != "1": folds = ["Fold1", "Fold2", "Fold3", "Fold4", "Fold5"] 122 | for dic in run_dics: 123 | for metric in ["NDCG@10" , "MAP"]: 124 | for fold in folds: 125 | for key in dic.keys(): 126 | if key not in ["Name","rankerNumber"]: 127 | for gridValue in dic[key]: 128 | simpleJobCounter += 1 129 | scriptfile = 'run_scripts{}{}_{}_{}_{}_{}_train.sh'.format(os.path.sep,data_label,dic["Name"],metric,fold,str(simpleJobCounter).zfill(6)) 130 | with open(scriptfile, "w") as bashfile: 131 | if cluster == "1": 132 | bashfile.write(sbatch_pre) 133 | 134 | outfile = 'echo "{} {} {} {}";\n'.format(data_label,dic["Name"],metric,fold,key) 135 | if ttest == "1": outfile += 'java -jar {} -silent -kcv 3'.format(jar_path) 136 | if ttest != "1": outfile += 'java -jar {} -silent -kcv 5'.format(jar_path) 137 | outfile += " -train {}{}{}{}train".format(data_dir,os.path.sep,fold,os.path.sep) 138 | outfile += " -test {}{}{}{}test".format(data_dir,os.path.sep,fold,os.path.sep) 139 | outfile += ' -ranker {}'.format(dic["rankerNumber"]) 140 | outfile += ' -metric2t {}'.format(metric) 141 | outfile += ' -metric2T {}'.format(metric) 142 | outfile += ' -save {}{}{}{}{}{}.txt'.format(results_dir,metric,fold,key,gridValue,dic["Name"]) 143 | outfile += " {} {}".format(key,gridValue) 144 | outfile += " > {}{}{}{}{}{}.out".format(results_dir,metric,fold,key,gridValue,dic["Name"]) 145 | outfile += " ;" 146 | # outfile += " &" 147 | bashfile.write(outfile) 148 | bashfile.write("\n") 149 | print('sbatch {}'.format(scriptfile)) 150 | 151 | if __name__ == '__main__': 152 | (options, args) = options() 153 | data_dir = options.data_dir 154 | data_label = options.data_label 155 | jar_path = '{}{}RankLib-2.10.jar'.format(options.jar_path,os.path.sep) 156 | results_dir = '{}{}{}{}'.format(options.results_dir,os.path.sep,data_label,os.path.sep) 157 | # data_dir 158 | if not os.path.exists(data_dir): 159 | print('Data path <<{}>> does not exist'.format(data_dir)) 160 | sys.exit(0) 161 | if not os.path.exists(jar_path): 162 | print('JAR path <<{}>> not found'.format(jar_path)) 163 | sys.exit(0) 164 | if not os.path.exists(results_dir): 165 | try: 166 | os.makedirs(results_dir) 167 | except Exception as e: 168 | print('Could not create output directory <<{}>>'.format(results_dir)) 169 | sys.exit(0) 170 | main(data_dir,data_label,jar_path,results_dir,options.cluster,options.test,options.ttest,options.model_names) 171 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Loading data 2 | For loading the data go into the folders `data` and run: 3 | 4 | ```bash 5 | source get_data.sh 6 | ``` 7 | For MQ2007/8 we remove all the zeros in the queries with the script `delete_zeros.py`. 8 | 9 | # Virtualenv & Compile AdaRank 10 | 11 | ```bash 12 | # install all the python requirements 13 | pip install virtualenv 14 | virtualenv DirectRanker 15 | source DirectRanker/bin/activate 16 | pip install -r /path/to/requirements.txt 17 | 18 | # install the Rust libary for AdaRank 19 | # first install Rust: https://www.rust-lang.org/learn/get-started 20 | pip install maturin 21 | # install it in the virtualenv 22 | cd rustlib 23 | maturin develop -r 24 | # install it in your global python env 25 | maturin build 26 | pip install target/wheels/*.whl 27 | ``` 28 | 29 | # Test the code 30 | 31 | ```bash 32 | ./test_all.sh 33 | ``` 34 | 35 | # Figure 4 36 | For generating the plots from Figure 2 one should run the following scripts in the folder `fig4`: 37 | 38 | 1. Generating the synthetic data: 39 | ```bash 40 | python gen_fig4a.py # for Fig. 4 a) 41 | python gen_fig4b.py # for Fig. 4 b) 42 | python gen_fig4c.py # for Fig. 4 c) 43 | python gen_fig4d.py # for Fig. 4 d) 44 | ``` 45 | 46 | # Figure 5 47 | For generating the plots from Figure 5 one should run the following scripts in the folder `fig5`: 48 | 49 | 1. Run the model tests for the synthetic data: 50 | ```bash 51 | python training_label.py 52 | python training_size.py 53 | ``` 54 | 55 | 2. Run for the MSLR-WEB10K data: 56 | ```bash 57 | python training_label_mslr.py 58 | python training_size_mslr.py 59 | ``` 60 | 61 | 3. Run the plot script: 62 | ```bash 63 | python plot_fig5.py 64 | ``` 65 | 66 | # Figure 6 67 | For generating the plots from Figure 6 one should run the following scripts in the folder `fig6`: 68 | 69 | 1. Generating the synthetic data: 70 | ```bash 71 | ln -s ../Rankers . 72 | python plot_sorted_list.py 73 | ``` 74 | 75 | # Table 1 and Figure 2/3 76 | For generating table 1 one should run the following scripts in the folder `table1`: 77 | 78 | 1. Running the ranklib models 79 | ```bash 80 | python create_runscripts.py --data ../data/MQ2007 --jarpath .. --resultsdir results_ranklib --datalabel MQ2007 # create run scripts for ranklib and MQ2007 81 | python create_runscripts.py --data ../data/MQ2008 --jarpath .. --resultsdir results_ranklib --datalabel MQ2008 # create run scripts for ranklib and MQ2008 82 | python create_runscripts.py --data ../data/MSLR-WEB10K --jarpath .. --resultsdir results_ranklib --datalabel MSLR-WEB10K # create run scripts for ranklib and MSLR-WEB10K 83 | python run_jobs.py # run the ranklib models 84 | ``` 85 | 86 | 2. Running the DirectRankerV1 model 87 | ```bash 88 | ln -s ../Rankers . 89 | python gridsearch.py --path ../data/MQ2007/ --model DirectRankerV1 --jobs 5 --data MQ2007 90 | python gridsearch.py --path ../data/MQ2008/ --model DirectRankerV1 --jobs 5 --data MQ2008 91 | python gridsearch.py --path ../data/MSLR-WEB10K/ --model DirectRankerV1 --jobs 5 --data MSLR10K 92 | ``` 93 | 94 | 3. Running the tensorflow V2 models 95 | ```bash 96 | python gridsearch.py --path ../data/MQ2007/ --model RankNet --jobs 5 --data MQ2007 --ttest 1 --binary 0 97 | python gridsearch.py --path ../data/MQ2007/ --model RankNet --jobs 5 --data MQ2007 --ttest 1 --binary 1 98 | python gridsearch.py --path ../data/MQ2008/ --model RankNet --jobs 5 --data MQ2008 --ttest 1 --binary 0 99 | python gridsearch.py --path ../data/MQ2008/ --model RankNet --jobs 5 --data MQ2008 --ttest 1 --binary 1 100 | python gridsearch.py --path ../data/MSLR-WEB10K/ --model RankNet --jobs 5 --data MSLR10K --ttest 1 --binary 0 101 | python gridsearch.py --path ../data/MSLR-WEB10K/ --model RankNet --jobs 5 --data MSLR10K --ttest 1 --binary 1 102 | 103 | python gridsearch.py --path ../data/MQ2007/ --model ListNet --jobs 5 --data MQ2007 --ttest 1 --binary 0 104 | python gridsearch.py --path ../data/MQ2007/ --model ListNetOri --jobs 5 --data MQ2007 --ttest 1 --binary 0 105 | python gridsearch.py --path ../data/MQ2007/ --model ListNet --jobs 5 --data MQ2007 --ttest 1 --binary 1 106 | python gridsearch.py --path ../data/MQ2007/ --model ListNetOri --jobs 5 --data MQ2007 --ttest 1 --binary 1 107 | python gridsearch.py --path ../data/MQ2008/ --model ListNet --jobs 5 --data MQ2008 --ttest 1 --binary 0 108 | python gridsearch.py --path ../data/MQ2008/ --model ListNetOri --jobs 5 --data MQ2008 --ttest 1 --binary 0 109 | python gridsearch.py --path ../data/MQ2008/ --model ListNet --jobs 5 --data MQ2008 --ttest 1 --binary 1 110 | python gridsearch.py --path ../data/MQ2008/ --model ListNetOri --jobs 5 --data MQ2008 --ttest 1 --binary 1 111 | python gridsearch.py --path ../data/MSLR-WEB10K/ --model ListNet --jobs 5 --data MSLR10K --ttest 1 --binary 0 112 | python gridsearch.py --path ../data/MSLR-WEB10K/ --model ListNetOri --jobs 5 --data MSLR10K --ttest 1 --binary 0 113 | python gridsearch.py --path ../data/MSLR-WEB10K/ --model ListNet --jobs 5 --data MSLR10K --ttest 1 --binary 1 114 | python gridsearch.py --path ../data/MSLR-WEB10K/ --model ListNetOri --jobs 5 --data MSLR10K --ttest 1 --binary 1 115 | 116 | python gridsearch.py --path ../data/MQ2007/ --model DirectRanker --jobs 5 --data MQ2007 --ttest 1 --binary 0 117 | python gridsearch.py --path ../data/MQ2007/ --model DirectRanker --jobs 5 --data MQ2007 --ttest 1 --binary 1 118 | python gridsearch.py --path ../data/MQ2008/ --model DirectRanker --jobs 5 --data MQ2008 --ttest 1 --binary 0 119 | python gridsearch.py --path ../data/MQ2008/ --model DirectRanker --jobs 5 --data MQ2008 --ttest 1 --binary 1 120 | python gridsearch.py --path ../data/MSLR-WEB10K/ --model DirectRanker --jobs 5 --data MSLR10K --ttest 1 --binary 0 121 | python gridsearch.py --path ../data/MSLR-WEB10K/ --model DirectRanker --jobs 5 --data MSLR10K --ttest 1 --binary 1 122 | 123 | python gridsearch.py --path ../data/MQ2007/ --model AdaRank --jobs 1 --data MQ2007 --ttest 1 --binary 0 124 | python gridsearch.py --path ../data/MQ2007/ --model AdaRank --jobs 1 --data MQ2007 --ttest 1 --binary 1 125 | python gridsearch.py --path ../data/MQ2008/ --model AdaRank --jobs 1 --data MQ2008 --ttest 1 --binary 0 126 | python gridsearch.py --path ../data/MQ2008/ --model AdaRank --jobs 1 --data MQ2008 --ttest 1 --binary 1 127 | python gridsearch.py --path ../data/MSLR-WEB10K/ --model AdaRank --jobs 1 --data MSLR10K --ttest 1 --binary 0 128 | python gridsearch.py --path ../data/MSLR-WEB10K/ --model AdaRank --jobs 1 --data MSLR10K --ttest 1 --binary 1 129 | 130 | python gridsearch.py --path ../data/MQ2007/ --model LambdaMart --jobs 5 --data MQ2007 --ttest 1 --binary 0 131 | python gridsearch.py --path ../data/MQ2007/ --model LambdaMart --jobs 5 --data MQ2007 --ttest 1 --binary 1 132 | python gridsearch.py --path ../data/MQ2008/ --model LambdaMart --jobs 5 --data MQ2008 --ttest 1 --binary 0 133 | python gridsearch.py --path ../data/MQ2008/ --model LambdaMart --jobs 5 --data MQ2008 --ttest 1 --binary 1 134 | python gridsearch.py --path ../data/MSLR-WEB10K/ --model LambdaMart --jobs 5 --data MSLR10K --ttest 1 --binary 0 135 | python gridsearch.py --path ../data/MSLR-WEB10K/ --model LambdaMart --jobs 5 --data MSLR10K --ttest 1 --binary 1 136 | ``` 137 | 138 | 4. Generating the Table / Friedman Test 139 | ```bash 140 | python get_results.py 141 | ``` 142 | 143 | # Table 2 144 | For generating table 2 one should run the following scripts in the folder `table2`: 145 | 146 | ```bash 147 | ./run_ranknet_eval.sh 148 | python ranknet_eval.py --plot 1 149 | ``` 150 | 151 | # Table B1 152 | For generating table B1 one should run the following scripts in the folder `tableB1`: 153 | 154 | ```bash 155 | ./run_time_ana.sh 156 | ``` 157 | The different time results for RankLib are printed in the terminal while the results for the 158 | DirectRanker and RankNet are stored in an output folder. 159 | -------------------------------------------------------------------------------- /Rankers/models/LambdaMart.py: -------------------------------------------------------------------------------- 1 | # code adapted from https://github.com/lezzago/LambdaMart/blob/master/lambdamart.py 2 | 3 | import numpy as np 4 | from sklearn.tree import DecisionTreeRegressor 5 | from sklearn.base import BaseEstimator 6 | from multiprocessing import Pool 7 | 8 | 9 | def dcg(scores): 10 | """ 11 | Returns the DCG value of the list of scores. 12 | Parameters 13 | ---------- 14 | scores : list 15 | Contains labels in a certain ranked order 16 | 17 | Returns 18 | ------- 19 | DCG_val: int 20 | This is the value of the DCG on the given scores 21 | """ 22 | return np.sum([ 23 | (np.power(2, scores[i]) - 1) / np.log2(i + 2) 24 | for i in range(len(scores)) 25 | ]) 26 | 27 | def dcg_k(scores, k): 28 | """ 29 | Returns the DCG value of the list of scores and truncates to k values. 30 | Parameters 31 | ---------- 32 | scores : list 33 | Contains labels in a certain ranked order 34 | k : int 35 | In the amount of values you want to only look at for computing DCG 36 | 37 | Returns 38 | ------- 39 | DCG_val: int 40 | This is the value of the DCG on the given scores 41 | """ 42 | return np.sum([ 43 | (np.power(2, scores[i]) - 1) / np.log2(i + 2) 44 | for i in range(len(scores[:k])) 45 | ]) 46 | 47 | def ideal_dcg(scores): 48 | """ 49 | Returns the Ideal DCG value of the list of scores. 50 | Parameters 51 | ---------- 52 | scores : list 53 | Contains labels in a certain ranked order 54 | 55 | Returns 56 | ------- 57 | Ideal_DCG_val: int 58 | This is the value of the Ideal DCG on the given scores 59 | """ 60 | scores = [score for score in sorted(scores)[::-1]] 61 | return dcg(scores) 62 | 63 | def ideal_dcg_k(scores, k): 64 | """ 65 | Returns the Ideal DCG value of the list of scores and truncates to k values. 66 | Parameters 67 | ---------- 68 | scores : list 69 | Contains labels in a certain ranked order 70 | k : int 71 | In the amount of values you want to only look at for computing DCG 72 | 73 | Returns 74 | ------- 75 | Ideal_DCG_val: int 76 | This is the value of the Ideal DCG on the given scores 77 | """ 78 | scores = [score for score in sorted(scores)[::-1]] 79 | return dcg_k(scores, k) 80 | 81 | def single_dcg(scores, i, j): 82 | """ 83 | Returns the DCG value at a single point. 84 | Parameters 85 | ---------- 86 | scores : list 87 | Contains labels in a certain ranked order 88 | i : int 89 | This points to the ith value in scores 90 | j : int 91 | This sets the ith value in scores to be the jth rank 92 | 93 | Returns 94 | ------- 95 | Single_DCG: int 96 | This is the value of the DCG at a single point 97 | """ 98 | return (np.power(2, scores[i]) - 1) / np.log2(j + 2) 99 | 100 | def compute_lambda(args): 101 | """ 102 | Returns the lambda and w values for a given query. 103 | Parameters 104 | ---------- 105 | args : zipped value of true_scores, predicted_scores, good_ij_pairs, idcg, query_key 106 | Contains a list of the true labels of documents, list of the predicted labels of documents, 107 | i and j pairs where true_score[i] > true_score[j], idcg values, and query keys. 108 | 109 | Returns 110 | ------- 111 | lambdas : numpy array 112 | This contains the calculated lambda values 113 | w : numpy array 114 | This contains the computed w values 115 | query_key : int 116 | This is the query id these values refer to 117 | """ 118 | 119 | true_scores, predicted_scores, good_ij_pairs, idcg, query_key = args 120 | num_docs = len(true_scores) 121 | sorted_indexes = np.argsort(predicted_scores)[::-1] 122 | rev_indexes = np.argsort(sorted_indexes) 123 | true_scores = true_scores[sorted_indexes] 124 | predicted_scores = predicted_scores[sorted_indexes] 125 | 126 | lambdas = np.zeros(num_docs) 127 | w = np.zeros(num_docs) 128 | 129 | single_dcgs = {} 130 | for i,j in good_ij_pairs: 131 | if (i,i) not in single_dcgs: 132 | single_dcgs[(i,i)] = single_dcg(true_scores, i, i) 133 | single_dcgs[(i,j)] = single_dcg(true_scores, i, j) 134 | if (j,j) not in single_dcgs: 135 | single_dcgs[(j,j)] = single_dcg(true_scores, j, j) 136 | single_dcgs[(j,i)] = single_dcg(true_scores, j, i) 137 | 138 | 139 | for i,j in good_ij_pairs: 140 | z_ndcg = abs(single_dcgs[(i,j)] - single_dcgs[(i,i)] + single_dcgs[(j,i)] - single_dcgs[(j,j)]) / idcg 141 | rho = 1 / (1 + np.exp(predicted_scores[i] - predicted_scores[j])) 142 | rho_complement = 1.0 - rho 143 | lambda_val = z_ndcg * rho 144 | lambdas[i] += lambda_val 145 | lambdas[j] -= lambda_val 146 | 147 | w_val = rho * rho_complement * z_ndcg 148 | w[i] += w_val 149 | w[j] += w_val 150 | 151 | return lambdas[rev_indexes], w[rev_indexes], query_key 152 | 153 | def group_queries(q): 154 | """ 155 | Returns a dictionary that groups the documents by their query ids. 156 | Parameters 157 | ---------- 158 | q : Numpy array of lists 159 | Contains a list query index 160 | 161 | Returns 162 | ------- 163 | query_indexes : dictionary 164 | The keys were the different query ids and teh values were the indexes in the training data that are associated of those keys. 165 | """ 166 | query_indexes = {} 167 | index = 0 168 | for record in q: 169 | query_indexes.setdefault(record, []) 170 | query_indexes[record].append(index) 171 | index += 1 172 | return query_indexes 173 | 174 | def get_pairs(scores): 175 | """ 176 | Returns pairs of indexes where the first value in the pair has a higher score than the second value in the pair. 177 | Parameters 178 | ---------- 179 | scores : list of int 180 | Contain a list of numbers 181 | 182 | Returns 183 | ------- 184 | query_pair : list of pairs 185 | This contains a list of pairs of indexes in scores. 186 | """ 187 | 188 | query_pair = [] 189 | for query_scores in scores: 190 | temp = sorted(query_scores, reverse=True) 191 | pairs = [] 192 | for i in range(len(temp)): 193 | for j in range(len(temp)): 194 | if temp[i] > temp[j]: 195 | pairs.append((i,j)) 196 | query_pair.append(pairs) 197 | return query_pair 198 | 199 | class LambdaMart(BaseEstimator): 200 | 201 | def __init__(self, number_of_trees=5, learning_rate=0.1, max_depth=50, min_samples_split=2): 202 | """ 203 | This is the constructor for the LambdaMART object. 204 | Parameters 205 | ---------- 206 | number_of_trees : int (default: 5) 207 | Number of trees LambdaMART goes through 208 | learning_rate : float (default: 0.1) 209 | Rate at which we update our prediction with each tree 210 | """ 211 | 212 | self.number_of_trees = number_of_trees 213 | self.learning_rate = learning_rate 214 | self.max_depth = max_depth 215 | self.min_samples_split = min_samples_split 216 | self.trees = [] 217 | 218 | def fit(self, x, y): 219 | """ 220 | Fits the model on the training data. 221 | """ 222 | 223 | q = x[:,0] 224 | x = x[:,1:] 225 | predicted_scores = np.zeros(len(x)) 226 | query_indexes = group_queries(q) 227 | query_keys = query_indexes.keys() 228 | true_scores = [y[query_indexes[query]] for query in query_keys] 229 | good_ij_pairs = get_pairs(true_scores) 230 | 231 | # ideal dcg calculation 232 | idcg = [ideal_dcg(scores) for scores in true_scores] 233 | 234 | for k in range(self.number_of_trees): 235 | print('Tree %d' % (k)) 236 | lambdas = np.zeros(len(predicted_scores)) 237 | w = np.zeros(len(predicted_scores)) 238 | pred_scores = [predicted_scores[query_indexes[query]] for query in query_keys] 239 | 240 | pool = Pool() 241 | for lambda_val, w_val, query_key in pool.map(compute_lambda, zip(true_scores, pred_scores, good_ij_pairs, idcg, query_keys), chunksize=1): 242 | indexes = query_indexes[query_key] 243 | lambdas[indexes] = lambda_val 244 | w[indexes] = w_val 245 | pool.close() 246 | 247 | # Sklearn implementation of the tree 248 | tree = DecisionTreeRegressor(max_depth=self.max_depth, min_samples_split=self.min_samples_split) 249 | tree.fit(x, lambdas) 250 | self.trees.append(tree) 251 | prediction = tree.predict(x) 252 | predicted_scores += prediction * self.learning_rate 253 | 254 | def predict_proba(self, data): 255 | """ 256 | Predicts the scores for the test dataset. 257 | Parameters 258 | ---------- 259 | data : Numpy array of documents of a single query 260 | 261 | Returns 262 | ------- 263 | predicted_scores : Numpy array of scores 264 | This contains an array or the predicted scores for the documents. 265 | """ 266 | data = data[:,1:] 267 | results = 0 268 | for tree in self.trees: 269 | results += self.learning_rate * tree.predict(data) 270 | return results 271 | -------------------------------------------------------------------------------- /Rankers/models/ListNet.py: -------------------------------------------------------------------------------- 1 | 2 | import tensorflow as tf 3 | 4 | import numpy as np 5 | from sklearn.base import BaseEstimator 6 | 7 | class ListNet(BaseEstimator): 8 | """ 9 | Tensorflow implementation of https://arxiv.org/pdf/1805.08716.pdf 10 | Inspired by: https://github.com/MilkaLichtblau/DELTR-Experiments 11 | 12 | Constructor 13 | :param hidden_layers: List containing the numbers of neurons in the layers for feature 14 | :param activation: tf function for the feature part of the net 15 | :param kernel_initializer: tf kernel_initializer 16 | :param start_batch_size: cost function of FairListNet 17 | :param min_doc: min size of docs in query if a list is given 18 | :param end_batch_size: max size of docs in query if a list is given 19 | :param start_len_qid: start size of the queries/batch 20 | :param end_len_qid: end size of the queries/batch 21 | :param learning_rate: learning rate for the optimizer 22 | :param max_steps: total training steps 23 | :param learning_rate_step_size: factor for increasing the learning rate 24 | :param learning_rate_decay_factor: factor for increasing the learning rate 25 | :param optimizer: tf optimizer object 26 | :param print_step: for which step the script should print out the cost for the current batch 27 | :param weight_regularization: float for weight regularization 28 | :param dropout: float amount of dropout 29 | :param input_dropout: float amount of input dropout 30 | :param name: name of the object 31 | :param num_features: number of input features 32 | :param protected_feature_deltr: column name of the protected attribute (index after query and document id) 33 | :param gamma_deltr: value of the gamma parameter 34 | :param iterations_deltr: number of iterations the training should run 35 | :param standardize_deltr: let's apply standardization to the features 36 | """ 37 | 38 | def __init__(self, 39 | # ListNet HPs 40 | hidden_layers_dr=[256, 128, 64, 20], 41 | feature_activation_dr='tanh', 42 | ranking_activation_dr='sigmoid', 43 | feature_bias_dr=True, 44 | kernel_initializer_dr=tf.random_normal_initializer, 45 | kernel_regularizer_dr=0.0, 46 | drop_out=0.0, 47 | # Common HPs 48 | batch_size=200, 49 | learning_rate=0.001, 50 | learning_rate_decay_rate=1, 51 | learning_rate_decay_steps=0, 52 | optimizer=tf.keras.optimizers.Adam,# 'Nadam' 'SGD' 53 | epoch=10, 54 | # other variables 55 | verbose=0, 56 | validation_size=0.0, 57 | num_features=0, 58 | name="ListNet", 59 | dtype=tf.float32, 60 | print_summary=False, 61 | ): 62 | 63 | # ListNet HPs 64 | self.hidden_layers_dr = hidden_layers_dr 65 | self.feature_activation_dr = feature_activation_dr 66 | self.ranking_activation_dr = ranking_activation_dr 67 | self.feature_bias_dr = feature_bias_dr 68 | self.kernel_initializer_dr = kernel_initializer_dr 69 | self.kernel_regularizer_dr = kernel_regularizer_dr 70 | self.drop_out = drop_out 71 | # Common HPs 72 | self.batch_size = batch_size 73 | self.learning_rate = learning_rate 74 | self.learning_rate_decay_rate = learning_rate_decay_rate 75 | self.learning_rate_decay_steps = learning_rate_decay_steps 76 | self.optimizer = optimizer 77 | self.epoch = epoch 78 | # other variables 79 | self.verbose = verbose 80 | self.validation_size = validation_size 81 | self.num_features = num_features 82 | self.name = name 83 | self.dtype = dtype 84 | self.print_summary = print_summary 85 | 86 | 87 | def _build_model(self): 88 | """ 89 | This function builds the ListNet with the values specified in the constructor 90 | :return: 91 | """ 92 | 93 | # Placeholders for the inputs 94 | input_layer = tf.keras.layers.Input( 95 | shape=self.num_features, 96 | dtype=self.dtype, 97 | name="input" 98 | ) 99 | 100 | nn = tf.keras.layers.Dense( 101 | units=self.hidden_layers_dr[0], 102 | activation=self.feature_activation_dr, 103 | use_bias=self.feature_bias_dr, 104 | kernel_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 105 | bias_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 106 | activity_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 107 | name="nn_hidden_0" 108 | )(input_layer) 109 | 110 | if self.drop_out > 0: 111 | nn = tf.keras.layers.Dropout(self.drop_out)(nn) 112 | 113 | for i in range(1, len(self.hidden_layers_dr)): 114 | nn = tf.keras.layers.Dense( 115 | units=self.hidden_layers_dr[i], 116 | activation=self.feature_activation_dr, 117 | use_bias=self.feature_bias_dr, 118 | kernel_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 119 | bias_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 120 | activity_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 121 | name="nn_hidden_" + str(i) 122 | )(nn) 123 | 124 | if self.drop_out > 0: 125 | nn = tf.keras.layers.Dropout(self.drop_out)(nn) 126 | 127 | nn = tf.keras.layers.Dense( 128 | units=1, # since we only have binary classes 129 | activation=self.ranking_activation_dr, 130 | kernel_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 131 | activity_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 132 | name="ranking_part" 133 | )(nn) 134 | 135 | self.model = tf.keras.models.Model(input_layer, nn, name='ListNet') 136 | 137 | if self.learning_rate_decay_steps > 0: 138 | lr_schedule = tf.keras.optimizers.schedules.InverseTimeDecay( 139 | self.learning_rate, 140 | decay_steps=self.learning_rate_decay_steps, 141 | decay_rate=self.learning_rate_decay_rate, 142 | staircase=False 143 | ) 144 | else: 145 | lr_schedule = self.learning_rate 146 | 147 | self.model.compile( 148 | optimizer=self.optimizer(lr_schedule), 149 | loss=self._def_cost, 150 | metrics=['acc'] 151 | ) 152 | 153 | if self.print_summary: 154 | self.model.summary() 155 | 156 | def _def_cost(self, y_actual, y_pred): 157 | """ 158 | The Top-1 approximated ListNet loss as in Cao et al (2006), Learning to 159 | Rank: From Pairwise Approach to Listwise Approach 160 | :param nn: activation of the previous layer 161 | :param y: target labels 162 | :return: The loss 163 | """ 164 | # ListNet top-1 reduces to a softmax and simple cross entropy 165 | st = tf.nn.softmax(tf.cast(y_actual, tf.float32), axis=0) 166 | sx = tf.nn.softmax(tf.cast(y_pred, tf.float32), axis=0) 167 | return -tf.math.reduce_sum(st * tf.math.log(sx)) 168 | 169 | def fit(self, x, y, **fit_params): 170 | """ 171 | :param features: list of queries for training the net 172 | :param real_classes: list of labels inside a query 173 | :return: 174 | """ 175 | self._build_model() 176 | 177 | self.model.fit( 178 | x=x, 179 | y=y, 180 | batch_size=self.batch_size, 181 | epochs=self.epoch, 182 | verbose=self.verbose, 183 | shuffle=True, 184 | validation_split=self.validation_size 185 | ) 186 | 187 | def predict_proba(self, features): 188 | """ 189 | :param features: list of features of the instance feed to the net 190 | :return: predicted class 191 | """ 192 | if len(features.shape) == 1: 193 | features = [features] 194 | 195 | res = self.model.predict(features, batch_size=self.batch_size, verbose=self.verbose) 196 | 197 | return np.array([0.5 * (value + 1) for value in res]) 198 | -------------------------------------------------------------------------------- /Rankers/models/DirectRanker.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | from sklearn.base import BaseEstimator 5 | import gc 6 | 7 | class DirectRanker(BaseEstimator): 8 | """ 9 | TODO 10 | """ 11 | 12 | def __init__(self, 13 | # DirectRanker HPs 14 | hidden_layers_dr=[256, 128, 64, 20], 15 | feature_activation_dr='tanh', 16 | ranking_activation_dr='sigmoid', 17 | feature_bias_dr=True, 18 | kernel_initializer_dr=tf.random_normal_initializer, 19 | kernel_regularizer_dr=0.0, 20 | drop_out=0, 21 | # Common HPs 22 | scale_factor_train_sample=1, 23 | batch_size=200, 24 | loss=tf.keras.losses.MeanSquaredError(),# 'binary_crossentropy' 25 | learning_rate=0.01, 26 | learning_rate_decay_rate=1, 27 | learning_rate_decay_steps=1000, 28 | optimizer=tf.keras.optimizers.Adam,# 'Nadam' 'SGD' 29 | epoch=10, 30 | steps_per_epoch=None, 31 | # other variables 32 | verbose=0, 33 | validation_size=0.0, 34 | num_features=0, 35 | name="DirectRanker", 36 | dtype=tf.float32, 37 | print_summary=False, 38 | ): 39 | 40 | # DirectRanker HPs 41 | self.hidden_layers_dr = hidden_layers_dr 42 | self.feature_activation_dr = feature_activation_dr 43 | self.ranking_activation_dr = ranking_activation_dr 44 | self.feature_bias_dr = feature_bias_dr 45 | self.kernel_initializer_dr = kernel_initializer_dr 46 | self.kernel_regularizer_dr = kernel_regularizer_dr 47 | self.drop_out = drop_out 48 | # Common HPs 49 | self.scale_factor_train_sample = scale_factor_train_sample 50 | self.batch_size = batch_size 51 | self.loss = loss 52 | self.learning_rate = learning_rate 53 | self.learning_rate_decay_rate = learning_rate_decay_rate 54 | self.learning_rate_decay_steps = learning_rate_decay_steps 55 | self.optimizer = optimizer 56 | self.epoch = epoch 57 | self.steps_per_epoch = steps_per_epoch 58 | # other variables 59 | self.verbose = verbose 60 | self.validation_size = validation_size 61 | self.num_features = num_features 62 | self.name = name 63 | self.dtype = dtype 64 | self.print_summary = print_summary 65 | 66 | def _build_model(self): 67 | """ 68 | TODO 69 | """ 70 | # Placeholders for the inputs 71 | self.x0 = tf.keras.layers.Input( 72 | shape=self.num_features, 73 | dtype=self.dtype, 74 | name="x0" 75 | ) 76 | 77 | self.x1 = tf.keras.layers.Input( 78 | shape=self.num_features, 79 | dtype=self.dtype, 80 | name="x1" 81 | ) 82 | 83 | input_layer = tf.keras.layers.Input( 84 | shape=self.num_features, 85 | dtype=self.dtype, 86 | name="input" 87 | ) 88 | 89 | nn = tf.keras.layers.Dense( 90 | units=self.hidden_layers_dr[0], 91 | activation=self.feature_activation_dr, 92 | use_bias=self.feature_bias_dr, 93 | kernel_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 94 | bias_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 95 | activity_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 96 | name="nn_hidden_0" 97 | )(input_layer) 98 | 99 | if self.drop_out > 0: 100 | nn = tf.keras.layers.Dropout(self.drop_out)(nn) 101 | 102 | for i in range(1, len(self.hidden_layers_dr)): 103 | nn = tf.keras.layers.Dense( 104 | units=self.hidden_layers_dr[i], 105 | activation=self.feature_activation_dr, 106 | use_bias=self.feature_bias_dr, 107 | kernel_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 108 | bias_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 109 | activity_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 110 | name="nn_hidden_" + str(i) 111 | )(nn) 112 | 113 | if self.drop_out > 0: 114 | nn = tf.keras.layers.Dropout(self.drop_out)(nn) 115 | 116 | feature_part = tf.keras.models.Model(input_layer, nn, name='feature_part') 117 | 118 | if self.print_summary: 119 | feature_part.summary() 120 | 121 | nn0 = feature_part(self.x0) 122 | nn1 = feature_part(self.x1) 123 | 124 | subtracted = tf.keras.layers.Subtract()([nn0, nn1]) 125 | 126 | out = tf.keras.layers.Dense( 127 | units=1, 128 | activation=self.ranking_activation_dr, 129 | use_bias=False, 130 | kernel_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 131 | activity_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 132 | name="ranking_part" 133 | )(subtracted) 134 | 135 | if self.name == "RankNet" and self.ranking_activation_dr == "tanh": 136 | out = (out + 1) / 2. 137 | 138 | self.model = tf.keras.models.Model( 139 | inputs=[self.x0, self.x1], 140 | outputs=out, 141 | name='Stacker' 142 | ) 143 | 144 | if self.learning_rate_decay_steps > 0: 145 | lr_schedule = tf.keras.optimizers.schedules.InverseTimeDecay( 146 | self.learning_rate, 147 | decay_steps=self.learning_rate_decay_steps, 148 | decay_rate=self.learning_rate_decay_rate, 149 | staircase=False 150 | ) 151 | else: 152 | lr_schedule = self.learning_rate 153 | 154 | self.model.compile( 155 | optimizer=self.optimizer(lr_schedule), 156 | loss=self.loss, 157 | metrics=['acc'] 158 | ) 159 | 160 | if self.print_summary: 161 | self.model.summary() 162 | 163 | def fit(self, x, y, **fit_params): 164 | """ 165 | TODO 166 | """ 167 | self._build_model() 168 | 169 | x_l = [] 170 | for c in np.unique(y): 171 | x_l.append(x[np.where(y == c)]) 172 | 173 | for i in range(self.epoch): 174 | 175 | x0_cur = [] 176 | x1_cur = [] 177 | y_len = 0 178 | for c in np.unique(y): 179 | if c == max(np.unique(y)): continue 180 | idx0 = np.random.randint(0, len(x_l[c+1]), int(self.scale_factor_train_sample*len(x)/(len(np.unique(y))-1))) 181 | idx1 = np.random.randint(0, len(x_l[c]), int(self.scale_factor_train_sample*len(x)/(len(np.unique(y))-1))) 182 | y_len += len(idx0) 183 | x0_cur.extend(x_l[c+1][idx0]) 184 | x1_cur.extend(x_l[c][idx1]) 185 | x0_cur = np.array(x0_cur) 186 | x1_cur = np.array(x1_cur) 187 | 188 | if self.verbose > 0: 189 | print('Epoch {}/{}'.format(i + 1, self.epoch)) 190 | 191 | self.model.fit( 192 | x=[x0_cur, x1_cur], 193 | y=np.ones(y_len), 194 | batch_size=self.batch_size, 195 | steps_per_epoch=self.steps_per_epoch, 196 | epochs=1, 197 | verbose=self.verbose, 198 | shuffle=True, 199 | validation_split=self.validation_size 200 | ) 201 | # https://github.com/tensorflow/tensorflow/issues/14181 202 | # https://github.com/tensorflow/tensorflow/issues/30324 203 | gc.collect() 204 | 205 | def predict_proba(self, features): 206 | """ 207 | TODO 208 | """ 209 | if len(features.shape) == 1: 210 | features = [features] 211 | 212 | res = self.model.predict([features, np.zeros(np.shape(features))], batch_size=self.batch_size, verbose=self.verbose) 213 | 214 | return res 215 | 216 | def predict(self, features, threshold): 217 | """ 218 | TODO 219 | """ 220 | if len(features.shape) == 1: 221 | features = [features] 222 | 223 | features_conv = np.expand_dims(features, axis=2) 224 | 225 | res = self.model.predict([features, np.zeros(np.shape(features))], batch_size=self.batch_size, verbose=self.verbose) 226 | 227 | return [1 if r > threshold else 0 for r in res] 228 | -------------------------------------------------------------------------------- /fig5/training_size_mslr.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tensorflow as tf 3 | 4 | from Rankers.models.DirectRanker import DirectRanker 5 | from Rankers.models.RankNet import RankNet 6 | from Rankers.models.ListNet import ListNet 7 | from Rankers.helpers import nDCG_cls, readData 8 | from Rankers.models.LambdaRank import LambdaRank 9 | 10 | import numpy as np 11 | from sklearn.preprocessing import QuantileTransformer 12 | from time import time 13 | 14 | import json 15 | import argparse 16 | 17 | 18 | def test_rankers(n_train=100000, overwrite=False, add_model=None, test=False): 19 | 20 | folds = 5 21 | if test: 22 | folds = 1 23 | 24 | if add_model == None: 25 | results = {"n_train": n_train} 26 | ndcg_dr_l = [] 27 | ndcg_lnet_l = [] 28 | ndcg_rnet_l = [] 29 | ndcg_lrank_l = [] 30 | for fold in range(folds): 31 | if os.path.isfile(results_path + "/results_{}.json".format(n_train)): 32 | if not (add_model == None): 33 | results = json.load(open(results_path + "/results_{}.json".format(n_train))) 34 | elif overwrite: 35 | os.remove(results_path + "/results_{}.json".format(n_train)) 36 | else: 37 | print("skip training for n_relevance {}".format(n_train)) 38 | return 0 39 | 40 | N = 50 41 | x = [] 42 | y = [] 43 | x_test = [] 44 | y_test = [] 45 | 46 | # read data 47 | print("Reading data") 48 | start = time() 49 | if os.path.isfile(output_path + "/data/x_train_fold{}.npy".format(fold+1)): 50 | x_train = np.load(output_path + "/data/x_train_fold{}.npy".format(fold+1)) 51 | y_train = np.load(output_path + "/data/y_train_fold{}.npy".format(fold+1)) 52 | 53 | x_test = np.load(output_path + "/data/x_test_fold{}.npy".format(fold+1)) 54 | y_test = np.load(output_path + "/data/y_test_fold{}.npy".format(fold+1)) 55 | q_test = np.load(output_path + "/data/q_test_fold{}.npy".format(fold+1)) 56 | else: 57 | x_train, y_train, _ = readData(path=data_path + "/Fold{}/train.txt".format(fold+1)) 58 | x_test, y_test, q_test = readData(path=data_path + "/Fold{}/test.txt".format(fold+1), binary=False) 59 | 60 | np.save(output_path + "/data/x_train_fold{}".format(fold+1), x_train) 61 | np.save(output_path + "/data/y_train_fold{}".format(fold+1), y_train) 62 | 63 | np.save(output_path + "/data/x_test_fold{}".format(fold+1), x_test) 64 | np.save(output_path + "/data/y_test_fold{}".format(fold+1), y_test) 65 | np.save(output_path + "/data/q_test_fold{}".format(fold+1), q_test) 66 | end = time() 67 | mins = int((end - start) / 60) 68 | secs = end - start - 60 * mins 69 | print("Finished in " + str(mins) + " min " + str(secs) + " s") 70 | 71 | print("Preparing data n_train {} on fold {}".format(n_train, fold)) 72 | if len(x_train) < n_train: 73 | n_train = len(x_train) 74 | x = x_train 75 | y = y_train 76 | else: 77 | idx = np.random.randint(0, len(x_train), n_train) 78 | x = x_train[idx] 79 | y = y_train[idx] 80 | 81 | scaler = QuantileTransformer(output_distribution="normal") 82 | x = scaler.fit_transform(x) / 3 83 | x_test = scaler.transform(x_test) / 3 84 | 85 | if test: 86 | epoch = 1 87 | else: 88 | epoch = 10 89 | 90 | dr = DirectRanker( 91 | hidden_layers_dr=[70, 5], 92 | num_features=len(x[0]), 93 | verbose=0, 94 | epoch=epoch, 95 | scale_factor_train_sample=1 96 | ) 97 | 98 | ranknet = RankNet( 99 | hidden_layers_dr=[70, 5], 100 | num_features=len(x[0]), 101 | verbose=0, 102 | epoch=epoch, 103 | scale_factor_train_sample=1 104 | ) 105 | 106 | lnet = ListNet( 107 | hidden_layers_dr=[70, 5, 1], 108 | num_features=len(x[0]), 109 | verbose=0, 110 | optimizer=tf.keras.optimizers.SGD, 111 | epoch=epoch 112 | ) 113 | 114 | lrank = LambdaRank( 115 | hidden_layers_dr=[70, 5], 116 | num_features=len(x[0]), 117 | verbose=2, 118 | epoch=epoch, 119 | scale_factor_train_sample=1 120 | ) 121 | 122 | print("Training") 123 | start = time() 124 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 125 | dr.fit(x, y) 126 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 127 | ranknet.fit(x, y) 128 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 129 | lnet.fit(x, tf.cast(y, tf.float32)) 130 | if not('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 131 | lrank.fit(x, y) 132 | end = time() 133 | mins = int((end - start) / 60) 134 | secs = end - start - 60 * mins 135 | print("Finished in " + str(mins) + " min " + str(secs) + " s") 136 | ndcg_dr_l_cur = [] 137 | ndcg_lnet_l_cur = [] 138 | ndcg_rnet_l_cur = [] 139 | nDCG_lrank_l_cur = [] 140 | 141 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 142 | dr_prediction = dr.predict_proba(x_test) 143 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 144 | ranknet_prediction = ranknet.predict_proba(x_test) 145 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 146 | lnet_prediction = np.array(lnet.predict_proba(x_test)) 147 | if not ('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 148 | lrank_prediction = lrank.predict_proba(x_test) 149 | 150 | for n in np.unique(q_test): 151 | xt = x_test[q_test == n] 152 | yt = y_test[q_test == n] 153 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 154 | nDCG_dr = nDCG_cls(dr_prediction[q_test == n], xt, yt, at=10, prediction=True) 155 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 156 | nDCG_rnet = nDCG_cls(ranknet_prediction[q_test == n], xt, yt, at=10, prediction=True) 157 | #print("nDCG@10 RankNet {} fold {} n_train {}".format(nDCG_rnet, fold, n_train)) 158 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 159 | nDCG_lnet = nDCG_cls(lnet_prediction[q_test == n], xt, yt, at=10, prediction=True) 160 | #print("nDCG@10 ListNet {} fold {} n_train {}".format(nDCG_lnet, fold, n_train)) 161 | if not ('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 162 | nDCG_lrank = nDCG_cls(lrank_prediction[q_test == n], xt, yt, at=10, prediction=True) 163 | #print("nDCG@10 LambdaRank {} fold {} n_train {}".format(nDCG_lrank, fold, n_train)) 164 | 165 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 166 | if str(nDCG_dr) != 'nan': 167 | ndcg_dr_l_cur.append(nDCG_dr) 168 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 169 | if str(nDCG_rnet) != 'nan': 170 | ndcg_rnet_l_cur.append(nDCG_rnet) 171 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 172 | if str(nDCG_lnet) != 'nan': 173 | ndcg_lnet_l_cur.append(nDCG_lnet) 174 | if not ('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 175 | if str(nDCG_lrank) != 'nan': 176 | nDCG_lrank_l_cur.append(nDCG_lrank) 177 | 178 | ndcg_dr_l.append(np.mean(ndcg_dr_l_cur)) 179 | ndcg_lnet_l.append(np.mean(ndcg_lnet_l_cur)) 180 | ndcg_rnet_l.append(np.mean(ndcg_rnet_l_cur)) 181 | ndcg_lrank_l.append(np.mean(nDCG_lrank_l_cur)) 182 | 183 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 184 | print("nDCG@10 DirectRanker {} fold {} n_train {}".format(np.mean(ndcg_dr_l_cur), fold, n_train)) 185 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 186 | print("nDCG@10 RankNet {} fold {} n_train {}".format(np.mean(ndcg_rnet_l_cur), fold, n_train)) 187 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 188 | print("nDCG@10 ListNet {} fold {} n_train {}".format(np.mean(ndcg_lnet_l_cur), fold, n_train)) 189 | if not ('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 190 | print("nDCG@10 LambdaRank {} fold {} n_train {}".format(np.mean(nDCG_lrank_l_cur), fold, n_train)) 191 | 192 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 193 | results['ndcg_DirectRanker'] = [np.mean(ndcg_dr_l), np.std(ndcg_dr_l)] 194 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 195 | results['ndcg_RankNet'] = [np.mean(ndcg_rnet_l), np.std(ndcg_rnet_l)] 196 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 197 | results['ndcg_ListNet'] = [np.mean(ndcg_lnet_l), np.std(ndcg_lnet_l)] 198 | if not ('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 199 | results['ndcg_LambdaRank'] = [np.mean(ndcg_lrank_l), np.std(ndcg_lrank_l)] 200 | 201 | with open(results_path + '/results_{}.json'.format(n_train), 'w', encoding='utf-8') as f: 202 | json.dump(results, f, indent=4) 203 | 204 | 205 | if __name__ == '__main__': 206 | 207 | parser = argparse.ArgumentParser(description='Get some run values.') 208 | parser.add_argument('--test', type=int, default=0, help='Run in test mode') 209 | parser.add_argument('--path', type=str, default="mslr_data", help='Path to data') 210 | 211 | args = parser.parse_args() 212 | 213 | os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 214 | 215 | if not os.path.exists("output"): 216 | os.makedirs("output") 217 | 218 | test = False 219 | if args.test == 1: 220 | test = True 221 | 222 | output_path = 'mslr_data' 223 | results_path = output_path + "/results" 224 | data_path = args.path 225 | 226 | if not os.path.exists(output_path): 227 | os.makedirs(output_path) 228 | if not os.path.exists(output_path + "/data"): 229 | os.makedirs(output_path + "/data") 230 | if not os.path.exists(results_path): 231 | os.makedirs(results_path) 232 | 233 | for n_train in np.logspace(2, 6, num=10): 234 | test_rankers(n_train=int(n_train), overwrite=False, test=test) 235 | if test: break 236 | -------------------------------------------------------------------------------- /fig5/training_size.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tensorflow as tf 3 | 4 | from Rankers.models.DirectRanker import DirectRanker 5 | from Rankers.models.RankNet import RankNet 6 | from Rankers.models.ListNet import ListNet 7 | from Rankers.helpers import nDCG_cls 8 | from Rankers.models.LambdaRank import LambdaRank 9 | 10 | import numpy as np 11 | from multiprocessing import Pool 12 | from sklearn.preprocessing import QuantileTransformer 13 | from sklearn.utils import shuffle 14 | from datetime import datetime 15 | from tqdm import tqdm 16 | import argparse 17 | from time import time 18 | 19 | import json 20 | import random 21 | import argparse 22 | 23 | 24 | def gen_set(n_relevance=5, n_train=100000, n_test=10000, overwrite=False, test=False): 25 | 26 | for fold in range(5): 27 | if os.path.isfile(data_path + "/train_{}_{}_{}".format(fold, n_train, n_test)): 28 | if overwrite: 29 | os.remove(data_path + "/train_{}_{}_{}".format(fold, n_train, n_test)) 30 | else: 31 | print("skip n_train file {}".format(n_train)) 32 | return 0 33 | n_feats = 70 34 | if test: 35 | n_train = 100 36 | n_test = 10 37 | else: 38 | n_train = n_train 39 | n_test = n_test 40 | n_relevance = 5 41 | max_qid_train = 10 42 | max_qid_test = 20 43 | means = np.random.rand(n_relevance, n_feats) * 100 44 | sigmas = np.random.rand(n_relevance, n_feats) * 150 + 50 45 | 46 | f = open(data_path + "/train_{}_{}_{}".format(fold, n_train, n_test),"w") 47 | print("Creating traing set...") 48 | for i in tqdm(range(n_train // n_relevance)): 49 | for j in range(n_relevance): 50 | f.write(str(j)) 51 | #f.write("\t" + str(int(random.uniform(0, max_qid_train)))) 52 | for n in np.random.randn(n_feats)*sigmas[j]+means[j]: 53 | f.write("\t"+str(n)) 54 | f.write("\n") 55 | f.close() 56 | 57 | f = open(data_path + "/test_{}_{}_{}".format(fold, n_train, n_test),"w") 58 | print("Creating test set...") 59 | for i in tqdm(range(n_test // n_relevance)): 60 | for j in range(n_relevance): 61 | f.write(str(j)) 62 | #f.write("\t" + str(int(random.uniform(0, max_qid_test)))) 63 | for n in np.random.randn(n_feats)*sigmas[j]+means[j]: 64 | f.write("\t"+str(n)) 65 | f.write("\n") 66 | f.close() 67 | 68 | def test_rankers(n_relevance=5, n_train=100000, n_test=10000, overwrite=False, add_model=None, test=False): 69 | 70 | folds = 5 71 | if test: 72 | n_train = 100 73 | n_test = 10 74 | folds = 1 75 | 76 | if add_model == None: 77 | results = {"n_train": n_train, "n_test": n_test} 78 | ndcg_dr_l = [] 79 | ndcg_lnet_l = [] 80 | ndcg_rnet_l = [] 81 | ndcg_lrank_l = [] 82 | for fold in range(folds): 83 | if os.path.isfile(results_path + "/results_{}_{}.json".format(n_train, n_test)): 84 | if not (add_model == None): 85 | results = json.load(open(results_path + "/results_{}_{}.json".format(n_train, n_test))) 86 | elif overwrite: 87 | os.remove(results_path + "/results_{}_{}.json".format(n_train, n_test)) 88 | else: 89 | print("skip training for n_relevance {}".format(n_train)) 90 | return 0 91 | 92 | size_min = 50 93 | size_max = 150 94 | N = 50 95 | x = [] 96 | y = [] 97 | y_dr = [] 98 | x_test = [] 99 | y_test = [] 100 | 101 | # read data 102 | print("Reading data") 103 | start = time() 104 | for line in open(data_path + "/train_{}_{}_{}".format(fold, n_train, n_test)): 105 | s = line.split() 106 | if int(s[0]) > (n_relevance-1)/2: 107 | y.append(1) 108 | else: 109 | y.append(0) 110 | y_dr.append(int(s[0])) 111 | x.append(np.zeros(len(s) - 1)) 112 | for i in range(len(s) - 1): 113 | x[-1][i] = float(s[i + 1]) 114 | 115 | for line in open(data_path + "/test_{}_{}_{}".format(fold, n_train, n_test)): 116 | s = line.split() 117 | y_test.append(int(str(s[0]))) 118 | x_test.append(np.zeros(len(s) - 1)) 119 | for i in range(len(s) - 1): 120 | x_test[-1][i] = float(s[i + 1]) 121 | 122 | end = time() 123 | mins = int((end - start) / 60) 124 | secs = end - start - 60 * mins 125 | print("Finished in " + str(mins) + " min " + str(secs) + " s") 126 | 127 | print("Preparing data n_train {} on fold {}".format(n_train, fold)) 128 | x = np.array(x) 129 | y = np.array(y) 130 | y_dr = np.array(y_dr) 131 | x, y, y_dr = shuffle(x, y, y_dr) 132 | x_test = np.array(x_test) 133 | y_test = np.array(y_test) 134 | x_test, y_test = shuffle(x_test, y_test) 135 | 136 | scaler = QuantileTransformer(output_distribution="normal") 137 | x = scaler.fit_transform(x) / 3 138 | x_test = scaler.transform(x_test) / 3 139 | 140 | if test: 141 | epoch = 1 142 | else: 143 | epoch = 10 144 | 145 | dr = DirectRanker( 146 | hidden_layers_dr=[70, 5], 147 | num_features=len(x[0]), 148 | verbose=0, 149 | epoch=epoch, 150 | scale_factor_train_sample=1 151 | ) 152 | 153 | ranknet = RankNet( 154 | hidden_layers_dr=[70, 5], 155 | num_features=len(x[0]), 156 | verbose=0, 157 | epoch=epoch, 158 | scale_factor_train_sample=1 159 | ) 160 | 161 | lnet = ListNet( 162 | hidden_layers_dr=[70, 5, 1], 163 | num_features=len(x[0]), 164 | verbose=0, 165 | epoch=epoch 166 | ) 167 | 168 | lrank = LambdaRank( 169 | hidden_layers_dr=[70, 5], 170 | num_features=len(x[0]), 171 | verbose=2, 172 | epoch=epoch, 173 | scale_factor_train_sample=1 174 | ) 175 | 176 | print("Training") 177 | start = time() 178 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 179 | dr.fit(x, y_dr) 180 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 181 | ranknet.fit(x, y_dr) 182 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 183 | lnet.fit(x, tf.cast(y, tf.float32)) 184 | if not('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 185 | lrank.fit(x, y_dr) 186 | end = time() 187 | mins = int((end - start) / 60) 188 | secs = end - start - 60 * mins 189 | print("Finished in " + str(mins) + " min " + str(secs) + " s") 190 | ndcg_dr_l_cur = [] 191 | ndcg_lnet_l_cur = [] 192 | ndcg_rnet_l_cur = [] 193 | nDCG_lrank_l_cur = [] 194 | for n in range(N): 195 | size = np.random.randint(size_min,size_max) 196 | ind = np.random.randint(0,len(x_test),size) 197 | 198 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 199 | nDCG_dr = nDCG_cls(dr, x_test[ind], y_test[ind], at=20) 200 | print("nDCG@20 DirectRanker {} fold {} n_train {}".format(nDCG_dr, fold, n_train)) 201 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 202 | nDCG_rnet = nDCG_cls(ranknet, x_test[ind], y_test[ind], at=20) 203 | print("nDCG@20 RankNet {} fold {} n_train {}".format(nDCG_rnet, fold, n_train)) 204 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 205 | nDCG_lnet = nDCG_cls(lnet, x_test[ind], y_test[ind], at=20) 206 | print("nDCG@20 ListNet {} fold {} n_train {}".format(nDCG_lnet, fold, n_train)) 207 | if not ('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 208 | nDCG_lrank = nDCG_cls(lrank, x_test[ind], y_test[ind], at=20) 209 | print("nDCG@20 LambdaRank {} fold {} n_train {}".format(nDCG_lrank, fold, n_train)) 210 | 211 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 212 | ndcg_dr_l_cur.append(nDCG_dr) 213 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 214 | ndcg_rnet_l_cur.append(nDCG_rnet) 215 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 216 | ndcg_lnet_l_cur.append(nDCG_lnet) 217 | if not ('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 218 | nDCG_lrank_l_cur.append(nDCG_lrank) 219 | 220 | ndcg_dr_l.append(np.mean(ndcg_dr_l_cur)) 221 | ndcg_lnet_l.append(np.mean(ndcg_lnet_l_cur)) 222 | ndcg_rnet_l.append(np.mean(ndcg_rnet_l_cur)) 223 | ndcg_lrank_l.append(np.mean(nDCG_lrank_l_cur)) 224 | 225 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 226 | results['ndcg_DirectRanker'] = [np.mean(ndcg_dr_l), np.std(ndcg_dr_l)] 227 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 228 | results['ndcg_RankNet'] = [np.mean(ndcg_rnet_l), np.std(ndcg_rnet_l)] 229 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 230 | results['ndcg_ListNet'] = [np.mean(ndcg_lnet_l), np.std(ndcg_lnet_l)] 231 | if not ('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 232 | results['ndcg_LambdaRank'] = [np.mean(ndcg_lrank_l), np.std(ndcg_lrank_l)] 233 | 234 | # hack to be able to plot the results in test mode 235 | if test: 236 | n_test = 10000 237 | 238 | with open(results_path + '/results_{}_{}.json'.format(n_train, n_test), 'w', encoding='utf-8') as f: 239 | json.dump(results, f, indent=4) 240 | 241 | 242 | if __name__ == '__main__': 243 | 244 | parser = argparse.ArgumentParser(description='Get some run values.') 245 | parser.add_argument('--test', type=int, default=0, help='Run in test mode') 246 | 247 | args = parser.parse_args() 248 | 249 | os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 250 | 251 | if not os.path.exists("output"): 252 | os.makedirs("output") 253 | 254 | test = False 255 | if args.test == 1: 256 | test = True 257 | 258 | path = 'synth_data' 259 | data_path = path + "/data" 260 | results_path = path + "/results" 261 | 262 | if not os.path.exists(path): 263 | os.makedirs(path) 264 | if not os.path.exists(data_path): 265 | os.makedirs(data_path) 266 | if not os.path.exists(results_path): 267 | os.makedirs(results_path) 268 | 269 | for n_train in np.logspace(2, 6, num=10): 270 | gen_set(n_train=int(n_train), n_test=10000, overwrite=True, test=test) 271 | test_rankers(n_train=int(n_train), n_test=10000, overwrite=True, test=test) 272 | if test: break 273 | -------------------------------------------------------------------------------- /tableB1/run_time_ana.sh: -------------------------------------------------------------------------------- 1 | ln -s ../Rankers . 2 | 3 | python timeana.py --model DirectRankerV1 --path ../data/MSLR-WEB10K 4 | python timeana.py --model RankNet --path ../data/MSLR-WEB10K 5 | 6 | mkdir .log 7 | 8 | echo Fold1 9 | 10 | (date +%s) > .log/start_time 11 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold1/train.txt -ranker 0 -metric2t NDCG@10 -silent 12 | (date +%s) > .log/cur_time 13 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 14 | 15 | (date +%s) > .log/start_time 16 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold1/train.txt -ranker 1 -metric2t NDCG@10 -silent 17 | (date +%s) > .log/cur_time 18 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 19 | 20 | (date +%s) > .log/start_time 21 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold1/train.txt -ranker 2 -metric2t NDCG@10 -silent 22 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 23 | (date +%s) > .log/cur_time 24 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 25 | 26 | (date +%s) > .log/start_time 27 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold1/train.txt -ranker 3 -metric2t NDCG@10 -silent 28 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 29 | (date +%s) > .log/cur_time 30 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 31 | 32 | (date +%s) > .log/start_time 33 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold1/train.txt -ranker 3 -metric2t NDCG@10 -silent 34 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 35 | (date +%s) > .log/cur_time 36 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 37 | 38 | (date +%s) > .log/start_time 39 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold1/train.txt -ranker 6 -metric2t NDCG@10 -silent 40 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 41 | (date +%s) > .log/cur_time 42 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 43 | 44 | (date +%s) > .log/start_time 45 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold1/train.txt -ranker 7 -metric2t NDCG@10 -silent 46 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 47 | (date +%s) > .log/cur_time 48 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 49 | 50 | echo Fold2 51 | 52 | (date +%s) > .log/start_time 53 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold2/train.txt -ranker 0 -metric2t NDCG@10 -silent 54 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 55 | (date +%s) > .log/cur_time 56 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 57 | 58 | (date +%s) > .log/start_time 59 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold2/train.txt -ranker 1 -metric2t NDCG@10 -silent 60 | (date +%s) > .log/cur_time 61 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 62 | 63 | (date +%s) > .log/start_time 64 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold2/train.txt -ranker 2 -metric2t NDCG@10 -silent 65 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 66 | (date +%s) > .log/cur_time 67 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 68 | 69 | (date +%s) > .log/start_time 70 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold2/train.txt -ranker 3 -metric2t NDCG@10 -silent 71 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 72 | (date +%s) > .log/cur_time 73 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 74 | 75 | (date +%s) > .log/start_time 76 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold2/train.txt -ranker 3 -metric2t NDCG@10 -silent 77 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 78 | (date +%s) > .log/cur_time 79 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 80 | 81 | (date +%s) > .log/start_time 82 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold2/train.txt -ranker 6 -metric2t NDCG@10 -silent 83 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 84 | (date +%s) > .log/cur_time 85 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 86 | 87 | (date +%s) > .log/start_time 88 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold2/train.txt -ranker 7 -metric2t NDCG@10 -silent 89 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 90 | (date +%s) > .log/cur_time 91 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 92 | 93 | echo Fold3 94 | 95 | (date +%s) > .log/start_time 96 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold3/train.txt -ranker 0 -metric2t NDCG@10 -silent 97 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 98 | (date +%s) > .log/cur_time 99 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 100 | 101 | (date +%s) > .log/start_time 102 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold3/train.txt -ranker 1 -metric2t NDCG@10 -silent 103 | (date +%s) > .log/cur_time 104 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 105 | 106 | (date +%s) > .log/start_time 107 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold3/train.txt -ranker 2 -metric2t NDCG@10 -silent 108 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 109 | (date +%s) > .log/cur_time 110 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 111 | 112 | (date +%s) > .log/start_time 113 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold3/train.txt -ranker 3 -metric2t NDCG@10 -silent 114 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 115 | (date +%s) > .log/cur_time 116 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 117 | 118 | (date +%s) > .log/start_time 119 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold3/train.txt -ranker 3 -metric2t NDCG@10 -silent 120 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 121 | (date +%s) > .log/cur_time 122 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 123 | 124 | (date +%s) > .log/start_time 125 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold3/train.txt -ranker 6 -metric2t NDCG@10 -silent 126 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 127 | (date +%s) > .log/cur_time 128 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 129 | 130 | (date +%s) > .log/start_time 131 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold3/train.txt -ranker 7 -metric2t NDCG@10 -silent 132 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 133 | (date +%s) > .log/cur_time 134 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 135 | 136 | echo Fold4 137 | 138 | (date +%s) > .log/start_time 139 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold4/train.txt -ranker 0 -metric2t NDCG@10 -silent 140 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 141 | (date +%s) > .log/cur_time 142 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 143 | 144 | (date +%s) > .log/start_time 145 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold4/train.txt -ranker 1 -metric2t NDCG@10 -silent 146 | (date +%s) > .log/cur_time 147 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 148 | 149 | (date +%s) > .log/start_time 150 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold4/train.txt -ranker 2 -metric2t NDCG@10 -silent 151 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 152 | (date +%s) > .log/cur_time 153 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 154 | 155 | (date +%s) > .log/start_time 156 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold4/train.txt -ranker 3 -metric2t NDCG@10 -silent 157 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 158 | (date +%s) > .log/cur_time 159 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 160 | 161 | (date +%s) > .log/start_time 162 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold4/train.txt -ranker 3 -metric2t NDCG@10 -silent 163 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 164 | (date +%s) > .log/cur_time 165 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 166 | 167 | (date +%s) > .log/start_time 168 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold4/train.txt -ranker 6 -metric2t NDCG@10 -silent 169 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 170 | (date +%s) > .log/cur_time 171 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 172 | 173 | (date +%s) > .log/start_time 174 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold4/train.txt -ranker 7 -metric2t NDCG@10 -silent 175 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 176 | (date +%s) > .log/cur_time 177 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 178 | 179 | echo Fold5 180 | 181 | (date +%s) > .log/start_time 182 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold5/train.txt -ranker 0 -metric2t NDCG@10 -silent 183 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 184 | (date +%s) > .log/cur_time 185 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 186 | 187 | (date +%s) > .log/start_time 188 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold5/train.txt -ranker 1 -metric2t NDCG@10 -silent 189 | (date +%s) > .log/cur_time 190 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 191 | 192 | (date +%s) > .log/start_time 193 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold5/train.txt -ranker 2 -metric2t NDCG@10 -silent 194 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 195 | (date +%s) > .log/cur_time 196 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 197 | 198 | (date +%s) > .log/start_time 199 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold5/train.txt -ranker 3 -metric2t NDCG@10 -silent 200 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 201 | (date +%s) > .log/cur_time 202 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 203 | 204 | (date +%s) > .log/start_time 205 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold5/train.txt -ranker 3 -metric2t NDCG@10 -silent 206 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 207 | (date +%s) > .log/cur_time 208 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 209 | 210 | (date +%s) > .log/start_time 211 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold5/train.txt -ranker 6 -metric2t NDCG@10 -silent 212 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 213 | (date +%s) > .log/cur_time 214 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 215 | 216 | (date +%s) > .log/start_time 217 | java -jar ../RankLib-2.10.jar -train ../data/MSLR-WEB10K/Fold5/train.txt -ranker 7 -metric2t NDCG@10 -silent 218 | ELAPSED_TIME=$(((date +%s) - $START_TIME)) 219 | (date +%s) > .log/cur_time 220 | python -c "print(int(open(\".log/cur_time\").read()) - int(open(\".log/start_time\").read()))" 221 | -------------------------------------------------------------------------------- /fig5/training_label.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from Rankers.models.DirectRanker import DirectRanker 4 | from Rankers.models.RankNet import RankNet 5 | from Rankers.models.ListNet import ListNet 6 | from Rankers.helpers import nDCG_cls 7 | from Rankers.models.LambdaRank import LambdaRank 8 | 9 | import numpy as np 10 | import tensorflow as tf 11 | from sklearn.preprocessing import QuantileTransformer 12 | from sklearn.utils import shuffle 13 | from tqdm import tqdm 14 | from time import time 15 | 16 | import json 17 | import random 18 | import argparse 19 | 20 | 21 | def gen_set(n_relevance=6, n_train=100000, n_test=10000, overwrite=False, test=False): 22 | 23 | for fold in range(5): 24 | if os.path.isfile(data_path + "/train_label_{}_{}_{}".format(fold, n_train, n_test)): 25 | if overwrite: 26 | os.remove(data_path + "/train_label_{}_{}_{}".format(fold, n_train, n_test)) 27 | else: 28 | print("skip n_train file {}".format(n_train)) 29 | return 0 30 | n_feats = 70 31 | if test: 32 | n_train = 100 33 | n_test = 10 34 | else: 35 | n_train = n_train 36 | n_test = n_test 37 | n_relevance = n_relevance 38 | means = np.random.rand(n_relevance, n_feats) * 100 39 | sigmas = np.random.rand(n_relevance, n_feats) * 150 + 50 40 | 41 | f = open(data_path + "/train_label_{}_{}_{}".format(fold, n_train, n_test),"w") 42 | print("Creating traing set...") 43 | for i in tqdm(range(n_train // n_relevance)): 44 | for j in range(n_relevance): 45 | f.write(str(j)) 46 | #f.write("\t" + str(int(random.uniform(0, max_qid_train)))) 47 | for n in np.random.randn(n_feats)*sigmas[j]+means[j]: 48 | f.write("\t"+str(n)) 49 | f.write("\n") 50 | f.close() 51 | 52 | f = open(data_path + "/test_label_{}_{}_{}".format(fold, n_train, n_test),"w") 53 | print("Creating test set...") 54 | for i in tqdm(range(n_test // n_relevance)): 55 | for j in range(n_relevance): 56 | f.write(str(j)) 57 | #f.write("\t" + str(int(random.uniform(0, max_qid_test)))) 58 | for n in np.random.randn(n_feats)*sigmas[j]+means[j]: 59 | f.write("\t"+str(n)) 60 | f.write("\n") 61 | f.close() 62 | 63 | def test_rankers(n_relevance=6, n_train=100000, n_test=10000, overwrite=False, add_model=None, test=False): 64 | 65 | if test: 66 | data_str = "100_10" 67 | folds = 1 68 | else: 69 | data_str = "100000_10000" 70 | folds = 5 71 | 72 | if add_model == None: 73 | results = {"n_train": n_train, "n_test": n_test} 74 | ndcg_dr_l = [] 75 | ndcg_lnet_l = [] 76 | ndcg_rnet_l = [] 77 | ndcg_lrank_l = [] 78 | for fold in range(folds): 79 | if os.path.isfile(results_path + "/results_label_{}_{}.json".format(n_train, n_test)): 80 | if not (add_model == None): 81 | results = json.load(open(results_path + "/results_label_{}_{}.json".format(n_train, n_test))) 82 | elif overwrite: 83 | os.remove(results_path + "/results_label_{}_{}.json".format(n_train, n_test)) 84 | else: 85 | print("skip training for n_relevance {}".format(n_train)) 86 | return 0 87 | 88 | size_min = 50 89 | size_max = 150 90 | N = 50 91 | x = [] 92 | y = [] 93 | x_test = [] 94 | y_test = [] 95 | 96 | # read data 97 | print("Reading data") 98 | start = time() 99 | if os.path.isfile(data_path + "/x_train_label_{}.npy".format(fold)): 100 | x = np.load(data_path + "/x_train_label_{}.npy".format(fold)) 101 | y = np.load(data_path + "/y_train_label_{}.npy".format(fold)) 102 | x_test = np.load(data_path + "/x_test_label_{}.npy".format(fold)) 103 | y_test = np.load(data_path + "/y_test_label_{}.npy".format(fold)) 104 | else: 105 | for line in open(data_path + "/train_label_{}_{}".format(fold, data_str)): 106 | s = line.split() 107 | y.append(int(str(s[0]))) 108 | x.append(np.zeros(len(s) - 1)) 109 | for i in range(len(s) - 1): 110 | x[-1][i] = float(s[i + 1]) 111 | 112 | for line in open(data_path + "/test_label_{}_{}".format(fold, data_str)): 113 | s = line.split() 114 | y_test.append(int(str(s[0]))) 115 | x_test.append(np.zeros(len(s) - 1)) 116 | for i in range(len(s) - 1): 117 | x_test[-1][i] = float(s[i + 1]) 118 | 119 | end = time() 120 | mins = int((end - start) / 60) 121 | secs = end - start - 60 * mins 122 | print("Finished in " + str(mins) + " min " + str(secs) + " s") 123 | x = np.array(x) 124 | y = np.array(y) 125 | x, y = shuffle(x, y) 126 | x_test = np.array(x_test) 127 | y_test = np.array(y_test) 128 | x_test, y_test = shuffle(x_test, y_test) 129 | np.save(data_path + "/x_train_label_{}.npy".format(fold), x) 130 | np.save(data_path + "/y_train_label_{}.npy".format(fold), y) 131 | np.save(data_path + "/x_test_label_{}.npy".format(fold), x_test) 132 | np.save(data_path + "/y_test_label_{}.npy".format(fold), y_test) 133 | 134 | print("Preparing data n_train {} on fold {}".format(n_train, fold)) 135 | # draw fartion 136 | if n_train >= 1e6: return 137 | # TODO: make me dynamic 138 | x5 = x[y==5] 139 | x4 = x[y==4] 140 | x3 = x[y==3] 141 | x0 = x[y<3] 142 | y0 = y[y<3] 143 | if n_train < 50000: 144 | x5 = np.array(random.sample(x5.tolist(), int(n_train/3))) 145 | x4 = np.array(random.sample(x4.tolist(), int(n_train/3))) 146 | x3 = np.array(random.sample(x3.tolist(), int(n_train/3))) 147 | x = np.concatenate([x0, x3, x4, x5]) 148 | y = np.concatenate( 149 | [ 150 | y0, 151 | np.ones(len(x3))*3, 152 | np.ones(len(x4))*4, 153 | np.ones(len(x5))*5 154 | ]).astype(int) 155 | x, y = shuffle(x, y) 156 | y_listnet = [] 157 | for yi in y: 158 | if yi > (n_relevance-1)/2: 159 | y_listnet.append(1) 160 | else: 161 | y_listnet.append(0) 162 | 163 | scaler = QuantileTransformer(output_distribution="normal") 164 | x = scaler.fit_transform(x) / 3 165 | x_test = scaler.transform(x_test) / 3 166 | 167 | if test: 168 | epoch = 1 169 | else: 170 | epoch = 10 171 | 172 | dr = DirectRanker( 173 | hidden_layers_dr=[70, 5], 174 | num_features=len(x[0]), 175 | verbose=0, 176 | epoch=epoch, 177 | scale_factor_train_sample=1 178 | ) 179 | 180 | ranknet = RankNet( 181 | hidden_layers_dr=[70, 5], 182 | num_features=len(x[0]), 183 | verbose=0, 184 | epoch=epoch, 185 | scale_factor_train_sample=1 186 | ) 187 | 188 | lnet = ListNet( 189 | hidden_layers_dr=[70, 5, 1], 190 | num_features=len(x[0]), 191 | verbose=0, 192 | epoch=epoch 193 | ) 194 | 195 | lrank = LambdaRank( 196 | hidden_layers_dr=[70, 5], 197 | num_features=len(x[0]), 198 | verbose=2, 199 | epoch=epoch, 200 | scale_factor_train_sample=1 201 | ) 202 | 203 | print("Training") 204 | start = time() 205 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 206 | dr.fit(x, y) 207 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 208 | ranknet.fit(x, y) 209 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 210 | lnet.fit(x, tf.cast(y_listnet, tf.float32)) 211 | if not('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 212 | lrank.fit(x, y) 213 | end = time() 214 | mins = int((end - start) / 60) 215 | secs = end - start - 60 * mins 216 | print("Finished in " + str(mins) + " min " + str(secs) + " s") 217 | ndcg_dr_l_cur = [] 218 | ndcg_lnet_l_cur = [] 219 | ndcg_rnet_l_cur = [] 220 | nDCG_lrank_l_cur = [] 221 | for n in range(N): 222 | size = np.random.randint(size_min,size_max) 223 | ind = np.random.randint(0,len(x_test),size) 224 | 225 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 226 | nDCG_dr = nDCG_cls(dr, x_test[ind], y_test[ind], at=20) 227 | print("nDCG@20 DirectRanker {} fold {} n_train {}".format(nDCG_dr, fold, n_train)) 228 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 229 | nDCG_rnet = nDCG_cls(ranknet, x_test[ind], y_test[ind], at=20) 230 | print("nDCG@20 RankNet {} fold {} n_train {}".format(nDCG_rnet, fold, n_train)) 231 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 232 | nDCG_lnet = nDCG_cls(lnet, x_test[ind], y_test[ind], at=20) 233 | print("nDCG@20 ListNet {} fold {} n_train {}".format(nDCG_lnet, fold, n_train)) 234 | if not ('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 235 | nDCG_lrank = nDCG_cls(lrank, x_test[ind], y_test[ind], at=20) 236 | print("nDCG@20 LambdaRank {} fold {} n_train {}".format(nDCG_lrank, fold, n_train)) 237 | 238 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 239 | ndcg_dr_l_cur.append(nDCG_dr) 240 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 241 | ndcg_rnet_l_cur.append(nDCG_rnet) 242 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 243 | ndcg_lnet_l_cur.append(nDCG_lnet) 244 | if not ('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 245 | nDCG_lrank_l_cur.append(nDCG_lrank) 246 | 247 | ndcg_dr_l.append(np.mean(ndcg_dr_l_cur)) 248 | ndcg_lnet_l.append(np.mean(ndcg_lnet_l_cur)) 249 | ndcg_rnet_l.append(np.mean(ndcg_rnet_l_cur)) 250 | ndcg_lrank_l.append(np.mean(nDCG_lrank_l_cur)) 251 | 252 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 253 | results['ndcg_DirectRanker'] = [np.mean(ndcg_dr_l), np.std(ndcg_dr_l)] 254 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 255 | results['ndcg_RankNet'] = [np.mean(ndcg_rnet_l), np.std(ndcg_rnet_l)] 256 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 257 | results['ndcg_ListNet'] = [np.mean(ndcg_lnet_l), np.std(ndcg_lnet_l)] 258 | if not ('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 259 | results['ndcg_LambdaRank'] = [np.mean(ndcg_lrank_l), np.std(ndcg_lrank_l)] 260 | 261 | with open(results_path + '/results_label_{}_{}.json'.format(n_train, n_test), 'w', encoding='utf-8') as f: 262 | json.dump(results, f, indent=4) 263 | 264 | 265 | if __name__ == '__main__': 266 | 267 | parser = argparse.ArgumentParser(description='Get some run values.') 268 | parser.add_argument('--test', type=int, default=0, help='Run in test mode') 269 | 270 | args = parser.parse_args() 271 | 272 | os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 273 | 274 | if not os.path.exists("output"): 275 | os.makedirs("output") 276 | 277 | test = False 278 | if args.test == 1: 279 | test = True 280 | 281 | path = 'synth_data' 282 | data_path = path + "/data" 283 | results_path = path + "/results" 284 | 285 | if not os.path.exists(path): 286 | os.makedirs(path) 287 | if not os.path.exists(data_path): 288 | os.makedirs(data_path) 289 | if not os.path.exists(results_path): 290 | os.makedirs(results_path) 291 | 292 | train_list = np.logspace(2, 6, num=10) 293 | train_list = np.concatenate([np.logspace(1, 2, num=3)[:2], train_list]) 294 | for n_train in train_list: 295 | gen_set(overwrite=False, test=test) 296 | test_rankers(n_train=int(n_train), n_test=10000, overwrite=True, test=test) 297 | if test: break 298 | -------------------------------------------------------------------------------- /Rankers/models/LambdaRank.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from sklearn.base import BaseEstimator 3 | import numpy as np 4 | import gc 5 | from ..helpers import ndcg_at_k 6 | 7 | 8 | class LambdaRank(BaseEstimator): 9 | 10 | def __init__(self, 11 | # DirectRanker HPs 12 | hidden_layers_dr=[64, 20], 13 | feature_activation_dr='sigmoid', 14 | ranking_activation_dr='sigmoid', 15 | feature_bias_dr=True, 16 | kernel_initializer_dr=tf.random_normal_initializer, 17 | kernel_regularizer_dr=0.0, 18 | drop_out=0, 19 | # Common HPs 20 | scale_factor_train_sample=1, 21 | batch_size=200, 22 | loss=tf.keras.losses.BinaryCrossentropy, 23 | learning_rate=0.001, 24 | learning_rate_decay_rate=1, 25 | learning_rate_decay_steps=1000, 26 | optimizer=tf.keras.optimizers.Adam, # 'Nadam' 'SGD' 27 | epoch=10, 28 | # other variables 29 | verbose=0, 30 | validation_size=0.0, 31 | num_features=0, 32 | name="DirectRanker", 33 | dtype=tf.float32, 34 | print_summary=False, 35 | grad_clip=True, 36 | clip_value=1.0, 37 | sigma=1 38 | ): 39 | 40 | # DirectRanker HPs 41 | self.hidden_layers_dr = hidden_layers_dr 42 | self.feature_activation_dr = feature_activation_dr 43 | self.ranking_activation_dr = ranking_activation_dr 44 | self.feature_bias_dr = feature_bias_dr 45 | self.kernel_initializer_dr = kernel_initializer_dr 46 | self.kernel_regularizer_dr = kernel_regularizer_dr 47 | self.drop_out = drop_out 48 | # Common HPs 49 | self.scale_factor_train_sample = scale_factor_train_sample 50 | self.batch_size = batch_size 51 | self.loss = loss 52 | self.learning_rate = learning_rate 53 | self.learning_rate_decay_rate = learning_rate_decay_rate 54 | self.learning_rate_decay_steps = learning_rate_decay_steps 55 | self.optimizer = optimizer 56 | self.epoch = epoch 57 | self.steps_per_epoch = None 58 | # other variables 59 | self.verbose = verbose 60 | self.validation_size = validation_size 61 | self.num_features = num_features 62 | self.name = name 63 | self.dtype = dtype 64 | self.print_summary = print_summary 65 | self.grad_clip = grad_clip 66 | self.sigma = sigma 67 | self.clip_value = clip_value 68 | 69 | def _build_model(self): 70 | """ 71 | TODO 72 | """ 73 | # Placeholders for the inputs 74 | self.x0 = tf.keras.layers.Input( 75 | shape=self.num_features, 76 | dtype=self.dtype, 77 | name="x0" 78 | ) 79 | 80 | self.x1 = tf.keras.layers.Input( 81 | shape=self.num_features, 82 | dtype=self.dtype, 83 | name="x1" 84 | ) 85 | 86 | input_layer = tf.keras.layers.Input( 87 | shape=self.num_features, 88 | dtype=self.dtype, 89 | name="input" 90 | ) 91 | 92 | nn = tf.keras.layers.Dense( 93 | units=self.hidden_layers_dr[0], 94 | activation=self.feature_activation_dr, 95 | use_bias=self.feature_bias_dr, 96 | kernel_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 97 | bias_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 98 | activity_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 99 | name="nn_hidden_0" 100 | )(input_layer) 101 | 102 | if self.drop_out > 0: 103 | nn = tf.keras.layers.Dropout(self.drop_out)(nn) 104 | 105 | for i in range(1, len(self.hidden_layers_dr)): 106 | nn = tf.keras.layers.Dense( 107 | units=self.hidden_layers_dr[i], 108 | activation=self.feature_activation_dr, 109 | use_bias=self.feature_bias_dr, 110 | kernel_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 111 | bias_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 112 | activity_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 113 | name="nn_hidden_" + str(i) 114 | )(nn) 115 | 116 | if self.drop_out > 0: 117 | nn = tf.keras.layers.Dropout(self.drop_out)(nn) 118 | 119 | feature_part = tf.keras.models.Model(input_layer, nn, name='feature_part') 120 | 121 | if self.print_summary: 122 | feature_part.summary() 123 | 124 | nn0 = feature_part(self.x0) 125 | nn1 = feature_part(self.x1) 126 | 127 | subtracted = tf.keras.layers.Subtract()([nn0, nn1]) 128 | 129 | out = tf.keras.layers.Dense( 130 | units=1, 131 | use_bias=False, 132 | kernel_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 133 | activity_regularizer=tf.keras.regularizers.l2(self.kernel_regularizer_dr), 134 | name="ranking_part", 135 | activation=tf.nn.sigmoid 136 | )(subtracted) 137 | 138 | self.model = tf.keras.models.Model( 139 | inputs=[self.x0, self.x1], 140 | outputs=out, 141 | name='LambdaRank' 142 | ) 143 | 144 | lr_schedule = tf.keras.optimizers.schedules.InverseTimeDecay( 145 | self.learning_rate, 146 | decay_steps=self.learning_rate_decay_steps, 147 | decay_rate=self.learning_rate_decay_rate, 148 | staircase=False 149 | ) 150 | 151 | self.optimizer = self.optimizer(lr_schedule) 152 | 153 | self.model.compile( 154 | optimizer=self.optimizer, 155 | loss=self.loss(), 156 | metrics=['acc'] 157 | ) 158 | 159 | if self.print_summary: 160 | self.model.summary() 161 | 162 | def fit(self, x, y, **fit_params): 163 | """ 164 | TODO 165 | """ 166 | tf.keras.backend.clear_session() 167 | 168 | 169 | self._build_model() 170 | self.steps_per_epoch = np.ceil(len(x) / self.batch_size) 171 | y = y.reshape(-1, 1) 172 | 173 | for i in range(self.epoch): 174 | idx0 = np.random.randint(0, len(x), self.scale_factor_train_sample*len(x)) 175 | idx1 = np.random.randint(0, len(x), self.scale_factor_train_sample*len(x)) 176 | 177 | x0_cur = [] 178 | x1_cur = [] 179 | y_cur = [] 180 | for i0, i1 in zip(idx0, idx1): 181 | if y[i0] - y[i1] != 0: 182 | if y[i0] - y[i1] > 0: y_cur.append([1]) 183 | if y[i0] - y[i1] < 0: y_cur.append([-1]) 184 | x0_cur.append(x[i0]) 185 | x1_cur.append(x[i1]) 186 | x0_cur = np.array(x0_cur).astype(np.float32) 187 | x1_cur = np.array(x1_cur).astype(np.float32) 188 | y_cur = np.array(y_cur).astype(np.float32) 189 | 190 | for j in range(int(self.steps_per_epoch)): 191 | if len(x0_cur) == 0: continue 192 | idx = np.random.randint(0, len(x0_cur), self.batch_size) 193 | step, train_loss, score = self._train_step((x0_cur[idx], x1_cur[idx]), y_cur[idx]) 194 | if step % 100 == 0: 195 | ndcg20, ndcg500 = self._get_ndcg(y, score) 196 | 197 | @tf.function 198 | def _train_step(self, x, y): 199 | with tf.GradientTape(persistent=True) as tape: 200 | #tape.watch(self.x0) 201 | #tape.watch(self.x1) 202 | pred_score = self.get_scores(x) 203 | loss = tf.reduce_mean(self._ranknet_loss(pred_score, y)) 204 | lambdas = self._get_lambdas(pred_score, y) 205 | pred_score = tf.reshape(pred_score, [-1]) 206 | 207 | with tf.name_scope("gradients"): 208 | gradients = [self._get_lambda_scaled_derivative(tape, pred_score, Wk, lambdas) 209 | for Wk in self.model.trainable_variables] 210 | if self.grad_clip: 211 | gradients = [(tf.clip_by_value(grad, -self.clip_value, self.clip_value)) 212 | for grad in gradients] 213 | self.optimizer.apply_gradients(zip(gradients, self.model.trainable_variables)) 214 | step = self.optimizer.iterations 215 | 216 | return step, loss, pred_score 217 | 218 | def _ranknet_loss(self, pred_scores, y): 219 | return tf.keras.backend.binary_crossentropy(y, pred_scores) 220 | 221 | def _get_lambda_scaled_derivative(self, tape, pred_score, Wk, lambdas): 222 | dsi_dWk = tape.jacobian(pred_score, Wk) # ∂si/∂wk 223 | dsi_dWk_minus_dsj_dWk = tf.expand_dims(dsi_dWk, 1) - tf.expand_dims(dsi_dWk, 0) # ∂si/∂wk−∂sj/∂wk 224 | shape = tf.concat([tf.shape(lambdas), 225 | tf.ones([tf.rank(dsi_dWk_minus_dsj_dWk) - tf.rank(lambdas)], 226 | dtype=tf.int32)], axis=0) 227 | # (1/2(1−Sij)−1/1+eσ(si−sj))(∂si/∂wk−∂sj/∂wk) 228 | grad = tf.reshape(lambdas, shape) * dsi_dWk_minus_dsj_dWk 229 | grad = tf.reduce_mean(grad, axis=[0, 1]) 230 | return grad 231 | 232 | def _get_lambdas(self, pred_score, labels): 233 | """https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/MSR-TR-2010-82.pdf 234 | As explained in equation 3 235 | (1/2(1-Sij)-1/1+e\sigma(si-sj))""" 236 | with tf.name_scope("lambdas"): 237 | batch_size = tf.shape(labels)[0] 238 | 239 | index = tf.reshape(tf.range(1.0, tf.cast(batch_size, 240 | dtype=tf.float32) + 1), 241 | tf.shape(labels)) 242 | 243 | sorted_labels = tf.sort(labels, 244 | direction="DESCENDING", 245 | axis=0) 246 | 247 | diff_matrix = labels - tf.transpose(labels) 248 | label_diff_matrix = tf.maximum(tf.minimum(1., diff_matrix), -1.) 249 | pred_diff_matrix = pred_score - tf.transpose(pred_score) 250 | lambdas = self.sigma * ((1 / 2) * (1 - label_diff_matrix) - 251 | tf.nn.sigmoid(-self.sigma * pred_diff_matrix)) 252 | 253 | with tf.name_scope("ndcg"): 254 | cg_discount = tf.math.log(1.0 + index) 255 | rel = 2 ** labels - 1 256 | sorted_rel = 2 ** sorted_labels - 1 257 | dcg_m = rel / cg_discount 258 | dcg = tf.reduce_sum(dcg_m) 259 | 260 | stale_ij = tf.tile(dcg_m, [1, batch_size]) 261 | new_ij = rel / tf.transpose(cg_discount) 262 | stale_ji = tf.transpose(stale_ij) 263 | new_ji = tf.transpose(new_ij) 264 | dcg_new = dcg - stale_ij + new_ij - stale_ji + new_ji 265 | dcg_max = tf.reduce_sum(sorted_rel / cg_discount) 266 | ndcg_delta = tf.math.abs(dcg_new - dcg) / dcg_max 267 | 268 | 269 | lambdas = lambdas * ndcg_delta 270 | 271 | return lambdas 272 | 273 | def predict_proba(self, features): 274 | """ 275 | TODO 276 | """ 277 | if len(features.shape) == 1: 278 | features = [features] 279 | 280 | res = self.model.predict([features, np.zeros(np.shape(features))], batch_size=self.batch_size, 281 | verbose=self.verbose) 282 | 283 | return res 284 | 285 | def get_scores(self, x_pair): 286 | scores = tf.nn.sigmoid(self.model(x_pair, training=False)) 287 | return scores 288 | 289 | def call(self, inputs, training=False, threshold=0.5): 290 | res = self.model(inputs) 291 | if training: 292 | return res 293 | else: 294 | res = self.model(inputs) 295 | return [1 if r > threshold else 0 for r in res] 296 | 297 | @staticmethod 298 | def _get_ndcg(target, pred_score): 299 | # print(tf.shape(pred_score)) 300 | target = tf.reshape(target, [-1]) 301 | 302 | # print(tf.shape(target)) 303 | zpd = list(zip(target.numpy(), pred_score.numpy())) 304 | zpd.sort(key=lambda x: x[1], reverse=True) 305 | pred_rank, _ = list(zip(*zpd)) 306 | 307 | test_ndcg_5 = ndcg_at_k(list(pred_rank), 5) 308 | test_ndcg_20 = ndcg_at_k(list(pred_rank), 20) 309 | 310 | return test_ndcg_5, test_ndcg_20 311 | -------------------------------------------------------------------------------- /fig5/training_label_mslr.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tensorflow as tf 3 | 4 | from Rankers.models.DirectRanker import DirectRanker 5 | from Rankers.models.RankNet import RankNet 6 | from Rankers.models.ListNet import ListNet 7 | from Rankers.helpers import nDCG_cls 8 | from Rankers.models.LambdaRank import LambdaRank 9 | 10 | import numpy as np 11 | from multiprocessing import Pool 12 | from sklearn.preprocessing import QuantileTransformer 13 | from sklearn.utils import shuffle 14 | from datetime import datetime 15 | from tqdm import tqdm 16 | import argparse 17 | from time import time 18 | 19 | import json 20 | import random 21 | 22 | 23 | def readData(path=None, binary=True, at=10): 24 | x = [] 25 | y = [] 26 | q = [] 27 | for line in open(path): 28 | s = line.split() 29 | if binary: 30 | if int(s[0]) > 1.5: 31 | y.append(1) 32 | else: 33 | y.append(0) 34 | else: 35 | y.append(int(s[0])) 36 | 37 | q.append(int(s[1].split(":")[1])) 38 | 39 | x.append(np.zeros(136)) 40 | for i in range(136): 41 | x[-1][i] = float(s[i + 2].split(":")[1]) 42 | 43 | x = np.array(x) 44 | y = np.array(y) 45 | q = np.array(q) 46 | 47 | xt = [] 48 | yt = [] 49 | qt = [] 50 | 51 | for qid in np.unique(q): 52 | if len(y[q == qid]) < at: continue 53 | xt.extend(x[q == qid].tolist()) 54 | yt.extend(y[q == qid].tolist()) 55 | qt.extend(q[q == qid].tolist()) 56 | 57 | return np.array(xt), np.array(yt), np.array(qt) 58 | 59 | def test_rankers(n_train=100000, overwrite=False, add_model=None, test=False): 60 | 61 | folds = 5 62 | if test: 63 | folds = 1 64 | 65 | if add_model == None: 66 | results = {"n_train": n_train} 67 | ndcg_dr_l = [] 68 | ndcg_lnet_l = [] 69 | ndcg_rnet_l = [] 70 | ndcg_lrank_l = [] 71 | for fold in range(folds): 72 | if os.path.isfile(results_path + "/results_label_{}.json".format(n_train)): 73 | if not (add_model == None): 74 | results = json.load(open(results_path + "/results_label_{}.json".format(n_train))) 75 | elif overwrite: 76 | os.remove(results_path + "/results_label_{}.json".format(n_train)) 77 | else: 78 | print("skip training for n_relevance {}".format(n_train)) 79 | return 0 80 | 81 | size_min = 50 82 | size_max = 150 83 | N = 50 84 | x = [] 85 | y = [] 86 | x_test = [] 87 | y_test = [] 88 | 89 | # read data 90 | print("Reading data") 91 | start = time() 92 | if os.path.isfile(output_path + "/data/x_train_label_fold{}.npy".format(fold+1)): 93 | x_train = np.load(output_path + "/data/x_train_label_fold{}.npy".format(fold+1)) 94 | y_train = np.load(output_path + "/data/y_train_label_fold{}.npy".format(fold+1)) 95 | 96 | x_test = np.load(output_path + "/data/x_test_label_fold{}.npy".format(fold+1)) 97 | y_test = np.load(output_path + "/data/y_test_label_fold{}.npy".format(fold+1)) 98 | q_test = np.load(output_path + "/data/q_test_label_fold{}.npy".format(fold+1)) 99 | else: 100 | x_train, y_train, _ = readData(path=data_path + "/Fold{}/train.txt".format(fold+1), binary=False) 101 | x_test, y_test, q_test = readData(path=data_path + "/Fold{}/test.txt".format(fold+1), binary=False) 102 | 103 | np.save(output_path + "/data/x_train_label_fold{}".format(fold+1), x_train) 104 | np.save(output_path + "/data/y_train_label_fold{}".format(fold+1), y_train) 105 | 106 | np.save(output_path + "/data/x_test_label_fold{}".format(fold+1), x_test) 107 | np.save(output_path + "/data/y_test_label_fold{}".format(fold+1), y_test) 108 | np.save(output_path + "/data/q_test_label_fold{}".format(fold+1), q_test) 109 | end = time() 110 | mins = int((end - start) / 60) 111 | secs = end - start - 60 * mins 112 | print("Finished in " + str(mins) + " min " + str(secs) + " s") 113 | 114 | print("Preparing data n_train {} on fold {}".format(n_train, fold)) 115 | # draw fartion 116 | if n_train >= 2e6: return 117 | # TODO: make me dynamic 118 | x4 = x_train[y_train==4] 119 | x3 = x_train[y_train==3] 120 | x2 = x_train[y_train==2] 121 | x0 = x_train[y_train<2] 122 | y0 = y_train[y_train<2] 123 | total = len(y_train[y_train>=2]) 124 | if n_train < 50000: 125 | x4 = np.array(random.sample(x4.tolist(), int(n_train*len(x4)/total))) 126 | x3 = np.array(random.sample(x3.tolist(), int(n_train*len(x3)/total))) 127 | x2 = np.array(random.sample(x2.tolist(), int(n_train*len(x2)/total))) 128 | x_list = [] 129 | y_list = [] 130 | for idx, xi in enumerate([x0, x2, x3, x4]): 131 | if len(xi) > 0: 132 | x_list.append(xi) 133 | y_list.append(np.ones(len(xi))*idx) 134 | x = np.concatenate(x_list) 135 | y = np.concatenate(y_list).astype(int) 136 | x, y = shuffle(x, y) 137 | y_listnet = [] 138 | for yi in y: 139 | if yi > 1.5: 140 | y_listnet.append(1) 141 | else: 142 | y_listnet.append(0) 143 | y_listnet = np.array(y_listnet) 144 | 145 | scaler = QuantileTransformer(output_distribution="normal") 146 | x = scaler.fit_transform(x) / 3 147 | x_test = scaler.transform(x_test) / 3 148 | 149 | if test: 150 | epoch = 1 151 | else: 152 | epoch = 10 153 | 154 | dr = DirectRanker( 155 | hidden_layers_dr=[70, 5], 156 | num_features=len(x[0]), 157 | verbose=0, 158 | epoch=epoch, 159 | scale_factor_train_sample=1 160 | ) 161 | 162 | ranknet = RankNet( 163 | hidden_layers_dr=[70, 5], 164 | num_features=len(x[0]), 165 | verbose=0, 166 | epoch=epoch, 167 | scale_factor_train_sample=1 168 | ) 169 | 170 | lnet = ListNet( 171 | hidden_layers_dr=[70, 5, 1], 172 | num_features=len(x[0]), 173 | verbose=0, 174 | optimizer=tf.keras.optimizers.SGD, 175 | epoch=epoch 176 | ) 177 | 178 | lrank = LambdaRank( 179 | hidden_layers_dr=[70, 5], 180 | num_features=len(x[0]), 181 | verbose=2, 182 | epoch=epoch, 183 | scale_factor_train_sample=1 184 | ) 185 | 186 | 187 | print("Training") 188 | start = time() 189 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 190 | dr.fit(x, y_listnet) 191 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 192 | ranknet.fit(x, y_listnet) 193 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 194 | lnet.fit(x, tf.cast(y_listnet, tf.float32)) 195 | if not('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 196 | lrank.fit(x, y_listnet) 197 | end = time() 198 | mins = int((end - start) / 60) 199 | secs = end - start - 60 * mins 200 | print("Finished in " + str(mins) + " min " + str(secs) + " s") 201 | ndcg_dr_l_cur = [] 202 | ndcg_lnet_l_cur = [] 203 | ndcg_rnet_l_cur = [] 204 | nDCG_lrank_l_cur = [] 205 | 206 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 207 | dr_prediction = dr.predict_proba(x_test) 208 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 209 | ranknet_prediction = ranknet.predict_proba(x_test) 210 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 211 | lnet_prediction = np.array(lnet.predict_proba(x_test)) 212 | if not ('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 213 | lrank_prediction = lrank.predict_proba(x_test) 214 | 215 | for n in np.unique(q_test): 216 | xt = x_test[q_test == n] 217 | yt = y_test[q_test == n] 218 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 219 | nDCG_dr = nDCG_cls(dr_prediction[q_test == n], xt, yt, at=10, prediction=True) 220 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 221 | nDCG_rnet = nDCG_cls(ranknet_prediction[q_test == n], xt, yt, at=10, prediction=True) 222 | #print("nDCG@10 RankNet {} fold {} n_train {}".format(nDCG_rnet, fold, n_train)) 223 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 224 | nDCG_lnet = nDCG_cls(lnet_prediction[q_test == n], xt, yt, at=10, prediction=True) 225 | #print("nDCG@10 ListNet {} fold {} n_train {}".format(nDCG_lnet, fold, n_train)) 226 | if not ('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 227 | nDCG_lrank = nDCG_cls(lrank_prediction[q_test == n], xt, yt, at=10, prediction=True) 228 | #print("nDCG@10 LambdaRank {} fold {} n_train {}".format(nDCG_lrank, fold, n_train)) 229 | 230 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 231 | if str(nDCG_dr) != 'nan': 232 | ndcg_dr_l_cur.append(nDCG_dr) 233 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 234 | if str(nDCG_rnet) != 'nan': 235 | ndcg_rnet_l_cur.append(nDCG_rnet) 236 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 237 | if str(nDCG_lnet) != 'nan': 238 | ndcg_lnet_l_cur.append(nDCG_lnet) 239 | if not ('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 240 | if str(nDCG_lrank) != 'nan': 241 | nDCG_lrank_l_cur.append(nDCG_lrank) 242 | 243 | ndcg_dr_l.append(np.mean(ndcg_dr_l_cur)) 244 | ndcg_lnet_l.append(np.mean(ndcg_lnet_l_cur)) 245 | ndcg_rnet_l.append(np.mean(ndcg_rnet_l_cur)) 246 | ndcg_lrank_l.append(np.mean(nDCG_lrank_l_cur)) 247 | 248 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 249 | print("nDCG@10 DirectRanker {} fold {} n_train {}".format(np.mean(ndcg_dr_l_cur), fold, n_train)) 250 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 251 | print("nDCG@10 RankNet {} fold {} n_train {}".format(np.mean(ndcg_rnet_l_cur), fold, n_train)) 252 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 253 | print("nDCG@10 ListNet {} fold {} n_train {}".format(np.mean(ndcg_lnet_l_cur), fold, n_train)) 254 | if not ('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 255 | print("nDCG@10 LambdaRank {} fold {} n_train {}".format(np.mean(nDCG_lrank_l_cur), fold, n_train)) 256 | 257 | if not ('ndcg_DirectRanker' in results) or add_model == 'DirectRanker': 258 | results['ndcg_DirectRanker'] = [np.mean(ndcg_dr_l), np.std(ndcg_dr_l)] 259 | if not ('ndcg_RankNet' in results) or add_model == 'RankNet': 260 | results['ndcg_RankNet'] = [np.mean(ndcg_rnet_l), np.std(ndcg_rnet_l)] 261 | if not ('ndcg_ListNet' in results) or add_model == 'ListNet': 262 | results['ndcg_ListNet'] = [np.mean(ndcg_lnet_l), np.std(ndcg_lnet_l)] 263 | if not ('ndcg_LambdaRank' in results) or add_model == 'LambdaRank': 264 | results['ndcg_LambdaRank'] = [np.mean(ndcg_lrank_l), np.std(ndcg_lrank_l)] 265 | 266 | with open(results_path + '/results_label_{}.json'.format(n_train), 'w', encoding='utf-8') as f: 267 | json.dump(results, f, indent=4) 268 | 269 | 270 | if __name__ == '__main__': 271 | 272 | parser = argparse.ArgumentParser(description='Get some run values.') 273 | parser.add_argument('--test', type=int, default=0, help='Run in test mode') 274 | parser.add_argument('--path', type=str, default="mslr_data", help='Path to data') 275 | 276 | args = parser.parse_args() 277 | 278 | os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 279 | 280 | if not os.path.exists("output"): 281 | os.makedirs("output") 282 | 283 | test = False 284 | if args.test == 1: 285 | test = True 286 | 287 | output_path = 'mslr_data' 288 | results_path = output_path + "/results" 289 | data_path = args.path 290 | 291 | if not os.path.exists(output_path): 292 | os.makedirs(output_path) 293 | if not os.path.exists(output_path + "/data"): 294 | os.makedirs(output_path + "/data") 295 | if not os.path.exists(results_path): 296 | os.makedirs(results_path) 297 | 298 | train_list = np.logspace(2, 6, num=10) 299 | train_list = np.concatenate([np.logspace(1, 2, num=3)[1:2], train_list]) 300 | for n_train in train_list: 301 | test_rankers(n_train=int(n_train), overwrite=True, test=test) 302 | if test: break 303 | -------------------------------------------------------------------------------- /Rankers/helpers.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import sys 4 | import os 5 | import numpy as np 6 | from functools import cmp_to_key, partial 7 | from sklearn.preprocessing import QuantileTransformer 8 | from sklearn import metrics 9 | from sklearn.metrics import average_precision_score 10 | from Rankers.models.DirectRanker import DirectRanker 11 | from tqdm import tqdm 12 | import random 13 | 14 | 15 | colors_points = [ 16 | (57 / 255, 106 / 255, 177 / 255), # blue 17 | (218 / 255, 124 / 255, 48 / 255), # orange 18 | (62 / 255, 150 / 255, 81 / 255), # green 19 | (204 / 255, 37 / 255, 41 / 255), # red 20 | (83 / 255, 81 / 255, 84 / 255), # black 21 | (107 / 255, 76 / 255, 154 / 255), # purple 22 | (146 / 255, 36 / 255, 40 / 255), # wine 23 | (148 / 255, 139 / 255, 61 / 255) # gold 24 | ] 25 | colors_bar = [ 26 | (114 / 255, 147 / 255, 203 / 255), # blue 27 | (225 / 255, 151 / 255, 76 / 255), # orange 28 | (132 / 255, 186 / 255, 91 / 255), # green 29 | (211 / 255, 94 / 255, 96 / 255), # red 30 | (128 / 255, 133 / 255, 133 / 255), # black 31 | (144 / 255, 103 / 255, 167 / 255), # purple 32 | (171 / 255, 104 / 255, 87 / 255), # wine 33 | (204 / 255, 194 / 255, 16 / 255) # gold 34 | ] 35 | 36 | 37 | def gen_set(n_feats=136, n_relevance=5, n_train=100000, n_test=10000, path="test", sigma_label=0.0, test=False): 38 | n_feats = n_feats 39 | if test: 40 | n_train = 100 41 | n_test = 100 42 | else: 43 | n_train = n_train 44 | n_test = n_test 45 | n_relevance = n_relevance 46 | max_qid_train = 10 47 | max_qid_test = 20 48 | means = np.random.rand(n_relevance, n_feats) * 100 49 | sigmas = np.random.rand(n_relevance, n_feats) * 150 + 50 50 | 51 | if not os.path.exists(path): 52 | os.makedirs(path) 53 | 54 | f = open(f"{path}/train", "w") 55 | print("Creating traing set...") 56 | for i in tqdm(range(n_train // n_relevance)): 57 | for j in range(n_relevance): 58 | if sigma_label == 0: 59 | f.write(str(j)) 60 | else: 61 | label = int(random.normalvariate(mu=j, sigma=sigma_label)) 62 | if label < 0: label = 0 63 | if label > 4: label = 4 64 | f.write(str(label)) 65 | f.write(" qid:" + str(int(random.uniform(0, max_qid_train)))) 66 | for idx, n in enumerate(np.random.randn(n_feats) * sigmas[j] + means[j]): 67 | f.write(" " + str(idx + 1) + ":" + str(n)) 68 | f.write("\n") 69 | f.close() 70 | 71 | f = open(f"{path}/test", "w") 72 | print("Creating test set...") 73 | for i in tqdm(range(n_test // n_relevance)): 74 | for j in range(n_relevance): 75 | if sigma_label == 0: 76 | f.write(str(j)) 77 | else: 78 | label = int(random.normalvariate(mu=j, sigma=sigma_label)) 79 | if label < 0: label = 0 80 | if label > 4: label = 4 81 | f.write(str(label)) 82 | f.write(" qid:" + str(int(random.uniform(max_qid_train, max_qid_test)))) 83 | for idx, n in enumerate(np.random.randn(n_feats) * sigmas[j] + means[j]): 84 | f.write(" " + str(idx + 1) + ":" + str(n)) 85 | f.write("\n") 86 | 87 | f.close() 88 | 89 | def auc_cls(estimator, X, y, w, cnn_bdt=False, linear=False, reorder=True, use_weights=True): 90 | if cnn_bdt: 91 | prediction = estimator.predict_proba(X)[:,1] 92 | else: 93 | if linear: 94 | prediction = estimator.predict(X) 95 | else: 96 | prediction = estimator.predict_proba(X) 97 | if use_weights: 98 | fpr, tpr, _ = metrics.roc_curve(y, prediction, sample_weight=w, pos_label=1) 99 | else: 100 | fpr, tpr, _ = metrics.roc_curve(y, prediction, pos_label=1) 101 | order = np.lexsort((tpr, fpr)) 102 | fpr, tpr = fpr[order], tpr[order] 103 | #print("AUC: " + str(metrics.auc(fpr, tpr))) 104 | return metrics.auc(fpr, tpr) 105 | 106 | 107 | def auc_value(prediction, y, w, reorder=True): 108 | fpr, tpr, _ = metrics.roc_curve(y, prediction, sample_weight=w, pos_label=1) 109 | return metrics.auc(fpr, tpr, reorder=reorder) 110 | 111 | 112 | def nDCG_cls(estimator, X, y, at=10, cnn_bdt=False, prediction=False): 113 | 114 | if prediction: 115 | prediction = estimator 116 | else: 117 | if cnn_bdt: 118 | prediction = estimator.predict_proba(X)[:,1] 119 | else: 120 | prediction = estimator.predict_proba(X) 121 | 122 | rand = np.random.random(len(prediction)) 123 | sorted_list = [yi for _, _, yi in sorted(zip(prediction, rand, y), reverse=True)] 124 | yref = sorted(y, reverse=True) 125 | 126 | DCG = 0. 127 | IDCG = 0. 128 | for i in range(min(at, len(sorted_list))): 129 | DCG += (2 ** sorted_list[i] - 1) / np.log2(i + 2) 130 | IDCG += (2 ** yref[i] - 1) / np.log2(i + 2) 131 | if IDCG == 0: 132 | return 0 133 | nDCG = DCG / IDCG 134 | return nDCG 135 | 136 | 137 | def comparator(x1, x2, estimator): 138 | """ 139 | :param x1: list of documents 140 | :param x2: list of documents 141 | :return: cmp value for sorting the query 142 | """ 143 | res = estimator.model.predict( 144 | [np.array([x1[:-1]]), np.array([x2[:-1]])], 145 | verbose=0 146 | ) 147 | if res < 0: 148 | return -1 149 | elif res > 0: 150 | return 1 151 | return 0 152 | 153 | 154 | def cmp_to_key(mycmp): 155 | """ 156 | Convert a cmp= function into a key= function 157 | """ 158 | 159 | class K: 160 | def __init__(self, obj, *args): 161 | self.obj = obj 162 | 163 | def __lt__(self, other): 164 | return mycmp(self.obj, other.obj) < 0 165 | 166 | def __gt__(self, other): 167 | return mycmp(self.obj, other.obj) > 0 168 | 169 | def __eq__(self, other): 170 | return mycmp(self.obj, other.obj) == 0 171 | 172 | def __le__(self, other): 173 | return mycmp(self.obj, other.obj) <= 0 174 | 175 | def __ge__(self, other): 176 | return mycmp(self.obj, other.obj) >= 0 177 | 178 | def __ne__(self, other): 179 | return mycmp(self.obj, other.obj) != 0 180 | 181 | return K 182 | 183 | 184 | def readData(path=None, binary=True, at=10, number_features=136, cut=1.5, synth_data=False): 185 | x = [] 186 | y = [] 187 | q = [] 188 | for line in open(path): 189 | s = line.split() 190 | if binary: 191 | if int(s[0]) > cut: 192 | y.append(1) 193 | else: 194 | y.append(0) 195 | else: 196 | y.append(int(s[0])) 197 | 198 | q.append(int(s[1].split(":")[1])) 199 | 200 | x.append(np.zeros(number_features)) 201 | for i in range(number_features): 202 | x[-1][i] = float(s[i + 2].split(":")[1]) 203 | 204 | x = np.array(x) 205 | y = np.array(y) 206 | q = np.array(q) 207 | 208 | if synth_data: 209 | return x, y, q 210 | 211 | xt = [] 212 | yt = [] 213 | qt = [] 214 | 215 | for qid in np.unique(q): 216 | if len(y[q == qid]) < at: continue 217 | xt.extend(x[q == qid].tolist()) 218 | yt.extend(y[q == qid].tolist()) 219 | qt.extend(q[q == qid].tolist()) 220 | 221 | return np.array(xt), np.array(yt), np.array(qt) 222 | 223 | 224 | def readDataV1(data_path=None, debug_data=False, binary=False, preprocessing="quantile_3", at=10, number_features=136, 225 | bin_cutoff=1.5, cut_zeros=False, synth_data=False): 226 | """ 227 | Function for reading the letor data 228 | :param binary: boolean if the labels of the data should be binary 229 | :param preprocessing: if set QuantileTransformer(output_distribution="normal") is used 230 | :param at: if the number of documents in the query is less than "at" they will not be 231 | taken into account. This is needed for calculating the ndcg@k until k=at. 232 | :return: list of queries, list of labels and list of query id 233 | """ 234 | if debug_data: 235 | path = "test_data.txt" 236 | elif data_path is not None: 237 | path = data_path 238 | 239 | x = [] 240 | y = [] 241 | q = [] 242 | for line in open(path): 243 | s = line.split() 244 | if binary: 245 | if int(s[0]) > bin_cutoff: 246 | y.append(1) 247 | else: 248 | y.append(0) 249 | else: 250 | y.append(int(s[0])) 251 | 252 | q.append(int(s[1].split(":")[1])) 253 | 254 | x.append(np.zeros(number_features)) 255 | for i in range(number_features): 256 | x[-1][i] = float(s[i + 2].split(":")[1]) 257 | 258 | if preprocessing == "quantile_3": 259 | x = QuantileTransformer( 260 | output_distribution="normal").fit_transform(x) / 3 261 | else: 262 | x = np.array(x) 263 | y = np.array([y]).transpose() 264 | q = np.array(q) 265 | 266 | if synth_data: 267 | return x, y, q 268 | 269 | xt = [] 270 | yt = [] 271 | 272 | for qid in np.unique(q): 273 | cs = [] 274 | if cut_zeros: 275 | for yy in y[q == qid][:, 0]: 276 | if yy not in cs: 277 | cs.append(yy) 278 | if len(cs) == 1: 279 | continue 280 | xt.append(x[q == qid]) 281 | yt.append(y[q == qid]) 282 | 283 | return np.array(xt), np.array(yt), q 284 | 285 | 286 | def AvgP_cls(estimator, X, y, prediction=False): 287 | if len(np.unique(y)) <= 1: 288 | return 0 289 | if prediction: 290 | avgp = average_precision_score(y, estimator) 291 | else: 292 | avgp = average_precision_score(y, estimator.predict_proba(X)) 293 | #print('AvgP: ' + str(avgp)) 294 | return avgp 295 | 296 | 297 | def MAP_cls(estimator, X, y): 298 | """ 299 | Function for evaluating the AvgP score over queries 300 | :param estimator: estimator class 301 | :param X: array of queries 302 | :param y: array of target values per query 303 | :return: AvgP score over the queries 304 | """ 305 | listOfAvgP = [] 306 | for query, y_query in zip(X, y): 307 | if sum(y_query) == 0: 308 | listOfAvgP.append(0.0) 309 | else: 310 | listOfAvgP.append(AvgP_cls(estimator, query, y_query)) 311 | 312 | return float(np.mean(listOfAvgP)) 313 | 314 | 315 | def dcg_at_k(r, k, method=0): 316 | r = np.asfarray(r)[:k] 317 | if r.size: 318 | if method == 0: 319 | return r[0] + np.sum(r[1:] / np.log2(np.arange(2, r.size + 1))) 320 | elif method == 1: 321 | return np.sum(r / np.log2(np.arange(2, r.size + 2))) 322 | else: 323 | raise ValueError('method must be 0 or 1.') 324 | return 0. 325 | 326 | 327 | def ndcg_at_k(r, k, method=0): 328 | dcg_max = dcg_at_k(sorted(r, reverse=True), k, method) 329 | if not dcg_max: 330 | return 0. 331 | return dcg_at_k(r, k, method) / dcg_max 332 | 333 | 334 | def nDCGScorer_cls(estimator, X, y, at=10): 335 | """ 336 | Function for evaluating the ndcg score over queries 337 | :param estimator: estimator class 338 | :param X: array of queries 339 | :param y: array of target values per query 340 | :return: ndcg score over the queries 341 | """ 342 | listOfnDCG = [] 343 | for query, y_query in zip(X, y): 344 | if len(query) < at: 345 | k = len(query) 346 | else: 347 | k = at 348 | listOfnDCG.append(nDCG_cls(estimator, query, y_query, k)) 349 | return float(np.mean(listOfnDCG)) 350 | 351 | 352 | def train_model(num_features, path="fig2", test=False): 353 | x_train, y_train, _ = readData( 354 | path=f"{path}/train", 355 | binary=False, 356 | at=20, 357 | number_features=num_features, 358 | synth_data=True 359 | ) 360 | 361 | x_test, y_test, q_test = readData( 362 | path=f"{path}/test", 363 | binary=False, 364 | at=20, 365 | number_features=num_features, 366 | synth_data=True 367 | ) 368 | 369 | scaler = QuantileTransformer(output_distribution="normal") 370 | x_train = scaler.fit_transform(x_train) / 3 371 | x_test = scaler.transform(x_test) / 3 372 | 373 | print("Train Model") 374 | if test: 375 | dr = DirectRanker( 376 | hidden_layers_dr=[70, 5], 377 | num_features=num_features, 378 | epoch=1, 379 | verbose=2 380 | ) 381 | else: 382 | dr = DirectRanker( 383 | hidden_layers_dr=[256, 128, 64, 20], 384 | drop_out=0.2, 385 | scale_factor_train_sample=3, 386 | num_features=num_features, 387 | epoch=20, 388 | verbose=2 389 | ) 390 | dr.fit(x_train, y_train) 391 | 392 | print("Test Model") 393 | nDCG_l = [] 394 | pred = dr.predict_proba(x_test) 395 | for n in np.unique(q_test): 396 | ndcg = nDCG_cls(pred[q_test==n], x_test[q_test==n], y_test[q_test==n], at=20, prediction=True) 397 | nDCG_l.append(ndcg) 398 | print(f"NDCG@20: {np.mean(nDCG_l)}") 399 | return np.mean(nDCG_l), np.std(nDCG_l) 400 | -------------------------------------------------------------------------------- /rustlib/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "autocfg" 7 | version = "1.1.0" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" 10 | 11 | [[package]] 12 | name = "bitflags" 13 | version = "1.3.2" 14 | source = "registry+https://github.com/rust-lang/crates.io-index" 15 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 16 | 17 | [[package]] 18 | name = "cfg-if" 19 | version = "1.0.0" 20 | source = "registry+https://github.com/rust-lang/crates.io-index" 21 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 22 | 23 | [[package]] 24 | name = "crossbeam-channel" 25 | version = "0.5.6" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" 28 | dependencies = [ 29 | "cfg-if", 30 | "crossbeam-utils", 31 | ] 32 | 33 | [[package]] 34 | name = "crossbeam-deque" 35 | version = "0.8.2" 36 | source = "registry+https://github.com/rust-lang/crates.io-index" 37 | checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" 38 | dependencies = [ 39 | "cfg-if", 40 | "crossbeam-epoch", 41 | "crossbeam-utils", 42 | ] 43 | 44 | [[package]] 45 | name = "crossbeam-epoch" 46 | version = "0.9.13" 47 | source = "registry+https://github.com/rust-lang/crates.io-index" 48 | checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" 49 | dependencies = [ 50 | "autocfg", 51 | "cfg-if", 52 | "crossbeam-utils", 53 | "memoffset 0.7.1", 54 | "scopeguard", 55 | ] 56 | 57 | [[package]] 58 | name = "crossbeam-utils" 59 | version = "0.8.14" 60 | source = "registry+https://github.com/rust-lang/crates.io-index" 61 | checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" 62 | dependencies = [ 63 | "cfg-if", 64 | ] 65 | 66 | [[package]] 67 | name = "either" 68 | version = "1.8.1" 69 | source = "registry+https://github.com/rust-lang/crates.io-index" 70 | checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" 71 | 72 | [[package]] 73 | name = "getrandom" 74 | version = "0.2.8" 75 | source = "registry+https://github.com/rust-lang/crates.io-index" 76 | checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" 77 | dependencies = [ 78 | "cfg-if", 79 | "libc", 80 | "wasi", 81 | ] 82 | 83 | [[package]] 84 | name = "hermit-abi" 85 | version = "0.2.6" 86 | source = "registry+https://github.com/rust-lang/crates.io-index" 87 | checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" 88 | dependencies = [ 89 | "libc", 90 | ] 91 | 92 | [[package]] 93 | name = "indoc" 94 | version = "1.0.9" 95 | source = "registry+https://github.com/rust-lang/crates.io-index" 96 | checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" 97 | 98 | [[package]] 99 | name = "itertools" 100 | version = "0.10.5" 101 | source = "registry+https://github.com/rust-lang/crates.io-index" 102 | checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" 103 | dependencies = [ 104 | "either", 105 | ] 106 | 107 | [[package]] 108 | name = "libc" 109 | version = "0.2.139" 110 | source = "registry+https://github.com/rust-lang/crates.io-index" 111 | checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" 112 | 113 | [[package]] 114 | name = "lock_api" 115 | version = "0.4.9" 116 | source = "registry+https://github.com/rust-lang/crates.io-index" 117 | checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" 118 | dependencies = [ 119 | "autocfg", 120 | "scopeguard", 121 | ] 122 | 123 | [[package]] 124 | name = "memoffset" 125 | version = "0.7.1" 126 | source = "registry+https://github.com/rust-lang/crates.io-index" 127 | checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" 128 | dependencies = [ 129 | "autocfg", 130 | ] 131 | 132 | [[package]] 133 | name = "memoffset" 134 | version = "0.8.0" 135 | source = "registry+https://github.com/rust-lang/crates.io-index" 136 | checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" 137 | dependencies = [ 138 | "autocfg", 139 | ] 140 | 141 | [[package]] 142 | name = "num_cpus" 143 | version = "1.15.0" 144 | source = "registry+https://github.com/rust-lang/crates.io-index" 145 | checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" 146 | dependencies = [ 147 | "hermit-abi", 148 | "libc", 149 | ] 150 | 151 | [[package]] 152 | name = "once_cell" 153 | version = "1.17.1" 154 | source = "registry+https://github.com/rust-lang/crates.io-index" 155 | checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" 156 | 157 | [[package]] 158 | name = "parking_lot" 159 | version = "0.12.1" 160 | source = "registry+https://github.com/rust-lang/crates.io-index" 161 | checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" 162 | dependencies = [ 163 | "lock_api", 164 | "parking_lot_core", 165 | ] 166 | 167 | [[package]] 168 | name = "parking_lot_core" 169 | version = "0.9.7" 170 | source = "registry+https://github.com/rust-lang/crates.io-index" 171 | checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" 172 | dependencies = [ 173 | "cfg-if", 174 | "libc", 175 | "redox_syscall", 176 | "smallvec", 177 | "windows-sys", 178 | ] 179 | 180 | [[package]] 181 | name = "ppv-lite86" 182 | version = "0.2.17" 183 | source = "registry+https://github.com/rust-lang/crates.io-index" 184 | checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" 185 | 186 | [[package]] 187 | name = "proc-macro2" 188 | version = "1.0.51" 189 | source = "registry+https://github.com/rust-lang/crates.io-index" 190 | checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" 191 | dependencies = [ 192 | "unicode-ident", 193 | ] 194 | 195 | [[package]] 196 | name = "pyo3" 197 | version = "0.18.1" 198 | source = "registry+https://github.com/rust-lang/crates.io-index" 199 | checksum = "06a3d8e8a46ab2738109347433cb7b96dffda2e4a218b03ef27090238886b147" 200 | dependencies = [ 201 | "cfg-if", 202 | "indoc", 203 | "libc", 204 | "memoffset 0.8.0", 205 | "parking_lot", 206 | "pyo3-build-config", 207 | "pyo3-ffi", 208 | "pyo3-macros", 209 | "unindent", 210 | ] 211 | 212 | [[package]] 213 | name = "pyo3-build-config" 214 | version = "0.18.1" 215 | source = "registry+https://github.com/rust-lang/crates.io-index" 216 | checksum = "75439f995d07ddfad42b192dfcf3bc66a7ecfd8b4a1f5f6f046aa5c2c5d7677d" 217 | dependencies = [ 218 | "once_cell", 219 | "target-lexicon", 220 | ] 221 | 222 | [[package]] 223 | name = "pyo3-ffi" 224 | version = "0.18.1" 225 | source = "registry+https://github.com/rust-lang/crates.io-index" 226 | checksum = "839526a5c07a17ff44823679b68add4a58004de00512a95b6c1c98a6dcac0ee5" 227 | dependencies = [ 228 | "libc", 229 | "pyo3-build-config", 230 | ] 231 | 232 | [[package]] 233 | name = "pyo3-macros" 234 | version = "0.18.1" 235 | source = "registry+https://github.com/rust-lang/crates.io-index" 236 | checksum = "bd44cf207476c6a9760c4653559be4f206efafb924d3e4cbf2721475fc0d6cc5" 237 | dependencies = [ 238 | "proc-macro2", 239 | "pyo3-macros-backend", 240 | "quote", 241 | "syn", 242 | ] 243 | 244 | [[package]] 245 | name = "pyo3-macros-backend" 246 | version = "0.18.1" 247 | source = "registry+https://github.com/rust-lang/crates.io-index" 248 | checksum = "dc1f43d8e30460f36350d18631ccf85ded64c059829208fe680904c65bcd0a4c" 249 | dependencies = [ 250 | "proc-macro2", 251 | "quote", 252 | "syn", 253 | ] 254 | 255 | [[package]] 256 | name = "quote" 257 | version = "1.0.23" 258 | source = "registry+https://github.com/rust-lang/crates.io-index" 259 | checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" 260 | dependencies = [ 261 | "proc-macro2", 262 | ] 263 | 264 | [[package]] 265 | name = "rand" 266 | version = "0.8.5" 267 | source = "registry+https://github.com/rust-lang/crates.io-index" 268 | checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" 269 | dependencies = [ 270 | "libc", 271 | "rand_chacha", 272 | "rand_core", 273 | ] 274 | 275 | [[package]] 276 | name = "rand_chacha" 277 | version = "0.3.1" 278 | source = "registry+https://github.com/rust-lang/crates.io-index" 279 | checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" 280 | dependencies = [ 281 | "ppv-lite86", 282 | "rand_core", 283 | ] 284 | 285 | [[package]] 286 | name = "rand_core" 287 | version = "0.6.4" 288 | source = "registry+https://github.com/rust-lang/crates.io-index" 289 | checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" 290 | dependencies = [ 291 | "getrandom", 292 | ] 293 | 294 | [[package]] 295 | name = "rayon" 296 | version = "1.6.1" 297 | source = "registry+https://github.com/rust-lang/crates.io-index" 298 | checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" 299 | dependencies = [ 300 | "either", 301 | "rayon-core", 302 | ] 303 | 304 | [[package]] 305 | name = "rayon-core" 306 | version = "1.10.2" 307 | source = "registry+https://github.com/rust-lang/crates.io-index" 308 | checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" 309 | dependencies = [ 310 | "crossbeam-channel", 311 | "crossbeam-deque", 312 | "crossbeam-utils", 313 | "num_cpus", 314 | ] 315 | 316 | [[package]] 317 | name = "redox_syscall" 318 | version = "0.2.16" 319 | source = "registry+https://github.com/rust-lang/crates.io-index" 320 | checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" 321 | dependencies = [ 322 | "bitflags", 323 | ] 324 | 325 | [[package]] 326 | name = "rustlib" 327 | version = "0.1.0" 328 | dependencies = [ 329 | "itertools", 330 | "pyo3", 331 | "rand", 332 | "rayon", 333 | ] 334 | 335 | [[package]] 336 | name = "scopeguard" 337 | version = "1.1.0" 338 | source = "registry+https://github.com/rust-lang/crates.io-index" 339 | checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" 340 | 341 | [[package]] 342 | name = "smallvec" 343 | version = "1.10.0" 344 | source = "registry+https://github.com/rust-lang/crates.io-index" 345 | checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" 346 | 347 | [[package]] 348 | name = "syn" 349 | version = "1.0.109" 350 | source = "registry+https://github.com/rust-lang/crates.io-index" 351 | checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" 352 | dependencies = [ 353 | "proc-macro2", 354 | "quote", 355 | "unicode-ident", 356 | ] 357 | 358 | [[package]] 359 | name = "target-lexicon" 360 | version = "0.12.6" 361 | source = "registry+https://github.com/rust-lang/crates.io-index" 362 | checksum = "8ae9980cab1db3fceee2f6c6f643d5d8de2997c58ee8d25fb0cc8a9e9e7348e5" 363 | 364 | [[package]] 365 | name = "unicode-ident" 366 | version = "1.0.6" 367 | source = "registry+https://github.com/rust-lang/crates.io-index" 368 | checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" 369 | 370 | [[package]] 371 | name = "unindent" 372 | version = "0.1.11" 373 | source = "registry+https://github.com/rust-lang/crates.io-index" 374 | checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" 375 | 376 | [[package]] 377 | name = "wasi" 378 | version = "0.11.0+wasi-snapshot-preview1" 379 | source = "registry+https://github.com/rust-lang/crates.io-index" 380 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 381 | 382 | [[package]] 383 | name = "windows-sys" 384 | version = "0.45.0" 385 | source = "registry+https://github.com/rust-lang/crates.io-index" 386 | checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" 387 | dependencies = [ 388 | "windows-targets", 389 | ] 390 | 391 | [[package]] 392 | name = "windows-targets" 393 | version = "0.42.1" 394 | source = "registry+https://github.com/rust-lang/crates.io-index" 395 | checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" 396 | dependencies = [ 397 | "windows_aarch64_gnullvm", 398 | "windows_aarch64_msvc", 399 | "windows_i686_gnu", 400 | "windows_i686_msvc", 401 | "windows_x86_64_gnu", 402 | "windows_x86_64_gnullvm", 403 | "windows_x86_64_msvc", 404 | ] 405 | 406 | [[package]] 407 | name = "windows_aarch64_gnullvm" 408 | version = "0.42.1" 409 | source = "registry+https://github.com/rust-lang/crates.io-index" 410 | checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" 411 | 412 | [[package]] 413 | name = "windows_aarch64_msvc" 414 | version = "0.42.1" 415 | source = "registry+https://github.com/rust-lang/crates.io-index" 416 | checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" 417 | 418 | [[package]] 419 | name = "windows_i686_gnu" 420 | version = "0.42.1" 421 | source = "registry+https://github.com/rust-lang/crates.io-index" 422 | checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" 423 | 424 | [[package]] 425 | name = "windows_i686_msvc" 426 | version = "0.42.1" 427 | source = "registry+https://github.com/rust-lang/crates.io-index" 428 | checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" 429 | 430 | [[package]] 431 | name = "windows_x86_64_gnu" 432 | version = "0.42.1" 433 | source = "registry+https://github.com/rust-lang/crates.io-index" 434 | checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" 435 | 436 | [[package]] 437 | name = "windows_x86_64_gnullvm" 438 | version = "0.42.1" 439 | source = "registry+https://github.com/rust-lang/crates.io-index" 440 | checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" 441 | 442 | [[package]] 443 | name = "windows_x86_64_msvc" 444 | version = "0.42.1" 445 | source = "registry+https://github.com/rust-lang/crates.io-index" 446 | checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" 447 | -------------------------------------------------------------------------------- /table1/gridsearch.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pathlib import Path 3 | file = Path(__file__).resolve() 4 | package_root_directory = file.parents[1] 5 | sys.path.append(str(package_root_directory)) 6 | 7 | import os 8 | from Rankers.models.RankNet import RankNet 9 | from Rankers.models.DirectRanker import DirectRanker 10 | from Rankers.models.ListNet import ListNet 11 | from Rankers.models.LambdaRank import LambdaRank 12 | from Rankers.models.LambdaMart2 import LambdaMart 13 | from Rankers.models.AdaRank import AdaRank 14 | from Rankers.helpers import nDCG_cls, readData, AvgP_cls, readDataV1, nDCGScorer_cls, MAP_cls 15 | 16 | import numpy as np 17 | import tensorflow as tf 18 | from functools import partial 19 | 20 | from sklearn.preprocessing import QuantileTransformer 21 | from sklearn.model_selection import GridSearchCV, train_test_split 22 | 23 | import argparse 24 | 25 | 26 | if __name__ == '__main__': 27 | 28 | parser = argparse.ArgumentParser(description='Get some run values.') 29 | parser.add_argument('--data', type=str, default='MSLR10K', help='Get data name') 30 | parser.add_argument('--model', type=str, default='DirectRanker', help='Get model name') 31 | parser.add_argument('--path', type=str, default='', help='Get path to data') 32 | parser.add_argument('--out_path', type=str, default='gridsearch', help='Get path to data') 33 | parser.add_argument('--cpu', type=int, default=1, help='Use CPU or GPU') 34 | parser.add_argument('--jobs', type=int, default=1, help='Use CPUs for running') 35 | parser.add_argument('--ttest', type=int, default=0, help='Using ttest') 36 | parser.add_argument('--runNum', type=int, default=0, help='Run Number') 37 | parser.add_argument('--binary', type=int, default=1, help='binary') 38 | parser.add_argument('--startFold', type=int, default=-1, help='skip the other folds') 39 | 40 | args = parser.parse_args() 41 | 42 | if args.cpu == 1: 43 | os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 44 | 45 | if args.binary == 1: 46 | binary = True 47 | else: 48 | binary = False 49 | 50 | if not os.path.exists(args.out_path): 51 | os.makedirs(args.out_path) 52 | os.makedirs(args.out_path + "/data") 53 | 54 | list_of_train = [f"{args.path}/Fold1/train.txt", f"{args.path}/Fold2/train.txt", 55 | f"{args.path}/Fold3/train.txt", f"{args.path}/Fold4/train.txt", 56 | f"{args.path}/Fold5/train.txt"] 57 | list_of_test = [f"{args.path}/Fold1/test.txt", f"{args.path}/Fold2/test.txt", 58 | f"{args.path}/Fold3/test.txt", f"{args.path}/Fold4/test.txt", 59 | f"{args.path}/Fold5/test.txt"] 60 | list_of_vali = [f"{args.path}/Fold1/vali.txt", f"{args.path}/Fold2/vali.txt", 61 | f"{args.path}/Fold3/vali.txt", f"{args.path}/Fold4/vali.txt", 62 | f"{args.path}/Fold5/vali.txt"] 63 | 64 | if args.ttest == 1: 65 | ttest = True 66 | numOutFolds = 15 67 | numInFolds = 3 68 | else: 69 | ttest = False 70 | numOutFolds = 5 71 | numInFolds = 5 72 | 73 | parameters = { 74 | 'hidden_layers_dr': [[10, 5], [20, 5], [30, 10, 5], [40, 30, 10, 5], [200, 50, 30, 10, 5]], 75 | 'feature_activation_dr': ["tanh", "sigmoid", "linear"], 76 | 'epoch': [5, 10, 30], 77 | 'ranking_activation_dr': ['sigmoid'], 78 | 'verbose': [2], 79 | 'optimizer': [tf.keras.optimizers.Adam], 80 | 'learning_rate': [0.001, 0.0001] 81 | } 82 | 83 | parameters_test = {'epoch': [1]} 84 | 85 | if args.model == "DirectRankerV1": 86 | nDCGScorer10 = partial(nDCGScorer_cls, at=10) 87 | scoring = {'NDGC@10': nDCGScorer10, 'MAP': MAP_cls} 88 | else: 89 | scoring = {'NDGC@10': nDCG_cls} 90 | 91 | list_of_ndcg = [] 92 | list_of_map = [] 93 | for foldidx in range(numOutFolds): 94 | print("FOLD ", foldidx) 95 | 96 | if args.startFold != -1 and args.startFold > foldidx: 97 | print("We skip") 98 | fold_dict = np.load(f"{args.out_path}/cv_results_{args.data}_{args.model}_fold{foldidx}_ttest{ttest}_binary{binary}_runNum{args.runNum}.npy", allow_pickle=True)[()] 99 | list_of_ndcg.append(fold_dict["ndcg"]) 100 | list_of_map.append(fold_dict["map"]) 101 | print(f"Test on Fold {foldidx}: NDCG@10 = {fold_dict['ndcg']} MAP = {fold_dict['map']}") 102 | continue 103 | 104 | if args.data == "MSLR10K": 105 | number_features = 136 106 | cut = 1.5 107 | if args.data == "test": 108 | number_features = 136 109 | cut = 0.5 110 | if args.data == "MQ2007" or args.data == "MQ2008": 111 | number_features = 46 112 | cut = 0.5 113 | 114 | if foldidx > 4: 115 | readFoldidx = 0 116 | else: 117 | readFoldidx = foldidx 118 | 119 | # read data 120 | print("Reading data") 121 | if os.path.isfile(f"{args.out_path}/data/{args.data}_q_vali_fold{readFoldidx+1}_binary{binary}.npy"): 122 | x_train = np.load(f"{args.out_path}/data/{args.data}_x_train_fold{readFoldidx+1}_binary{binary}.npy") 123 | y_train = np.load(f"{args.out_path}/data/{args.data}_y_train_fold{readFoldidx+1}_binary{binary}.npy") 124 | q_train = np.load(f"{args.out_path}/data/{args.data}_q_train_fold{readFoldidx+1}_binary{binary}.npy") 125 | 126 | x_test = np.load(f"{args.out_path}/data/{args.data}_x_test_fold{readFoldidx+1}_binary{binary}.npy") 127 | y_test = np.load(f"{args.out_path}/data/{args.data}_y_test_fold{readFoldidx+1}_binary{binary}.npy") 128 | q_test = np.load(f"{args.out_path}/data/{args.data}_q_test_fold{readFoldidx+1}_binary{binary}.npy") 129 | 130 | x_vali = np.load(f"{args.out_path}/data/{args.data}_x_vali_fold{readFoldidx+1}_binary{binary}.npy") 131 | y_vali = np.load(f"{args.out_path}/data/{args.data}_y_vali_fold{readFoldidx+1}_binary{binary}.npy") 132 | q_vali = np.load(f"{args.out_path}/data/{args.data}_q_vali_fold{readFoldidx+1}_binary{binary}.npy") 133 | else: 134 | x_train, y_train, q_train = readData( 135 | path=list_of_train[readFoldidx], 136 | binary=binary, 137 | at=10, 138 | number_features=number_features, 139 | cut=cut 140 | ) 141 | 142 | x_test, y_test, q_test = readData( 143 | path=list_of_test[readFoldidx], 144 | binary=binary, 145 | at=10, 146 | number_features=number_features, 147 | cut=cut 148 | ) 149 | 150 | x_vali, y_vali, q_vali = readData( 151 | path=list_of_vali[readFoldidx], 152 | binary=binary, 153 | at=10, 154 | number_features=number_features, 155 | cut=cut 156 | ) 157 | 158 | np.save(f"{args.out_path}/data/{args.data}_x_train_fold{readFoldidx+1}_binary{binary}", x_train) 159 | np.save(f"{args.out_path}/data/{args.data}_y_train_fold{readFoldidx+1}_binary{binary}", y_train) 160 | np.save(f"{args.out_path}/data/{args.data}_q_train_fold{readFoldidx+1}_binary{binary}", q_train) 161 | 162 | np.save(f"{args.out_path}/data/{args.data}_x_test_fold{readFoldidx+1}_binary{binary}", x_test) 163 | np.save(f"{args.out_path}/data/{args.data}_y_test_fold{readFoldidx+1}_binary{binary}", y_test) 164 | np.save(f"{args.out_path}/data/{args.data}_q_test_fold{readFoldidx+1}_binary{binary}", q_test) 165 | 166 | np.save(f"{args.out_path}/data/{args.data}_x_vali_fold{readFoldidx+1}_binary{binary}", x_vali) 167 | np.save(f"{args.out_path}/data/{args.data}_y_vali_fold{readFoldidx+1}_binary{binary}", y_vali) 168 | np.save(f"{args.out_path}/data/{args.data}_q_vali_fold{readFoldidx+1}_binary{binary}", q_vali) 169 | 170 | if ttest: 171 | x = np.concatenate([x_train, x_test, x_vali]) 172 | y = np.concatenate([y_train, y_test, y_vali]) 173 | q = np.concatenate([q_train, q_test, q_vali]) 174 | x_train, x_test, y_train, y_test, q_train, q_test = train_test_split(x, y, q, test_size=0.33, shuffle=True) 175 | 176 | print("Preprocessing") 177 | scaler = QuantileTransformer(output_distribution="normal") 178 | x_train = scaler.fit_transform(x_train) / 3 179 | x_test = scaler.transform(x_test) / 3 180 | binary_y_test = [] 181 | # if we have multi class in training we do binary for eval 182 | if not binary: 183 | for yi in y_test: 184 | if int(yi) > cut: 185 | binary_y_test.append(1) 186 | else: 187 | binary_y_test.append(0) 188 | y_test = np.array(binary_y_test) 189 | 190 | print("Training") 191 | if args.model == "RankNet": 192 | parameters['optimizer'] = [tf.keras.optimizers.SGD] 193 | ranker = RankNet(num_features=x_train.shape[1]) 194 | 195 | if args.model == "DirectRanker": 196 | parameters['ranking_activation_dr'] = ['tanh'] 197 | parameters['drop_out'] = [0., 0.2, 0.5] 198 | parameters['kernel_regularizer_dr'] = [0., 0.001, 0.01] 199 | ranker = DirectRanker(num_features=x_train.shape[1]) 200 | 201 | if args.model == "DirectRankerV1": 202 | from Rankers.models.DirectRankerV1 import DirectRankerV1 203 | parameters = { 204 | 'hidden_layers': [[10], [20], [30], [50], [100], [50,5], [100, 5], [100,50,5], [50,25,5], [100,50,25,5], [200,100,50,25,5], [100,100,50,25,5]], 205 | 'weight_regularization': [0., 0.0001, 0.001, 0.01, 0.1, 0.999], 206 | 'early_stopping': [False, True], 207 | 'dropout': [0., 0.5] 208 | } 209 | x_train, y_train, q_train = readDataV1( 210 | data_path=list_of_train[readFoldidx], 211 | binary=True, 212 | at=10, 213 | number_features=number_features, 214 | bin_cutoff=cut, 215 | synth_data=True, 216 | cut_zeros=True 217 | ) 218 | x_test, y_test, q_test = readDataV1( 219 | data_path=list_of_test[readFoldidx], 220 | binary=True, 221 | at=10, 222 | number_features=number_features, 223 | bin_cutoff=cut, 224 | synth_data=True, 225 | cut_zeros=True 226 | ) 227 | 228 | if ttest: 229 | x = np.concatenate([x_train, x_test, x_vali]) 230 | y = np.concatenate([y_train, y_test, y_vali]) 231 | q = np.concatenate([q_train, q_test, q_vali]) 232 | x_train, x_test, y_train, y_test, q_train, q_test = train_test_split(x, y, q, test_size=0.33, shuffle=True) 233 | 234 | parameters_test = {"max_steps": [100], "print_step": [10], "start_batch_size": [10], "end_batch_size": [100]} 235 | ranker = DirectRankerV1() 236 | 237 | if args.model == "ListNet": 238 | parameters['drop_out'] = [0., 0.2, 0.5] 239 | parameters['kernel_regularizer_dr'] = [0., 0.001, 0.01] 240 | ranker = ListNet(num_features=x_train.shape[1]) 241 | 242 | if args.model == "ListNetOri": 243 | parameters['optimizer'] = [tf.keras.optimizers.SGD] 244 | ranker = ListNet(num_features=x_train.shape[1]) 245 | 246 | if args.model == "LambdaRank": 247 | ranker = LambdaRank(num_features=x_train.shape[1]) 248 | 249 | if args.model == "LambdaMart": 250 | parameters_test = { 251 | "number_of_trees": [5], 252 | "learning_rate": [0.1] 253 | } 254 | parameters = { 255 | "number_of_trees": [5, 10], 256 | "max_depth": [4], 257 | "learning_rate": [0.1, 0.01], 258 | } 259 | ranker = LambdaMart() 260 | x_train = np.concatenate([q_train.reshape(len(q_train), 1), x_train], axis=1) 261 | x_test = np.concatenate([q_test.reshape(len(q_test), 1), x_test], axis=1) 262 | 263 | if args.model == "AdaRank": 264 | parameters_test = { 265 | "T": [100], 266 | "verbose": [True], 267 | "estop": [10] 268 | } 269 | parameters = { 270 | "T": [100], 271 | "verbose": [True], 272 | "estop": [0, 10, 20] 273 | } 274 | ranker = AdaRank() 275 | x_train = np.concatenate([q_train.reshape(len(q_train), 1), x_train], axis=1) 276 | x_test = np.concatenate([q_test.reshape(len(q_test), 1), x_test], axis=1) 277 | 278 | if args.data == "test": 279 | parameters = parameters_test 280 | 281 | if args.data == "test": 282 | ranker = GridSearchCV(ranker, parameters, cv=2, n_jobs=args.jobs, verbose=1, scoring=scoring, refit='NDGC@10', return_train_score=False) 283 | else: 284 | ranker = GridSearchCV(ranker, parameters, cv=numInFolds, n_jobs=args.jobs, verbose=1, scoring=scoring, refit='NDGC@10', return_train_score=False) 285 | 286 | if args.model == "DirectRankerV1": 287 | ranker.fit(x_train, y_train, ranking=False) 288 | else: 289 | ranker.fit(x_train, y_train) 290 | 291 | print("Eval") 292 | if args.model == "DirectRankerV1": 293 | map_l = MAP_cls(ranker.best_estimator_, x_test, y_test) 294 | nDCG_l = nDCGScorer10(ranker.best_estimator_, x_test, y_test) 295 | else: 296 | nDCG_l = [] 297 | map_l = [] 298 | pred = ranker.best_estimator_.predict_proba(x_test) 299 | for n in np.unique(q_test): 300 | ndcg = nDCG_cls(pred[q_test==n], x_test[q_test==n], y_test[q_test==n], at=10, prediction=True) 301 | mapv = AvgP_cls(pred[q_test==n], x_test[q_test==n], y_test[q_test==n], prediction=True) 302 | # exclude if there was a querry with only one class 303 | if ndcg == 0 and (args.data == "MQ2008" or args.data == "MQ2007" or args.data == "MSLR10K"): 304 | ndcg == 1 305 | if mapv == 0 and (args.data == "MQ2008" or args.data == "MQ2007" or args.data == "MSLR10K"): 306 | mapv == 1 307 | nDCG_l.append(ndcg) 308 | map_l.append(mapv) 309 | 310 | ranker.cv_results_["ndcg"] = np.mean(nDCG_l) 311 | ranker.cv_results_["map"] = np.mean(map_l) 312 | 313 | print(f"Test on Fold {foldidx}: NDCG@10 = {np.mean(nDCG_l)} MAP = {np.mean(map_l)}") 314 | #print(ranker.cv_results_) 315 | 316 | list_of_ndcg.append(np.mean(nDCG_l)) 317 | list_of_map.append(np.mean(map_l)) 318 | 319 | np.save(f"{args.out_path}/cv_results_{args.data}_{args.model}_fold{foldidx}_ttest{ttest}_binary{binary}_runNum{args.runNum}", ranker.cv_results_) 320 | 321 | np.save(f"{args.out_path}/{args.data}_{args.model}_list_of_ndcg_ttest{ttest}_binary{binary}_runNum{args.runNum}", list_of_ndcg) 322 | np.save(f"{args.out_path}/{args.data}_{args.model}_list_of_map_ttest{ttest}_binary{binary}_runNum{args.runNum}", list_of_map) 323 | -------------------------------------------------------------------------------- /table1/get_results.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import scipy.stats as ss 4 | import scikit_posthocs as sp 5 | import matplotlib.pyplot as plt 6 | import matplotlib as mpl 7 | from matplotlib.colors import ListedColormap 8 | import matplotlib.patches as mpatches 9 | import os 10 | 11 | 12 | def get_ranklib(ttest=False): 13 | # get ranklib results 14 | for algo in ["LambdaMart", "AdaRank", "ListNet", "RankNet"]: 15 | for pathname in ["MQ2007", "MQ2008", "MSLR10K"]: 16 | if ttest: 17 | folds = [f"Fold{i}" for i in range(1,16)] 18 | best_map = [0 for i in range(1,16)] 19 | best_map_para = [0 for i in range(1,16)] 20 | best_ndcg = [0 for i in range(1,16)] 21 | best_ndcg_para = [0 for i in range(1,16)] 22 | if not ttest: 23 | folds = [f"Fold{i}" for i in range(1,6)] 24 | best_map = [0 for i in range(1,6)] 25 | best_map_para = [0 for i in range(1,6)] 26 | best_ndcg = [0 for i in range(1,6)] 27 | best_ndcg_para = [0 for i in range(1,6)] 28 | for idx_fold, fold in enumerate(folds): 29 | if not os.path.exists("results_ranklib/" + pathname + "/"): continue 30 | for filename in os.listdir("results_ranklib/" + pathname + "/"): 31 | if algo not in filename: 32 | continue 33 | if fold not in filename: 34 | continue 35 | with open("results_ranklib/" + pathname + "/" + filename, "r") as file: 36 | for line in file: 37 | if "Avg." in line: 38 | #print(line, (line.replace("\n","").split()[2])) 39 | if "MAP" in filename: 40 | if best_map[idx_fold] < float(line.replace("\n","").split()[2]): 41 | best_map[idx_fold] = float(line.replace("\n","").split()[2]) 42 | best_map_para[idx_fold] = filename 43 | else: 44 | if best_ndcg[idx_fold] < float(line.replace("\n","").split()[2]): 45 | best_ndcg[idx_fold] = float(line.replace("\n","").split()[2]) 46 | best_ndcg_para[idx_fold] = filename 47 | print(algo, pathname) 48 | print("MAP " + str(best_map), f"{round(np.mean(best_map),3)}({int(round(np.std(best_map),3)*1000)})") 49 | print("NDCG@10 " + str(best_ndcg), f"{round(np.mean(best_ndcg),3)}({int(round(np.std(best_ndcg),3)*1000)})") 50 | 51 | #get_ranklib(ttest=True) 52 | 53 | # get tensorflow V2 results 54 | results = {} 55 | for model in ["DirectRanker", "ListNet", "RankNet", "DirectRankerV1"]: 56 | cur_dict = {} 57 | for data in ["MSLR10K", "MQ2008", "MQ2007"]: 58 | print(model, data, "NDCG", np.load(f"gridsearch/{data}_{model}_list_of_ndcg.npy", allow_pickle=True)) 59 | print(model, data, "MAP", np.load(f"gridsearch/{data}_{model}_list_of_map.npy", allow_pickle=True)) 60 | cur_dict[data] = { 61 | "NDCG-mean": np.mean(np.load(f"gridsearch/{data}_{model}_list_of_ndcg.npy", allow_pickle=True)), 62 | "NDCG-std": np.std(np.load(f"gridsearch/{data}_{model}_list_of_ndcg.npy", allow_pickle=True)), 63 | "MAP-mean": np.mean(np.load(f"gridsearch/{data}_{model}_list_of_map.npy", allow_pickle=True)), 64 | "MAP-std": np.std(np.load(f"gridsearch/{data}_{model}_list_of_map.npy", allow_pickle=True)) 65 | } 66 | results[model] = cur_dict 67 | 68 | print(cur_dict) 69 | 70 | # get ttest results 71 | def bengio_nadeau_test(model1, model2, alpha=0.05): 72 | n = len(model1) 73 | assert len(model1) == len(model2) 74 | differences = [model1[i] - model2[i] for i in range(n)] 75 | divisor = 1 / n * np.sum(differences) 76 | # critical to change this if values change in run_experiment 77 | test_train_ratio = 0.33 78 | denominator = np.sqrt(1 / n + test_train_ratio) * np.std(differences) 79 | t_stat = divisor / denominator 80 | # degrees of freedom 81 | df = n - 1 82 | # critical value 83 | cv = ss.t.ppf(1.0 - alpha, df) 84 | # p-value 85 | p = (1.0 - ss.t.cdf(abs(t_stat), df)) * 2 86 | return p, np.mean(model1), np.mean(model2), np.std(model1), np.std(model2) 87 | 88 | for binary in [True, False]: 89 | restuls_ttest = {} 90 | for model in ["DirectRanker", "ListNet*", "ListNet", "RankNet", "DirectRankerV1", "LambdaMart", "AdaRank"]: 91 | cur_dict = {} 92 | runNum = "_runNum0" 93 | if model == "LambdaMart" or model == "AdaRank": 94 | runNum = "_runNum1" 95 | for data in ["MSLR10K", "MQ2008", "MQ2007"]: 96 | model_file = model 97 | if model == "RankNet" and data == "MSLR10K": 98 | runNum = "_runNum99" 99 | elif model == "RankNet": 100 | runNum = "_runNum99" 101 | if model == "ListNet": 102 | runNum = "_runNum0" 103 | elif model == "ListNet*": 104 | model_file = "ListNet" 105 | runNum = "_runNum99" 106 | if os.path.isfile(f"gridsearch/{data}_{model_file}_list_of_ndcg_ttestTrue_binary{binary}{runNum}.npy"): 107 | cur_dict[data] = { 108 | "NDCG": np.load(f"gridsearch/{data}_{model_file}_list_of_ndcg_ttestTrue_binary{binary}{runNum}.npy", allow_pickle=True), 109 | "MAP": np.load(f"gridsearch/{data}_{model_file}_list_of_map_ttestTrue_binary{binary}{runNum}.npy", allow_pickle=True), 110 | } 111 | else: 112 | cur_dict[data] = {"NDCG":[0 for i in range(15)],"MAP":[0 for i in range(15)]} 113 | restuls_ttest[model] = cur_dict 114 | 115 | ttest_table_dict = {"Algorithm / Data": ["MSLR10K-NDCG", "MSLR10K-MAP", "MQ2008-NDCG", "MQ2008-MAP", "MQ2007-NDCG", "MQ2007-MAP"]} 116 | ttest_value_dict = {} 117 | for model1 in ["DirectRanker", "ListNet*", "ListNet", "RankNet", "LambdaMart", "AdaRank"]: 118 | cur_list = [] 119 | for model2 in ["DirectRanker", "ListNet*", "ListNet", "RankNet", "LambdaMart", "AdaRank"]: 120 | for data in ["MSLR10K", "MQ2008", "MQ2007"]: 121 | for metric in ["NDCG", "MAP"]: 122 | if model1 == model2: 123 | mean = np.mean(restuls_ttest[model1][data][metric]) 124 | std = np.std(restuls_ttest[model1][data][metric]) 125 | cur_list.append(f"{round(mean,3)}({int(round(std,3)*1000)})") 126 | ttest = [1] 127 | else: 128 | ttest = bengio_nadeau_test(restuls_ttest[model1][data][metric], restuls_ttest[model2][data][metric]) 129 | ttest_value_dict[f"{data}-{metric}-{model1}-{model2}"] = ttest[0] 130 | ttest_table_dict[model1] = cur_list 131 | print(f"Binary {binary}") 132 | print(pd.DataFrame(ttest_table_dict)) 133 | 134 | # plot ttest results 135 | model_names = ["DirectRanker", "ListNet*", "ListNet", "RankNet", "LambdaMart", "AdaRank"] 136 | model_names2 = [r"RankNet$^*$", "ListNet$^*$", "ListNet", "RankNet", "LambdaMart", "AdaRank"] 137 | fig, ax = plt.subplots(2, 3, constrained_layout=True) 138 | # define colormap 139 | white = np.array([255/255, 255/255, 255/255, 1]) 140 | purple = np.array([107/255, 76/255, 154/255, 1]) 141 | blue = np.array([57/255, 106/255, 177/255, 1]) 142 | orange = np.array([218/255, 124/255, 48/255, 1]) 143 | red = np.array([204/255, 37/255, 41/255, 1]) 144 | black = np.array([83 / 255, 81 / 255, 84 / 255, 1]) 145 | viridis = mpl.colormaps['viridis'].resampled(1000) 146 | newcolors = viridis(np.linspace(0, 1, 10000)) 147 | newcolors[9500:, :] = white # self correlation 148 | newcolors[500:9500, :] = black # NS 149 | newcolors[:500, :] = red # 0.05 150 | # newcolors[10:100, :] = orange # 0.01 151 | # newcolors[:10, :] = red # 0.001 152 | newcmp = ListedColormap(newcolors) 153 | for col, data in enumerate(["MSLR10K", "MQ2008", "MQ2007"]): 154 | for row, metric in enumerate(["NDCG", "MAP"]): 155 | df_list = [] 156 | df_list_bigger = [] 157 | for model1 in model_names: 158 | cur_list = [] 159 | cur_list_bigger = [] 160 | for model2 in model_names: 161 | if np.mean(restuls_ttest[model1][data][metric]) > np.mean(restuls_ttest[model2][data][metric]): 162 | cur_list_bigger.append(1) 163 | else: 164 | cur_list_bigger.append(0) 165 | cur_list.append(ttest_value_dict[f"{data}-{metric}-{model1}-{model2}"]) 166 | df_list.append(cur_list) 167 | df_list_bigger.append(cur_list_bigger) 168 | pc = pd.DataFrame(df_list) 169 | for coli, _ in enumerate(model_names): 170 | for rowi, _ in enumerate(model_names): 171 | if rowi > coli: 172 | pc[coli].loc[rowi] = 1 173 | else: 174 | value = round(pc[coli].loc[rowi], 3) 175 | if round(pc[coli].loc[rowi], 3) == 0: 176 | value = "< 0.001" 177 | 178 | if df_list_bigger[coli][rowi] == 1 and round(pc[coli].loc[rowi], 3) <= 0.05: 179 | ax[row][col].annotate("", xy=(coli+0.3, rowi), xytext=(coli+0.3, rowi+0.5), arrowprops=dict(arrowstyle="->", color=white)) 180 | ax[row][col].text(coli, rowi, value, va='center', ha='center', size=4, color="white", fontstyle='italic') 181 | elif df_list_bigger[coli][rowi] == 0 and round(pc[coli].loc[rowi], 3) <= 0.05: 182 | ax[row][col].annotate("", xy=(coli, rowi+0.3), xytext=(coli+0.5, rowi+0.3), arrowprops=dict(arrowstyle="->", color=white)) 183 | ax[row][col].text(coli, rowi, value, va='center', ha='center', size=4, color="white", fontstyle='italic') 184 | else: 185 | ax[row][col].text(coli, rowi, value, va='center', ha='center', size=4, color="white") 186 | ax[row][col].matshow(pc, cmap=newcmp, vmin=0, vmax=1) 187 | data_name = data 188 | if data == "MSLR10K": 189 | data_name = "MSLR-WEB10K" 190 | ax[row][col].set_title(f"{data_name} {metric}") 191 | ax[row][col].set_xticks(np.arange(len(model_names))) 192 | ax[row][col].set_xticklabels(model_names2, fontsize=6, rotation=45) 193 | ax[row][col].set_yticks(np.arange(len(model_names))) 194 | ax[row][col].set_yticklabels(model_names2, fontsize=6, rotation=45) 195 | 196 | plt.savefig(f"15cv_ttest_binary{binary}.pdf") 197 | 198 | # get tensorflow V1 results for DirectRanker 199 | if results["DirectRanker"]["MSLR10K"]["NDCG-mean"] < np.mean(np.load("gridsearch/MSLR10K_DirectRankerV1_list_of_ndcg.npy")): 200 | results["DirectRanker"]["MSLR10K"]["NDCG-mean"] = np.mean(np.load("gridsearch/MSLR10K_DirectRankerV1_list_of_ndcg.npy")) 201 | results["DirectRanker"]["MSLR10K"]["MAP-mean"] = np.mean(np.load("gridsearch/MSLR10K_DirectRankerV1_list_of_map.npy")) 202 | results["DirectRanker"]["MSLR10K"]["NDCG-std"] = np.std(np.load("gridsearch/MSLR10K_DirectRankerV1_list_of_ndcg.npy")) 203 | results["DirectRanker"]["MSLR10K"]["MAP-std"] = np.std(np.load("gridsearch/MSLR10K_DirectRankerV1_list_of_map.npy")) 204 | if results["DirectRanker"]["MQ2008"]["NDCG-mean"] < np.mean(np.load("gridsearch/MQ2008_DirectRankerV1_list_of_ndcg.npy")): 205 | results["DirectRanker"]["MQ2008"]["NDCG-mean"] = np.mean(np.load("gridsearch/MQ2008_DirectRankerV1_list_of_ndcg.npy")) 206 | results["DirectRanker"]["MQ2008"]["MAP-mean"] = np.mean(np.load("gridsearch/MQ2008_DirectRankerV1_list_of_map.npy")) 207 | results["DirectRanker"]["MQ2008"]["NDCG-std"] = np.std(np.load("gridsearch/MQ2008_DirectRankerV1_list_of_ndcg.npy")) 208 | results["DirectRanker"]["MQ2008"]["MAP-std"] = np.std(np.load("gridsearch/MQ2008_DirectRankerV1_list_of_map.npy")) 209 | if results["DirectRanker"]["MQ2007"]["NDCG-mean"] < np.mean(np.load("gridsearch/MQ2007_DirectRankerV1_list_of_ndcg.npy")): 210 | results["DirectRanker"]["MQ2007"]["NDCG-mean"] = np.mean(np.load("gridsearch/MQ2007_DirectRankerV1_list_of_ndcg.npy")) 211 | results["DirectRanker"]["MQ2007"]["MAP-mean"] = np.mean(np.load("gridsearch/MQ2007_DirectRankerV1_list_of_map.npy")) 212 | results["DirectRanker"]["MQ2007"]["NDCG-std"] = np.std(np.load("gridsearch/MQ2007_DirectRankerV1_list_of_ndcg.npy")) 213 | results["DirectRanker"]["MQ2007"]["MAP-std"] = np.std(np.load("gridsearch/MQ2007_DirectRankerV1_list_of_map.npy")) 214 | 215 | # set ranklib results from paper 216 | if results["RankNet"]["MSLR10K"]["NDCG-mean"] < 0.157: 217 | results["RankNet"]["MSLR10K"]["NDCG-mean"] = 0.157 218 | results["RankNet"]["MSLR10K"]["NDCG-std"] = 0.003 219 | results["RankNet"]["MSLR10K"]["MAP-mean"] = 0.195 220 | results["RankNet"]["MSLR10K"]["MAP-std"] = 0.002 221 | if results["RankNet"]["MQ2008"]["NDCG-mean"] < 0.716: 222 | results["RankNet"]["MQ2008"]["NDCG-mean"] = 0.716 223 | results["RankNet"]["MQ2008"]["NDCG-std"] = 0.011 224 | results["RankNet"]["MQ2008"]["MAP-mean"] = 0.642 225 | results["RankNet"]["MQ2008"]["MAP-std"] = 0.010 226 | if results["RankNet"]["MQ2007"]["NDCG-mean"] < 0.525: 227 | results["RankNet"]["MQ2007"]["NDCG-mean"] = 0.525 228 | results["RankNet"]["MQ2007"]["NDCG-std"] = 0.011 229 | results["RankNet"]["MQ2007"]["MAP-mean"] = 0.525 230 | results["RankNet"]["MQ2007"]["MAP-std"] = 0.007 231 | 232 | if results["ListNet"]["MSLR10K"]["NDCG-mean"] < 0.157: 233 | results["ListNet"]["MSLR10K"]["NDCG-mean"] = 0.157 234 | results["ListNet"]["MSLR10K"]["NDCG-std"] = 0.003 235 | results["ListNet"]["MSLR10K"]["MAP-mean"] = 0.192 236 | results["ListNet"]["MSLR10K"]["MAP-std"] = 0.002 237 | if results["ListNet"]["MQ2008"]["NDCG-mean"] < 0.719: 238 | results["ListNet"]["MQ2008"]["NDCG-mean"] = 0.719 239 | results["ListNet"]["MQ2008"]["NDCG-std"] = 0.010 240 | results["ListNet"]["MQ2008"]["MAP-mean"] = 0.647 241 | results["ListNet"]["MQ2008"]["MAP-std"] = 0.006 242 | if results["ListNet"]["MQ2007"]["NDCG-mean"] < 0.526: 243 | results["ListNet"]["MQ2007"]["NDCG-mean"] = 0.526 244 | results["ListNet"]["MQ2007"]["NDCG-std"] = 0.010 245 | results["ListNet"]["MQ2007"]["MAP-mean"] = 0.525 246 | results["ListNet"]["MQ2007"]["MAP-std"] = 0.009 247 | 248 | for model in ["LambdaMart", "AdaRank", "ES-Rank", "IESR-Rank", "IESVM-Rank"]: 249 | cur_dict = {} 250 | for data in ["MSLR10K", "MQ2008", "MQ2007"]: 251 | cur_dict[data] = { 252 | "NDCG-mean": 0, 253 | "MAP-mean": 0, 254 | "NDCG-std": 0, 255 | "MAP-std": 0 256 | } 257 | results[model] = cur_dict 258 | 259 | results["LambdaMart"]["MSLR10K"]["NDCG-mean"] = 0.476 260 | results["LambdaMart"]["MSLR10K"]["NDCG-std"] = 0.003 261 | results["LambdaMart"]["MSLR10K"]["MAP-mean"] = 0.366 262 | results["LambdaMart"]["MSLR10K"]["MAP-std"] = 0.003 263 | 264 | results["LambdaMart"]["MQ2008"]["NDCG-mean"] = 0.723 265 | results["LambdaMart"]["MQ2008"]["NDCG-std"] = 0.007 266 | results["LambdaMart"]["MQ2008"]["MAP-mean"] = 0.624 267 | results["LambdaMart"]["MQ2008"]["MAP-std"] = 0.006 268 | 269 | results["LambdaMart"]["MQ2007"]["NDCG-mean"] = 0.531 270 | results["LambdaMart"]["MQ2007"]["NDCG-std"] = 0.012 271 | results["LambdaMart"]["MQ2007"]["MAP-mean"] = 0.510 272 | results["LambdaMart"]["MQ2007"]["MAP-std"] = 0.011 273 | 274 | results["AdaRank"]["MSLR10K"]["NDCG-mean"] = 0.400 275 | results["AdaRank"]["MSLR10K"]["NDCG-std"] = 0.016 276 | results["AdaRank"]["MSLR10K"]["MAP-mean"] = 0.322 277 | results["AdaRank"]["MSLR10K"]["MAP-std"] = 0.010 278 | 279 | results["AdaRank"]["MQ2008"]["NDCG-mean"] = 0.722 280 | results["AdaRank"]["MQ2008"]["NDCG-std"] = 0.010 281 | results["AdaRank"]["MQ2008"]["MAP-mean"] = 0.653 282 | results["AdaRank"]["MQ2008"]["MAP-std"] = 0.009 283 | 284 | results["AdaRank"]["MQ2007"]["NDCG-mean"] = 0.526 285 | results["AdaRank"]["MQ2007"]["NDCG-std"] = 0.010 286 | results["AdaRank"]["MQ2007"]["MAP-mean"] = 0.527 287 | results["AdaRank"]["MQ2007"]["MAP-std"] = 0.010 288 | 289 | results["ES-Rank"]["MSLR10K"]["NDCG-mean"] = 0.382 290 | results["ES-Rank"]["MSLR10K"]["MAP-mean"] = 0.570 291 | results["ES-Rank"]["MQ2008"]["NDCG-mean"] = 0.507 292 | results["ES-Rank"]["MQ2008"]["MAP-mean"] = 0.483 293 | results["ES-Rank"]["MQ2007"]["NDCG-mean"] = 0.451 294 | results["ES-Rank"]["MQ2007"]["MAP-mean"] = 0.470 295 | 296 | results["IESR-Rank"]["MSLR10K"]["NDCG-mean"] = 0.415 297 | results["IESR-Rank"]["MSLR10K"]["MAP-mean"] = 0.603 298 | results["IESR-Rank"]["MQ2008"]["NDCG-mean"] = 0.517 299 | results["IESR-Rank"]["MQ2008"]["MAP-mean"] = 0.494 300 | results["IESR-Rank"]["MQ2007"]["NDCG-mean"] = 0.455 301 | results["IESR-Rank"]["MQ2007"]["MAP-mean"] = 0.473 302 | 303 | results["IESVM-Rank"]["MSLR10K"]["NDCG-mean"] = 0.224 304 | results["IESVM-Rank"]["MSLR10K"]["MAP-mean"] = 0.457 305 | results["IESVM-Rank"]["MQ2008"]["NDCG-mean"] = 0.498 306 | results["IESVM-Rank"]["MQ2008"]["MAP-mean"] = 0.473 307 | results["IESVM-Rank"]["MQ2007"]["NDCG-mean"] = 0.436 308 | results["IESVM-Rank"]["MQ2007"]["MAP-mean"] = 0.456 309 | 310 | print(results) 311 | 312 | model_names = ["DirectRanker", "RankNet", "ListNet", "LambdaMart", "AdaRank", "ES-Rank", "IESR-Rank", "IESVM-Rank"] 313 | 314 | # performe Friedman Nemenyi Test 315 | data_list = [] 316 | std_list = [] 317 | for data in ["MSLR10K", "MQ2008", "MQ2007"]: 318 | for metric in ["NDCG", "MAP"]: 319 | cur_row = [] 320 | cur_row_std = [] 321 | for model in model_names: 322 | cur_row.append(results[model][data][f"{metric}-mean"]) 323 | cur_row_std.append(results[model][data][f"{metric}-std"]) 324 | data_list.append(cur_row) 325 | std_list.append(cur_row_std) 326 | data_list = np.array(data_list) 327 | std_list = np.array(std_list) 328 | 329 | print("Table-Data: ", data_list.T, std_list.T) 330 | 331 | friedman_result = ss.friedmanchisquare(*data_list.T) 332 | 333 | print(friedman_result[1]) 334 | 335 | model_names = [r"RankNet^$*$", "RankNet", "ListNet", "LambdaMart", "AdaRank", "ES-Rank", "IESR-Rank", "IESVM-Rank"] 336 | # if pvalue < 0.05 we can reject the null hypothesis and do a post hoc order test 337 | if friedman_result[1] < 0.05: 338 | pc = sp.posthoc_nemenyi_friedman(data_list) 339 | print(pc) 340 | fig, ax_kwargs = plt.subplots(figsize=(3.3, 2.5), constrained_layout=True) 341 | heatmap_args = {'linewidths': 0.25, 'linecolor': '0.5', 'clip_on': False, 'square': True, 'cbar_ax_bbox': [0.80, 0.35, 0.04, 0.3]} 342 | ax, cbar = sp.sign_plot(pc, **heatmap_args) 343 | ax.set_xticks(np.arange(len(model_names))+0.5) 344 | ax.set_xticklabels(model_names, fontsize=5, rotation=45) 345 | ax.set_yticks(np.arange(len(model_names))+0.5) 346 | ax.set_yticklabels(model_names, fontsize=5, rotation=45) 347 | cbar.ax.tick_params(size=0, labelsize=5) 348 | cbar.orientation = "horizontal" 349 | plt.savefig('friedman_test.pdf') 350 | --------------------------------------------------------------------------------