├── .gitignore ├── LICENSE ├── README.md ├── examples └── demo.ipynb ├── scripts ├── ds_zero2.json ├── ds_zero3.json ├── eval_bright.sh ├── eval_example.sh ├── eval_mair.sh ├── train_listwise.sh └── train_pointwise.sh ├── setup.py ├── src └── llm4ranking │ ├── __init__.py │ ├── evaluation │ ├── evaluate_bright.py │ ├── evaluate_mair.py │ ├── evaluator.py │ ├── trec_eval.py │ └── utils.py │ ├── lm │ ├── __init__.py │ ├── base.py │ ├── huggingface.py │ ├── openai.py │ └── vllm.py │ ├── model │ ├── __init__.py │ ├── base.py │ ├── first.py │ ├── prp.py │ ├── query_generation.py │ ├── rankgpt.py │ ├── relevance_generation.py │ └── selection.py │ ├── ranker │ ├── __init__.py │ └── base.py │ └── training │ ├── pointwise │ ├── arguments.py │ ├── data.py │ └── train.py │ └── sft │ ├── arguments.py │ ├── data.py │ └── train.py └── toy_data ├── dl19_bm25_top20.jsonl ├── listwise_train.jsonl └── pointwise_train.jsonl /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/README.md -------------------------------------------------------------------------------- /examples/demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/examples/demo.ipynb -------------------------------------------------------------------------------- /scripts/ds_zero2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/scripts/ds_zero2.json -------------------------------------------------------------------------------- /scripts/ds_zero3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/scripts/ds_zero3.json -------------------------------------------------------------------------------- /scripts/eval_bright.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/scripts/eval_bright.sh -------------------------------------------------------------------------------- /scripts/eval_example.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/scripts/eval_example.sh -------------------------------------------------------------------------------- /scripts/eval_mair.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/scripts/eval_mair.sh -------------------------------------------------------------------------------- /scripts/train_listwise.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/scripts/train_listwise.sh -------------------------------------------------------------------------------- /scripts/train_pointwise.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/scripts/train_pointwise.sh -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/setup.py -------------------------------------------------------------------------------- /src/llm4ranking/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/__init__.py -------------------------------------------------------------------------------- /src/llm4ranking/evaluation/evaluate_bright.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/evaluation/evaluate_bright.py -------------------------------------------------------------------------------- /src/llm4ranking/evaluation/evaluate_mair.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/evaluation/evaluate_mair.py -------------------------------------------------------------------------------- /src/llm4ranking/evaluation/evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/evaluation/evaluator.py -------------------------------------------------------------------------------- /src/llm4ranking/evaluation/trec_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/evaluation/trec_eval.py -------------------------------------------------------------------------------- /src/llm4ranking/evaluation/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/evaluation/utils.py -------------------------------------------------------------------------------- /src/llm4ranking/lm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/lm/__init__.py -------------------------------------------------------------------------------- /src/llm4ranking/lm/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/lm/base.py -------------------------------------------------------------------------------- /src/llm4ranking/lm/huggingface.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/lm/huggingface.py -------------------------------------------------------------------------------- /src/llm4ranking/lm/openai.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/lm/openai.py -------------------------------------------------------------------------------- /src/llm4ranking/lm/vllm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/lm/vllm.py -------------------------------------------------------------------------------- /src/llm4ranking/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/model/__init__.py -------------------------------------------------------------------------------- /src/llm4ranking/model/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/model/base.py -------------------------------------------------------------------------------- /src/llm4ranking/model/first.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/model/first.py -------------------------------------------------------------------------------- /src/llm4ranking/model/prp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/model/prp.py -------------------------------------------------------------------------------- /src/llm4ranking/model/query_generation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/model/query_generation.py -------------------------------------------------------------------------------- /src/llm4ranking/model/rankgpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/model/rankgpt.py -------------------------------------------------------------------------------- /src/llm4ranking/model/relevance_generation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/model/relevance_generation.py -------------------------------------------------------------------------------- /src/llm4ranking/model/selection.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/model/selection.py -------------------------------------------------------------------------------- /src/llm4ranking/ranker/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/ranker/__init__.py -------------------------------------------------------------------------------- /src/llm4ranking/ranker/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/ranker/base.py -------------------------------------------------------------------------------- /src/llm4ranking/training/pointwise/arguments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/training/pointwise/arguments.py -------------------------------------------------------------------------------- /src/llm4ranking/training/pointwise/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/training/pointwise/data.py -------------------------------------------------------------------------------- /src/llm4ranking/training/pointwise/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/training/pointwise/train.py -------------------------------------------------------------------------------- /src/llm4ranking/training/sft/arguments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/training/sft/arguments.py -------------------------------------------------------------------------------- /src/llm4ranking/training/sft/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/training/sft/data.py -------------------------------------------------------------------------------- /src/llm4ranking/training/sft/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/src/llm4ranking/training/sft/train.py -------------------------------------------------------------------------------- /toy_data/dl19_bm25_top20.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/toy_data/dl19_bm25_top20.jsonl -------------------------------------------------------------------------------- /toy_data/listwise_train.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/toy_data/listwise_train.jsonl -------------------------------------------------------------------------------- /toy_data/pointwise_train.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuqi6777/llm4ranking/HEAD/toy_data/pointwise_train.jsonl --------------------------------------------------------------------------------