├── LICENSE ├── MANIFEST.in ├── README.md ├── pyproject.toml ├── requirements.txt ├── scripts └── download_msmarco.sh └── src ├── eval └── test.py └── umbrela ├── __init__.py ├── ensemble_judge.py ├── gemini_judge.py ├── gpt_judge.py ├── hgfllm_judge.py ├── llm_judge.py ├── osllm_judge.py ├── prompts ├── __init__.py ├── qrel_fewshot_basic.txt ├── qrel_fewshot_bing.txt ├── qrel_zeroshot_basic.txt └── qrel_zeroshot_bing.txt └── utils ├── __init__.py ├── common_utils.py └── qrel_utils.py /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/LICENSE -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/README.md -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/pyproject.toml -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/download_msmarco.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/scripts/download_msmarco.sh -------------------------------------------------------------------------------- /src/eval/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/src/eval/test.py -------------------------------------------------------------------------------- /src/umbrela/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/umbrela/ensemble_judge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/src/umbrela/ensemble_judge.py -------------------------------------------------------------------------------- /src/umbrela/gemini_judge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/src/umbrela/gemini_judge.py -------------------------------------------------------------------------------- /src/umbrela/gpt_judge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/src/umbrela/gpt_judge.py -------------------------------------------------------------------------------- /src/umbrela/hgfllm_judge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/src/umbrela/hgfllm_judge.py -------------------------------------------------------------------------------- /src/umbrela/llm_judge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/src/umbrela/llm_judge.py -------------------------------------------------------------------------------- /src/umbrela/osllm_judge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/src/umbrela/osllm_judge.py -------------------------------------------------------------------------------- /src/umbrela/prompts/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/umbrela/prompts/qrel_fewshot_basic.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/src/umbrela/prompts/qrel_fewshot_basic.txt -------------------------------------------------------------------------------- /src/umbrela/prompts/qrel_fewshot_bing.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/src/umbrela/prompts/qrel_fewshot_bing.txt -------------------------------------------------------------------------------- /src/umbrela/prompts/qrel_zeroshot_basic.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/src/umbrela/prompts/qrel_zeroshot_basic.txt -------------------------------------------------------------------------------- /src/umbrela/prompts/qrel_zeroshot_bing.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/src/umbrela/prompts/qrel_zeroshot_bing.txt -------------------------------------------------------------------------------- /src/umbrela/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/umbrela/utils/common_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/src/umbrela/utils/common_utils.py -------------------------------------------------------------------------------- /src/umbrela/utils/qrel_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/castorini/umbrela/HEAD/src/umbrela/utils/qrel_utils.py --------------------------------------------------------------------------------