├── .github └── workflows │ ├── deploy.yml │ ├── docs.yml │ ├── style.yml │ └── test.yml ├── .gitignore ├── .readthedocs.yaml ├── LICENSE.txt ├── MANIFEST.in ├── README.md ├── docs ├── .DS_Store ├── CNAME ├── Makefile ├── _static │ └── main.css ├── advanced.rst ├── api.rst ├── conf.py ├── dblp_cache.json.gz ├── extras │ ├── __init__.py │ └── cite.py ├── getting-started.rst ├── index.rst ├── make.bat └── requirements.txt ├── examples └── demo.ipynb ├── ir_measures ├── __init__.py ├── __main__.py ├── bin │ ├── __init__.py │ ├── gdeval.pl │ └── msmarco_eval.py ├── lazylibs.py ├── measures │ ├── __init__.py │ ├── accuracy.py │ ├── ap.py │ ├── base.py │ ├── bpm.py │ ├── bpref.py │ ├── compat.py │ ├── diversity.py │ ├── err.py │ ├── infap.py │ ├── inst.py │ ├── iprec.py │ ├── judged.py │ ├── ndcg.py │ ├── nerr.py │ ├── numq.py │ ├── numrel.py │ ├── numret.py │ ├── p.py │ ├── r.py │ ├── rbp.py │ ├── rprec.py │ ├── rr.py │ ├── sdcg.py │ ├── set_measures.py │ └── success.py ├── providers │ ├── __init__.py │ ├── accuracy_provider.py │ ├── base.py │ ├── compat_provider.py │ ├── cwl_eval.py │ ├── fallback_provider.py │ ├── gdeval_provider.py │ ├── judged_provider.py │ ├── msmarco_provider.py │ ├── pyndeval_provider.py │ ├── pytrec_eval_provider.py │ ├── ranx_provider.py │ ├── runtime_provider.py │ └── trectools_provider.py ├── py.typed └── util.py ├── pyproject.toml ├── requirements-dev.txt ├── requirements.txt └── tests ├── __init__.py ├── base.py ├── compat.qrels ├── compat.run ├── cwl.qrels ├── cwl.run ├── test_accuracy.py ├── test_aggregate.py ├── test_compat.py ├── test_cwl_eval.py ├── test_gdeval.py ├── test_judged.py ├── test_measures.py ├── test_providers.py ├── test_pyndeval.py ├── test_pytrec_eval.py ├── test_ranx.py ├── test_runtime.py └── test_util.py /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/.github/workflows/deploy.yml -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/.github/workflows/docs.yml -------------------------------------------------------------------------------- /.github/workflows/style.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/.github/workflows/style.yml -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/.github/workflows/test.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/.gitignore -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/.readthedocs.yaml -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/LICENSE.txt -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/README.md -------------------------------------------------------------------------------- /docs/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/docs/.DS_Store -------------------------------------------------------------------------------- /docs/CNAME: -------------------------------------------------------------------------------- 1 | ir-measur.es -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/docs/Makefile -------------------------------------------------------------------------------- /docs/_static/main.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/docs/_static/main.css -------------------------------------------------------------------------------- /docs/advanced.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/docs/advanced.rst -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/docs/api.rst -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/docs/conf.py -------------------------------------------------------------------------------- /docs/dblp_cache.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/docs/dblp_cache.json.gz -------------------------------------------------------------------------------- /docs/extras/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/extras/cite.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/docs/extras/cite.py -------------------------------------------------------------------------------- /docs/getting-started.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/docs/getting-started.rst -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/docs/index.rst -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/docs/make.bat -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/docs/requirements.txt -------------------------------------------------------------------------------- /examples/demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/examples/demo.ipynb -------------------------------------------------------------------------------- /ir_measures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/__init__.py -------------------------------------------------------------------------------- /ir_measures/__main__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/__main__.py -------------------------------------------------------------------------------- /ir_measures/bin/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ir_measures/bin/gdeval.pl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/bin/gdeval.pl -------------------------------------------------------------------------------- /ir_measures/bin/msmarco_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/bin/msmarco_eval.py -------------------------------------------------------------------------------- /ir_measures/lazylibs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/lazylibs.py -------------------------------------------------------------------------------- /ir_measures/measures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/__init__.py -------------------------------------------------------------------------------- /ir_measures/measures/accuracy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/accuracy.py -------------------------------------------------------------------------------- /ir_measures/measures/ap.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/ap.py -------------------------------------------------------------------------------- /ir_measures/measures/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/base.py -------------------------------------------------------------------------------- /ir_measures/measures/bpm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/bpm.py -------------------------------------------------------------------------------- /ir_measures/measures/bpref.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/bpref.py -------------------------------------------------------------------------------- /ir_measures/measures/compat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/compat.py -------------------------------------------------------------------------------- /ir_measures/measures/diversity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/diversity.py -------------------------------------------------------------------------------- /ir_measures/measures/err.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/err.py -------------------------------------------------------------------------------- /ir_measures/measures/infap.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/infap.py -------------------------------------------------------------------------------- /ir_measures/measures/inst.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/inst.py -------------------------------------------------------------------------------- /ir_measures/measures/iprec.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/iprec.py -------------------------------------------------------------------------------- /ir_measures/measures/judged.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/judged.py -------------------------------------------------------------------------------- /ir_measures/measures/ndcg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/ndcg.py -------------------------------------------------------------------------------- /ir_measures/measures/nerr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/nerr.py -------------------------------------------------------------------------------- /ir_measures/measures/numq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/numq.py -------------------------------------------------------------------------------- /ir_measures/measures/numrel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/numrel.py -------------------------------------------------------------------------------- /ir_measures/measures/numret.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/numret.py -------------------------------------------------------------------------------- /ir_measures/measures/p.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/p.py -------------------------------------------------------------------------------- /ir_measures/measures/r.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/r.py -------------------------------------------------------------------------------- /ir_measures/measures/rbp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/rbp.py -------------------------------------------------------------------------------- /ir_measures/measures/rprec.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/rprec.py -------------------------------------------------------------------------------- /ir_measures/measures/rr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/rr.py -------------------------------------------------------------------------------- /ir_measures/measures/sdcg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/sdcg.py -------------------------------------------------------------------------------- /ir_measures/measures/set_measures.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/set_measures.py -------------------------------------------------------------------------------- /ir_measures/measures/success.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/measures/success.py -------------------------------------------------------------------------------- /ir_measures/providers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/providers/__init__.py -------------------------------------------------------------------------------- /ir_measures/providers/accuracy_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/providers/accuracy_provider.py -------------------------------------------------------------------------------- /ir_measures/providers/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/providers/base.py -------------------------------------------------------------------------------- /ir_measures/providers/compat_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/providers/compat_provider.py -------------------------------------------------------------------------------- /ir_measures/providers/cwl_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/providers/cwl_eval.py -------------------------------------------------------------------------------- /ir_measures/providers/fallback_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/providers/fallback_provider.py -------------------------------------------------------------------------------- /ir_measures/providers/gdeval_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/providers/gdeval_provider.py -------------------------------------------------------------------------------- /ir_measures/providers/judged_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/providers/judged_provider.py -------------------------------------------------------------------------------- /ir_measures/providers/msmarco_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/providers/msmarco_provider.py -------------------------------------------------------------------------------- /ir_measures/providers/pyndeval_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/providers/pyndeval_provider.py -------------------------------------------------------------------------------- /ir_measures/providers/pytrec_eval_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/providers/pytrec_eval_provider.py -------------------------------------------------------------------------------- /ir_measures/providers/ranx_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/providers/ranx_provider.py -------------------------------------------------------------------------------- /ir_measures/providers/runtime_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/providers/runtime_provider.py -------------------------------------------------------------------------------- /ir_measures/providers/trectools_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/providers/trectools_provider.py -------------------------------------------------------------------------------- /ir_measures/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ir_measures/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/ir_measures/util.py -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/pyproject.toml -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/requirements-dev.txt -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytrec-eval-terrier>=0.5.5 2 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/base.py -------------------------------------------------------------------------------- /tests/compat.qrels: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/compat.qrels -------------------------------------------------------------------------------- /tests/compat.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/compat.run -------------------------------------------------------------------------------- /tests/cwl.qrels: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/cwl.qrels -------------------------------------------------------------------------------- /tests/cwl.run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/cwl.run -------------------------------------------------------------------------------- /tests/test_accuracy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/test_accuracy.py -------------------------------------------------------------------------------- /tests/test_aggregate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/test_aggregate.py -------------------------------------------------------------------------------- /tests/test_compat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/test_compat.py -------------------------------------------------------------------------------- /tests/test_cwl_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/test_cwl_eval.py -------------------------------------------------------------------------------- /tests/test_gdeval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/test_gdeval.py -------------------------------------------------------------------------------- /tests/test_judged.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/test_judged.py -------------------------------------------------------------------------------- /tests/test_measures.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/test_measures.py -------------------------------------------------------------------------------- /tests/test_providers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/test_providers.py -------------------------------------------------------------------------------- /tests/test_pyndeval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/test_pyndeval.py -------------------------------------------------------------------------------- /tests/test_pytrec_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/test_pytrec_eval.py -------------------------------------------------------------------------------- /tests/test_ranx.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/test_ranx.py -------------------------------------------------------------------------------- /tests/test_runtime.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/test_runtime.py -------------------------------------------------------------------------------- /tests/test_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terrierteam/ir_measures/HEAD/tests/test_util.py --------------------------------------------------------------------------------