├── 01-collect-grades.ipynb ├── 02-compute-metrics.ipynb ├── 03-bootstrap.ipynb ├── LICENSE ├── README.md ├── codex.ipynb ├── data ├── datasets │ ├── codex_context.jsonl │ ├── conala-dataset.json │ ├── conala-dataset.jsonl │ ├── conala-test.json │ ├── conala-train.json │ ├── django.json │ ├── hs.json │ └── hs_cards │ │ ├── 0.png │ │ ├── 1.png │ │ ├── 10.png │ │ ├── 11.png │ │ ├── 12.png │ │ ├── 13.png │ │ ├── 14.png │ │ ├── 15.png │ │ ├── 16.png │ │ ├── 17.png │ │ ├── 18.png │ │ ├── 19.png │ │ ├── 2.png │ │ ├── 20.png │ │ ├── 21.png │ │ ├── 22.png │ │ ├── 23.png │ │ ├── 24.png │ │ ├── 25.png │ │ ├── 26.png │ │ ├── 27.png │ │ ├── 28.png │ │ ├── 29.png │ │ ├── 3.png │ │ ├── 30.png │ │ ├── 31.png │ │ ├── 32.png │ │ ├── 33.png │ │ ├── 34.png │ │ ├── 35.png │ │ ├── 36.png │ │ ├── 37.png │ │ ├── 38.png │ │ ├── 39.png │ │ ├── 4.png │ │ ├── 40.png │ │ ├── 41.png │ │ ├── 42.png │ │ ├── 43.png │ │ ├── 44.png │ │ ├── 45.png │ │ ├── 46.png │ │ ├── 47.png │ │ ├── 48.png │ │ ├── 49.png │ │ ├── 5.png │ │ ├── 50.png │ │ ├── 51.png │ │ ├── 52.png │ │ ├── 53.png │ │ ├── 54.png │ │ ├── 55.png │ │ ├── 56.png │ │ ├── 57.png │ │ ├── 58.png │ │ ├── 59.png │ │ ├── 6.png │ │ ├── 60.png │ │ ├── 61.png │ │ ├── 62.png │ │ ├── 63.png │ │ ├── 64.png │ │ ├── 65.png │ │ ├── 7.jpg │ │ ├── 7.png │ │ ├── 8.png │ │ └── 9.png ├── plots │ └── graders-distrib.pdf └── to-grade │ ├── conala-model-pairs.json │ ├── conala-model-scores.json │ ├── conala │ ├── conala-aggregated-grades.json │ ├── conala-all-grades.json │ ├── conala-human-grades.json │ ├── conala.json │ ├── grader1.conala.json │ ├── grader10.conala.json │ ├── grader11.conala.json │ ├── grader12.conala.json │ ├── grader13.conala.json │ ├── grader14.conala.json │ ├── grader15.conala.json │ ├── grader16.conala.json │ ├── grader2.conala.json │ ├── grader3.conala.json │ ├── grader4.conala.json │ ├── grader5.conala.json │ ├── grader6.conala.json │ ├── grader7.conala.json │ ├── grader8.conala.json │ └── grader9.conala.json │ ├── hs-model-pairs.json │ ├── hs-model-scores.json │ └── hs │ ├── grader1.hs.json │ ├── grader2.hs.json │ ├── grader3.hs.json │ ├── grader4.hs.json │ ├── hs-aggregated-grades.json │ ├── hs-all-grades.json │ ├── hs-human-grades.json │ └── hs.json ├── metrics_evaluation ├── __init__.py ├── analysis │ ├── __init__.py │ └── bootstrap.py ├── grading │ ├── __init__.py │ ├── hs-rate-notk.py │ ├── hs-rate.py │ └── rate.py └── metrics │ ├── README.md │ ├── __init__.py │ ├── codebleu │ ├── __init__.py │ ├── bleu_score.py │ ├── codebleu.py │ ├── graph_generator │ │ ├── __init__.py │ │ ├── dataflowpass.py │ │ ├── extract_graphs.py │ │ ├── graphgenerator.py │ │ ├── graphgenutils.py │ │ ├── plot_histogram.py │ │ ├── run.py │ │ ├── type_lattice_generator.py │ │ └── typeparsing │ │ │ ├── __init__.py │ │ │ ├── aliasreplacement.py │ │ │ ├── erasure.py │ │ │ ├── inheritancerewrite.py │ │ │ ├── nodes.py │ │ │ ├── pruneannotations.py │ │ │ ├── rewriterules │ │ │ ├── __init__.py │ │ │ ├── removegenericwithany.py │ │ │ ├── removerecursivegenerics.py │ │ │ ├── removestandalones.py │ │ │ ├── removeunionwithanys.py │ │ │ └── rewriterule.py │ │ │ ├── rewriterulevisitor.py │ │ │ └── visitor.py │ └── test_codebleu.py │ ├── ruby │ ├── __init__.py │ ├── nx_graphs.py │ ├── similarity.py │ ├── test.py │ └── util.py │ └── sacrebleu_code │ ├── CHANGELOG.md │ ├── DATASETS.md │ ├── LICENSE.txt │ ├── Makefile │ ├── README.md │ ├── __init__.py │ ├── mypy.ini │ ├── pytest.ini │ ├── sacrebleu_methods │ ├── __init__.py │ ├── __main__.py │ ├── compat.py │ ├── dataset.py │ ├── metrics │ │ ├── __init__.py │ │ ├── base.py │ │ ├── bleu.py │ │ ├── chrf.py │ │ ├── helpers.py │ │ ├── lib_ter.py │ │ └── ter.py │ ├── py.typed │ ├── sacrebleu.py │ ├── significance.py │ ├── tokenizers │ │ ├── __init__.py │ │ ├── tokenizer_13a.py │ │ ├── tokenizer_base.py │ │ ├── tokenizer_char.py │ │ ├── tokenizer_code.py │ │ ├── tokenizer_intl.py │ │ ├── tokenizer_ja_mecab.py │ │ ├── tokenizer_re.py │ │ ├── tokenizer_spm.py │ │ ├── tokenizer_ter.py │ │ └── tokenizer_zh.py │ └── utils.py │ ├── scripts │ ├── __init__.py │ ├── add_wmt.sh │ └── perf_test.py │ ├── setup.cfg │ ├── setup.py │ ├── test.sh │ ├── test │ ├── __init__.py │ ├── test_api.py │ ├── test_bleu.py │ ├── test_chrf.py │ ├── test_sentence_bleu.py │ ├── test_significance.py │ ├── test_ter.py │ ├── test_tokenizer_ter.py │ └── wmt17_en_de_systems.pkl.bz2 │ └── tox.ini ├── poetry.lock └── pyproject.toml /01-collect-grades.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/01-collect-grades.ipynb -------------------------------------------------------------------------------- /02-compute-metrics.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/02-compute-metrics.ipynb -------------------------------------------------------------------------------- /03-bootstrap.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/03-bootstrap.ipynb -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/README.md -------------------------------------------------------------------------------- /codex.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/codex.ipynb -------------------------------------------------------------------------------- /data/datasets/codex_context.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/codex_context.jsonl -------------------------------------------------------------------------------- /data/datasets/conala-dataset.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/conala-dataset.json -------------------------------------------------------------------------------- /data/datasets/conala-dataset.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/conala-dataset.jsonl -------------------------------------------------------------------------------- /data/datasets/conala-test.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/conala-test.json -------------------------------------------------------------------------------- /data/datasets/conala-train.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/conala-train.json -------------------------------------------------------------------------------- /data/datasets/django.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/django.json -------------------------------------------------------------------------------- /data/datasets/hs.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs.json -------------------------------------------------------------------------------- /data/datasets/hs_cards/0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/0.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/1.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/10.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/11.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/12.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/13.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/14.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/15.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/16.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/17.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/18.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/18.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/19.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/19.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/2.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/20.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/20.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/21.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/21.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/22.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/22.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/23.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/23.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/24.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/24.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/25.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/25.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/26.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/26.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/27.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/27.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/28.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/28.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/29.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/29.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/3.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/30.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/30.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/31.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/31.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/32.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/33.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/33.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/34.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/34.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/35.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/35.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/36.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/36.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/37.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/37.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/38.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/38.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/39.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/39.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/4.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/40.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/40.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/41.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/41.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/42.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/42.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/43.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/43.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/44.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/44.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/45.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/45.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/46.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/46.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/47.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/47.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/48.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/48.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/49.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/49.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/5.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/50.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/50.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/51.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/51.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/52.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/52.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/53.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/53.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/54.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/54.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/55.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/55.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/56.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/56.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/57.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/57.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/58.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/58.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/59.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/59.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/6.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/60.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/60.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/61.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/61.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/62.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/62.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/63.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/63.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/64.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/65.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/65.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/7.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/7.jpg -------------------------------------------------------------------------------- /data/datasets/hs_cards/7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/7.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/8.png -------------------------------------------------------------------------------- /data/datasets/hs_cards/9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/datasets/hs_cards/9.png -------------------------------------------------------------------------------- /data/plots/graders-distrib.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/plots/graders-distrib.pdf -------------------------------------------------------------------------------- /data/to-grade/conala-model-pairs.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala-model-pairs.json -------------------------------------------------------------------------------- /data/to-grade/conala-model-scores.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala-model-scores.json -------------------------------------------------------------------------------- /data/to-grade/conala/conala-aggregated-grades.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/conala-aggregated-grades.json -------------------------------------------------------------------------------- /data/to-grade/conala/conala-all-grades.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/conala-all-grades.json -------------------------------------------------------------------------------- /data/to-grade/conala/conala-human-grades.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/conala-human-grades.json -------------------------------------------------------------------------------- /data/to-grade/conala/conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/conala.json -------------------------------------------------------------------------------- /data/to-grade/conala/grader1.conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/grader1.conala.json -------------------------------------------------------------------------------- /data/to-grade/conala/grader10.conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/grader10.conala.json -------------------------------------------------------------------------------- /data/to-grade/conala/grader11.conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/grader11.conala.json -------------------------------------------------------------------------------- /data/to-grade/conala/grader12.conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/grader12.conala.json -------------------------------------------------------------------------------- /data/to-grade/conala/grader13.conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/grader13.conala.json -------------------------------------------------------------------------------- /data/to-grade/conala/grader14.conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/grader14.conala.json -------------------------------------------------------------------------------- /data/to-grade/conala/grader15.conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/grader15.conala.json -------------------------------------------------------------------------------- /data/to-grade/conala/grader16.conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/grader16.conala.json -------------------------------------------------------------------------------- /data/to-grade/conala/grader2.conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/grader2.conala.json -------------------------------------------------------------------------------- /data/to-grade/conala/grader3.conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/grader3.conala.json -------------------------------------------------------------------------------- /data/to-grade/conala/grader4.conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/grader4.conala.json -------------------------------------------------------------------------------- /data/to-grade/conala/grader5.conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/grader5.conala.json -------------------------------------------------------------------------------- /data/to-grade/conala/grader6.conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/grader6.conala.json -------------------------------------------------------------------------------- /data/to-grade/conala/grader7.conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/grader7.conala.json -------------------------------------------------------------------------------- /data/to-grade/conala/grader8.conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/grader8.conala.json -------------------------------------------------------------------------------- /data/to-grade/conala/grader9.conala.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/conala/grader9.conala.json -------------------------------------------------------------------------------- /data/to-grade/hs-model-pairs.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/hs-model-pairs.json -------------------------------------------------------------------------------- /data/to-grade/hs-model-scores.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/hs-model-scores.json -------------------------------------------------------------------------------- /data/to-grade/hs/grader1.hs.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/hs/grader1.hs.json -------------------------------------------------------------------------------- /data/to-grade/hs/grader2.hs.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/hs/grader2.hs.json -------------------------------------------------------------------------------- /data/to-grade/hs/grader3.hs.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/hs/grader3.hs.json -------------------------------------------------------------------------------- /data/to-grade/hs/grader4.hs.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/hs/grader4.hs.json -------------------------------------------------------------------------------- /data/to-grade/hs/hs-aggregated-grades.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/hs/hs-aggregated-grades.json -------------------------------------------------------------------------------- /data/to-grade/hs/hs-all-grades.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/hs/hs-all-grades.json -------------------------------------------------------------------------------- /data/to-grade/hs/hs-human-grades.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/hs/hs-human-grades.json -------------------------------------------------------------------------------- /data/to-grade/hs/hs.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/data/to-grade/hs/hs.json -------------------------------------------------------------------------------- /metrics_evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics_evaluation/analysis/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics_evaluation/analysis/bootstrap.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/analysis/bootstrap.py -------------------------------------------------------------------------------- /metrics_evaluation/grading/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics_evaluation/grading/hs-rate-notk.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/grading/hs-rate-notk.py -------------------------------------------------------------------------------- /metrics_evaluation/grading/hs-rate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/grading/hs-rate.py -------------------------------------------------------------------------------- /metrics_evaluation/grading/rate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/grading/rate.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/README.md -------------------------------------------------------------------------------- /metrics_evaluation/metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/__init__.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/bleu_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/bleu_score.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/codebleu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/codebleu.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/dataflowpass.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/dataflowpass.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/extract_graphs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/extract_graphs.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/graphgenerator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/graphgenerator.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/graphgenutils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/graphgenutils.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/plot_histogram.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/plot_histogram.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/run.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/type_lattice_generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/type_lattice_generator.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/__init__.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/aliasreplacement.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/aliasreplacement.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/erasure.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/erasure.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/inheritancerewrite.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/inheritancerewrite.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/nodes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/nodes.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/pruneannotations.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/pruneannotations.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/rewriterules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/rewriterules/__init__.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/rewriterules/removegenericwithany.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/rewriterules/removegenericwithany.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/rewriterules/removerecursivegenerics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/rewriterules/removerecursivegenerics.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/rewriterules/removestandalones.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/rewriterules/removestandalones.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/rewriterules/removeunionwithanys.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/rewriterules/removeunionwithanys.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/rewriterules/rewriterule.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/rewriterules/rewriterule.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/rewriterulevisitor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/rewriterulevisitor.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/visitor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/codebleu/graph_generator/typeparsing/visitor.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/codebleu/test_codebleu.py: -------------------------------------------------------------------------------- 1 | import codebleu 2 | -------------------------------------------------------------------------------- /metrics_evaluation/metrics/ruby/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics_evaluation/metrics/ruby/nx_graphs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/ruby/nx_graphs.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/ruby/similarity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/ruby/similarity.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/ruby/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/ruby/test.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/ruby/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/ruby/util.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/CHANGELOG.md -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/DATASETS.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/DATASETS.md -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/LICENSE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/LICENSE.txt -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/Makefile -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/README.md -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/mypy.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/mypy.ini -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/pytest.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/pytest.ini -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/__init__.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/__main__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/__main__.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/compat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/compat.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/dataset.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/metrics/__init__.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/metrics/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/metrics/base.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/metrics/bleu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/metrics/bleu.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/metrics/chrf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/metrics/chrf.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/metrics/helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/metrics/helpers.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/metrics/lib_ter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/metrics/lib_ter.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/metrics/ter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/metrics/ter.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/sacrebleu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/sacrebleu.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/significance.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/significance.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/__init__.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_13a.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_13a.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_base.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_char.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_char.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_code.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_code.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_intl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_intl.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_ja_mecab.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_ja_mecab.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_re.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_re.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_spm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_spm.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_ter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_ter.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_zh.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/tokenizers/tokenizer_zh.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/sacrebleu_methods/utils.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/scripts/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/scripts/add_wmt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/scripts/add_wmt.sh -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/scripts/perf_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/scripts/perf_test.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/setup.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/setup.cfg -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/setup.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/test.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/test.sh -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/test/test_api.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/test/test_bleu.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/test/test_chrf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/test/test_chrf.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/test/test_sentence_bleu.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/test/test_significance.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/test/test_ter.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/test/test_tokenizer_ter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/test/test_tokenizer_ter.py -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/test/wmt17_en_de_systems.pkl.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/metrics_evaluation/metrics/sacrebleu_code/test/wmt17_en_de_systems.pkl.bz2 -------------------------------------------------------------------------------- /metrics_evaluation/metrics/sacrebleu_code/tox.ini: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E501,E265 3 | -------------------------------------------------------------------------------- /poetry.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/poetry.lock -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JetBrains-Research/codegen-metrics/HEAD/pyproject.toml --------------------------------------------------------------------------------