├── .gitignore ├── LICENSE ├── README.md ├── docs ├── dataset_cards │ └── en │ │ ├── bps.md │ │ ├── chegeka.md │ │ ├── lcs.md │ │ ├── mathlogicqa.md │ │ ├── multiq.md │ │ ├── parus.md │ │ ├── rcb.md │ │ ├── rudetox.md │ │ ├── ruethics.md │ │ ├── ruhatespeech.md │ │ ├── ruhhh.md │ │ ├── ruhumaneval.md │ │ ├── rummlu.md │ │ ├── rumodar.md │ │ ├── rumultiar.md │ │ ├── ruopenbookqa.md │ │ ├── rutie.md │ │ ├── ruworldtree.md │ │ ├── rwsd.md │ │ ├── simplear.md │ │ └── use.md ├── mera-logo.svg └── mera.png ├── examples └── instruction.ipynb ├── humanbenchmarks ├── BPS │ ├── agg.ipynb │ ├── assignments_from_pool_41554142__29-09-2023.tsv │ └── workerSkills.csv ├── LCS │ ├── agg.ipynb │ ├── assignments_from_pool_41565705__29-09-2023.tsv │ └── workerSkills.csv ├── MathLogicQA │ ├── agg.ipynb │ ├── assignments_from_pool_41847389__20-10-2023.tsv │ └── workerSkills.csv ├── RCB │ ├── agg.ipynb │ ├── assignments_from_pool_42365874__11-12-2023.tsv │ └── workerSkills.csv ├── README.md ├── RWSD │ ├── agg.ipynb │ ├── assignments_from_pool_41009024__19-12-2023.tsv │ ├── assignments_from_pool_41266267__19-12-2023.tsv │ └── workerSkills.csv ├── SimpleAr │ ├── agg.ipynb │ ├── assignments_from_pool_41864367__20-10-2023.tsv │ └── workerSkills.csv ├── ruDetox │ ├── agg.ipynb │ ├── assignments_from_pool_41793133__17-10-2023.tsv │ ├── assignments_from_pool_41793355__21-10-2023.tsv │ ├── assignments_from_pool_41793535__21-10-2023.tsv │ ├── dataset.csv │ ├── eval.py │ ├── score_calibrations_ru.pkl │ ├── workerSkills_fluency.csv │ ├── workerSkills_meaning.csv │ └── workerSkills_toxic.csv ├── ruEthics │ ├── agg.ipynb │ ├── data.json │ ├── pool1.csv │ ├── pool2.csv │ └── pool3.csv ├── ruHHH.multiple_prompts │ ├── agg.ipynb │ ├── assignments_from_pool_40892383__11-09-2023.tsv │ ├── assignments_from_pool_40892601__24-08-2023.tsv │ ├── hhh_answers.csv │ └── workerSkills.csv ├── ruHHH.single_prompt │ ├── agg.ipynb │ ├── assignments_from_pool_40893001__11-09-2023.tsv │ ├── assignments_from_pool_40893071__23-08-2023.tsv │ ├── hhh_answers.csv │ └── workerSkills.csv ├── ruHateSpeech │ ├── agg.ipynb │ ├── assignments_from_pool_42002038__30-10-2023.tsv │ ├── data.csv │ └── workerSkills.csv ├── ruMMLU │ ├── agg.ipynb │ ├── assignments_from_pool_42366581__10-12-2023.tsv │ └── workerSkills.csv ├── ruModAr │ ├── agg.ipynb │ ├── assignments_from_pool_41461361__28-09-2023.tsv │ └── workerSkills.csv ├── ruMultiAr │ ├── agg.ipynb │ ├── assignments_from_pool_41562232__04-10-2023.tsv │ └── workerSkills.csv ├── ruTiE │ ├── agg.ipynb │ ├── pool1.tsv │ └── pool2.csv └── ruWorldTree │ ├── agg.ipynb │ ├── assignments_from_pool_42366885__28-11-2023.tsv │ └── workerSkills.csv ├── lm-evaluation-harness ├── .flake8 ├── .gitignore ├── .pre-commit-config.yaml ├── CITATION.bib ├── README.md ├── fasilitate.py ├── ignore.txt ├── lm_eval │ ├── __init__.py │ ├── base.py │ ├── decontamination │ │ ├── __init__.py │ │ ├── archiver.py │ │ ├── decontaminate.py │ │ └── janitor.py │ ├── evaluator.py │ ├── metrics.py │ ├── models │ │ ├── __init__.py │ │ ├── anthropic_llms.py │ │ ├── dummy.py │ │ ├── gigachat.py │ │ ├── gpt2.py │ │ ├── gpt3.py │ │ └── huggingface.py │ ├── tasks │ │ ├── __init__.py │ │ ├── bps.py │ │ ├── json.py │ │ ├── lcs.py │ │ ├── mathlogicqa.py │ │ ├── rsg.py │ │ ├── rudetox │ │ │ ├── __init__.py │ │ │ ├── rudetox.py │ │ │ └── score_calibrations_ru.pkl │ │ ├── ruethics.py │ │ ├── ruhatespeech.py │ │ ├── ruhhh.py │ │ ├── ruhumaneval │ │ │ ├── __init__.py │ │ │ ├── execute.py │ │ │ └── ruhumaneval.py │ │ ├── rummlu.py │ │ ├── rumodar.py │ │ ├── rumultiar.py │ │ ├── rutie.py │ │ ├── simplear.py │ │ ├── tape.py │ │ └── use.py │ └── utils.py ├── main.py ├── pyproject.toml ├── requirements.txt ├── run_mera.sh ├── run_mera_seq2seq.sh ├── scripts │ └── log_to_submission.py └── setup.py └── modules ├── __init__.py └── scoring ├── README.md ├── __init__.py ├── configs ├── bps.yaml ├── chegeka.yaml ├── errors_comments.yaml ├── lcs.yaml ├── main.yaml ├── mathlogicqa.yaml ├── multiq.yaml ├── parus.yaml ├── rcb.yaml ├── rudetox.yaml ├── ruethics.yaml ├── ruhatespeech.yaml ├── ruhhh.yaml ├── ruhumaneval.yaml ├── rummlu.yaml ├── rumodar.yaml ├── rumultiar.yaml ├── ruopenbookqa.yaml ├── rutie.yaml ├── ruworldtree.yaml ├── rwsd.yaml ├── simplear.yaml └── use.yaml ├── evaluate_submission.py ├── examples ├── sample_response.json ├── sample_response_bad.json ├── sample_submission.zip └── sample_submission_bad.zip ├── generage_sample_submission.py ├── requirements.txt └── src ├── __init__.py ├── base.py ├── dataset ├── __init__.py └── dataset.py ├── enums.py ├── logger_config.py ├── metrics.py ├── registry └── __init__.py ├── tasks ├── __init__.py ├── bps.py ├── chegeka.py ├── lcs.py ├── mathlogicqa.py ├── multiq.py ├── parus.py ├── rcb.py ├── rudetox.py ├── ruethics.py ├── ruhatespeech.py ├── ruhhh.py ├── ruhumaneval.py ├── rummlu.py ├── rumodar.py ├── rumultiar.py ├── ruopenbookqa.py ├── rutie.py ├── ruworldtree.py ├── rwsd.py ├── simplear.py ├── task.py └── use.py ├── utils.py └── worker.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/README.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/bps.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/bps.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/chegeka.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/chegeka.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/lcs.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/lcs.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/mathlogicqa.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/mathlogicqa.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/multiq.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/multiq.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/parus.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/parus.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/rcb.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/rcb.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/rudetox.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/rudetox.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/ruethics.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/ruethics.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/ruhatespeech.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/ruhatespeech.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/ruhhh.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/ruhhh.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/ruhumaneval.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/ruhumaneval.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/rummlu.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/rummlu.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/rumodar.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/rumodar.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/rumultiar.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/rumultiar.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/ruopenbookqa.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/ruopenbookqa.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/rutie.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/rutie.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/ruworldtree.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/ruworldtree.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/rwsd.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/rwsd.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/simplear.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/simplear.md -------------------------------------------------------------------------------- /docs/dataset_cards/en/use.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/dataset_cards/en/use.md -------------------------------------------------------------------------------- /docs/mera-logo.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/mera-logo.svg -------------------------------------------------------------------------------- /docs/mera.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/docs/mera.png -------------------------------------------------------------------------------- /examples/instruction.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/examples/instruction.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/BPS/agg.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/BPS/agg.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/BPS/assignments_from_pool_41554142__29-09-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/BPS/assignments_from_pool_41554142__29-09-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/BPS/workerSkills.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/BPS/workerSkills.csv -------------------------------------------------------------------------------- /humanbenchmarks/LCS/agg.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/LCS/agg.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/LCS/assignments_from_pool_41565705__29-09-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/LCS/assignments_from_pool_41565705__29-09-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/LCS/workerSkills.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/LCS/workerSkills.csv -------------------------------------------------------------------------------- /humanbenchmarks/MathLogicQA/agg.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/MathLogicQA/agg.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/MathLogicQA/assignments_from_pool_41847389__20-10-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/MathLogicQA/assignments_from_pool_41847389__20-10-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/MathLogicQA/workerSkills.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/MathLogicQA/workerSkills.csv -------------------------------------------------------------------------------- /humanbenchmarks/RCB/agg.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/RCB/agg.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/RCB/assignments_from_pool_42365874__11-12-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/RCB/assignments_from_pool_42365874__11-12-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/RCB/workerSkills.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/RCB/workerSkills.csv -------------------------------------------------------------------------------- /humanbenchmarks/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/README.md -------------------------------------------------------------------------------- /humanbenchmarks/RWSD/agg.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/RWSD/agg.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/RWSD/assignments_from_pool_41009024__19-12-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/RWSD/assignments_from_pool_41009024__19-12-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/RWSD/assignments_from_pool_41266267__19-12-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/RWSD/assignments_from_pool_41266267__19-12-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/RWSD/workerSkills.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/RWSD/workerSkills.csv -------------------------------------------------------------------------------- /humanbenchmarks/SimpleAr/agg.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/SimpleAr/agg.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/SimpleAr/assignments_from_pool_41864367__20-10-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/SimpleAr/assignments_from_pool_41864367__20-10-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/SimpleAr/workerSkills.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/SimpleAr/workerSkills.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruDetox/agg.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruDetox/agg.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/ruDetox/assignments_from_pool_41793133__17-10-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruDetox/assignments_from_pool_41793133__17-10-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/ruDetox/assignments_from_pool_41793355__21-10-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruDetox/assignments_from_pool_41793355__21-10-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/ruDetox/assignments_from_pool_41793535__21-10-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruDetox/assignments_from_pool_41793535__21-10-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/ruDetox/dataset.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruDetox/dataset.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruDetox/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruDetox/eval.py -------------------------------------------------------------------------------- /humanbenchmarks/ruDetox/score_calibrations_ru.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruDetox/score_calibrations_ru.pkl -------------------------------------------------------------------------------- /humanbenchmarks/ruDetox/workerSkills_fluency.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruDetox/workerSkills_fluency.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruDetox/workerSkills_meaning.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruDetox/workerSkills_meaning.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruDetox/workerSkills_toxic.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruDetox/workerSkills_toxic.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruEthics/agg.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruEthics/agg.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/ruEthics/data.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruEthics/data.json -------------------------------------------------------------------------------- /humanbenchmarks/ruEthics/pool1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruEthics/pool1.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruEthics/pool2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruEthics/pool2.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruEthics/pool3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruEthics/pool3.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruHHH.multiple_prompts/agg.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruHHH.multiple_prompts/agg.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/ruHHH.multiple_prompts/assignments_from_pool_40892383__11-09-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruHHH.multiple_prompts/assignments_from_pool_40892383__11-09-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/ruHHH.multiple_prompts/assignments_from_pool_40892601__24-08-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruHHH.multiple_prompts/assignments_from_pool_40892601__24-08-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/ruHHH.multiple_prompts/hhh_answers.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruHHH.multiple_prompts/hhh_answers.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruHHH.multiple_prompts/workerSkills.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruHHH.multiple_prompts/workerSkills.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruHHH.single_prompt/agg.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruHHH.single_prompt/agg.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/ruHHH.single_prompt/assignments_from_pool_40893001__11-09-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruHHH.single_prompt/assignments_from_pool_40893001__11-09-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/ruHHH.single_prompt/assignments_from_pool_40893071__23-08-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruHHH.single_prompt/assignments_from_pool_40893071__23-08-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/ruHHH.single_prompt/hhh_answers.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruHHH.single_prompt/hhh_answers.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruHHH.single_prompt/workerSkills.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruHHH.single_prompt/workerSkills.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruHateSpeech/agg.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruHateSpeech/agg.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/ruHateSpeech/assignments_from_pool_42002038__30-10-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruHateSpeech/assignments_from_pool_42002038__30-10-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/ruHateSpeech/data.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruHateSpeech/data.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruHateSpeech/workerSkills.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruHateSpeech/workerSkills.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruMMLU/agg.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruMMLU/agg.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/ruMMLU/assignments_from_pool_42366581__10-12-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruMMLU/assignments_from_pool_42366581__10-12-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/ruMMLU/workerSkills.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruMMLU/workerSkills.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruModAr/agg.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruModAr/agg.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/ruModAr/assignments_from_pool_41461361__28-09-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruModAr/assignments_from_pool_41461361__28-09-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/ruModAr/workerSkills.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruModAr/workerSkills.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruMultiAr/agg.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruMultiAr/agg.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/ruMultiAr/assignments_from_pool_41562232__04-10-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruMultiAr/assignments_from_pool_41562232__04-10-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/ruMultiAr/workerSkills.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruMultiAr/workerSkills.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruTiE/agg.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruTiE/agg.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/ruTiE/pool1.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruTiE/pool1.tsv -------------------------------------------------------------------------------- /humanbenchmarks/ruTiE/pool2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruTiE/pool2.csv -------------------------------------------------------------------------------- /humanbenchmarks/ruWorldTree/agg.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruWorldTree/agg.ipynb -------------------------------------------------------------------------------- /humanbenchmarks/ruWorldTree/assignments_from_pool_42366885__28-11-2023.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruWorldTree/assignments_from_pool_42366885__28-11-2023.tsv -------------------------------------------------------------------------------- /humanbenchmarks/ruWorldTree/workerSkills.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/humanbenchmarks/ruWorldTree/workerSkills.csv -------------------------------------------------------------------------------- /lm-evaluation-harness/.flake8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/.flake8 -------------------------------------------------------------------------------- /lm-evaluation-harness/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/.gitignore -------------------------------------------------------------------------------- /lm-evaluation-harness/.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/.pre-commit-config.yaml -------------------------------------------------------------------------------- /lm-evaluation-harness/CITATION.bib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/CITATION.bib -------------------------------------------------------------------------------- /lm-evaluation-harness/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/README.md -------------------------------------------------------------------------------- /lm-evaluation-harness/fasilitate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/fasilitate.py -------------------------------------------------------------------------------- /lm-evaluation-harness/ignore.txt: -------------------------------------------------------------------------------- 1 | ROUGE 2 | rouge 3 | nin 4 | maka 5 | mor 6 | te 7 | ond 8 | extraversion 9 | ans 10 | fo 11 | nd 12 | wit 13 | -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/base.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/decontamination/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/decontamination/archiver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/decontamination/archiver.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/decontamination/decontaminate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/decontamination/decontaminate.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/decontamination/janitor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/decontamination/janitor.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/evaluator.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/metrics.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/models/__init__.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/models/anthropic_llms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/models/anthropic_llms.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/models/dummy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/models/dummy.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/models/gigachat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/models/gigachat.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/models/gpt2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/models/gpt2.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/models/gpt3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/models/gpt3.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/models/huggingface.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/models/huggingface.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/__init__.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/bps.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/bps.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/json.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/json.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/lcs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/lcs.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/mathlogicqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/mathlogicqa.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/rsg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/rsg.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/rudetox/__init__.py: -------------------------------------------------------------------------------- 1 | from .rudetox import * 2 | -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/rudetox/rudetox.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/rudetox/rudetox.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/rudetox/score_calibrations_ru.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/rudetox/score_calibrations_ru.pkl -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/ruethics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/ruethics.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/ruhatespeech.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/ruhatespeech.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/ruhhh.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/ruhhh.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/ruhumaneval/__init__.py: -------------------------------------------------------------------------------- 1 | from .ruhumaneval import * 2 | -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/ruhumaneval/execute.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/ruhumaneval/execute.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/ruhumaneval/ruhumaneval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/ruhumaneval/ruhumaneval.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/rummlu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/rummlu.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/rumodar.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/rumodar.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/rumultiar.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/rumultiar.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/rutie.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/rutie.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/simplear.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/simplear.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/tape.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/tape.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/tasks/use.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/tasks/use.py -------------------------------------------------------------------------------- /lm-evaluation-harness/lm_eval/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/lm_eval/utils.py -------------------------------------------------------------------------------- /lm-evaluation-harness/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/main.py -------------------------------------------------------------------------------- /lm-evaluation-harness/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/pyproject.toml -------------------------------------------------------------------------------- /lm-evaluation-harness/requirements.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | -------------------------------------------------------------------------------- /lm-evaluation-harness/run_mera.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/run_mera.sh -------------------------------------------------------------------------------- /lm-evaluation-harness/run_mera_seq2seq.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/run_mera_seq2seq.sh -------------------------------------------------------------------------------- /lm-evaluation-harness/scripts/log_to_submission.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/scripts/log_to_submission.py -------------------------------------------------------------------------------- /lm-evaluation-harness/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/lm-evaluation-harness/setup.py -------------------------------------------------------------------------------- /modules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /modules/scoring/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/README.md -------------------------------------------------------------------------------- /modules/scoring/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /modules/scoring/configs/bps.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true 4 | -------------------------------------------------------------------------------- /modules/scoring/configs/chegeka.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true 4 | -------------------------------------------------------------------------------- /modules/scoring/configs/errors_comments.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/configs/errors_comments.yaml -------------------------------------------------------------------------------- /modules/scoring/configs/lcs.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true 4 | -------------------------------------------------------------------------------- /modules/scoring/configs/main.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/configs/main.yaml -------------------------------------------------------------------------------- /modules/scoring/configs/mathlogicqa.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true -------------------------------------------------------------------------------- /modules/scoring/configs/multiq.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true -------------------------------------------------------------------------------- /modules/scoring/configs/parus.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true -------------------------------------------------------------------------------- /modules/scoring/configs/rcb.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true -------------------------------------------------------------------------------- /modules/scoring/configs/rudetox.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/configs/rudetox.yaml -------------------------------------------------------------------------------- /modules/scoring/configs/ruethics.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: false -------------------------------------------------------------------------------- /modules/scoring/configs/ruhatespeech.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: false -------------------------------------------------------------------------------- /modules/scoring/configs/ruhhh.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: false -------------------------------------------------------------------------------- /modules/scoring/configs/ruhumaneval.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true 4 | ks: [1, 5, 10] 5 | -------------------------------------------------------------------------------- /modules/scoring/configs/rummlu.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true 4 | -------------------------------------------------------------------------------- /modules/scoring/configs/rumodar.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true -------------------------------------------------------------------------------- /modules/scoring/configs/rumultiar.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true -------------------------------------------------------------------------------- /modules/scoring/configs/ruopenbookqa.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true -------------------------------------------------------------------------------- /modules/scoring/configs/rutie.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true -------------------------------------------------------------------------------- /modules/scoring/configs/ruworldtree.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true -------------------------------------------------------------------------------- /modules/scoring/configs/rwsd.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true -------------------------------------------------------------------------------- /modules/scoring/configs/simplear.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true -------------------------------------------------------------------------------- /modules/scoring/configs/use.yaml: -------------------------------------------------------------------------------- 1 | extension: .json 2 | split: test 3 | use_in_total: true 4 | max_grade_point: 34 -------------------------------------------------------------------------------- /modules/scoring/evaluate_submission.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/evaluate_submission.py -------------------------------------------------------------------------------- /modules/scoring/examples/sample_response.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/examples/sample_response.json -------------------------------------------------------------------------------- /modules/scoring/examples/sample_response_bad.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/examples/sample_response_bad.json -------------------------------------------------------------------------------- /modules/scoring/examples/sample_submission.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/examples/sample_submission.zip -------------------------------------------------------------------------------- /modules/scoring/examples/sample_submission_bad.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/examples/sample_submission_bad.zip -------------------------------------------------------------------------------- /modules/scoring/generage_sample_submission.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/generage_sample_submission.py -------------------------------------------------------------------------------- /modules/scoring/requirements.txt: -------------------------------------------------------------------------------- 1 | transformers==4.30.2 2 | omegaconf==2.0.6 3 | boto3 4 | scikit-learn==1.0.2 -------------------------------------------------------------------------------- /modules/scoring/src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /modules/scoring/src/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/base.py -------------------------------------------------------------------------------- /modules/scoring/src/dataset/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/dataset/__init__.py -------------------------------------------------------------------------------- /modules/scoring/src/dataset/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/dataset/dataset.py -------------------------------------------------------------------------------- /modules/scoring/src/enums.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/enums.py -------------------------------------------------------------------------------- /modules/scoring/src/logger_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/logger_config.py -------------------------------------------------------------------------------- /modules/scoring/src/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/metrics.py -------------------------------------------------------------------------------- /modules/scoring/src/registry/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/registry/__init__.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/__init__.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/bps.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/bps.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/chegeka.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/chegeka.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/lcs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/lcs.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/mathlogicqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/mathlogicqa.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/multiq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/multiq.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/parus.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/parus.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/rcb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/rcb.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/rudetox.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/rudetox.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/ruethics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/ruethics.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/ruhatespeech.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/ruhatespeech.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/ruhhh.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/ruhhh.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/ruhumaneval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/ruhumaneval.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/rummlu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/rummlu.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/rumodar.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/rumodar.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/rumultiar.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/rumultiar.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/ruopenbookqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/ruopenbookqa.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/rutie.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/rutie.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/ruworldtree.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/ruworldtree.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/rwsd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/rwsd.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/simplear.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/simplear.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/task.py -------------------------------------------------------------------------------- /modules/scoring/src/tasks/use.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/tasks/use.py -------------------------------------------------------------------------------- /modules/scoring/src/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/utils.py -------------------------------------------------------------------------------- /modules/scoring/src/worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ai-forever/MERA/HEAD/modules/scoring/src/worker.py --------------------------------------------------------------------------------