├── mle-bench ├── environment │ ├── __init__.py │ ├── config │ │ └── container_configs │ │ │ └── default.json │ ├── defaults.py │ ├── validate_submission.sh │ └── entrypoint.sh ├── mlebench │ ├── __init__.py │ └── competitions │ │ ├── __init__.py │ │ ├── multi-modal-gesture-recognition │ │ ├── kernels.txt │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── the-icml-2013-whale-challenge-right-whale-redux │ │ ├── kernels.txt │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── spooky-author-identification │ │ ├── classes.py │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── billion-word-imputation │ │ ├── kernels.txt │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── ml2021spring-hw2 │ │ ├── kernels.txt │ │ ├── description_obfuscated.md │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ ├── config.yaml │ │ └── description.md │ │ ├── jigsaw-toxic-comment-classification-challenge │ │ ├── classes.py │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── AI4Code │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── hotel-id-2021-fgvc8 │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── imet-2020-fgvc7 │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── iwildcam-2020-fgvc7 │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── mlsp-2013-birds │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── kernels.txt │ │ ├── config.yaml │ │ └── grade.py │ │ ├── spaceship-titanic │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ ├── config.yaml │ │ └── prepare.py │ │ ├── champs-scalar-coupling │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── google-quest-challenge │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── classes.py │ │ ├── herbarium-2020-fgvc7 │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── herbarium-2021-fgvc8 │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── kernels.txt │ │ ├── herbarium-2022-fgvc9 │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── inaturalist-2019-fgvc6 │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── kernels.txt │ │ ├── iwildcam-2019-fgvc6 │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── kuzushiji-recognition │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── leaf-classification │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── lmsys-chatbot-arena │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── playground-series-s3e18 │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── random-acts-of-pizza │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ ├── config.yaml │ │ └── kernels.txt │ │ ├── siim-covid19-detection │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── stanford-covid-vaccine │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── aerial-cactus-identification │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── alaska2-image-steganalysis │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── bms-molecular-translation │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── denoising-dirty-documents │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── dog-breed-identification │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── freesound-audio-tagging-2019 │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── hubmap-kidney-segmentation │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── invasive-species-monitoring │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── nfl-player-contact-detection │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── paddy-disease-classification │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── petfinder-pawpularity-score │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── plant-pathology-2020-fgvc7 │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── plant-pathology-2021-fgvc8 │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── rsna-breast-cancer-detection │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── seti-breakthrough-listen │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ └── config.yaml │ │ ├── smartphone-decimeter-2022 │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── tweet-sentiment-extraction │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── aptos2019-blindness-detection │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── histopathologic-cancer-detection │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── icecube-neutrinos-in-deep-ice │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── plant-seedlings-classification │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── tensorflow2-question-answering │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── ventilator-pressure-prediction │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── vesuvius-challenge-ink-detection │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── whale-categorization-playground │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── cassava-leaf-disease-classification │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ ├── description_obfuscated.md │ │ └── config.yaml │ │ ├── detecting-insults-in-social-commentary │ │ ├── leaderboard.csv │ │ ├── kernels.txt │ │ ├── checksums.yaml │ │ ├── grade.py │ │ └── config.yaml │ │ ├── dogs-vs-cats-redux-kernels-edition │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── description_obfuscated.md │ │ └── config.yaml │ │ ├── new-york-city-taxi-fare-prediction │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── grade.py │ │ ├── osic-pulmonary-fibrosis-progression │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── predict-volcanic-eruptions-ingv-oe │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── siim-isic-melanoma-classification │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ └── config.yaml │ │ ├── statoil-iceberg-classifier-challenge │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── tabular-playground-series-dec-2021 │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── tabular-playground-series-may-2022 │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── tgs-salt-identification-challenge │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── us-patent-phrase-to-phrase-matching │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── uw-madison-gi-tract-image-segmentation │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── 3d-object-detection-for-autonomous-vehicles │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── cdiscount-image-classification-challenge │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ └── config.yaml │ │ ├── chaii-hindi-and-tamil-question-answering │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── facebook-recruiting-iii-keyword-extraction │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ ├── description_obfuscated.md │ │ └── kernels.txt │ │ ├── hms-harmful-brain-activity-classification │ │ ├── leaderboard.csv │ │ ├── constants.py │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── nomad2018-predict-transparent-conductors │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── ranzcr-clip-catheter-line-classification │ │ ├── leaderboard.csv │ │ ├── classes.py │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── rsna-2022-cervical-spine-fracture-detection │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── tensorflow-speech-recognition-challenge │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ └── config.yaml │ │ ├── h-and-m-personalized-fashion-recommendations │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── learning-agency-lab-automated-essay-scoring-2 │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── config.yaml │ │ └── description_obfuscated.md │ │ ├── movie-review-sentiment-analysis-kernels-only │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ ├── description_obfuscated.md │ │ └── config.yaml │ │ ├── text-normalization-challenge-english-language │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── text-normalization-challenge-russian-language │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── kernels.txt │ │ └── config.yaml │ │ ├── vinbigdata-chest-xray-abnormalities-detection │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml │ │ ├── jigsaw-unintended-bias-in-toxicity-classification │ │ ├── leaderboard.csv │ │ └── checksums.yaml │ │ ├── rsna-miccai-brain-tumor-radiogenomic-classification │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ ├── grade.py │ │ └── config.yaml │ │ └── google-research-identify-contrails-reduce-global-warming │ │ ├── leaderboard.csv │ │ ├── checksums.yaml │ │ └── config.yaml ├── .dockerignore ├── extras │ └── plagiarism_detector │ │ ├── .gitignore │ │ ├── package.json │ │ └── run.py ├── experiments │ ├── familiarity │ │ ├── .gitignore │ │ ├── .gitattributes │ │ ├── requirements.txt │ │ └── comps_to_urls.json │ ├── splits │ │ ├── aptos2019-blindness-detection.txt │ │ ├── hard.txt │ │ ├── easy.txt │ │ ├── medium.txt │ │ ├── automind.txt │ │ └── low.txt │ └── competition_categories.csv ├── agents │ ├── dummy │ │ ├── config.yaml │ │ ├── requirements.txt │ │ ├── start.sh │ │ ├── Dockerfile │ │ └── main.py │ └── aide │ │ ├── run.sh │ │ ├── src │ │ ├── aide │ │ │ └── example_tasks │ │ │ │ ├── bitcoin_price │ │ │ │ └── BTC-USD.csv │ │ │ │ ├── house_prices │ │ │ │ ├── train.csv │ │ │ │ └── sample_submission.csv │ │ │ │ └── bitcoin_price.md │ │ └── LICENSE │ │ └── additional_notes.txt ├── .gitattributes ├── SECURITY.md ├── .pre-commit-config.yaml ├── additional_notes.txt └── Dockerfile ├── automind ├── backend │ └── all-MiniLM-L6-v2 │ │ └── .gitkeep ├── setup.py └── utils │ └── additional_notes.txt ├── assets ├── framework.png └── main_result.png ├── scripts ├── build_base_env.sh └── run_mlebench.sh └── LICENSE /mle-bench/environment/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mle-bench/mlebench/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /automind/backend/all-MiniLM-L6-v2/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mle-bench/.dockerignore: -------------------------------------------------------------------------------- 1 | /data/ 2 | /*runs/ 3 | /runs*/ -------------------------------------------------------------------------------- /mle-bench/extras/plagiarism_detector/.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules/ 2 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/multi-modal-gesture-recognition/kernels.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assets/framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjunlp/AutoMind/HEAD/assets/framework.png -------------------------------------------------------------------------------- /mle-bench/experiments/familiarity/.gitignore: -------------------------------------------------------------------------------- 1 | meta-kaggle 2 | meta-kaggle.zip 3 | discussions -------------------------------------------------------------------------------- /mle-bench/experiments/splits/aptos2019-blindness-detection.txt: -------------------------------------------------------------------------------- 1 | aptos2019-blindness-detection -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/the-icml-2013-whale-challenge-right-whale-redux/kernels.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assets/main_result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjunlp/AutoMind/HEAD/assets/main_result.png -------------------------------------------------------------------------------- /mle-bench/agents/dummy/config.yaml: -------------------------------------------------------------------------------- 1 | dummy: 2 | start: dummy/start.sh 3 | dockerfile: dummy/Dockerfile 4 | -------------------------------------------------------------------------------- /mle-bench/experiments/familiarity/.gitattributes: -------------------------------------------------------------------------------- 1 | comps_to_urls.json filter=lfs diff=lfs merge=lfs -text 2 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/spooky-author-identification/classes.py: -------------------------------------------------------------------------------- 1 | CLASSES = ["EAP", "HPL", "MWS"] 2 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/billion-word-imputation/kernels.txt: -------------------------------------------------------------------------------- 1 | mpwolke/billion-word-imputation-zip-file 2 | -------------------------------------------------------------------------------- /mle-bench/agents/dummy/requirements.txt: -------------------------------------------------------------------------------- 1 | # Include any packages here that your agent requires, e.g. 2 | # torch 3 | -------------------------------------------------------------------------------- /mle-bench/extras/plagiarism_detector/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "@dodona/dolos-lib": "^3.3.1" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /mle-bench/agents/aide/run.sh: -------------------------------------------------------------------------------- 1 | aide data_dir="src/aide/example_tasks/house_prices" desc_file="src/aide/example_tasks/house_prices.md" 2 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/ml2021spring-hw2/kernels.txt: -------------------------------------------------------------------------------- 1 | tamakoyl/2021hw02phoneme 2 | harrytheorange/2021springhw2 3 | lizupeng/notebook4bf3cf8e90 4 | -------------------------------------------------------------------------------- /mle-bench/experiments/familiarity/requirements.txt: -------------------------------------------------------------------------------- 1 | openai==0.28.1 2 | tiktoken==0.7.0 3 | tqdm 4 | numpy 5 | pandas 6 | matplotlib 7 | tenacity 8 | playwright 9 | scipy -------------------------------------------------------------------------------- /mle-bench/environment/config/container_configs/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "mem_limit": null, 3 | "shm_size": "4G", 4 | "nano_cpus": 4e9, 5 | "runtime": "runc" 6 | } 7 | -------------------------------------------------------------------------------- /mle-bench/experiments/splits/hard.txt: -------------------------------------------------------------------------------- 1 | denoising-dirty-documents 2 | new-york-city-taxi-fare-prediction 3 | tgs-salt-identification-challenge 4 | ventilator-pressure-prediction -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/jigsaw-toxic-comment-classification-challenge/classes.py: -------------------------------------------------------------------------------- 1 | CLASSES = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"] 2 | -------------------------------------------------------------------------------- /mle-bench/environment/defaults.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | DEFAULT_CONTAINER_CONFIG_PATH = ( 4 | Path(__file__).parent / "config" / "container_configs" / "default.json" 5 | ) 6 | -------------------------------------------------------------------------------- /mle-bench/experiments/competition_categories.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:5b6967e944e6b105f54f943cd13042158f49fec5206c0ff536d16d42cf39b634 3 | size 6322 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/ml2021spring-hw2/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Framewise phoneme classification of TIMIT dataset 4 | 5 | # Metric 6 | 7 | Catagorization accuracy -------------------------------------------------------------------------------- /mle-bench/experiments/familiarity/comps_to_urls.json: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:9204e162f467ca40c667c799994c936c0de6d15e7983053730d65565917784e9 3 | size 24376 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/AI4Code/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:086ebfd7d2bc3093587b134109f7e98aedde0301813485f64d0414d7947970b9 3 | size 53367 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/hotel-id-2021-fgvc8/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:3fa2e06dd116ec73025b7c7ebcbab9e046793a527f1085e0ce4840d607ad1344 3 | size 4899 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/imet-2020-fgvc7/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:0791c9718c9ba1c2d8456a7d65a8c993ae07fe53e64a075f342e4f811138e824 3 | size 4863 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/iwildcam-2020-fgvc7/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:919603c1a3bf75eafc7c2bbf88fd0a91f686ce5f6ea584255931dc3e3ac37224 3 | size 6163 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/ml2021spring-hw2/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:5b05251108c31782432318cbe25f60b59562e08a0e3fd734953f04c8477d1549 3 | size 82359 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/mlsp-2013-birds/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:5372359ad885c57c801561dd676a3d0e2a5e70eed4c55051c28371ff06b8b430 3 | size 4275 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/spaceship-titanic/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:e071aca01cb968fa45344d94a60553e97d1429d3b0de05910e72012248168619 3 | size 125841 4 | -------------------------------------------------------------------------------- /mle-bench/agents/aide/src/aide/example_tasks/bitcoin_price/BTC-USD.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:ca388783a9c655f256dc1ce5af5d6b876846e1ca428979a1d096f7b25324e487 3 | size 41966 4 | -------------------------------------------------------------------------------- /mle-bench/agents/aide/src/aide/example_tasks/house_prices/train.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:1e18addf81e5e4d347cc17ee6075bbe4a42b7fa26b9e5b063e8f692a5f929d41 3 | size 460676 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/billion-word-imputation/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:8813cb79ed487ca7ea7b0e6e6bf33bcd6e97e032048f15be4a35d99cdbf85e6a 3 | size 4716 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/champs-scalar-coupling/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:07de6017c6f173ed7d5bf3d78b8058db5c45ce59f95eb44eb21424207b77a8be 3 | size 151605 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/google-quest-challenge/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:87795e3f621dd8408f89fc55609cfd6263f5aa1ea9133ab4f39a3ce57811345f 3 | size 85077 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/herbarium-2020-fgvc7/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:68042738a0adc84e77d06c48c0ac480558c93cfa79d58e5c067cc4b1fb36f6d3 3 | size 8379 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/herbarium-2021-fgvc8/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:bd3beffc58858da0a0917af1f43eb08e1d2a194202c11834a687acad1016aae4 3 | size 4383 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/herbarium-2022-fgvc9/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:87c7d9deed55521e4d689d1a3d4d8ff8a5bdea909b1d0360de3031d1c87294d0 3 | size 7461 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/inaturalist-2019-fgvc6/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:eea782f17eb97f3463ffe256d4dcc3d41e5a73e31d1db334e7d0128b2551c512 3 | size 11673 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/iwildcam-2019-fgvc6/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:9b216876399a0dfcfd114df871c1e9b69d124e1090e85a27b412270be06bbe30 3 | size 16913 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/kuzushiji-recognition/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:59935b074c73bd3778e1db5a3c02119e0fe24f65e0fe534ff905398b3de33174 3 | size 14763 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/leaf-classification/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:d5f3dc57f971ea476f1e43ea68ec5f184f0ecb41f56528335d0a1add5eeb19a9 3 | size 84693 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/lmsys-chatbot-arena/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:245a02f29c7e02623abeefcd600484973cfe610630f76f5a479ac35ff1d371e6 3 | size 92945 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/playground-series-s3e18/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:3111d041b3de42c18f9d4abcca2e13a0831edaf9c687f382da52176fdb144983 3 | size 57758 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/random-acts-of-pizza/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:ec6a5554c698d891e78b01a5da2a824f61956b1684b3d79b0ed8debb6abaefcf 3 | size 24526 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/siim-covid19-detection/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:0da2802f168c561d7d81582d9bbbb98688681d514eeb0e4505a2d2f7858102f7 3 | size 65313 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/stanford-covid-vaccine/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:5c3700fe1f674aef5c506de558711174fe15b085860ce2d1563e4b119ed581c1 3 | size 88407 4 | -------------------------------------------------------------------------------- /mle-bench/experiments/splits/easy.txt: -------------------------------------------------------------------------------- 1 | aptos2019-blindness-detection 2 | random-acts-of-pizza 3 | spooky-author-identification 4 | google-quest-challenge 5 | stanford-covid-vaccine 6 | predict-volcanic-eruptions-ingv-oe -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/aerial-cactus-identification/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:cdb280161eca63c9ab1f937dcfb715b710fd222afe4c65f13a763d64e4c48318 3 | size 63555 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/alaska2-image-steganalysis/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:2b434df9cff58efc89099f6bd66fa87c81711337bf50e089de9ad3209e058cac 3 | size 54813 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/bms-molecular-translation/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:2630dfbfac6f392452a531a276cf5be31c4f3f47a38557d3b5521ef9981133ce 3 | size 42795 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/denoising-dirty-documents/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:382fff781ec8849fb07da6f25b868fa9ced905cd4ae848240895ef985bdca790 3 | size 8649 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/dog-breed-identification/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:819d5a5551355c6bbb1288988d4a4219e0d1c1e563867997cca5e27f04a26758 3 | size 69165 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/freesound-audio-tagging-2019/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:38bc4fee589c34f233c18185dda4bb6a726f9448d92f7c6d3b892de5c623811a 3 | size 22635 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/hubmap-kidney-segmentation/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:ce983b14b43447e0947a80d8923a687316a29a3715d00e85e0947188cb3366d9 3 | size 62463 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/invasive-species-monitoring/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:ab63e290502ba7de760d0800f2ffaeb8a583ef35ab90b44e05f669dddfc4c14c 3 | size 27199 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/nfl-player-contact-detection/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:cbe3c5142e238064f4fd2f4236a23cc05588e37b8c20102eca047770ec74da32 3 | size 50825 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/paddy-disease-classification/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:19b3e78b5cd838b6e8e9e52e12ca31ed2ac206a3bf8a1fe96db59b93691573df 3 | size 35541 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/petfinder-pawpularity-score/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:87503e558edfb61caf4b33ac6601b514adcfdf41648e4b01864c9897298693f1 3 | size 198199 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/plant-pathology-2020-fgvc7/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:2f40bed8e0cace49fb3202e3a1d55f9924e6591f528537bb029426294fcf6e55 3 | size 71235 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/plant-pathology-2021-fgvc8/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:5c511350e83ac758041d70d532617bee8e250a3e89438c0615d1733b04f6757b 3 | size 33867 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/rsna-breast-cancer-detection/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:b582ed651866fddd6a77e2eaeb656e9aa40d0f531a27722c9ddd8f750c7d9966 3 | size 81087 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/seti-breakthrough-listen/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:e29f79353f26bf38fde1d36ba0128744436852f41ea79e11e55d3341de39637d 3 | size 41589 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/smartphone-decimeter-2022/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:c62530f39e51f47a0f7f1d42948155cef4e062061f4aea73ac34ecb5d555b4de 3 | size 29023 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/spooky-author-identification/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:1087afc671c8eddcca7f8dc792a4d55877213ce86ac9d1f8c7b280edd0eb2514 3 | size 67155 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tweet-sentiment-extraction/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:5fcc119877022b737a085149812686f08016d110063a644d5e378f8322d331ba 3 | size 120213 4 | -------------------------------------------------------------------------------- /mle-bench/agents/aide/src/aide/example_tasks/house_prices/sample_submission.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:f19e99699b925aa3085c8d2ae43c62f15fbcd35e4a953b98025f8392c3ffa807 3 | size 31939 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/aptos2019-blindness-detection/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:a63976a7913abbf857a375d0d35359d167f06f620cc24abe9ab39bf6feaea5af 3 | size 164137 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/histopathologic-cancer-detection/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:6d98b168a42f292c67d800a8943b25de62abca9ca0d28921aab2dcdd9a798544 3 | size 59811 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/icecube-neutrinos-in-deep-ice/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:08211a3ab31b5374b5c028489e505420e9ed5773c4a84dc6faef10d8abc5b50f 3 | size 45815 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/multi-modal-gesture-recognition/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:0909152ea34842d6827b29abe9efaf6009664b2f8a683e5b4cc7f0f3c32c7d5f 3 | size 1051 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/plant-seedlings-classification/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:9dd14c3449055b2913cb98ed5fb826129c0ec41ccd00440e24ad94d12c365959 3 | size 45153 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tensorflow2-question-answering/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:f7adc06eeabb0b06983579d0919ec1ee776ceb6381af9642967a16c0358b0146 3 | size 66699 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/ventilator-pressure-prediction/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:aa45b4c1c8eb82d0f8b0bd58e2ccea7d7981623fcf171a5158782ee4d3cc3e4a 3 | size 135621 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/vesuvius-challenge-ink-detection/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:0db140c534a32cf369d5e02d1f61ddaf537ecccb0739f2fbc8969835fa783bc7 3 | size 71313 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/whale-categorization-playground/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:7c4fc6c57f6e8725a195eb1ecaa52a1df95e26ca7d809e953b189db4ca8b9f6e 3 | size 28629 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/cassava-leaf-disease-classification/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:140a49c0cbacc8b1858fb7903bdcc9ded486889617899359c00169ebd543b625 3 | size 202915 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/detecting-insults-in-social-commentary/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:1a5eadf7d4ce0fa626f407131a1efa7bf8aacedbe835820d448adc599428f936 3 | size 2663 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/dogs-vs-cats-redux-kernels-edition/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:ffc1b021c803f331c25d1492fec3a028e9c2badf45744661241e8f232d41ae42 3 | size 69880 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/new-york-city-taxi-fare-prediction/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:54c294495d43f0ed48992fe6412862f334cfe3b2cced9b978b40639bdc5ba072 3 | size 80317 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/osic-pulmonary-fibrosis-progression/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:4a7849dbd657a2a1c86a11026a563f55e1d7c87de44fb040de7dadd9bb718481 3 | size 113435 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/predict-volcanic-eruptions-ingv-oe/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:fc3dc17d344fc5d4a394f937b85a43ce2fe93a54e284755cb751e5910c44e112 3 | size 33861 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/siim-isic-melanoma-classification/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:2040b19b3f2e557427dc84dc87ea5fb10d7e3f46cd622e39061313fc06d4ed59 3 | size 172079 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/statoil-iceberg-classifier-challenge/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:56a65da03a5a8f25d8609bcdaf584b0145f2b0211161c62f0c365ad9875bf63a 3 | size 179947 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tabular-playground-series-dec-2021/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:9e96cefce2af17eb98ac08a94ddbd4249db8a0028cb1f6084bf5f905ca6b39a6 3 | size 64269 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tabular-playground-series-may-2022/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:e8e6ebcd72e1c0b9bbda04f6ac39b76eeabaa9eb483eefde4b0ec3c3afba3cc6 3 | size 62271 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tgs-salt-identification-challenge/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:5f42ce100d4e6051c84bf1270d9e480e97501aef96b23eed146f06691e40c9b8 3 | size 173889 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/us-patent-phrase-to-phrase-matching/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:a987a36d3cfd003127da6be32a9050cb435f65844f1bb7c0797d51a3b26c596f 3 | size 98369 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/uw-madison-gi-tract-image-segmentation/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:b5a3d083238ca6b9b97f12796c9cd6f9108bbf73a80e3671fc3d6ac4ca24772b 3 | size 83655 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/3d-object-detection-for-autonomous-vehicles/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:709a38d6eeaa7357a0b523020096f8b2f3c9cfd24cde95080a47e3cc00b4de81 3 | size 27363 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/cdiscount-image-classification-challenge/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:c4fc781bea0566fa5e73ba3bd9747a782da24caaf970c691e0fd634c29f20d22 3 | size 33501 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/chaii-hindi-and-tamil-question-answering/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:b7145be0472d5722d1798268b996f9706b7e180d4f3a5bed5da39be9a8caaef3 3 | size 50985 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/facebook-recruiting-iii-keyword-extraction/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:cb9f48a977b44b14ebaccd7380fd82e140c8ca48b56b4271bcc3ad45f992c92f 3 | size 19355 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/hms-harmful-brain-activity-classification/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:eeb99b9600d54bbe9dd3265efabf4d70d91b1f3e4b8974c2b5a8b6ca7c3bf788 3 | size 157802 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/nomad2018-predict-transparent-conductors/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:e076672052acbaf323557cce20ab40cdde6310b66cf680c90982a29d8cb28ab5 3 | size 47529 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/ranzcr-clip-catheter-line-classification/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:5ffa7a1161f71c4418a8496136853864e1a71d76fbbb6e895df493a4bae736e6 3 | size 83601 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/rsna-2022-cervical-spine-fracture-detection/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:b357013d95d2537736e2cdbc6ed07044910ab49b12063e6ca7347689436115a8 3 | size 46045 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tensorflow-speech-recognition-challenge/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:5f0d198ca31e37837574b6185c30557cc60c353048a875d7a25f5aa11d135898 3 | size 71019 4 | -------------------------------------------------------------------------------- /mle-bench/experiments/splits/medium.txt: -------------------------------------------------------------------------------- 1 | msys-chatbot-arena 2 | us-patent-phrase-to-phrase-matching 3 | mlsp-2013-birds 4 | statoil-iceberg-classifier-challenge 5 | petfinder-pawpularity-score 6 | tensorflow-speech-recognition-challenge -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/h-and-m-personalized-fashion-recommendations/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:520d8f1a64cb211538cfc51509c38dea90a89249c5b1752b63d7a7568a4d9c0a 3 | size 159579 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/jigsaw-toxic-comment-classification-challenge/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:5fbf44ea205eaa97f976603528dff762549a727e627a386c7c87304d696ea9ea 3 | size 245169 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/learning-agency-lab-automated-essay-scoring-2/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:afc8a5c8a38fad00cf1e12c389cb2ce17df78488e01c9ba9e363584632f1305a 3 | size 149013 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/movie-review-sentiment-analysis-kernels-only/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:4ed7989f8ed071a4747382ebd6f853b59cf23d311d71bb8d645ee6da36a539c5 3 | size 22203 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/text-normalization-challenge-english-language/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:05799b6839968769c1a40f8b6ca03478c455eb7bf8ad51067e77a2bbfc7cfede 3 | size 13996 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/text-normalization-challenge-russian-language/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:3fab6cb146c951ad97fdca572272c2e08c087ac3fd371845970fca5d415af1ab 3 | size 8787 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/the-icml-2013-whale-challenge-right-whale-redux/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:3d3e561386606db8b4d3c943f8e16bcd1fd4315fafe7def8ea0a894e98f7d8d5 3 | size 6771 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/vinbigdata-chest-xray-abnormalities-detection/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:a96ba07d49421a0a2e2d712cf0977325fc98aa8d90aedcf3e3b72e0c49e7e403 3 | size 63813 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/jigsaw-unintended-bias-in-toxicity-classification/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:30418976c3a829bce007f7c968834fb7bc20726fc8255228428de41d2b9459a6 3 | size 142299 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/ml2021spring-hw2/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 9fde0306ed6a02eea1b27686c54ce2bf 3 | public: 4 | sampleSubmission.csv: 4fe69a042cfe9e066c088e603917eec4 5 | zip: 4bce22f1c15d09d6f224d99853dcb79f 6 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/mlsp-2013-birds/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: f40c0692caff26ca8045b64109371144 3 | public: 4 | sample_submission.csv: 6c0430ab5959d2e718911c3bd7a26a2f 5 | zip: e3f07a83a78a5d270a40fb1f4e8753f6 6 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/rsna-miccai-brain-tumor-radiogenomic-classification/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:fb66b73925674f2e2f5eef30c6c27dc48bbc2c54cdeca6a59e7403e97c31d263 3 | size 84087 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/google-research-identify-contrails-reduce-global-warming/leaderboard.csv: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:c9aa2c1482fb2e36ffc5dc70fa5569098a3b9b5d129910ad223569979f32f56a 3 | size 52533 4 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/herbarium-2020-fgvc7/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 5e7355890f648e4f8be7a68e6c9c71c5 3 | public: 4 | sample_submission.csv: a1bdb830f6582fcb9cffe8cec363a0b1 5 | zip: 30eafe0eda5d047bce1488c9a3470d57 6 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/herbarium-2021-fgvc8/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: ba6fdf67c5f53a1a39bef7a129b38108 3 | public: 4 | sample_submission.csv: e53508ab941ce7f219c6e82fde9a6baa 5 | zip: c4632c48c0f74ee925ef0e99883ac673 6 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/alaska2-image-steganalysis/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: f90b06076f750fb56ae35c3df715ada1 3 | public: 4 | sample_submission.csv: f4623502821bbd1399ecc8f5d66fc243 5 | zip: 774e19a747b3d48d742334b9021c9915 6 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/billion-word-imputation/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | sample_submission.csv: b734f937cbff4fbc88b2cfccc5d12d58 3 | test.csv: 5340d30e22b84bf14de19b9c877b16e3 4 | public: {} 5 | zip: 70a08536f59df46f1d2c9d4bfeae4b0b 6 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/denoising-dirty-documents/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 5701cc4b5e1d59aedbb6aea4434d093a 3 | public: 4 | sampleSubmission.csv: b9ae2607ee98bd6b9903de500a53bf90 5 | zip: bc31eea6ff37861abe7d1c9e3411ac1f 6 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/detecting-insults-in-social-commentary/kernels.txt: -------------------------------------------------------------------------------- 1 | rishabhgarg1023/detecting-insults-from-social-commentary 2 | suraj520/cyberbully-detection-basics-to-advanced 3 | jozefkond/silvo-sematicky 4 | pabuwalnikhil/insults 5 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/smartphone-decimeter-2022/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 8c2ca9cd51dcea17a3943bbf20336254 3 | public: 4 | sample_submission.csv: 0cf35d457c068e151816fd36f93ce036 5 | zip: 69602e4dc220a6da765c048bdfd84229 6 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/invasive-species-monitoring/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 931c56b826f984b6e8e38de046240166 3 | sample_submission.csv: 3fc85f5b579bbf85aa605899887df903 4 | public: {} 5 | zip: 6e32b8c25b22dc4653540891871bc853 6 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/plant-seedlings-classification/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 89d5dba3d48b22669f108f6a3bc10e0f 3 | public: 4 | sample_submission.csv: e993f1a8a6b576a5401e027154a5cbac 5 | zip: 89007a5c7fcec56d3eb66afa89376c76 6 | -------------------------------------------------------------------------------- /mle-bench/.gitattributes: -------------------------------------------------------------------------------- 1 | *.csv filter=lfs diff=lfs merge=lfs -text 2 | mlebench/competitions/*/top_solutions/** filter=lfs diff=lfs merge=lfs -text 3 | runs/**/*.json filter=lfs diff=lfs merge=lfs -text 4 | runs/**/*.jsonl filter=lfs diff=lfs merge=lfs -text 5 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/dogs-vs-cats-redux-kernels-edition/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 98f293fd3f37f967c17aa9263e866c30 3 | public: 4 | sample_submission.csv: cfe8f41567220b6fcbbdc4eb4df7230f 5 | zip: 73dbbb43cbd91b769244408ff2a4fd6d 6 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/statoil-iceberg-classifier-challenge/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | sample_submission.csv: eeade5905d2b08fe7efd353de37ac58b 3 | test.csv: 0b3eba8691a3cc72dcb7998baa672446 4 | public: {} 5 | zip: 20847b5ba9d0165bac850eb10d8fc234 6 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tensorflow-speech-recognition-challenge/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 5f4d739491ce2cc9acecaffca9787367 3 | public: 4 | sample_submission.csv: 0b22e4798056a81416614ee8aa3c6ae5 5 | zip: 8e459afaaf22efd6411d545a690a7452 6 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/hms-harmful-brain-activity-classification/constants.py: -------------------------------------------------------------------------------- 1 | ID_COL = "eeg_id" 2 | TARGET_COLS = [ 3 | "seizure_vote", 4 | "lpd_vote", 5 | "gpd_vote", 6 | "lrda_vote", 7 | "grda_vote", 8 | "other_vote", 9 | ] 10 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/movie-review-sentiment-analysis-kernels-only/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 82f44bb7f0a1b075afdadfd356c3644b 3 | public: 4 | sampleSubmission.csv: 2da66116279038cc280e8cafcb27a6ea 5 | zip: 297ae2983c4a07603ed2fd31613c1b5e 6 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/the-icml-2013-whale-challenge-right-whale-redux/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 199fdb3b74d7e1bc30fabc4fe4c66743 3 | public: 4 | sampleSubmission.csv: 76160a892fbd2db6d847bd444f576cee 5 | zip: 31d4fe71434eaa84fb0003374e6a30cc 6 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/text-normalization-challenge-english-language/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 9cdf7a9f064c7476c74b8e7b90c72398 3 | sample_submission.csv: e4d1b5d86abed38e6ef37cb7996c3f2c 4 | public: {} 5 | zip: 49579a7cd2984841d1f09bea3824b22a 6 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/text-normalization-challenge-russian-language/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 22913afbc4db4599fa201655efb4906b 3 | sample_submission.csv: d815d621314754ed7b886b5d16f91d93 4 | public: {} 5 | zip: cbe64d0eb83f22f1838806121fc5c6e2 6 | -------------------------------------------------------------------------------- /mle-bench/agents/aide/src/aide/example_tasks/bitcoin_price.md: -------------------------------------------------------------------------------- 1 | ## Goal 2 | Build a timeseries forcasting model for bitcoin close price. 3 | 4 | ## Evaluation metric 5 | Root-Mean-Squared-Error (RMSE) between the logarithm of the predicted value and the logarithm of the observed price. -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/hotel-id-2021-fgvc8/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 6c90a309e95ac51e4a5cb4d3d894e1fa 3 | public: 4 | sample_submission.csv: 79ae3a798fbeb435b880c312fcf6f357 5 | train.csv: 0263cce94164ab82b7f8e9948b85f990 6 | zip: ecf1ac608b38c5bcfd81b4f9224b45b1 7 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/aerial-cactus-identification/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: ebdb02d62dbaf60270a9b3a8617840af 3 | public: 4 | sample_submission.csv: 14c675bde0de98994efafac8e008277e 5 | train.csv: 4f5f752b84f8836c852f4f70e988a545 6 | zip: 72018554bfbb009f2145f56121380078 7 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/dog-breed-identification/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 83d0c48056db066d07102ee13aaecc08 3 | public: 4 | labels.csv: 0ab1a234cd6b80e9efd617e305a4eda5 5 | sample_submission.csv: 30c85225fcd6e22dc51a7d94a333575f 6 | zip: e01b86f22c3573af92b0682a13d5b8d2 7 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/paddy-disease-classification/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: ad25b008d56c0a716d1566da6d8f55bd 3 | public: 4 | sample_submission.csv: c14ec25b7d3d20715830e2b34b3c0d4a 5 | train.csv: 893609e41711931bc46caadb00327a49 6 | zip: c3bdea9db24a456b2ca5383ecd8d2df5 7 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/plant-pathology-2021-fgvc8/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: cfe891cbe7e6b58e77919acb517125ea 3 | public: 4 | sample_submission.csv: fc2941d5d4aedafb25332b57342688bf 5 | train.csv: 0c4ca06e59e35f1e00f55aa79a5dadc5 6 | zip: c1b568a240bbf0c039b2d5c3fb84a4d5 7 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/seti-breakthrough-listen/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: bde6668df6af0c858a8eea028a140bfb 3 | public: 4 | sample_submission.csv: 6bec1773dc78bbe72a70a539e86a8bac 5 | train_labels.csv: c2dc9aa0bb1a60ba40215f979d81df6c 6 | zip: 0004b231e3a9dcff5bf82b59a52b9160 7 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/whale-categorization-playground/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: c86022f990b9fccf644806e023d24b51 3 | public: 4 | sample_submission.csv: 6ca155a2486ee7455662e159318b09df 5 | train.csv: fe97ca1df1ac09c8c3c6a24853481f79 6 | zip: 1fc460b6656261ddafcc21a44248844e 7 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/predict-volcanic-eruptions-ingv-oe/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 1034df7a3f38ccd7bf9e5d6357db86f3 3 | public: 4 | sample_submission.csv: 99631b72e1d2019b0b498f0dd6eec1cb 5 | train.csv: b7c10ca530487fb7e3d1f1669a6e5059 6 | zip: eb45476258edbdbf2b57a30e77484904 7 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/3d-object-detection-for-autonomous-vehicles/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 1972edf2739f010c260822761b94b1df 3 | public: 4 | sample_submission.csv: 4057929857ceb634ecb3875f93ac1688 5 | train.csv: 687b1288965d6efa35ea29c3adff5557 6 | zip: 48824e3ca27955e71bf0c30938e560f7 7 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/histopathologic-cancer-detection/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 949d1acc553f3a04b89262f6e24e9fc7 3 | public: 4 | sample_submission.csv: 8016b8be0087d487ff38ac46f646a01d 5 | train_labels.csv: 806d8b128caeeaf4d0091bd6034cd0fb 6 | zip: a49ad4e5cf84b7df1ccb1f58526d2e21 7 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/vesuvius-challenge-ink-detection/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | gold_submission.csv: 448955259fbb2d5399d4d6b0944b255d 3 | inklabels_rle.csv: 319de381a2477837da0ba44679c6227e 4 | public: 5 | sample_submission.csv: 034491951a7c40fbcac61ebe97ec4629 6 | zip: 01b67ade839d9f0c68baee2c6025b478 7 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/rsna-miccai-brain-tumor-radiogenomic-classification/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 9069c7813ae83cd1dc962efed1875074 3 | public: 4 | sample_submission.csv: 893556127819a5b9428102dbbc0e0ed3 5 | train_labels.csv: 891e1c30bfe288e25c931759a746f254 6 | zip: 148f363879c193b0c510b840832833e3 7 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/leaf-classification/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 028b4798af4eeafc69ab98b22dc2c70a 3 | public: 4 | sample_submission.csv: e9fbff4eafce74b4905feffdaa9e5859 5 | test.csv: 52a1b1a2b6fb392986b9579dd52036c7 6 | train.csv: 02920a969976f89192a3d1c03a30cd2c 7 | zip: 043bdec7fadb7922978b7484217b6007 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/spaceship-titanic/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 1cb99c65efecaba968128577722a83ef 3 | public: 4 | sample_submission.csv: 5414b5da6ebcff454de1c9ed39b73774 5 | test.csv: 8186bd4048aece25f1ed45708f8f34e1 6 | train.csv: 564a1614ecef9d9611cec4efdce4c412 7 | zip: 67c177886736583aff9b947e121395a9 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/google-quest-challenge/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 2bc3cc2878f0c50a05858f6562b12469 3 | public: 4 | sample_submission.csv: ba75781e8b1a5936533c4de82e515c98 5 | test.csv: 06f482b597547c3c975ff1fc59314011 6 | train.csv: f41aa13133d70b94bfa53801762937aa 7 | zip: 9b2e21108f8c0f690457ec23c0d7d249 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/imet-2020-fgvc7/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: c708435becf1c7679bf09c1746942992 3 | public: 4 | labels.csv: a62c433510585b96a558564fa8f530f4 5 | sample_submission.csv: 879aa23cfa7947e6c39737615fa7f429 6 | train.csv: ce560adcffe6ee6e9b6da89981f8da2e 7 | zip: 5e91584b7f667ed2c267fbc2ee83b335 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/lmsys-chatbot-arena/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 43ef7dc300c731231796afc046cf4230 3 | public: 4 | sample_submission.csv: e67f89b25eb382df62d821e72bed8d4b 5 | test.csv: fa5d6d46adf7cfa5d7f7dc3ef3e358c8 6 | train.csv: e11dec14f6fc839a97fb21abcaebdaa4 7 | zip: 73e311b0e622c3f82fb22c7ee24ae91e 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/playground-series-s3e18/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: f744297d99692b67dc82c54f672a4745 3 | public: 4 | sample_submission.csv: 9bb782a52ccbfd6873f9a1e9cc3e97d3 5 | test.csv: 1a77aa7922b712e3085a771b8fe73c5f 6 | train.csv: e19b27b614d5a3bb74b930c52d659799 7 | zip: 7e5aa885e5138b6b5668f426347c7602 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/random-acts-of-pizza/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: b7a9c13b5f063dcfff7102f39f8c95ca 3 | public: 4 | sampleSubmission.csv: 6d7556c9a2f353b5d83ab0424a1bc914 5 | test.json: f86bbac7e5eb9658632ac51ccef06042 6 | train.json: 67a72ab0b01bb8fabb81f8406cc485f1 7 | zip: 16319db32e81271a30f106f4e86f1b3d 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/stanford-covid-vaccine/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 41f284c21e1f7e112f29e1d7b7c1fd73 3 | public: 4 | sample_submission.csv: 5d6a8da1eae26187ec3158b68ec7d666 5 | test.json: 86e2201a0168275ccadd49cafa6cab66 6 | train.json: 0081f791ac8730984c147955a0c7bba9 7 | zip: 16ec2b3aa849804b7ab68a3bd043d00c 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/aptos2019-blindness-detection/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: a5c28ffe71a45d1fadf4bdda3c52d957 3 | public: 4 | sample_submission.csv: 4a80cd1603bdd3423b602b609cc074c2 5 | test.csv: 395258c0476fa058de79fa8691a12723 6 | train.csv: 1a1f11fbeb7f20afd334e28421825725 7 | zip: fff5440648ab9ee707bc56c97c40dc2f 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/petfinder-pawpularity-score/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: da4c520ad07c54f8a67b8b414bc368e9 3 | public: 4 | sample_submission.csv: 569c9d854c4242c647c421e0b003b915 5 | test.csv: 7c709809dbba57d5fd14a9a3f9abc240 6 | train.csv: 619372a40df4773bcae08197ede2fab7 7 | zip: 35dd50bacd33ebba746738508e20e276 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/plant-pathology-2020-fgvc7/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 8835c3fd71013bd15ea81566d4f85d7b 3 | public: 4 | sample_submission.csv: 7a67fc684abd73d9d1afc554e8d5cf07 5 | test.csv: 4a6cc8292989880ba253a124634efff1 6 | train.csv: f99c8adb1be7d3f0a5c96fe804fcf3f3 7 | zip: 741d4c394bd458d56b8ebf905bb3918a 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/spooky-author-identification/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: fb1381c6c9a7e9efd236c625807a0c8a 3 | public: 4 | sample_submission.csv: 3b754ff50b2b4416150bb00c86c34889 5 | test.csv: 2ea5096a8e8cf340b13a7a18d9bdd914 6 | train.csv: 1124166bf7b8b1e337a32fe568732447 7 | zip: 8f12d4f1f138f3867e881293b9bc54ee 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tweet-sentiment-extraction/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: f67a6c60b4b93ad6a7a248a0dfcae16c 3 | public: 4 | sample_submission.csv: 0341dc99c0063baba150e1e64d8b5000 5 | test.csv: b52b4f32e127ea49f8f0d218527fa93f 6 | train.csv: 4248531bac2a97518a553a38d05355ac 7 | zip: 561dfe47375e6fdd7b36cd507509f1c2 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/AI4Code/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test_orders.csv: 58e1e03e110aa2d805cd84f6e6a6477f 3 | public: 4 | sample_submission.csv: dc03df9d2f67ead372ce0a244360268a 5 | train_ancestors.csv: 5a4c8b08ea96ac4039515cba1832da4f 6 | train_orders.csv: b537c96c54e435385363b670b7672cba 7 | zip: 0c118bd40925cdc0ca35e5e22460f836 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/multi-modal-gesture-recognition/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 3bf5058c005b39736340bca90f69d4d9 3 | public: 4 | randomPredictions.csv: f91424deb15d00eb184f5a1447cf9907 5 | test.csv: 1d20628560ba011f2866a054a7551780 6 | training.csv: 8cfa71cc5e9865ef8912e8e6efbb3acb 7 | zip: daaa0b3c56891c54f91494fe3783dc56 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/rsna-breast-cancer-detection/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: f6e2830725827a87b1b85de66b35c656 3 | public: 4 | sample_submission.csv: 66e5e72a439b76b1f38b105aeec770b8 5 | test.csv: 6116661d0f3457cc3d9655ba942a9916 6 | train.csv: c61eac186ab4fd952f2783974e7b189d 7 | zip: 94179aefcefcf67828b6caa12af00d94 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/siim-isic-melanoma-classification/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 89aae5dac5881e960258883de51152fe 3 | public: 4 | sample_submission.csv: c8fa2aaa934d434e5b87e148ca91d42e 5 | test.csv: faeb3cdaf1ee9c47120bcbf46787fe81 6 | train.csv: dc473826f850a25d5e62f981b5e62d54 7 | zip: ed8a6c59a3677cada6365ecdb42d8d23 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tabular-playground-series-dec-2021/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 0819faab160bc678ff4e04b799caf748 3 | public: 4 | sample_submission.csv: d7b1e1f35677df7cce6942a8c053633f 5 | test.csv: 3427fd7ec4244230795e37dac986916a 6 | train.csv: f5b8080079184e0eb10ef7b6b2f1fec7 7 | zip: c7f38370ee2609e5602814a55cb26555 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/ventilator-pressure-prediction/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: ef4ebee21345154a7cdfbec0606db01a 3 | public: 4 | sample_submission.csv: 6ab5162be2a03922cfc6068befc31c22 5 | test.csv: 39c126e1ac7b2e6641ec54f698b1ebdf 6 | train.csv: 4b0aaa1c637a452d5d1109aa585bca9f 7 | zip: dd04f45fe8aaa3608c7cfb17adeaa98a 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/new-york-city-taxi-fare-prediction/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: f51d45e95bcd7565f1175206e604506e 3 | public: 4 | labels.csv: b1aa60e8d6ca817c18b23bd00e4a2500 5 | sample_submission.csv: 10c791174db974943c1c80f38fc86391 6 | test.csv: a7b9d9b7208e9b7cff6891cfd56a0053 7 | zip: fe01f4ab11ebb0ee2851754516883070 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/osic-pulmonary-fibrosis-progression/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 0487bd97c387b5e6c28d5060026462d9 3 | public: 4 | sample_submission.csv: 6204ba4dd60cbf2629cd43c60144b9a7 5 | test.csv: 4d8bf1fd3b55feae1a1e3b5088a2f268 6 | train.csv: dd984a7b700fcd6f167c89ebbbe8619e 7 | zip: c158d5991c84fa57940d8be0eb53272f 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tgs-salt-identification-challenge/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: eedcb00ab3595210c2663838578745c3 3 | public: 4 | depths.csv: aa56896090e86d7b42dbe036921d11bc 5 | sample_submission.csv: e94197284c1c392e79d539b89158cf4a 6 | train.csv: cb31a738448596dfe36b1e3a67d656c8 7 | zip: b5e3f38ebdae156974f7389a6e402c08 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/us-patent-phrase-to-phrase-matching/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 34f67932a2c9f49169cdccf5e7bbc57f 3 | public: 4 | sample_submission.csv: 282590a9bcbf32edf5d351a40a30c6a0 5 | test.csv: ea1dccd31666f938f0ae7632c69c95c6 6 | train.csv: 48042478b2d9a8f36d614589b7ee86e4 7 | zip: e3eae0b5c78ed58de04e1f32e499188a 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/uw-madison-gi-tract-image-segmentation/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: ab8cd8e48b4c72df3e4933c04dc08740 3 | public: 4 | sample_submission.csv: 96d669ecf7556353b7203a31f54edd1e 5 | test.csv: fe823b73f539d27fd2e5c16f2775940f 6 | train.csv: 289c63c9dfb86e7d776cda60f2f1313b 7 | zip: 53a2958bbd106d05f34fb9b97c641548 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/chaii-hindi-and-tamil-question-answering/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 3ae8b59f0ae28791170f45313fdf0432 3 | public: 4 | sample_submission.csv: 772a81a2f11029153ebe965572fd9410 5 | test.csv: f0fe24d3f56320b3d1dae5a66aef0fac 6 | train.csv: b032f44429b21d64a103a3e7f8d61ad0 7 | zip: 2766e34372404b2563fae09ddcc95161 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/facebook-recruiting-iii-keyword-extraction/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 1aeaa15b5bce7088a7d5423c2c404eca 3 | public: 4 | sample_submission.csv: e4ea6ff928dff4bf2d2a5c3b0089402c 5 | test.csv: 10335b0f34418bb6a523506290aff802 6 | train.csv: 3e16dc3e4df1697f70c93cf6bb473eaf 7 | zip: 68ceda9a963e745ab4fe02d0939638f7 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/freesound-audio-tagging-2019/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 1c2444d48bddb7853be3637b30425c6f 3 | public: 4 | sample_submission.csv: 9e937378f06d20d2bbe5816ec3d7ae11 5 | train_curated.csv: f4144b2a64918cf6fa53f4a4640f397a 6 | train_noisy.csv: 3bda3cf565fd3a9082b6405e9698f61f 7 | zip: 8945190179afbdd7643764e073335be3 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/herbarium-2022-fgvc9/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: d7988cc4decb444ed99cbae59381298d 3 | public: 4 | sample_submission.csv: c2b0d6f95036aaa177a4e14a0d627054 5 | test_metadata.json: 465f92cf49fb94a793648066989fe817 6 | train_metadata.json: 22394926583a35658f6dd26ea0dcc683 7 | zip: d99e3df24b03dcb2f07fbc4c81b76a5a 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/nomad2018-predict-transparent-conductors/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: fc17cf7adf422874ce65f2b1b1265d98 3 | public: 4 | sample_submission.csv: f6ca859f7a81041cad0594cf31f171d9 5 | test.csv: be4e478c19d962eae8ffde95163fe8b1 6 | train.csv: 27ef5faad359d2dd4ac872fab5c9010d 7 | zip: faf192f27aa208d8da5ab99d41bd9e39 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/ranzcr-clip-catheter-line-classification/classes.py: -------------------------------------------------------------------------------- 1 | CLASSES = [ 2 | "ETT - Abnormal", 3 | "ETT - Borderline", 4 | "ETT - Normal", 5 | "NGT - Abnormal", 6 | "NGT - Borderline", 7 | "NGT - Incompletely Imaged", 8 | "NGT - Normal", 9 | "CVC - Abnormal", 10 | "CVC - Borderline", 11 | ] 12 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/bms-molecular-translation/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 9d0cf13e5b604a31bb4dd77faaead8ae 3 | public: 4 | extra_approved_InChIs.csv: 24301ba135996bd369cc83740e01f8c8 5 | sample_submission.csv: fb64d50adadf358d780b001dc14daa47 6 | train_labels.csv: 35513e9a1d7f42bfa97f93b0476abc4c 7 | zip: b8268fc1e7d4be3b172fb7d90a86bdf6 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/jigsaw-toxic-comment-classification-challenge/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: c23029ecede85b3adfbc2527e7812b84 3 | public: 4 | sample_submission.csv: 2af5ed06d19b00c13c0573b474edb165 5 | test.csv: d21e68ec9e3728be456aa0f477be230c 6 | train.csv: 7ed6a529f1671bd558cf581831d3462b 7 | zip: a0159f30626e4c6a590f09d597c147d6 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/jigsaw-unintended-bias-in-toxicity-classification/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 57d838e932f0d2a66785f7b719adfd91 3 | public: 4 | sample_submission.csv: 6f6dce475d8d7d068e965f57dd491e03 5 | test.csv: a887551e003361bad9e054d7501333cd 6 | train.csv: 7a687c299462dd009e90db97ac053eef 7 | zip: e7ca6425042681da2a3d85c55880626e 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/learning-agency-lab-automated-essay-scoring-2/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 834401684a4041881fe174146dfb54c5 3 | public: 4 | sample_submission.csv: e62f0d933304a49b8af9c110ad0214e2 5 | test.csv: 04a299576e54a568db57c9a252e9ea0b 6 | train.csv: 8288235ab42c523d4a3c32498e307807 7 | zip: 392ed5c55e47ebac98f6d4d529b97f50 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/siim-covid19-detection/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 5a5802f4cd7cec932bb824533cb5b57c 3 | public: 4 | sample_submission.csv: b9bdbd95e7b93f64f54d6f8f2530899c 5 | train_image_level.csv: 87c704caa46ea0df4a50eae97092bcc4 6 | train_study_level.csv: 6026bc39b28881d65a85b51e1e991160 7 | zip: 3176c9c084fbf50e767e514f5989a71f 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/ranzcr-clip-catheter-line-classification/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: e2959bb0ca0ca9fc983b02e01caf31fa 3 | public: 4 | sample_submission.csv: fade90160bce8a12138711b20fa89584 5 | train.csv: 620fa6759729f96539ed66ae860a84f9 6 | train_annotations.csv: 1929aab17c1206719cc7edf6d7e9e57b 7 | zip: f8a117e7ba1b5527c99c80b54beddeb5 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/cassava-leaf-disease-classification/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 34b2b412be39aba1edac9c04990e4fb8 3 | public: 4 | label_num_to_disease_map.json: 133bf5c6a60c901792937eaad034b188 5 | sample_submission.csv: b27270a483141fa217d990d55b4b3d49 6 | train.csv: fede2927f0d412590f195abfbd07b578 7 | zip: e1dc225f0fc8a4161e48a4f9015e16c0 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/vinbigdata-chest-xray-abnormalities-detection/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 7f52e6634e15cf925fd73049d4f7cd91 3 | gold_submission.csv: c3dc4669e63958e16c6b76e0caa7cbf1 4 | public: 5 | sample_submission.csv: b72a880d1f2d14c9398498e06bc0cc10 6 | train.csv: f87c05d09de910f075b198dbbe831874 7 | zip: 8e2b05ace7e4c8d0bbe702454a099d53 8 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/mlsp-2013-birds/kernels.txt: -------------------------------------------------------------------------------- 1 | shreyasajal/birdclef-librosa-audio-feature-extraction 2 | shreyasajal/audio-albumentations-torchaudio-audiomentations 3 | sagniksanyal/birdclef-2022-torchaudio-audiomentations-skimpy 4 | titankinansalaatsa/mlsp-2013-bird-classification-challenge-1301180283 5 | muhammadzubairkhan92/birdclef-2021-librosa-audio-feature-extraction 6 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/iwildcam-2019-fgvc6/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 83006769438332df65e5106e31c5d1fd 3 | test.csv: e81e5aefbfbb046ae5cc5493012f303c 4 | public: 5 | sample_submission.csv: 2c3d7681f93545d6c811de41fbe2013a 6 | test.csv: 7b83b8c770dc4b820ab2c35484f3fc55 7 | train.csv: 3e7e01b0c342da6aebda97efae2de0e4 8 | zip: a47f19c8dfb20d33aeb65fd3975af2c7 9 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/google-research-identify-contrails-reduce-global-warming/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 19bcf4d920a7b6edb1ce5bbda2299437 3 | public: 4 | sample_submission.csv: 76f1e9892ad523bd23909fd41bc47734 5 | train_metadata.json: c1904919eceebb2ff92e05e384e52eba 6 | validation_metadata.json: fbba4a0196aecfea9749f347a407519d 7 | zip: 55480a1ef98fd61becf36d8608ffeb74 -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tabular-playground-series-may-2022/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | gold_submission.csv: fd2e8a542de49e4f4b46ea1a174db71a 3 | test.csv: 0ad7146d50d6a3043b98e9471835db59 4 | public: 5 | sample_submission.csv: 353457517aa847e701141e00a7a774eb 6 | test.csv: a15b39583b2c38e02f986251d0292851 7 | train.csv: 40ef2671fc5bde862789ff7cfcc48c20 8 | zip: 2cad1cd0e611cc4e7a1e780e81d3e2ea 9 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/inaturalist-2019-fgvc6/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 6a6ca89c88327fe0071c350e846141d3 3 | public: 4 | kaggle_sample_submission.csv: 7cc29744e99a9f5ed8299c5e2979a2cc 5 | test2019.json: c572fca9d6b042b49039d0b87e2a4c98 6 | train2019.json: 558b56eb13ad126127de75e379e299f6 7 | val2019.json: e5a1d2eb4fd50f2d1ae5ef571be96eb3 8 | zip: 9d8bb906b3e2740c10b9df89e6a234d1 9 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/kuzushiji-recognition/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | gold_submission.csv: f764948b29a9b7bd45c8fcd7fc8af117 3 | test.csv: 1d2c597a03e298e2f03eec22692271ec 4 | public: 5 | sample_submission.csv: 9ebf151965c85f43249479f38392e00c 6 | train.csv: cb6a4b3dc1f2f2921394cf60ff8083d2 7 | unicode_translation.csv: 3b48a1cd37de399cf20dd018057900cf 8 | zip: dd6ab4e97c8aff48b38a423c992d8a6c 9 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/detecting-insults-in-social-commentary/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | gold_submission.csv: 937dc9172f07ab3f72f123c79e37652c 3 | test.csv: 76d6e15e475a2fd634bdd8cdfb4746c0 4 | public: 5 | sample_submission_null.csv: a0dcbebb9f3aa642325ec3b83dacafa1 6 | test.csv: 6dd3e4189309b7b513fcf124be21c790 7 | train.csv: 60f8ff55967759d144efd4f055f37673 8 | zip: 719b313286d397ddc6d8e454135249de 9 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/hms-harmful-brain-activity-classification/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | gold_submission.csv: ba90c5dda42c504e3638af6ee7dc46fb 3 | test.csv: 205ee658cf7e8cfd317e7bee6045dffd 4 | public: 5 | sample_submission.csv: 5e8cb4e5956283462cbc704e1f09af4d 6 | test.csv: dfe1f087a65ffa7c7341cdcdfc143554 7 | train.csv: c82b8b6f36e0a20932c5566957f6eb63 8 | zip: bbf763e3e95da2e325ab82624462d342 9 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/icecube-neutrinos-in-deep-ice/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: 6b3eef7a836409e5c99557763d1d2c7d 3 | public: 4 | sample_submission.csv: df2a0e29adf304f4fa521a43ba9031f1 5 | sensor_geometry.csv: acad541886071600c427b405a70841e8 6 | test_meta.parquet: 855451176da14ba1114640544462ebbb 7 | train_meta.parquet: 9d8bd35236ee452f8baed612b9f5fd0c 8 | zip: 8663264342ff575230df4d29f20eff7c 9 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/hubmap-kidney-segmentation/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | gold_submission.csv: 04e4a0b5efa7d41491a1cdd01990b507 3 | test.csv: 17d0985b7ced8c4e23fee515af8e381f 4 | public: 5 | HuBMAP-20-dataset_information.csv: 6a19818d50bb232dbe51fbc375bc89cb 6 | sample_submission.csv: 135a2b41c9aa16f03880908a7171dc28 7 | train.csv: a5760de09c0fcc1f78a32bb8f6459f45 8 | zip: 3c3cba984d64b78ccf2f747d02f7178b 9 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/rsna-2022-cervical-spine-fracture-detection/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 9d13e08d12211a39b8c5d2c29a9e7e8f 3 | public: 4 | sample_submission.csv: 2690b6c1b8c2fbc8d090e8fb00c1d79f 5 | test.csv: d5c91b7191186743a2aa74b904207931 6 | train.csv: 812aecfc08948d621decc684759e243a 7 | train_bounding_boxes.csv: d74270bdb517e79f9f2ac6a0f90825fd 8 | zip: 2b38d6c8f3f357d56a247b09769e862b 9 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/h-and-m-personalized-fashion-recommendations/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 46123d4726564502bb3bb546f647fb42 3 | public: 4 | articles.csv: f193ef0f582e8476ebb613d143742d0f 5 | customers.csv: 3aa1713831590115fffc667ba6c187a7 6 | sample_submission.csv: 004eaa20317d414d62f23165293c4a7b 7 | transactions_train.csv: 2a6e1b3ad8f550894341e4bc78a8279d 8 | zip: b7f44b0fc78b749967d4711fe6497121 9 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tensorflow2-question-answering/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | gold_submission.csv: 544ba606cfb641eed0d222c3d94ffee6 3 | test.jsonl: 86393efdae1c319a8d1d18e36f490e97 4 | public: 5 | sample_submission.csv: 483bdebc6d8e55344c3e0124061eb0ff 6 | simplified-nq-test.jsonl: 3abc9e42896e04f4aeaa493e5944e49a 7 | simplified-nq-train.jsonl: 47ef1135698eb044006d1e65b68916b5 8 | zip: 4d3bbcfad1ff7ea12f3c1c3347c4f821 9 | -------------------------------------------------------------------------------- /mle-bench/SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | For a more in-depth look at our security policy, please check out our [Coordinated Vulnerability Disclosure Policy](https://openai.com/security/disclosure/#:~:text=Disclosure%20Policy,-Security%20is%20essential&text=OpenAI%27s%20coordinated%20vulnerability%20disclosure%20policy,expect%20from%20us%20in%20return.). 3 | 4 | Our PGP key can located [at this address.](https://cdn.openai.com/security.txt) 5 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/iwildcam-2020-fgvc7/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 88adac005790552cbd7530e14c88525a 3 | public: 4 | iwildcam2020_megadetector_results.json: 1b9ece07ab63b920bf8a9ae8f05bd38c 5 | iwildcam2020_test_information.json: 88143d613e565e6ce875c222b9d49ebc 6 | iwildcam2020_train_annotations.json: d67abd02833d86fcf7f08e5655c34cae 7 | sample_submission.csv: 0b664574b369e81790f7f092fbb8ae11 8 | zip: 904c5f3195cc47eae83d4334ad9c20e6 9 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/cdiscount-image-classification-challenge/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: 40700f1566c1be9eda129ec310ce6fe8 3 | public: 4 | category_names.csv: f3942c134e6010e531709f56fcf63ce4 5 | sample_submission.csv: 386d237e1e17a667040d9a8a8e753371 6 | test.bson: b2b39dbcd5e04c1c27aea258e103e1a4 7 | train.bson: af1df955ac1abfab0b9c3fa5ad732760 8 | train_example.bson: e715bde1bc797b076f21349674898e4d 9 | zip: dce8429382191257a67421fd3914e257 10 | -------------------------------------------------------------------------------- /scripts/build_base_env.sh: -------------------------------------------------------------------------------- 1 | export SUBMISSION_DIR=/home/submission 2 | export LOGS_DIR=/home/logs 3 | export CODE_DIR=/home/code 4 | export AGENT_DIR=/home/agent 5 | 6 | docker build --platform=linux/amd64 -t mlebench-env -f mle-bench/environment/Dockerfile ./mle-bench 7 | docker build --no-cache --platform=linux/amd64 -t automind-base -f mle-bench/BaseDockerfile . --build-arg SUBMISSION_DIR=$SUBMISSION_DIR --build-arg LOGS_DIR=$LOGS_DIR --build-arg CODE_DIR=$CODE_DIR --build-arg AGENT_DIR=$AGENT_DIR 8 | -------------------------------------------------------------------------------- /mle-bench/.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/psf/black 3 | rev: 22.8.0 4 | hooks: 5 | - id: black 6 | args: [--line-length=100, --exclude=""] 7 | 8 | # this is not technically always safe but usually is 9 | # use comments `# isort: off` and `# isort: on` to disable/re-enable isort 10 | - repo: https://github.com/pycqa/isort 11 | rev: 5.12.0 12 | hooks: 13 | - id: isort 14 | args: [--line-length=100, --profile=black] 15 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/iwildcam-2020-fgvc7/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import accuracy_score 3 | 4 | from mlebench.competitions.utils import prepare_for_accuracy_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | accuracy_inputs = prepare_for_accuracy_metric( 9 | submission=submission, answers=answers, target_column="Category", id_column="Id" 10 | ) 11 | return accuracy_score(**accuracy_inputs) 12 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tensorflow-speech-recognition-challenge/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import accuracy_score 3 | 4 | from mlebench.competitions.utils import prepare_for_accuracy_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | accuracy_inputs = prepare_for_accuracy_metric( 9 | submission=submission, answers=answers, target_column="label", id_column="fname" 10 | ) 11 | return accuracy_score(**accuracy_inputs) 12 | -------------------------------------------------------------------------------- /mle-bench/experiments/splits/automind.txt: -------------------------------------------------------------------------------- 1 | aptos2019-blindness-detection 2 | random-acts-of-pizza 3 | spooky-author-identification 4 | google-quest-challenge 5 | stanford-covid-vaccine 6 | predict-volcanic-eruptions-ingv-oe 7 | lmsys-chatbot-arena 8 | us-patent-phrase-to-phrase-matching 9 | mlsp-2013-birds 10 | statoil-iceberg-classifier-challenge 11 | tensorflow-speech-recognition-challenge 12 | denoising-dirty-documents 13 | new-york-city-taxi-fare-prediction 14 | tgs-salt-identification-challenge 15 | ventilator-pressure-prediction -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/spaceship-titanic/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import accuracy_score 3 | 4 | from mlebench.competitions.utils import prepare_for_accuracy_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | accuracy_inputs = prepare_for_accuracy_metric( 9 | submission=submission, 10 | answers=answers, 11 | target_column="Transported", 12 | id_column="PassengerId", 13 | ) 14 | return accuracy_score(**accuracy_inputs) 15 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tabular-playground-series-may-2022/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import roc_auc_score 3 | 4 | from mlebench.competitions.utils import prepare_for_auroc_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | roc_auc_inputs = prepare_for_auroc_metric( 9 | submission, answers, id_col="id", target_col="target" 10 | ) 11 | return roc_auc_score( 12 | y_true=roc_auc_inputs["y_true"], y_score=roc_auc_inputs["y_score"] 13 | ) 14 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tabular-playground-series-dec-2021/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import accuracy_score 3 | 4 | from mlebench.competitions.utils import prepare_for_accuracy_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | accuracy_inputs = prepare_for_accuracy_metric( 9 | submission=submission, 10 | answers=answers, 11 | target_column="Cover_Type", 12 | id_column="Id", 13 | ) 14 | return accuracy_score(**accuracy_inputs) 15 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/seti-breakthrough-listen/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import roc_auc_score 3 | 4 | from mlebench.competitions.utils import prepare_for_auroc_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | roc_auc_inputs = prepare_for_auroc_metric( 9 | submission=submission, answers=answers, id_col="id", target_col="target" 10 | ) 11 | return roc_auc_score( 12 | y_true=roc_auc_inputs["y_true"], y_score=roc_auc_inputs["y_score"] 13 | ) 14 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/siim-isic-melanoma-classification/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import roc_auc_score 3 | 4 | from mlebench.competitions.utils import prepare_for_auroc_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | roc_auc_inputs = prepare_for_auroc_metric( 9 | submission, answers, id_col="image_name", target_col="target" 10 | ) 11 | return roc_auc_score( 12 | y_true=roc_auc_inputs["y_true"], y_score=roc_auc_inputs["y_score"] 13 | ) 14 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/cdiscount-image-classification-challenge/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import accuracy_score 3 | 4 | from mlebench.competitions.utils import prepare_for_accuracy_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | accuracy_inputs = prepare_for_accuracy_metric( 9 | submission=submission, 10 | answers=answers, 11 | target_column="category_id", 12 | id_column="_id", 13 | ) 14 | return accuracy_score(**accuracy_inputs) 15 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/invasive-species-monitoring/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import roc_auc_score 3 | 4 | from mlebench.competitions.utils import prepare_for_auroc_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | roc_auc_inputs = prepare_for_auroc_metric( 9 | submission=submission, answers=answers, id_col="name", target_col="invasive" 10 | ) 11 | return roc_auc_score( 12 | y_true=roc_auc_inputs["y_true"], y_score=roc_auc_inputs["y_score"] 13 | ) 14 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/aerial-cactus-identification/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import roc_auc_score 3 | 4 | from mlebench.competitions.utils import prepare_for_auroc_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | roc_auc_inputs = prepare_for_auroc_metric( 9 | submission=submission, answers=answers, id_col="id", target_col="has_cactus" 10 | ) 11 | return roc_auc_score( 12 | y_true=roc_auc_inputs["y_true"], y_score=roc_auc_inputs["y_score"] 13 | ) 14 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/ml2021spring-hw2/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import accuracy_score 3 | 4 | from mlebench.competitions.utils import prepare_for_accuracy_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | accuracy_inputs = prepare_for_accuracy_metric( 9 | submission=submission, answers=answers, target_column="ClassId", id_column="Id" 10 | ) 11 | return accuracy_score( 12 | y_true=accuracy_inputs["y_true"], y_pred=accuracy_inputs["y_pred"] 13 | ) 14 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/movie-review-sentiment-analysis-kernels-only/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import accuracy_score 3 | 4 | from mlebench.competitions.utils import prepare_for_accuracy_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | accuracy_inputs = prepare_for_accuracy_metric( 9 | submission=submission, 10 | answers=answers, 11 | target_column="Sentiment", 12 | id_column="PhraseId", 13 | ) 14 | return accuracy_score(**accuracy_inputs) 15 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/the-icml-2013-whale-challenge-right-whale-redux/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import roc_auc_score 3 | 4 | from mlebench.competitions.utils import prepare_for_auroc_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | roc_auc_inputs = prepare_for_auroc_metric( 9 | submission, answers, id_col="clip", target_col="probability" 10 | ) 11 | return roc_auc_score( 12 | y_true=roc_auc_inputs["y_true"], y_score=roc_auc_inputs["y_score"] 13 | ) 14 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/cassava-leaf-disease-classification/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import accuracy_score 3 | 4 | from mlebench.competitions.utils import prepare_for_accuracy_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | accuracy_inputs = prepare_for_accuracy_metric( 9 | submission=submission, 10 | answers=answers, 11 | target_column="label", 12 | id_column="image_id", 13 | ) 14 | score = accuracy_score(**accuracy_inputs) 15 | return score 16 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/detecting-insults-in-social-commentary/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import roc_auc_score 3 | 4 | from mlebench.competitions.utils import prepare_for_auroc_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | roc_auc_inputs = prepare_for_auroc_metric( 9 | submission=submission, answers=answers, id_col="Comment", target_col="Insult" 10 | ) 11 | return roc_auc_score( 12 | y_true=roc_auc_inputs["y_true"], y_score=roc_auc_inputs["y_score"] 13 | ) 14 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/histopathologic-cancer-detection/grade.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | from sklearn.metrics import roc_auc_score 4 | 5 | from mlebench.competitions.utils import prepare_for_auroc_metric 6 | 7 | 8 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 9 | roc_auc_inputs = prepare_for_auroc_metric( 10 | submission=submission, answers=answers, id_col="id", target_col="label" 11 | ) 12 | return roc_auc_score( 13 | y_true=roc_auc_inputs["y_true"], y_score=roc_auc_inputs["y_score"] 14 | ) 15 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/ml2021spring-hw2/config.yaml: -------------------------------------------------------------------------------- 1 | id: ml2021spring-hw2 2 | name: ML2021Spring-hw2 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/ml2021spring-hw2/description.md 7 | 8 | dataset: 9 | answers: ml2021spring-hw2/prepared/private/answers.csv 10 | sample_submission: ml2021spring-hw2/prepared/public/sampleSubmission.csv 11 | 12 | grader: 13 | name: accuracy 14 | grade_fn: mlebench.competitions.ml2021spring-hw2.grade:grade 15 | 16 | preparer: mlebench.competitions.ml2021spring-hw2.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/spaceship-titanic/config.yaml: -------------------------------------------------------------------------------- 1 | id: spaceship-titanic 2 | name: Spaceship Titanic 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/spaceship-titanic/description.md 7 | 8 | dataset: 9 | answers: spaceship-titanic/prepared/private/test.csv 10 | sample_submission: spaceship-titanic/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: accuracy 14 | grade_fn: mlebench.competitions.spaceship-titanic.grade:grade 15 | 16 | preparer: mlebench.competitions.spaceship-titanic.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/imet-2020-fgvc7/config.yaml: -------------------------------------------------------------------------------- 1 | id: imet-2020-fgvc7 2 | name: iMet Collection 2020 - FGVC7 3 | competition_type: code 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/imet-2020-fgvc7/description.md 7 | 8 | dataset: 9 | answers: imet-2020-fgvc7/prepared/private/answers.csv 10 | sample_submission: imet-2020-fgvc7/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: micro-f1-score 14 | grade_fn: mlebench.competitions.imet-2020-fgvc7.grade:grade 15 | 16 | preparer: mlebench.competitions.imet-2020-fgvc7.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/random-acts-of-pizza/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import roc_auc_score 3 | 4 | from mlebench.competitions.utils import prepare_for_auroc_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | roc_auc_inputs = prepare_for_auroc_metric( 9 | submission=submission, 10 | answers=answers, 11 | id_col="request_id", 12 | target_col="requester_received_pizza", 13 | ) 14 | return roc_auc_score( 15 | y_true=roc_auc_inputs["y_true"], y_score=roc_auc_inputs["y_score"] 16 | ) 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/iwildcam-2020-fgvc7/config.yaml: -------------------------------------------------------------------------------- 1 | id: iwildcam-2020-fgvc7 2 | name: iWildCam 2020 - FGVC7 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/iwildcam-2020-fgvc7/description.md 7 | 8 | dataset: 9 | answers: iwildcam-2020-fgvc7/prepared/private/answers.csv 10 | sample_submission: iwildcam-2020-fgvc7/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: accuracy 14 | grade_fn: mlebench.competitions.iwildcam-2020-fgvc7.grade:grade 15 | 16 | preparer: mlebench.competitions.iwildcam-2020-fgvc7.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/random-acts-of-pizza/config.yaml: -------------------------------------------------------------------------------- 1 | id: random-acts-of-pizza 2 | name: Random Acts of Pizza 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/random-acts-of-pizza/description.md 7 | 8 | dataset: 9 | answers: random-acts-of-pizza/prepared/private/test.csv 10 | sample_submission: random-acts-of-pizza/prepared/public/sampleSubmission.csv 11 | 12 | grader: 13 | name: auc-roc 14 | grade_fn: mlebench.competitions.random-acts-of-pizza.grade:grade 15 | 16 | preparer: mlebench.competitions.random-acts-of-pizza.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/environment/validate_submission.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script is a tool for agents to submit a file to the grading server for validation. 4 | 5 | # validate_submission.sh 6 | # Usage: ./validate_submission.sh user_submission.csv 7 | 8 | SUBMISSION_FILE=$1 9 | SERVER_URL="http://localhost:5000/validate" 10 | 11 | if [ -z "$SUBMISSION_FILE" ]; then 12 | echo "Usage: $0 " 13 | exit 1 14 | fi 15 | 16 | if [ ! -f "$SUBMISSION_FILE" ]; then 17 | echo "File $SUBMISSION_FILE does not exist." 18 | exit 1 19 | fi 20 | 21 | curl -X POST -F "file=@${SUBMISSION_FILE}" ${SERVER_URL} -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/herbarium-2020-fgvc7/config.yaml: -------------------------------------------------------------------------------- 1 | id: herbarium-2020-fgvc7 2 | name: Herbarium 2020 - FGVC7 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/herbarium-2020-fgvc7/description.md 7 | 8 | dataset: 9 | answers: herbarium-2020-fgvc7/prepared/private/answers.csv 10 | sample_submission: herbarium-2020-fgvc7/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: macro-f1-score 14 | grade_fn: mlebench.competitions.herbarium-2020-fgvc7.grade:grade 15 | 16 | preparer: mlebench.competitions.herbarium-2020-fgvc7.prepare:prepare -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/herbarium-2021-fgvc8/config.yaml: -------------------------------------------------------------------------------- 1 | id: herbarium-2021-fgvc8 2 | name: Herbarium 2021 - FGVC8 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/herbarium-2021-fgvc8/description.md 7 | 8 | dataset: 9 | answers: herbarium-2021-fgvc8/prepared/private/answers.csv 10 | sample_submission: herbarium-2021-fgvc8/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: macro-f1-score 14 | grade_fn: mlebench.competitions.herbarium-2021-fgvc8.grade:grade 15 | 16 | preparer: mlebench.competitions.herbarium-2021-fgvc8.prepare:prepare -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/herbarium-2022-fgvc9/config.yaml: -------------------------------------------------------------------------------- 1 | id: herbarium-2022-fgvc9 2 | name: Herbarium 2022 - FGVC9 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/herbarium-2022-fgvc9/description.md 7 | 8 | dataset: 9 | answers: herbarium-2022-fgvc9/prepared/private/answers.csv 10 | sample_submission: herbarium-2022-fgvc9/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: macro-f1-score 14 | grade_fn: mlebench.competitions.herbarium-2022-fgvc9.grade:grade 15 | 16 | preparer: mlebench.competitions.herbarium-2022-fgvc9.prepare:prepare -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/iwildcam-2019-fgvc6/config.yaml: -------------------------------------------------------------------------------- 1 | id: iwildcam-2019-fgvc6 2 | name: iWildCam 2019 - FGVC6 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/iwildcam-2019-fgvc6/description.md 7 | 8 | dataset: 9 | answers: iwildcam-2019-fgvc6/prepared/private/answers.csv 10 | sample_submission: iwildcam-2019-fgvc6/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: macro-f1-score 14 | grade_fn: mlebench.competitions.iwildcam-2019-fgvc6.grade:grade 15 | 16 | preparer: mlebench.competitions.iwildcam-2019-fgvc6.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/leaf-classification/config.yaml: -------------------------------------------------------------------------------- 1 | id: leaf-classification 2 | name: Leaf Classification 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/leaf-classification/description.md 7 | 8 | dataset: 9 | answers: leaf-classification/prepared/private/test.csv 10 | sample_submission: leaf-classification/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: multi-class-log-loss 14 | grade_fn: mlebench.competitions.leaf-classification.grade:grade 15 | 16 | preparer: mlebench.competitions.leaf-classification.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/hotel-id-2021-fgvc8/config.yaml: -------------------------------------------------------------------------------- 1 | id: hotel-id-2021-fgvc8 2 | name: Hotel-ID to Combat Human Trafficking 2021 - FGVC8 3 | competition_type: code 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/hotel-id-2021-fgvc8/description.md 7 | 8 | dataset: 9 | answers: hotel-id-2021-fgvc8/prepared/private/test.csv 10 | sample_submission: hotel-id-2021-fgvc8/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: map-at-5 14 | grade_fn: mlebench.competitions.hotel-id-2021-fgvc8.grade:grade 15 | 16 | preparer: mlebench.competitions.hotel-id-2021-fgvc8.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/billion-word-imputation/config.yaml: -------------------------------------------------------------------------------- 1 | id: billion-word-imputation 2 | name: Billion Word Imputation 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/billion-word-imputation/description.md 7 | 8 | dataset: 9 | answers: billion-word-imputation/prepared/private/test.csv 10 | sample_submission: billion-word-imputation/prepared/private/sample_submission.csv 11 | 12 | grader: 13 | name: levenshtein-distance 14 | grade_fn: mlebench.competitions.billion-word-imputation.grade:grade 15 | 16 | preparer: mlebench.competitions.billion-word-imputation.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/dog-breed-identification/config.yaml: -------------------------------------------------------------------------------- 1 | id: dog-breed-identification 2 | name: Dog Breed Identification 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/dog-breed-identification/description.md 7 | 8 | dataset: 9 | answers: dog-breed-identification/prepared/private/test.csv 10 | sample_submission: dog-breed-identification/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: multi-class-log-loss 14 | grade_fn: mlebench.competitions.dog-breed-identification.grade:grade 15 | 16 | preparer: mlebench.competitions.dog-breed-identification.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/inaturalist-2019-fgvc6/config.yaml: -------------------------------------------------------------------------------- 1 | id: inaturalist-2019-fgvc6 2 | name: iNaturalist 2019 at FGVC6 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/inaturalist-2019-fgvc6/description.md 7 | 8 | dataset: 9 | answers: inaturalist-2019-fgvc6/prepared/private/answers.csv 10 | sample_submission: inaturalist-2019-fgvc6/prepared/public/kaggle_sample_submission.csv 11 | 12 | grader: 13 | name: top-1-classification-error 14 | grade_fn: mlebench.competitions.inaturalist-2019-fgvc6.grade:grade 15 | 16 | preparer: mlebench.competitions.inaturalist-2019-fgvc6.prepare:prepare -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/denoising-dirty-documents/config.yaml: -------------------------------------------------------------------------------- 1 | id: denoising-dirty-documents 2 | name: Denoising Dirty Documents 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/denoising-dirty-documents/description.md 7 | 8 | dataset: 9 | answers: denoising-dirty-documents/prepared/private/answers.csv 10 | sample_submission: denoising-dirty-documents/prepared/public/sampleSubmission.csv 11 | 12 | grader: 13 | name: root_mean_squared_error 14 | grade_fn: mlebench.competitions.denoising-dirty-documents.grade:grade 15 | 16 | preparer: mlebench.competitions.denoising-dirty-documents.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/invasive-species-monitoring/config.yaml: -------------------------------------------------------------------------------- 1 | id: invasive-species-monitoring 2 | name: Invasive Species Monitoring 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/invasive-species-monitoring/description.md 7 | 8 | dataset: 9 | answers: invasive-species-monitoring/prepared/private/answers.csv 10 | sample_submission: invasive-species-monitoring/prepared/private/sample_submission.csv 11 | 12 | grader: 13 | name: auc-roc 14 | grade_fn: mlebench.competitions.invasive-species-monitoring.grade:grade 15 | 16 | preparer: mlebench.competitions.invasive-species-monitoring.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/plant-pathology-2021-fgvc8/config.yaml: -------------------------------------------------------------------------------- 1 | id: plant-pathology-2021-fgvc8 2 | name: Plant Pathology 2021 - FGVC8 3 | competition_type: code 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/plant-pathology-2021-fgvc8/description.md 7 | 8 | dataset: 9 | answers: plant-pathology-2021-fgvc8/prepared/private/answers.csv 10 | sample_submission: plant-pathology-2021-fgvc8/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: micro-f1-score 14 | grade_fn: mlebench.competitions.plant-pathology-2021-fgvc8.grade:grade 15 | 16 | preparer: mlebench.competitions.plant-pathology-2021-fgvc8.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/text-normalization-challenge-russian-language/kernels.txt: -------------------------------------------------------------------------------- 1 | zfturbo/simple-ru-baseline-lb-0-9627 2 | tunguz/ru-baseline-lb-0-9632 3 | arccosmos/ru-baseline-lb-0-9799-from-en-thread 4 | jtoffler/baseline-improvement-lb-9750-to-9751 5 | kambarakun/what-shoud-we-do-in-russian-challenge 6 | tunguz/ru-baselane-stage-2-0-9644-on-lb 7 | danavg/explore-russian-text-normilization-challenge 8 | tunguz/test-2 9 | dequadras/simple-aproach-0-9627-create-a-dictionary 10 | leogan/explore-and-analyse-data 11 | ivanov1917/dictionary-based-most-commons-predictor 12 | jihyeseo/russian-normalize-text 13 | innakt/russianclassificationchallenge 14 | vykhand/some-eda 15 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/aerial-cactus-identification/config.yaml: -------------------------------------------------------------------------------- 1 | id: aerial-cactus-identification 2 | name: Aerial Cactus Identification 3 | competition_type: code 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/aerial-cactus-identification/description.md 7 | 8 | dataset: 9 | answers: aerial-cactus-identification/prepared/private/test.csv 10 | sample_submission: aerial-cactus-identification/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: auc-roc 14 | grade_fn: mlebench.competitions.aerial-cactus-identification.grade:grade 15 | 16 | preparer: mlebench.competitions.aerial-cactus-identification.prepare:prepare 17 | -------------------------------------------------------------------------------- /scripts/run_mlebench.sh: -------------------------------------------------------------------------------- 1 | export SUBMISSION_DIR=/home/submission 2 | export LOGS_DIR=/home/logs 3 | export CODE_DIR=/home/code 4 | export AGENT_DIR=/home/agent 5 | 6 | IMAGE_ID=$(docker build -q --no-cache --platform=linux/amd64 -t automind -f mle-bench/Dockerfile . --build-arg SUBMISSION_DIR=$SUBMISSION_DIR --build-arg LOGS_DIR=$LOGS_DIR --build-arg CODE_DIR=$CODE_DIR --build-arg AGENT_DIR=$AGENT_DIR) 7 | echo $IMAGE_ID 8 | python mle-bench/run_agent.py --agent-id automind --agent-name automind --agent-dir automind --agent-config configs/mlebench.yaml --competition-set mle-bench/experiments/splits/aptos2019-blindness-detection.txt --data-dir /path/to/data --gpu-device 0 9 | docker rmi $IMAGE_ID -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/plant-pathology-2020-fgvc7/config.yaml: -------------------------------------------------------------------------------- 1 | id: plant-pathology-2020-fgvc7 2 | name: Plant Pathology 2020 - FGVC7 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/plant-pathology-2020-fgvc7/description.md 7 | 8 | dataset: 9 | answers: plant-pathology-2020-fgvc7/prepared/private/test.csv 10 | sample_submission: plant-pathology-2020-fgvc7/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: mean-column-wise-roc-auc 14 | grade_fn: mlebench.competitions.plant-pathology-2020-fgvc7.grade:grade 15 | 16 | preparer: mlebench.competitions.plant-pathology-2020-fgvc7.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/nfl-player-contact-detection/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | test.csv: fa42cdab2f042c5bd0665d7739a715fb 3 | public: 4 | sample_submission.csv: e4fe9463edad278506f9911fd86b17f3 5 | test_baseline_helmets.csv: 2981cca0abdb9305780edfea905a7ff4 6 | test_player_tracking.csv: dfa825f54143e63aa4b7ccba897e4d3f 7 | test_video_metadata.csv: 6978a882a8cf8ec2141e5df6959c2617 8 | train_baseline_helmets.csv: 063ba03e0ca7acf3c17bd74d704983f0 9 | train_labels.csv: 8721fe87ec4fb244ecf217522045ab8c 10 | train_player_tracking.csv: 5f8604c4c541db8d1f0247de34e99cd2 11 | train_video_metadata.csv: 5ba91169bc8df45c0334db954c86cbc3 12 | zip: 9fc02688ee4e93ab144338a90f8d07a8 13 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/paddy-disease-classification/config.yaml: -------------------------------------------------------------------------------- 1 | id: paddy-disease-classification 2 | name: 'Paddy Doctor: Paddy Disease Classification' 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/paddy-disease-classification/description.md 7 | 8 | dataset: 9 | answers: paddy-disease-classification/prepared/private/test.csv 10 | sample_submission: paddy-disease-classification/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: accuracy 14 | grade_fn: mlebench.competitions.paddy-disease-classification.grade:grade 15 | 16 | preparer: mlebench.competitions.paddy-disease-classification.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/playground-series-s3e18/config.yaml: -------------------------------------------------------------------------------- 1 | id: playground-series-s3e18 2 | name: Explore Multi-Label Classification with an Enzyme Substrate Dataset 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/playground-series-s3e18/description.md 7 | 8 | dataset: 9 | answers: playground-series-s3e18/prepared/private/test.csv 10 | sample_submission: playground-series-s3e18/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: auroc-multi-label 14 | grade_fn: mlebench.competitions.playground-series-s3e18.grade:grade 15 | 16 | preparer: mlebench.competitions.playground-series-s3e18.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/spooky-author-identification/config.yaml: -------------------------------------------------------------------------------- 1 | id: spooky-author-identification 2 | name: Spooky Author Identification 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/spooky-author-identification/description.md 7 | 8 | dataset: 9 | answers: spooky-author-identification/prepared/private/test.csv 10 | sample_submission: spooky-author-identification/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: multi-class-log-loss 14 | grade_fn: mlebench.competitions.spooky-author-identification.grade:grade 15 | 16 | preparer: mlebench.competitions.spooky-author-identification.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/plant-seedlings-classification/config.yaml: -------------------------------------------------------------------------------- 1 | id: plant-seedlings-classification 2 | name: Plant Seedlings Classification 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/plant-seedlings-classification/description.md 7 | 8 | dataset: 9 | answers: plant-seedlings-classification/prepared/private/answers.csv 10 | sample_submission: plant-seedlings-classification/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: micro-f1-score 14 | grade_fn: mlebench.competitions.plant-seedlings-classification.grade:grade 15 | 16 | preparer: mlebench.competitions.plant-seedlings-classification.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/dogs-vs-cats-redux-kernels-edition/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Given a dataset of images of dogs and cats, predict if an image is a dog or a cat. 4 | 5 | # Metric 6 | 7 | Log loss. 8 | 9 | # Submission Format 10 | 11 | For each image in the test set, you must submit a probability that image is a dog. The file should have a header and be in the following format: 12 | 13 | ``` 14 | id,label 15 | 1,0.5 16 | 2,0.5 17 | 3,0.5 18 | ... 19 | ``` 20 | 21 | # Dataset 22 | 23 | The train folder contains 25,000 images of dogs and cats. Each image in this folder has the label as part of the filename. The test folder contains 12,500 images, named according to a numeric id. -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/whale-categorization-playground/config.yaml: -------------------------------------------------------------------------------- 1 | id: whale-categorization-playground 2 | name: Humpback Whale Identification Challenge 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/whale-categorization-playground/description.md 7 | 8 | dataset: 9 | answers: whale-categorization-playground/prepared/private/test.csv 10 | sample_submission: whale-categorization-playground/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: MAP@5 14 | grade_fn: mlebench.competitions.whale-categorization-playground.grade:grade 15 | 16 | preparer: mlebench.competitions.whale-categorization-playground.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/histopathologic-cancer-detection/config.yaml: -------------------------------------------------------------------------------- 1 | id: histopathologic-cancer-detection 2 | name: Histopathologic Cancer Detection 3 | competition_type: code 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/histopathologic-cancer-detection/description.md 7 | 8 | dataset: 9 | answers: histopathologic-cancer-detection/prepared/private/answers.csv 10 | sample_submission: histopathologic-cancer-detection/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: auc-roc 14 | grade_fn: mlebench.competitions.histopathologic-cancer-detection.grade:grade 15 | 16 | preparer: mlebench.competitions.histopathologic-cancer-detection.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/rsna-miccai-brain-tumor-radiogenomic-classification/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import roc_auc_score 3 | 4 | from mlebench.competitions.utils import prepare_for_auroc_metric 5 | from mlebench.grade_helpers import InvalidSubmissionError 6 | 7 | 8 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 9 | roc_auc_inputs = prepare_for_auroc_metric( 10 | answers=answers, 11 | submission=submission, 12 | id_col="BraTS21ID", 13 | target_col="MGMT_value", 14 | ) 15 | score = roc_auc_score( 16 | y_true=roc_auc_inputs["y_true"], y_score=roc_auc_inputs["y_score"] 17 | ) 18 | return score 19 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/mlsp-2013-birds/config.yaml: -------------------------------------------------------------------------------- 1 | id: mlsp-2013-birds 2 | name: MLSP 2013 Bird Classification Challenge 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 700 8 | - position: 2 9 | value: 600 10 | - position: 3 11 | value: 500 12 | description: mlebench/competitions/mlsp-2013-birds/description.md 13 | 14 | dataset: 15 | answers: mlsp-2013-birds/prepared/private/answers.csv 16 | sample_submission: mlsp-2013-birds/prepared/public/sample_submission.csv 17 | 18 | grader: 19 | name: auc-roc 20 | grade_fn: mlebench.competitions.mlsp-2013-birds.grade:grade 21 | 22 | preparer: mlebench.competitions.mlsp-2013-birds.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/champs-scalar-coupling/checksums.yaml: -------------------------------------------------------------------------------- 1 | private: 2 | answers.csv: d41126a1d909d1e5c33d0a4828f2694d 3 | public: 4 | dipole_moments.csv: e9227539164f9d43b1150121ae727940 5 | magnetic_shielding_tensors.csv: 9a495d3d556092a7bf96f087187ee181 6 | mulliken_charges.csv: 3bdc6ff0f3f451796923d4659eb6766d 7 | potential_energy.csv: b30f3b3b2889e433487c3abdab6b6814 8 | sample_submission.csv: fbb4313c3052857027152c871900a0c4 9 | scalar_coupling_contributions.csv: 233037169b2a8f8ee1519c19c6eb7b6e 10 | structures.csv: 6082e6a941b90bcabb7325f3bee64622 11 | test.csv: 2c8b2fd3060697f3be78489290f40768 12 | train.csv: bc4222feab583532854367a124d93aaa 13 | zip: 2e7bb40d8e2b3488250f15430b8cf947 14 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/dogs-vs-cats-redux-kernels-edition/config.yaml: -------------------------------------------------------------------------------- 1 | id: dogs-vs-cats-redux-kernels-edition 2 | name: "Dogs vs. Cats Redux: Kernels Edition" 3 | competition_type: code 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/dogs-vs-cats-redux-kernels-edition/description.md 7 | 8 | dataset: 9 | answers: dogs-vs-cats-redux-kernels-edition/prepared/private/answers.csv 10 | sample_submission: dogs-vs-cats-redux-kernels-edition/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: log-loss 14 | grade_fn: mlebench.competitions.dogs-vs-cats-redux-kernels-edition.grade:grade 15 | 16 | preparer: mlebench.competitions.dogs-vs-cats-redux-kernels-edition.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/predict-volcanic-eruptions-ingv-oe/config.yaml: -------------------------------------------------------------------------------- 1 | id: predict-volcanic-eruptions-ingv-oe 2 | name: INGV - Volcanic Eruption Prediction 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/predict-volcanic-eruptions-ingv-oe/description.md 7 | 8 | dataset: 9 | answers: predict-volcanic-eruptions-ingv-oe/prepared/private/test.csv 10 | sample_submission: predict-volcanic-eruptions-ingv-oe/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: mean-absolute-error 14 | grade_fn: mlebench.competitions.predict-volcanic-eruptions-ingv-oe.grade:grade 15 | 16 | preparer: mlebench.competitions.predict-volcanic-eruptions-ingv-oe.prepare:prepare -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/new-york-city-taxi-fare-prediction/config.yaml: -------------------------------------------------------------------------------- 1 | id: new-york-city-taxi-fare-prediction 2 | name: New York City Taxi Fare Prediction 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/new-york-city-taxi-fare-prediction/description.md 7 | 8 | dataset: 9 | answers: new-york-city-taxi-fare-prediction/prepared/private/test.csv 10 | sample_submission: new-york-city-taxi-fare-prediction/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: root-mean-squared-error 14 | grade_fn: mlebench.competitions.new-york-city-taxi-fare-prediction.grade:grade 15 | 16 | preparer: mlebench.competitions.new-york-city-taxi-fare-prediction.prepare:prepare 17 | 18 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tabular-playground-series-dec-2021/config.yaml: -------------------------------------------------------------------------------- 1 | id: tabular-playground-series-dec-2021 2 | name: Tabular Playground Series - Dec 2021 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/tabular-playground-series-dec-2021/description.md 7 | 8 | dataset: 9 | answers: tabular-playground-series-dec-2021/prepared/private/test.csv 10 | sample_submission: tabular-playground-series-dec-2021/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: multi-class-classification-accuracy 14 | grade_fn: mlebench.competitions.tabular-playground-series-dec-2021.grade:grade 15 | 16 | preparer: mlebench.competitions.tabular-playground-series-dec-2021.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/AI4Code/config.yaml: -------------------------------------------------------------------------------- 1 | id: AI4Code 2 | name: Google AI4Code - Understand Code in Python Notebooks 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 50000 8 | - position: 2 9 | value: 40000 10 | - position: 3 11 | value: 30000 12 | - position: 4 13 | value: 20000 14 | - position: 5 15 | value: 10000 16 | description: mlebench/competitions/AI4Code/description.md 17 | 18 | dataset: 19 | answers: AI4Code/prepared/private/test_orders.csv 20 | sample_submission: AI4Code/prepared/public/sample_submission.csv 21 | 22 | grader: 23 | name: kendall-tau 24 | grade_fn: mlebench.competitions.AI4Code.grade:grade 25 | 26 | preparer: mlebench.competitions.AI4Code.prepare:prepare 27 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tabular-playground-series-dec-2021/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Predict the class of a given image from a synthetic dataset. 4 | 5 | # Metric 6 | Multi-class classification accuracy. 7 | 8 | # Submission Format 9 | For each `Id` in the test set, you must predict the `Cover_Type` class. The file should contain a header and have the following format: 10 | ``` 11 | Id,Cover_Type 12 | 4000000,2 13 | 4000001,1 14 | 4000001,3 15 | etc. 16 | ``` 17 | 18 | # Dataset 19 | 20 | - train.csv - the training data with the target `Cover_Type` column 21 | - test.csv - the test set; you will be predicting the `Cover_Type` for each row in this file (the target integer class) 22 | - sample_submission.csv - a sample submission file in the correct format -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/alaska2-image-steganalysis/config.yaml: -------------------------------------------------------------------------------- 1 | id: alaska2-image-steganalysis 2 | name: ALASKA2 Image Steganalysis 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 12000 8 | - position: 2 9 | value: 8000 10 | - position: 3 11 | value: 5000 12 | description: mlebench/competitions/alaska2-image-steganalysis/description.md 13 | 14 | dataset: 15 | answers: alaska2-image-steganalysis/prepared/private/test.csv 16 | sample_submission: alaska2-image-steganalysis/prepared/public/sample_submission.csv 17 | 18 | grader: 19 | name: weighted-auroc 20 | grade_fn: mlebench.competitions.alaska2-image-steganalysis.grade:grade 21 | 22 | preparer: mlebench.competitions.alaska2-image-steganalysis.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/movie-review-sentiment-analysis-kernels-only/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Predict the sentiment of phrases. 4 | 5 | # Metric 6 | 7 | Classification accuracy. 8 | 9 | # Submission Format 10 | 11 | For each phrase in the test set, predict a label for the sentiment. Your submission should have a header and look like the following: 12 | 13 | ``` 14 | PhraseId,Sentiment 15 | 156061,2 16 | 156062,2 17 | 156063,2 18 | ... 19 | ``` 20 | 21 | # Dataset 22 | 23 | The dataset is comprised of tab-separated files with phrases. Each phrase has a PhraseId. Each sentence has a SentenceId. 24 | 25 | The sentiment labels are: 26 | 27 | 0 - negative 28 | 29 | 1 - somewhat negative 30 | 31 | 2 - neutral 32 | 33 | 3 - somewhat positive 34 | 35 | 4 - positive -------------------------------------------------------------------------------- /mle-bench/agents/dummy/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x # Print commands and their arguments as they are executed 3 | 4 | eval "$(conda shell.bash hook)" # make conda available to the shell 5 | conda activate agent 6 | 7 | python ${AGENT_DIR}/main.py 8 | 9 | # Check that the agent doesn't have permissions to read private dir 10 | ls /private 11 | # ls: cannot open directory '/private': Permission denied 12 | 13 | # Check that the agent does have permissions to read/write everything in /home 14 | ls /home/data 15 | touch $CODE_DIR/code.py 16 | touch $LOGS_DIR/run.log 17 | touch $AGENT_DIR/agent_was_here.txt 18 | cat /home/instructions.txt 19 | 20 | # Use the environment-provided grading server to validate our submission 21 | bash /home/validate_submission.sh /home/submission/submission.csv 22 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tweet-sentiment-extraction/config.yaml: -------------------------------------------------------------------------------- 1 | id: tweet-sentiment-extraction 2 | name: Tweet Sentiment Extraction 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 7000 8 | - position: 2 9 | value: 5000 10 | - position: 3 11 | value: 3000 12 | description: mlebench/competitions/tweet-sentiment-extraction/description.md 13 | 14 | dataset: 15 | answers: tweet-sentiment-extraction/prepared/private/test.csv 16 | sample_submission: tweet-sentiment-extraction/prepared/public/sample_submission.csv 17 | 18 | grader: 19 | name: jaccard-similarity 20 | grade_fn: mlebench.competitions.tweet-sentiment-extraction.grade:grade 21 | 22 | preparer: mlebench.competitions.tweet-sentiment-extraction.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/facebook-recruiting-iii-keyword-extraction/config.yaml: -------------------------------------------------------------------------------- 1 | id: facebook-recruiting-iii-keyword-extraction 2 | name: Facebook Recruiting III - Keyword Extraction 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: null 6 | description: mlebench/competitions/facebook-recruiting-iii-keyword-extraction/description.md 7 | 8 | dataset: 9 | answers: facebook-recruiting-iii-keyword-extraction/prepared/private/test.csv 10 | sample_submission: facebook-recruiting-iii-keyword-extraction/prepared/public/sample_submission.csv 11 | 12 | grader: 13 | name: micro-f1-score 14 | grade_fn: mlebench.competitions.facebook-recruiting-iii-keyword-extraction.grade:grade 15 | 16 | preparer: mlebench.competitions.facebook-recruiting-iii-keyword-extraction.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/seti-breakthrough-listen/config.yaml: -------------------------------------------------------------------------------- 1 | id: seti-breakthrough-listen 2 | name: SETI Breakthrough Listen - E.T. Signal Search 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 6000 8 | - position: 2 9 | value: 5000 10 | - position: 3 11 | value: 4000 12 | description: mlebench/competitions/seti-breakthrough-listen/description.md 13 | 14 | dataset: 15 | answers: seti-breakthrough-listen/prepared/private/test.csv 16 | sample_submission: seti-breakthrough-listen/prepared/public/sample_submission.csv 17 | 18 | grader: 19 | name: auc-roc 20 | grade_fn: mlebench.competitions.seti-breakthrough-listen.grade:grade 21 | 22 | preparer: mlebench.competitions.seti-breakthrough-listen.prepare:prepare 23 | 24 | 25 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/stanford-covid-vaccine/config.yaml: -------------------------------------------------------------------------------- 1 | id: stanford-covid-vaccine 2 | name: "OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction" 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 12000 8 | - position: 2 9 | value: 8000 10 | - position: 3 11 | value: 5000 12 | description: mlebench/competitions/stanford-covid-vaccine/description.md 13 | 14 | dataset: 15 | answers: stanford-covid-vaccine/prepared/private/test.csv 16 | sample_submission: stanford-covid-vaccine/prepared/public/sample_submission.csv 17 | 18 | grader: 19 | name: multi-class-log-loss 20 | grade_fn: mlebench.competitions.stanford-covid-vaccine.grade:grade 21 | 22 | preparer: mlebench.competitions.stanford-covid-vaccine.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/bms-molecular-translation/config.yaml: -------------------------------------------------------------------------------- 1 | id: bms-molecular-translation 2 | name: Bristol-Myers Squibb - Molecular Translation 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 25000 8 | - position: 2 9 | value: 15000 10 | - position: 3 11 | value: 10000 12 | description: mlebench/competitions/bms-molecular-translation/description.md 13 | 14 | dataset: 15 | answers: bms-molecular-translation/prepared/private/test.csv 16 | sample_submission: bms-molecular-translation/prepared/public/sample_submission.csv 17 | 18 | grader: 19 | name: levenshtein-distance 20 | grade_fn: mlebench.competitions.bms-molecular-translation.grade:grade 21 | 22 | preparer: mlebench.competitions.bms-molecular-translation.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/inaturalist-2019-fgvc6/kernels.txt: -------------------------------------------------------------------------------- 1 | ateplyuk/inat2019-starter-keras-efficientnet 2 | khursani8/fast-ai-ootb-cutout-efficientnet 3 | feichin/inception3-last-years-baseline 4 | hsinwenchang/keras-data-augmentation-visualize 5 | macaodha/basic-inat2019-data-exploration 6 | hsinwenchang/keras-mobilenet-data-augmentation-visualize 7 | jas10022/cnn-hypernetwork 8 | s3chwartz/inaturalist-2019-at-fgvc6 9 | sujoykg/xception-keras 10 | interneuron/previous-benchmark-in-a-kernel-v0-0-0 11 | hedibendaoud/efficientnet-keras 12 | praxitelisk/inaturalist-2019-eda-dl 13 | alainminda/kernel11ab6576ac 14 | lowecoryr/learn-from-other-kernels-fork-from-me 15 | anjalisingh065/species-classification 16 | byhwdy/inat-utils 17 | chrisevans/kernelfea8793d3e 18 | cedriclacrambe/inaturalist-xception-512 19 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/movie-review-sentiment-analysis-kernels-only/config.yaml: -------------------------------------------------------------------------------- 1 | id: movie-review-sentiment-analysis-kernels-only 2 | name: Movie Review Sentiment Analysis (Kernels Only) 3 | competition_type: code 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/movie-review-sentiment-analysis-kernels-only/description.md 7 | 8 | dataset: 9 | answers: movie-review-sentiment-analysis-kernels-only/prepared/private/answers.csv 10 | sample_submission: movie-review-sentiment-analysis-kernels-only/prepared/public/sampleSubmission.csv 11 | 12 | grader: 13 | name: accuracy 14 | grade_fn: mlebench.competitions.movie-review-sentiment-analysis-kernels-only.grade:grade 15 | 16 | preparer: mlebench.competitions.movie-review-sentiment-analysis-kernels-only.prepare:prepare 17 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/smartphone-decimeter-2022/config.yaml: -------------------------------------------------------------------------------- 1 | id: smartphone-decimeter-2022 2 | name: Google Smartphone Decimeter Challenge 2022 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 5000 8 | - position: 2 9 | value: 3000 10 | - position: 3 11 | value: 2000 12 | description: mlebench/competitions/smartphone-decimeter-2022/description.md 13 | 14 | dataset: 15 | answers: smartphone-decimeter-2022/prepared/private/test.csv 16 | sample_submission: smartphone-decimeter-2022/prepared/public/sample_submission.csv 17 | 18 | grader: 19 | name: average-haversine-distance 20 | grade_fn: mlebench.competitions.smartphone-decimeter-2022.grade:grade 21 | 22 | preparer: mlebench.competitions.smartphone-decimeter-2022.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/petfinder-pawpularity-score/config.yaml: -------------------------------------------------------------------------------- 1 | id: petfinder-pawpularity-score 2 | name: PetFinder.my - Pawpularity Contest 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 12000 8 | - position: 2 9 | value: 8000 10 | - position: 3 11 | value: 5000 12 | description: mlebench/competitions/petfinder-pawpularity-score/description.md 13 | 14 | dataset: 15 | answers: petfinder-pawpularity-score/prepared/private/test.csv 16 | sample_submission: petfinder-pawpularity-score/prepared/public/sample_submission.csv 17 | 18 | grader: 19 | name: root-mean-squared-error 20 | grade_fn: mlebench.competitions.petfinder-pawpularity-score.grade:grade 21 | 22 | preparer: mlebench.competitions.petfinder-pawpularity-score.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tabular-playground-series-may-2022/config.yaml: -------------------------------------------------------------------------------- 1 | id: tabular-playground-series-may-2022 2 | name: Tabular Playground Series - May 2022 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: null 6 | description: mlebench/competitions/tabular-playground-series-may-2022/description.md 7 | 8 | dataset: 9 | answers: tabular-playground-series-may-2022/prepared/private/test.csv 10 | gold_submission: tabular-playground-series-may-2022/prepared/private/gold_submission.csv 11 | sample_submission: tabular-playground-series-may-2022/prepared/public/sample_submission.csv 12 | 13 | grader: 14 | name: auc-roc 15 | grade_fn: mlebench.competitions.tabular-playground-series-may-2022.grade:grade 16 | 17 | preparer: mlebench.competitions.tabular-playground-series-may-2022.prepare:prepare 18 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/ventilator-pressure-prediction/config.yaml: -------------------------------------------------------------------------------- 1 | id: ventilator-pressure-prediction 2 | name: "Google Brain - Ventilator Pressure Prediction" 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 2500 8 | - position: 2 9 | value: 2500 10 | - position: 3 11 | value: 2500 12 | description: mlebench/competitions/ventilator-pressure-prediction/description.md 13 | 14 | dataset: 15 | answers: ventilator-pressure-prediction/prepared/private/test.csv 16 | sample_submission: ventilator-pressure-prediction/prepared/public/sample_submission.csv 17 | 18 | grader: 19 | name: dice-hausdorff-combo 20 | grade_fn: mlebench.competitions.ventilator-pressure-prediction.grade:grade 21 | 22 | preparer: mlebench.competitions.ventilator-pressure-prediction.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/google-quest-challenge/config.yaml: -------------------------------------------------------------------------------- 1 | id: google-quest-challenge 2 | name: Google QUEST Q&A Labeling 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 10000 8 | - position: 2 9 | value: 7000 10 | - position: 3 11 | value: 5000 12 | - position: 4 13 | value: 2000 14 | - position: 5 15 | value: 1000 16 | description: mlebench/competitions/google-quest-challenge/description.md 17 | 18 | dataset: 19 | answers: google-quest-challenge/prepared/private/test.csv 20 | sample_submission: google-quest-challenge/prepared/public/sample_submission.csv 21 | 22 | grader: 23 | name: column-wise-spearman 24 | grade_fn: mlebench.competitions.google-quest-challenge.grade:grade 25 | 26 | preparer: mlebench.competitions.google-quest-challenge.prepare:prepare 27 | -------------------------------------------------------------------------------- /mle-bench/experiments/splits/low.txt: -------------------------------------------------------------------------------- 1 | aerial-cactus-identification 2 | aptos2019-blindness-detection 3 | denoising-dirty-documents 4 | detecting-insults-in-social-commentary 5 | dog-breed-identification 6 | dogs-vs-cats-redux-kernels-edition 7 | histopathologic-cancer-detection 8 | jigsaw-toxic-comment-classification-challenge 9 | leaf-classification 10 | mlsp-2013-birds 11 | new-york-city-taxi-fare-prediction 12 | nomad2018-predict-transparent-conductors 13 | plant-pathology-2020-fgvc7 14 | random-acts-of-pizza 15 | ranzcr-clip-catheter-line-classification 16 | siim-isic-melanoma-classification 17 | spooky-author-identification 18 | tabular-playground-series-dec-2021 19 | tabular-playground-series-may-2022 20 | text-normalization-challenge-english-language 21 | text-normalization-challenge-russian-language 22 | the-icml-2013-whale-challenge-right-whale-redux -------------------------------------------------------------------------------- /mle-bench/environment/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Print commands and their arguments as they are executed 4 | set -x 5 | 6 | { 7 | # log into /home/logs 8 | LOGS_DIR=/home/logs 9 | mkdir -p $LOGS_DIR 10 | 11 | # chmod the /home directory such that nonroot users can work on everything within it. We do this at container start 12 | # time so that anything added later in agent-specific Dockerfiles will also receive the correct permissions. 13 | # (this command does `chmod a+rw /home` but with the exception of /home/data, which is a read-only volume) 14 | find /home -path /home/data -prune -o -exec chmod a+rw {} \; 15 | ls -l /home 16 | 17 | # Launch grading server, stays alive throughout container lifetime to service agent requests. 18 | /opt/conda/bin/python /private/grading_server.py 19 | } 2>&1 | tee $LOGS_DIR/entrypoint.log 20 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/lmsys-chatbot-arena/config.yaml: -------------------------------------------------------------------------------- 1 | id: lmsys-chatbot-arena 2 | name: LMSYS - Chatbot Arena Human Preference Predictions 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 25000 8 | - position: 2 9 | value: 20000 10 | - position: 3 11 | value: 20000 12 | - position: 4 13 | value: 20000 14 | - position: 5 15 | value: 15000 16 | description: mlebench/competitions/lmsys-chatbot-arena/description.md 17 | 18 | dataset: 19 | answers: lmsys-chatbot-arena/prepared/private/answers.csv 20 | sample_submission: lmsys-chatbot-arena/prepared/public/sample_submission.csv 21 | 22 | grader: 23 | name: multi-class-log-loss 24 | grade_fn: mlebench.competitions.lmsys-chatbot-arena.grade:grade 25 | 26 | preparer: mlebench.competitions.lmsys-chatbot-arena.prepare:prepare 27 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/champs-scalar-coupling/config.yaml: -------------------------------------------------------------------------------- 1 | id: champs-scalar-coupling 2 | name: Predicting Molecular Properties 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 12500 8 | - position: 2 9 | value: 7500 10 | - position: 3 11 | value: 5000 12 | - position: 4 13 | value: 3000 14 | - position: 5 15 | value: 2000 16 | description: mlebench/competitions/champs-scalar-coupling/description.md 17 | 18 | dataset: 19 | answers: champs-scalar-coupling/prepared/private/answers.csv 20 | sample_submission: champs-scalar-coupling/prepared/public/sample_submission.csv 21 | 22 | grader: 23 | name: log-mean-absolute-error 24 | grade_fn: mlebench.competitions.champs-scalar-coupling.grade:grade 25 | 26 | preparer: mlebench.competitions.champs-scalar-coupling.prepare:prepare 27 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/aptos2019-blindness-detection/config.yaml: -------------------------------------------------------------------------------- 1 | id: aptos2019-blindness-detection 2 | name: APTOS 2019 Blindness Detection 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 25000 8 | - position: 2 9 | value: 12000 10 | - position: 3 11 | value: 8000 12 | - position: 4 13 | value: 5000 14 | description: mlebench/competitions/aptos2019-blindness-detection/description.md 15 | 16 | dataset: 17 | answers: aptos2019-blindness-detection/prepared/private/test.csv 18 | sample_submission: aptos2019-blindness-detection/prepared/public/sample_submission.csv 19 | 20 | grader: 21 | name: quadratic-weighted-kappa 22 | grade_fn: mlebench.competitions.aptos2019-blindness-detection.grade:grade 23 | 24 | preparer: mlebench.competitions.aptos2019-blindness-detection.prepare:prepare 25 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/statoil-iceberg-classifier-challenge/config.yaml: -------------------------------------------------------------------------------- 1 | id: statoil-iceberg-classifier-challenge 2 | name: Statoil/C-CORE Iceberg Classifier Challenge 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 25000 8 | - position: 2 9 | value: 15000 10 | - position: 3 11 | value: 10000 12 | description: mlebench/competitions/statoil-iceberg-classifier-challenge/description.md 13 | 14 | dataset: 15 | answers: statoil-iceberg-classifier-challenge/prepared/private/test.csv 16 | sample_submission: statoil-iceberg-classifier-challenge/prepared/private/sample_submission.csv 17 | 18 | grader: 19 | name: log-loss 20 | grade_fn: mlebench.competitions.statoil-iceberg-classifier-challenge.grade:grade 21 | 22 | preparer: mlebench.competitions.statoil-iceberg-classifier-challenge.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/freesound-audio-tagging-2019/config.yaml: -------------------------------------------------------------------------------- 1 | id: freesound-audio-tagging-2019 2 | name: Freesound Audio Tagging 2019 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 2200 8 | - position: 2 9 | value: 1200 10 | - position: 3 11 | value: 800 12 | - position: Judges' Award Prize 13 | value: 800 14 | description: mlebench/competitions/freesound-audio-tagging-2019/description.md 15 | 16 | dataset: 17 | answers: freesound-audio-tagging-2019/prepared/private/test.csv 18 | sample_submission: freesound-audio-tagging-2019/prepared/public/sample_submission.csv 19 | 20 | grader: 21 | name: label-ranking-average-precision 22 | grade_fn: mlebench.competitions.freesound-audio-tagging-2019.grade:grade 23 | 24 | preparer: mlebench.competitions.freesound-audio-tagging-2019.prepare:prepare 25 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/osic-pulmonary-fibrosis-progression/config.yaml: -------------------------------------------------------------------------------- 1 | id: osic-pulmonary-fibrosis-progression 2 | name: OSIC Pulmonary Fibrosis Progression 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 30000 8 | - position: 2 9 | value: 15000 10 | - position: 3 11 | value: 10000 12 | description: mlebench/competitions/osic-pulmonary-fibrosis-progression/description.md 13 | 14 | dataset: 15 | answers: osic-pulmonary-fibrosis-progression/prepared/private/test.csv 16 | sample_submission: osic-pulmonary-fibrosis-progression/prepared/public/sample_submission.csv 17 | 18 | grader: 19 | name: modified-laplace-log-likelihood 20 | grade_fn: mlebench.competitions.osic-pulmonary-fibrosis-progression.grade:grade 21 | 22 | preparer: mlebench.competitions.osic-pulmonary-fibrosis-progression.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/us-patent-phrase-to-phrase-matching/config.yaml: -------------------------------------------------------------------------------- 1 | id: us-patent-phrase-to-phrase-matching 2 | name: U.S. Patent Phrase to Phrase Matching 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 1200 8 | - position: 2 9 | value: 8000 10 | - position: 3 11 | value: 5000 12 | description: mlebench/competitions/us-patent-phrase-to-phrase-matching/description.md 13 | 14 | dataset: 15 | answers: us-patent-phrase-to-phrase-matching/prepared/private/test.csv 16 | sample_submission: us-patent-phrase-to-phrase-matching/prepared/public/sample_submission.csv 17 | 18 | grader: 19 | name: pearson-correlation-coefficient 20 | grade_fn: mlebench.competitions.us-patent-phrase-to-phrase-matching.grade:grade 21 | 22 | preparer: mlebench.competitions.us-patent-phrase-to-phrase-matching.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/additional_notes.txt: -------------------------------------------------------------------------------- 1 | ADDITIONAL NOTES 2 | ------ 3 | 4 | - **Compute**: You have access to ${HARDWARE} with the appropriate drivers installed. 5 | - **Total Runtime**: You have a maximum of ${TIME_LIMIT} hours to attempt this task. You may iteratively develop your solution within this time frame, but bear in mind that program runtime also counts towards this limit. 6 | - **Use as much of the available time as possible to refine and optimize your submission**: Your goal is to achieve the best possible score, not just to submit your first working solution. Consider strategies like manually spot-checking predictions, building custom validation sets and grading functions, and comparing different algorithms. You may keep working until the time limit expires -- as long as your best submission is stored at the designated location at the end of your run, it will be picked up for grading. 7 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/uw-madison-gi-tract-image-segmentation/config.yaml: -------------------------------------------------------------------------------- 1 | id: uw-madison-gi-tract-image-segmentation 2 | name: UW-Madison GI Tract Image Segmentation 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 1200 8 | - position: 2 9 | value: 8000 10 | - position: 3 11 | value: 5000 12 | description: mlebench/competitions/uw-madison-gi-tract-image-segmentation/description.md 13 | 14 | dataset: 15 | answers: uw-madison-gi-tract-image-segmentation/prepared/private/test.csv 16 | sample_submission: uw-madison-gi-tract-image-segmentation/prepared/public/sample_submission.csv 17 | 18 | grader: 19 | name: dice-hausdorff-combo 20 | grade_fn: mlebench.competitions.uw-madison-gi-tract-image-segmentation.grade:grade 21 | 22 | preparer: mlebench.competitions.uw-madison-gi-tract-image-segmentation.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tabular-playground-series-may-2022/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Given simulated manufacturing control data, predict whether the machine is in state `0` or state `1`. 4 | 5 | # Metric 6 | 7 | Area under the ROC curve. 8 | 9 | # Submission Format 10 | 11 | For each `id` in the test set, you must predict a probability for the `target` variable. The file should contain a header and have the following format: 12 | 13 | ``` 14 | id,target 15 | 900000,0.65 16 | 900001,0.97 17 | 900002,0.02 18 | etc. 19 | ``` 20 | 21 | # Dataset 22 | 23 | - **train.csv** - the training data, which includes normalized continuous data and categorical data 24 | - **test.csv** - the test set; your task is to predict binary `target` variable which represents the state of a manufacturing process 25 | - **sample_submission.csv** - a sample submission file in the correct format 26 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/cdiscount-image-classification-challenge/config.yaml: -------------------------------------------------------------------------------- 1 | id: cdiscount-image-classification-challenge 2 | name: Cdiscount’s Image Classification Challenge 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 20000 8 | - position: 2 9 | value: 10000 10 | - position: 3 11 | value: 5000 12 | description: mlebench/competitions/cdiscount-image-classification-challenge/description.md 13 | 14 | dataset: 15 | answers: cdiscount-image-classification-challenge/prepared/private/answers.csv 16 | sample_submission: cdiscount-image-classification-challenge/prepared/public/sample_submission.csv 17 | 18 | grader: 19 | name: accuracy 20 | grade_fn: mlebench.competitions.cdiscount-image-classification-challenge.grade:grade 21 | 22 | preparer: mlebench.competitions.cdiscount-image-classification-challenge.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/dog-breed-identification/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Given a dataset of images of dogs, predict the breed of each image. 4 | 5 | # Metric 6 | 7 | Multi Class Log Loss. 8 | 9 | # Submission Format 10 | 11 | For each image in the test set, you must predict a probability for each of the different breeds. The file should contain a header and have the following format: 12 | ``` 13 | id,affenpinscher,afghan_hound,..,yorkshire_terrier 14 | 000621fb3cbb32d8935728e48679680e,0.0083,0.0,...,0.0083 15 | etc. 16 | ``` 17 | 18 | # Dataset Description 19 | 20 | - `train.zip` - the training set, you are provided the breed for these dogs 21 | - `test.zip` - the test set, you must predict the probability of each breed for each image 22 | - `sample_submission.csv` - a sample submission file in the correct format 23 | - `labels.csv` - the breeds for the images in the train set -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/the-icml-2013-whale-challenge-right-whale-redux/config.yaml: -------------------------------------------------------------------------------- 1 | id: the-icml-2013-whale-challenge-right-whale-redux 2 | name: The ICML 2013 Whale Challenge - Right Whale Redux 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 350 8 | - position: 2 9 | value: 150 10 | description: mlebench/competitions/the-icml-2013-whale-challenge-right-whale-redux/description.md 11 | 12 | dataset: 13 | answers: the-icml-2013-whale-challenge-right-whale-redux/prepared/private/test.csv 14 | sample_submission: the-icml-2013-whale-challenge-right-whale-redux/prepared/public/sampleSubmission.csv 15 | 16 | grader: 17 | name: auc-roc 18 | grade_fn: mlebench.competitions.the-icml-2013-whale-challenge-right-whale-redux.grade:grade 19 | 20 | preparer: mlebench.competitions.the-icml-2013-whale-challenge-right-whale-redux.prepare:prepare 21 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/kuzushiji-recognition/config.yaml: -------------------------------------------------------------------------------- 1 | id: kuzushiji-recognition 2 | name: Kuzushiji Recognition 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: 6 | - position: 1 7 | value: 3000 8 | - position: 2 9 | value: 3000 10 | - position: 3 11 | value: 3000 12 | - position: 4 13 | value: 3000 14 | - position: 5 15 | value: 3000 16 | description: mlebench/competitions/kuzushiji-recognition/description.md 17 | 18 | dataset: 19 | answers: kuzushiji-recognition/prepared/private/test.csv 20 | gold_submission: kuzushiji-recognition/prepared/private/gold_submission.csv 21 | sample_submission: kuzushiji-recognition/prepared/public/sample_submission.csv 22 | 23 | grader: 24 | name: f1-score 25 | grade_fn: mlebench.competitions.kuzushiji-recognition.grade:grade 26 | 27 | preparer: mlebench.competitions.kuzushiji-recognition.prepare:prepare 28 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/nomad2018-predict-transparent-conductors/config.yaml: -------------------------------------------------------------------------------- 1 | id: nomad2018-predict-transparent-conductors 2 | name: Nomad2018 Predicting Transparent Conductors 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: 6 | - position: 1 7 | value: 2496 8 | - position: 2 9 | value: 1872 10 | - position: 3 11 | value: 1248 12 | description: mlebench/competitions/nomad2018-predict-transparent-conductors/description.md 13 | 14 | dataset: 15 | answers: nomad2018-predict-transparent-conductors/prepared/private/test.csv 16 | sample_submission: nomad2018-predict-transparent-conductors/prepared/public/sample_submission.csv 17 | 18 | grader: 19 | name: mean-column-wise-rmsle 20 | grade_fn: mlebench.competitions.nomad2018-predict-transparent-conductors.grade:grade 21 | 22 | preparer: mlebench.competitions.nomad2018-predict-transparent-conductors.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/nfl-player-contact-detection/config.yaml: -------------------------------------------------------------------------------- 1 | id: nfl-player-contact-detection 2 | name: 1st and Future - Player Contact Detection 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 50000 8 | - position: 2 9 | value: 25000 10 | - position: 3 11 | value: 13000 12 | - position: 4 13 | value: 7000 14 | - position: 5 15 | value: 5000 16 | description: mlebench/competitions/nfl-player-contact-detection/description.md 17 | 18 | dataset: 19 | answers: nfl-player-contact-detection/prepared/private/test.csv 20 | sample_submission: nfl-player-contact-detection/prepared/public/sample_submission.csv 21 | 22 | grader: 23 | name: matthews-correlation-coefficient 24 | grade_fn: mlebench.competitions.nfl-player-contact-detection.grade:grade 25 | 26 | preparer: mlebench.competitions.nfl-player-contact-detection.prepare:prepare 27 | -------------------------------------------------------------------------------- /mle-bench/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM automind-base 2 | 3 | # where to put submission.csv, will be extracted 4 | ARG SUBMISSION_DIR 5 | ENV SUBMISSION_DIR=${SUBMISSION_DIR} 6 | # where to put any logs, will be extracted 7 | ARG LOGS_DIR 8 | ENV LOGS_DIR=${LOGS_DIR} 9 | # where to put any code, will be extracted 10 | ARG CODE_DIR 11 | ENV CODE_DIR=${CODE_DIR} 12 | # where to put any other agent-specific files, will not be necessarily extracted 13 | ARG AGENT_DIR 14 | ENV AGENT_DIR=${AGENT_DIR} 15 | 16 | # [Optional] Set Hugging Face mirror 17 | ENV HF_ENDPOINT=https://hf-mirror.com 18 | ENV TRANSFORMERS_CACHE=/tmp/transformers_cache 19 | ENV HF_HOME=/tmp/huggingface_cache 20 | 21 | # put all the agent files in the expected location 22 | RUN mkdir ${AGENT_DIR}/automind 23 | COPY ./automind ${AGENT_DIR}/automind 24 | COPY ./mle-bench/start.sh ${AGENT_DIR}/start.sh 25 | COPY ./mle-bench/additional_notes.txt ${AGENT_DIR}/additional_notes.txt -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/3d-object-detection-for-autonomous-vehicles/config.yaml: -------------------------------------------------------------------------------- 1 | id: 3d-object-detection-for-autonomous-vehicles 2 | name: Lyft 3D Object Detection for Autonomous Vehicles 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 1200 8 | - position: 2 9 | value: 8000 10 | - position: 3 11 | value: 5000 12 | description: mlebench/competitions/3d-object-detection-for-autonomous-vehicles/description.md 13 | 14 | dataset: 15 | answers: 3d-object-detection-for-autonomous-vehicles/prepared/private/test.csv 16 | sample_submission: 3d-object-detection-for-autonomous-vehicles/prepared/public/sample_submission.csv 17 | 18 | grader: 19 | name: mean-average-precision 20 | grade_fn: mlebench.competitions.3d-object-detection-for-autonomous-vehicles.grade:grade 21 | 22 | preparer: mlebench.competitions.3d-object-detection-for-autonomous-vehicles.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tgs-salt-identification-challenge/config.yaml: -------------------------------------------------------------------------------- 1 | id: tgs-salt-identification-challenge 2 | name: TGS Salt Identification Challenge 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 50000 8 | - position: 2 9 | value: 25000 10 | - position: 3 11 | value: 15000 12 | - position: 4 13 | value: 10000 14 | description: mlebench/competitions/tgs-salt-identification-challenge/description.md 15 | 16 | dataset: 17 | answers: tgs-salt-identification-challenge/prepared/private/test.csv 18 | sample_submission: tgs-salt-identification-challenge/prepared/public/sample_submission.csv 19 | 20 | grader: 21 | name: mean-precision-intersection-over-union-at-different-thresholds 22 | grade_fn: mlebench.competitions.tgs-salt-identification-challenge.grade:grade 23 | 24 | preparer: mlebench.competitions.tgs-salt-identification-challenge.prepare:prepare 25 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/jigsaw-toxic-comment-classification-challenge/config.yaml: -------------------------------------------------------------------------------- 1 | id: jigsaw-toxic-comment-classification-challenge 2 | name: Toxic Comment Classification Challenge 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 18000 8 | - position: 2 9 | value: 12000 10 | - position: 3 11 | value: 5000 12 | description: mlebench/competitions/jigsaw-toxic-comment-classification-challenge/description.md 13 | 14 | dataset: 15 | answers: jigsaw-toxic-comment-classification-challenge/prepared/private/test.csv 16 | sample_submission: jigsaw-toxic-comment-classification-challenge/prepared/public/sample_submission.csv 17 | 18 | grader: 19 | name: column-wise ROC AUC 20 | grade_fn: mlebench.competitions.jigsaw-toxic-comment-classification-challenge.grade:grade 21 | 22 | preparer: mlebench.competitions.jigsaw-toxic-comment-classification-challenge.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/text-normalization-challenge-english-language/config.yaml: -------------------------------------------------------------------------------- 1 | id: text-normalization-challenge-english-language 2 | name: Text Normalization Challenge - English Language 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 12000 8 | - position: 2 9 | value: 8000 10 | - position: 3 11 | value: 5000 12 | description: mlebench/competitions/text-normalization-challenge-english-language/description.md 13 | 14 | dataset: 15 | answers: text-normalization-challenge-english-language/prepared/private/answers.csv 16 | sample_submission: text-normalization-challenge-english-language/prepared/private/sample_submission.csv 17 | 18 | grader: 19 | name: accuracy 20 | grade_fn: mlebench.competitions.text-normalization-challenge-english-language.grade:grade 21 | 22 | preparer: mlebench.competitions.text-normalization-challenge-english-language.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/text-normalization-challenge-russian-language/config.yaml: -------------------------------------------------------------------------------- 1 | id: text-normalization-challenge-russian-language 2 | name: Text Normalization Challenge - Russian Language 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 12000 8 | - position: 2 9 | value: 8000 10 | - position: 3 11 | value: 5000 12 | description: mlebench/competitions/text-normalization-challenge-russian-language/description.md 13 | 14 | dataset: 15 | answers: text-normalization-challenge-russian-language/prepared/private/answers.csv 16 | sample_submission: text-normalization-challenge-russian-language/prepared/private/sample_submission.csv 17 | 18 | grader: 19 | name: accuracy 20 | grade_fn: mlebench.competitions.text-normalization-challenge-russian-language.grade:grade 21 | 22 | preparer: mlebench.competitions.text-normalization-challenge-russian-language.prepare:prepare 23 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tensorflow-speech-recognition-challenge/config.yaml: -------------------------------------------------------------------------------- 1 | id: tensorflow-speech-recognition-challenge 2 | name: TensorFlow Speech Recognition Challenge 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 8000 8 | - position: 2 9 | value: 6000 10 | - position: 3 11 | value: 3000 12 | - position: Special TensorFlow Prize 13 | value: 8000 14 | description: mlebench/competitions/tensorflow-speech-recognition-challenge/description.md 15 | 16 | dataset: 17 | answers: tensorflow-speech-recognition-challenge/prepared/private/test.csv 18 | sample_submission: tensorflow-speech-recognition-challenge/prepared/public/sample_submission.csv 19 | 20 | grader: 21 | name: accuracy 22 | grade_fn: mlebench.competitions.tensorflow-speech-recognition-challenge.grade:grade 23 | 24 | preparer: mlebench.competitions.tensorflow-speech-recognition-challenge.prepare:prepare 25 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/mlsp-2013-birds/grade.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.metrics import roc_auc_score 3 | 4 | from mlebench.competitions.utils import prepare_for_auroc_metric 5 | 6 | 7 | def grade(submission: pd.DataFrame, answers: pd.DataFrame) -> float: 8 | if ( 9 | "Id" not in submission.columns 10 | and "rec_id" in submission.columns 11 | and "species" in submission.columns 12 | ): 13 | submission["Id"] = submission["rec_id"] * 100 + submission["species"] 14 | if "Probability" not in submission.columns and "probability" in submission.columns: 15 | submission = submission.rename(columns={"probability": "Probability"}) 16 | roc_auc_inputs = prepare_for_auroc_metric( 17 | submission=submission, answers=answers, id_col="Id", target_col="Probability" 18 | ) 19 | return roc_auc_score( 20 | y_true=roc_auc_inputs["y_true"], y_score=roc_auc_inputs["y_score"] 21 | ) 22 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/cassava-leaf-disease-classification/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Classify each cassava image into four disease categories or a fifth category indicating a healthy leaf. 4 | 5 | # Metric 6 | 7 | Categorization accuracy. 8 | 9 | # Submission Format 10 | 11 | ``` 12 | image_id,label 13 | 1000471002.jpg,4 14 | 1000840542.jpg,4 15 | etc. 16 | ``` 17 | 18 | # Dataset 19 | 20 | **[train/test]_images** the image files. 21 | 22 | **train.csv** 23 | 24 | - `image_id` the image file name. 25 | 26 | - `label` the ID code for the disease. 27 | 28 | **sample_submission.csv** A properly formatted sample submission, given the disclosed test set content. 29 | 30 | - `image_id` the image file name. 31 | 32 | - `label` the predicted ID code for the disease. 33 | 34 | **[train/test]_tfrecords** the image files in tfrecord format. 35 | 36 | **label_num_to_disease_map.json** The mapping between each disease code and the real disease name. 37 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/denoising-dirty-documents/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Given a dataset of images of scanned text that is noisy, remove the noise. 4 | 5 | # Metric 6 | 7 | Root mean squared error between the cleaned pixel intensities and the actual grayscale pixel intensities. 8 | 9 | # Submission Format 10 | 11 | Form the submission file by melting each images into a set of pixels, assigning each pixel an id of image_row_col (e.g. 1_2_1 is image 1, row 2, column 1). Intensity values range from 0 (black) to 1 (white). The file should contain a header and have the following format: 12 | 13 | ``` 14 | id,value 15 | 1_1_1,1 16 | 1_2_1,1 17 | 1_3_1,1 18 | etc. 19 | ``` 20 | 21 | # Dataset 22 | 23 | You are provided two sets of images, train and test. These images contain various styles of text, to which synthetic noise has been added to simulate real-world, messy artifacts. The training set includes the test without the noise (train_cleaned). 24 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/playground-series-s3e18/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Predict values for synthetic data. 4 | 5 | ## Description 6 | 7 | # Metric 8 | 9 | Area under the ROC curve for each target, with the final score being the average of the individual AUCs of each predicted column. 10 | 11 | # Submission Format 12 | 13 | For each `id` in the test set, you must predict the value for the targets `EC1` and `EC2`. The file should contain a header and have the following format: 14 | 15 | ``` 16 | id,EC1,EC2 17 | 14838,0.22,0.71 18 | 14839,0.78,0.43 19 | 14840,0.53,0.11 20 | etc. 21 | ``` 22 | 23 | # Dataset 24 | 25 | - **train.csv** - the training dataset; `[EC1 - EC6]` are the (binary) targets, although you are only asked to predict `EC1` and `EC2`. 26 | - **test.csv** - the test dataset; your objective is to predict the probability of the two targets `EC1` and `EC2` 27 | - **sample_submission.csv** - a sample submission file in the correct format -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/siim-isic-melanoma-classification/config.yaml: -------------------------------------------------------------------------------- 1 | id: siim-isic-melanoma-classification 2 | name: SIIM-ISIC Melanoma Classification 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 10000 8 | - position: 2 9 | value: 6000 10 | - position: 3 11 | value: 4000 12 | - position: With Context Special Prize 13 | value: 5000 14 | - position: Without Context Special Prize 15 | value: 5000 16 | description: mlebench/competitions/siim-isic-melanoma-classification/description.md 17 | 18 | dataset: 19 | answers: siim-isic-melanoma-classification/prepared/private/test.csv 20 | sample_submission: siim-isic-melanoma-classification/prepared/public/sample_submission.csv 21 | 22 | grader: 23 | name: auc-roc 24 | grade_fn: mlebench.competitions.siim-isic-melanoma-classification.grade:grade 25 | 26 | preparer: mlebench.competitions.siim-isic-melanoma-classification.prepare:prepare 27 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/chaii-hindi-and-tamil-question-answering/config.yaml: -------------------------------------------------------------------------------- 1 | id: chaii-hindi-and-tamil-question-answering 2 | name: Hindi and Tamil Question Answering 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 2000 8 | - position: 2 9 | value: 2000 10 | - position: 3 11 | value: 2000 12 | - position: 4 13 | value: 2000 14 | - position: 5 15 | value: 2000 16 | description: mlebench/competitions/chaii-hindi-and-tamil-question-answering/description.md 17 | 18 | dataset: 19 | answers: chaii-hindi-and-tamil-question-answering/prepared/private/test.csv 20 | sample_submission: chaii-hindi-and-tamil-question-answering/prepared/public/sample_submission.csv 21 | 22 | grader: 23 | name: word-level-jaccard-score 24 | grade_fn: mlebench.competitions.chaii-hindi-and-tamil-question-answering.grade:grade 25 | 26 | preparer: mlebench.competitions.chaii-hindi-and-tamil-question-answering.prepare:prepare 27 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/the-icml-2013-whale-challenge-right-whale-redux/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Given a recording, predict if it contains a right whale call (1) or noise (0). 4 | 5 | # Metric 6 | 7 | Area under the ROC curve. 8 | 9 | # Submission Format 10 | 11 | For every recording in the test set, submission files should contain two columns: clip and probability. The submission file should have a header. 12 | 13 | ``` 14 | clip,probability 15 | 20090404_000000_012s0ms_Test0.aif,0 16 | 20090404_000000_042s6ms_Test1.aif,0 17 | 20090404_000000_064s8ms_Test2.aif,0 18 | 20090404_000000_082s3ms_Test3.aif,0 19 | 20090404_000000_095s0ms_Test4.aif,0 20 | ``` 21 | 22 | # Dataset 23 | 24 | - **train2.zip** - all of the training data as .aif clips. If the file ends in "_1.aif" it was labeled a whale call, if it ends in "_0.aif", it was labeled noise. 25 | - **test2.zip** - all of the testing data as .aif clips. You should predict a probability for each of these files. -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/detecting-insults-in-social-commentary/config.yaml: -------------------------------------------------------------------------------- 1 | id: detecting-insults-in-social-commentary 2 | name: Detecting Insults in Social Commentary 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 7000 8 | - position: 2 9 | value: 2500 10 | - position: Visualization Prospect 11 | value: 500 12 | description: mlebench/competitions/detecting-insults-in-social-commentary/description.md 13 | 14 | dataset: 15 | answers: detecting-insults-in-social-commentary/prepared/private/test.csv 16 | gold_submission: detecting-insults-in-social-commentary/prepared/private/gold_submission.csv 17 | sample_submission: detecting-insults-in-social-commentary/prepared/public/sample_submission_null.csv 18 | 19 | grader: 20 | name: auc-roc 21 | grade_fn: mlebench.competitions.detecting-insults-in-social-commentary.grade:grade 22 | 23 | preparer: mlebench.competitions.detecting-insults-in-social-commentary.prepare:prepare 24 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/herbarium-2020-fgvc7/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Build a model to classify the species of plants from images. 4 | 5 | # Metric 6 | 7 | Macro F1 score. A separate F1 score is calculated for each `species` value and then averaged. 8 | 9 | # Submission Format 10 | 11 | For each image `Id`, you should predict the corresponding image label ("category_id") in the `Predicted` column. The submission file should have the following format: 12 | 13 | ``` 14 | Id,Predicted 15 | 0,0 16 | 1,27 17 | 2,42 18 | ... 19 | ``` 20 | 21 | # Dataset 22 | 23 | This dataset uses the [COCO dataset format](http://cocodataset.org/#format-data) with additional annotation fields. In addition to the species category labels, we also provide region and supercategory information. 24 | 25 | The training set images are organized in subfolders `train///.jpg`. 26 | 27 | The test set images are organized in subfolders `test//.jpg`. -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/cassava-leaf-disease-classification/config.yaml: -------------------------------------------------------------------------------- 1 | id: cassava-leaf-disease-classification 2 | name: Cassava Leaf Disease Classification 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 8000 8 | - position: 2 9 | value: 4000 10 | - position: 3 11 | value: 3000 12 | - position: TPU Star 1 13 | value: 1000 14 | - position: TPU Star 2 15 | value: 1000 16 | - position: TPU Star 3 17 | value: 1000 18 | description: mlebench/competitions/cassava-leaf-disease-classification/description.md 19 | 20 | dataset: 21 | answers: cassava-leaf-disease-classification/prepared/private/test.csv 22 | sample_submission: cassava-leaf-disease-classification/prepared/public/sample_submission.csv 23 | 24 | grader: 25 | name: accuracy 26 | grade_fn: mlebench.competitions.cassava-leaf-disease-classification.grade:grade 27 | 28 | preparer: mlebench.competitions.cassava-leaf-disease-classification.prepare:prepare 29 | -------------------------------------------------------------------------------- /mle-bench/extras/plagiarism_detector/run.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | from pathlib import Path 3 | 4 | from analyze import run_plagiarism_detector 5 | 6 | parser = ArgumentParser( 7 | description="Script for analyzing submissions and detecting plagiarism." 8 | ) 9 | 10 | parser.add_argument( 11 | "--submission", 12 | help="Path to the submission JSONL file", 13 | type=str, 14 | required=True, 15 | ) 16 | parser.add_argument( 17 | "--output-dir", 18 | help="Path to the output directory", 19 | type=str, 20 | required=True, 21 | ) 22 | parser.add_argument( 23 | "--timeout", 24 | help="Timeout for each submission processing in seconds", 25 | type=int, 26 | default=300, 27 | required=False, 28 | ) 29 | 30 | args = parser.parse_args() 31 | 32 | 33 | if __name__ == "__main__": 34 | run_plagiarism_detector( 35 | submission=Path(args.submission), 36 | output_dir=Path(args.output_dir), 37 | timeout=args.timeout, 38 | ) 39 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/multi-modal-gesture-recognition/config.yaml: -------------------------------------------------------------------------------- 1 | id: multi-modal-gesture-recognition 2 | name: Multi-modal Gesture Recognition 3 | competition_type: simple 4 | awards_medals: false 5 | prizes: 6 | - position: Quantitative 1 7 | value: 1500 8 | - position: Quantitative 2 9 | value: 1000 10 | - position: Quantitative 3 11 | value: 500 12 | - position: Qualitative 1 13 | value: 1500 14 | - position: Qualitative 2 15 | value: 1000 16 | - position: Qualitative 3 17 | value: 500 18 | description: mlebench/competitions/multi-modal-gesture-recognition/description.md 19 | 20 | dataset: 21 | answers: multi-modal-gesture-recognition/prepared/private/test.csv 22 | sample_submission: multi-modal-gesture-recognition/prepared/public/randomPredictions.csv 23 | 24 | grader: 25 | name: levenhstein-distance 26 | grade_fn: mlebench.competitions.multi-modal-gesture-recognition.grade:grade 27 | 28 | preparer: mlebench.competitions.multi-modal-gesture-recognition.prepare:prepare 29 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/histopathologic-cancer-detection/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Given a dataset of images from digital pathology scans, predict if the center 32x32px region of a patch contains at least one pixel of tumor tissue. Tumor tissue in the outer region of the patch does not influence the label. 4 | 5 | # Metric 6 | 7 | Area under the ROC curve. 8 | 9 | # Submission Format 10 | 11 | For each `id` in the test set, you must predict a probability that center 32x32px region of a patch contains at least one pixel of tumor tissue. The file should contain a header and have the following format: 12 | 13 | ``` 14 | id,label 15 | 0b2ea2a822ad23fdb1b5dd26653da899fbd2c0d5,0 16 | 95596b92e5066c5c52466c90b69ff089b39f2737,0 17 | 248e6738860e2ebcf6258cdc1f32f299e0c76914,0 18 | etc. 19 | ``` 20 | 21 | # Dataset 22 | 23 | Files are named with an image `id`. The `train_labels.csv` file provides the ground truth for the images in the `train` folder. You are predicting the labels for the images in the `test` folder. -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/spaceship-titanic/prepare.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from sklearn.model_selection import train_test_split 4 | 5 | from mlebench.utils import read_csv 6 | 7 | 8 | def prepare(raw: Path, public: Path, private: Path): 9 | # Create train and test splits from train set 10 | old_train = read_csv(raw / "train.csv") 11 | new_train, new_test = train_test_split(old_train, test_size=0.1, random_state=0) 12 | 13 | # Create random example submission 14 | example_submission = new_test[["PassengerId", "Transported"]].copy() 15 | example_submission["Transported"] = False 16 | example_submission.to_csv(public / "sample_submission.csv", index=False) 17 | 18 | # Create private files 19 | new_test.to_csv(private / "test.csv", index=False) 20 | 21 | # Create public files visible to agents 22 | new_train.to_csv(public / "train.csv", index=False) 23 | new_test.drop("Transported", axis="columns").to_csv( 24 | public / "test.csv", index=False 25 | ) 26 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/ml2021spring-hw2/description.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | ## Description 4 | 5 | ### Task Description 6 | 7 | Framewise phoneme classification of TIMIT dataset 8 | 9 | - 1229932 training samples 10 | - 451552 testing samples 11 | - Evaluation metric: catagorization accuracy 12 | 13 | ### Deadlines 14 | 15 | - **Kaggle: 2021/04/02 23:59 (UTC+8)** 16 | - **NTU COOL: 2021/04/04 23:59 (UTC+8)** 17 | 18 | ### Notes 19 | 20 | - Team Name: `_` 21 | - If there are any questions, you may discuss with us on NTU COOL or email ( [ntu-ml-2021spring-ta@googlegroups.com](mailto:ntu-ml-2021spring-ta@googlegroups.com) with the title begining with [hw2] ) 22 | 23 | ## Citation 24 | 25 | yenmeng. (2021). ML2021Spring-hw2. Kaggle. https://kaggle.com/competitions/ml2021spring-hw2 26 | 27 | # Data 28 | 29 | ## Dataset Description 30 | 31 | complete data : `timit_11/` 32 | 33 | - training data: `training_11.npy` 34 | - training label: `trainin_label_11.npy` 35 | - testing data: `testing_label_11.npy` -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/rsna-breast-cancer-detection/config.yaml: -------------------------------------------------------------------------------- 1 | id: rsna-breast-cancer-detection 2 | name: RSNA Screening Mammography Breast Cancer Detection 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 10000 8 | - position: 2 9 | value: 8000 10 | - position: 3 11 | value: 7000 12 | - position: 4 13 | value: 5000 14 | - position: 5 15 | value: 5000 16 | - position: 6 17 | value: 5000 18 | - position: 7 19 | value: 5000 20 | - position: 8 21 | value: 5000 22 | description: mlebench/competitions/rsna-breast-cancer-detection/description.md 23 | 24 | dataset: 25 | answers: rsna-breast-cancer-detection/prepared/private/answers.csv 26 | sample_submission: rsna-breast-cancer-detection/prepared/public/sample_submission.csv 27 | 28 | grader: 29 | name: probabilistic-f1-score 30 | grade_fn: mlebench.competitions.rsna-breast-cancer-detection.grade:grade 31 | 32 | preparer: mlebench.competitions.rsna-breast-cancer-detection.prepare:prepare 33 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/h-and-m-personalized-fashion-recommendations/config.yaml: -------------------------------------------------------------------------------- 1 | id: h-and-m-personalized-fashion-recommendations 2 | name: H&M Personalized Fashion Recommendations 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 15000 8 | - position: 2 9 | value: 10000 10 | - position: 3 11 | value: 8000 12 | - position: 4 13 | value: 7000 14 | - position: 5 15 | value: 5000 16 | - position: 6 17 | value: 5000 18 | description: mlebench/competitions/h-and-m-personalized-fashion-recommendations/description.md 19 | 20 | dataset: 21 | answers: h-and-m-personalized-fashion-recommendations/prepared/private/answers.csv 22 | sample_submission: h-and-m-personalized-fashion-recommendations/prepared/public/sample_submission.csv 23 | 24 | grader: 25 | name: MAP@12 26 | grade_fn: mlebench.competitions.h-and-m-personalized-fashion-recommendations.grade:grade 27 | 28 | preparer: mlebench.competitions.h-and-m-personalized-fashion-recommendations.prepare:prepare 29 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/iwildcam-2020-fgvc7/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Classify species of images of animals from camera traps. 4 | 5 | # Metric 6 | 7 | Categorization accuracy. 8 | 9 | # Submission Format 10 | 11 | ``` 12 | Id,Predicted 13 | 58857ccf-23d2-11e8-a6a3-ec086b02610b,1 14 | 591e4006-23d2-11e8-a6a3-ec086b02610b,5 15 | ``` 16 | 17 | The `Id` column corresponds to the test image id. The `Category` is an integer value that indicates the class of the animal, or `0` to represent the absence of an animal. 18 | 19 | # Dataset 20 | 21 | Supplementary training data can be downloaded from https://github.com/visipedia/iwildcam_comp. 22 | 23 | ## Camera Trap Animal Detection Model 24 | 25 | We are also providing a general animal detection model. 26 | 27 | The model is a TensorFlow Faster-RCNN model with Inception-Resnet-v2 backbone and atrous convolution. 28 | 29 | The model and sample code for running the detector over a folder of images can be found [here](https://github.com/microsoft/CameraTraps/blob/master/megadetector.md). -------------------------------------------------------------------------------- /mle-bench/agents/dummy/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mlebench-env 2 | 3 | # where to put submission.csv, will be extracted 4 | ARG SUBMISSION_DIR 5 | ENV SUBMISSION_DIR=${SUBMISSION_DIR} 6 | # where to put any logs, will be extracted 7 | ARG LOGS_DIR 8 | ENV LOGS_DIR=${LOGS_DIR} 9 | # where to put any code, will be extracted 10 | ARG CODE_DIR 11 | ENV CODE_DIR=${CODE_DIR} 12 | # where to put any other agent-specific files, will not be necessarily extracted 13 | ARG AGENT_DIR 14 | ENV AGENT_DIR=${AGENT_DIR} 15 | 16 | RUN mkdir -p ${LOGS_DIR} ${CODE_DIR} ${AGENT_DIR} 17 | 18 | ARG CONDA_ENV_NAME=agent 19 | ARG REQUIREMENTS=${AGENT_DIR}/requirements.txt 20 | 21 | # copy just the requirements file, so that we can cache conda separately from the agent files 22 | COPY requirements.txt ${AGENT_DIR}/requirements.txt 23 | 24 | # create conda environment and install the requirements to it 25 | RUN conda run -n ${CONDA_ENV_NAME} pip install -r ${AGENT_DIR}/requirements.txt && \ 26 | conda clean -afy 27 | 28 | # put all the agent files in the expected location 29 | COPY . ${AGENT_DIR} -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/paddy-disease-classification/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Develop a model to classify paddy leaf images into one of the nine disease categories or normal leaf. 4 | 5 | # Metric 6 | 7 | Categorization accuracy. 8 | 9 | # Submission Format 10 | 11 | ``` 12 | image_id,label 13 | 200001.jpg,normal 14 | 200002.jpg,blast 15 | etc. 16 | ``` 17 | 18 | # Dataset 19 | 20 | **train.csv** - The training set 21 | 22 | - `image_id` - Unique image identifier corresponds to image file names (.jpg) found in the train_images directory. 23 | - `label` - Type of paddy disease, also the target class. There are ten categories, including the normal leaf. 24 | - `variety` - The name of the paddy variety. 25 | - `age` - Age of the paddy in days. 26 | 27 | **sample_submission.csv** - Sample submission file. 28 | 29 | **train_images** - Training images stored under different sub-directories corresponding to ten target classes. Filename corresponds to the `image_id` column of `train.csv`. 30 | 31 | **test_images** - Test set images. 32 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/siim-covid19-detection/config.yaml: -------------------------------------------------------------------------------- 1 | id: siim-covid19-detection 2 | name: SIIM-FISABIO-RSNA COVID-19 Detection 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 30000 8 | - position: 2 9 | value: 20000 10 | - position: 3 11 | value: 10000 12 | - position: 4 13 | value: 8000 14 | - position: 5 15 | value: 7000 16 | - position: 6 17 | value: 5000 18 | - position: 7 19 | value: 5000 20 | - position: 8 21 | value: 5000 22 | - position: 9 23 | value: 5000 24 | - position: 10 25 | value: 5000 26 | description: mlebench/competitions/siim-covid19-detection/description.md 27 | 28 | dataset: 29 | answers: siim-covid19-detection/prepared/private/test.csv 30 | sample_submission: siim-covid19-detection/prepared/public/sample_submission.csv 31 | 32 | grader: 33 | name: mean-average-precision 34 | grade_fn: mlebench.competitions.siim-covid19-detection.grade:grade 35 | 36 | preparer: mlebench.competitions.siim-covid19-detection.prepare:prepare 37 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/ranzcr-clip-catheter-line-classification/config.yaml: -------------------------------------------------------------------------------- 1 | id: ranzcr-clip-catheter-line-classification 2 | name: RANZCR CLiP - Catheter and Line Position Challenge 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 16000 8 | - position: 2 9 | value: 12000 10 | - position: 3 11 | value: 6000 12 | - position: 4 13 | value: 4000 14 | - position: 5 15 | value: 4000 16 | - position: 6 17 | value: 4000 18 | - position: 7 19 | value: 4000 20 | 21 | description: mlebench/competitions/ranzcr-clip-catheter-line-classification/description.md 22 | 23 | dataset: 24 | answers: ranzcr-clip-catheter-line-classification/prepared/private/test.csv 25 | sample_submission: ranzcr-clip-catheter-line-classification/prepared/public/sample_submission.csv 26 | 27 | grader: 28 | name: auc-roc 29 | grade_fn: mlebench.competitions.ranzcr-clip-catheter-line-classification.grade:grade 30 | 31 | preparer: mlebench.competitions.ranzcr-clip-catheter-line-classification.prepare:prepare 32 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/plant-seedlings-classification/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Classify plant seedlings into their respective species. 4 | 5 | # Metric 6 | 7 | Micro-averaged F1-score. 8 | 9 | # Submission Format 10 | 11 | For each `file` in the test set, you must predict a probability for the `species` variable. The file should contain a header and have the following format: 12 | 13 | ``` 14 | file,species 15 | 0021e90e4.png,Maize 16 | 003d61042.png,Sugar beet 17 | 007b3da8b.png,Common wheat 18 | etc. 19 | ``` 20 | 21 | # Dataset 22 | 23 | The list of species is as follows: 24 | 25 | ``` 26 | Black-grass 27 | Charlock 28 | Cleavers 29 | Common Chickweed 30 | Common wheat 31 | Fat Hen 32 | Loose Silky-bent 33 | Maize 34 | Scentless Mayweed 35 | Shepherds Purse 36 | Small-flowered Cranesbill 37 | Sugar beet 38 | ``` 39 | 40 | - **train.csv** - the training set, with plant species organized by folder 41 | - **test.csv** - the test set, you need to predict the species of each image 42 | - **sample_submission.csv** - a sample submission file in the correct format -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/hms-harmful-brain-activity-classification/config.yaml: -------------------------------------------------------------------------------- 1 | id: hms-harmful-brain-activity-classification 2 | name: HMS - Harmful Brain Activity Classification 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 20000 8 | - position: 2 9 | value: 12000 10 | - position: 3 11 | value: 7000 12 | - position: 4 13 | value: 6000 14 | - position: 5 15 | value: 5000 16 | description: mlebench/competitions/hms-harmful-brain-activity-classification/description.md 17 | 18 | dataset: 19 | answers: hms-harmful-brain-activity-classification/prepared/private/test.csv 20 | gold_submission: hms-harmful-brain-activity-classification/prepared/private/gold_submission.csv 21 | sample_submission: hms-harmful-brain-activity-classification/prepared/public/sample_submission.csv 22 | 23 | grader: 24 | name: KL-divergence 25 | grade_fn: mlebench.competitions.hms-harmful-brain-activity-classification.grade:grade 26 | 27 | preparer: mlebench.competitions.hms-harmful-brain-activity-classification.prepare:prepare 28 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/hubmap-kidney-segmentation/config.yaml: -------------------------------------------------------------------------------- 1 | id: hubmap-kidney-segmentation 2 | name: HuBMAP - Hacking the Kidney 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: Accuracy Prize 1 7 | value: 18000 8 | - position: Accuracy Prize 2 9 | value: 10000 10 | - position: Accuracy Prize 3 11 | value: 4000 12 | - position: Scientific Prize 13 | value: 15000 14 | - position: Innovation Prize 15 | value: 10000 16 | - position: Diversity Prize 17 | value: 3000 18 | description: mlebench/competitions/hubmap-kidney-segmentation/description.md 19 | 20 | dataset: 21 | answers: hubmap-kidney-segmentation/prepared/private/test.csv 22 | gold_submission: hubmap-kidney-segmentation/prepared/private/gold_submission.csv 23 | sample_submission: hubmap-kidney-segmentation/prepared/public/sample_submission.csv 24 | 25 | grader: 26 | name: dice-coefficient 27 | grade_fn: mlebench.competitions.hubmap-kidney-segmentation.grade:grade 28 | 29 | preparer: mlebench.competitions.hubmap-kidney-segmentation.prepare:prepare 30 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/aerial-cactus-identification/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Create a classifier to predict whether an image contains a cactus. 4 | 5 | # Metric 6 | 7 | Area under the ROC curve. 8 | 9 | # Submission Format 10 | 11 | For each ID in the test set, you must predict a probability for the `has_cactus` variable. The file should contain a header and have the following format: 12 | 13 | ``` 14 | id,has_cactus 15 | 000940378805c44108d287872b2f04ce.jpg,0.5 16 | 0017242f54ececa4512b4d7937d1e21e.jpg,0.5 17 | 001ee6d8564003107853118ab87df407.jpg,0.5 18 | etc. 19 | ``` 20 | 21 | # Dataset 22 | 23 | This dataset contains a large number of 32 x 32 thumbnail images containing aerial photos of a cactus. The file name of an image corresponds to its `id`. 24 | 25 | - **train/** - the training set images 26 | - **test/** - the test set images (you must predict the labels of these) 27 | - **train.csv** - the training set labels, indicates whether the image has a cactus (`has_cactus = 1`) 28 | - **sample_submission.csv** - a sample submission file in the correct format 29 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tensorflow2-question-answering/config.yaml: -------------------------------------------------------------------------------- 1 | id: tensorflow2-question-answering 2 | name: TensorFlow 2.0 Question Answering 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 12000 8 | - position: 2 9 | value: 8000 10 | - position: 3 11 | value: 5000 12 | - position: TensorFlow 2.0 Prize 1 13 | value: 12000 14 | - position: TensorFlow 2.0 Prize 2 15 | value: 8000 16 | - position: TensorFlow 2.0 Prize 3 17 | value: 5000 18 | description: mlebench/competitions/tensorflow2-question-answering/description.md 19 | 20 | dataset: 21 | answers: tensorflow2-question-answering/prepared/private/test.jsonl 22 | gold_submission: tensorflow2-question-answering/prepared/private/gold_submission.csv 23 | sample_submission: tensorflow2-question-answering/prepared/public/sample_submission.csv 24 | 25 | grader: 26 | name: micro-f1-score 27 | grade_fn: mlebench.competitions.tensorflow2-question-answering.grade:grade 28 | 29 | preparer: mlebench.competitions.tensorflow2-question-answering.prepare:prepare 30 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/vinbigdata-chest-xray-abnormalities-detection/config.yaml: -------------------------------------------------------------------------------- 1 | id: vinbigdata-chest-xray-abnormalities-detection 2 | name: VinBigData Chest X-ray Abnormalities Detection 3 | competition_type: simple 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 20000 8 | - position: 2 9 | value: 14000 10 | - position: 3 11 | value: 8000 12 | - position: Special Prize 13 | value: 8000 14 | description: mlebench/competitions/vinbigdata-chest-xray-abnormalities-detection/description.md 15 | 16 | dataset: 17 | answers: vinbigdata-chest-xray-abnormalities-detection/prepared/private/answers.csv 18 | gold_submission: vinbigdata-chest-xray-abnormalities-detection/prepared/private/gold_submission.csv 19 | sample_submission: vinbigdata-chest-xray-abnormalities-detection/prepared/public/sample_submission.csv 20 | 21 | grader: 22 | name: mAP-at-IoU>0.4 23 | grade_fn: mlebench.competitions.vinbigdata-chest-xray-abnormalities-detection.grade:grade 24 | 25 | preparer: mlebench.competitions.vinbigdata-chest-xray-abnormalities-detection.prepare:prepare 26 | -------------------------------------------------------------------------------- /mle-bench/agents/dummy/main.py: -------------------------------------------------------------------------------- 1 | """ 2 | A dummy agent which copies the `sample_submission.csv` from the workspace/data directory 3 | and uses that as its `submission.csv`. 4 | """ 5 | 6 | import getpass 7 | import os 8 | import shutil 9 | import sys 10 | from pathlib import Path 11 | 12 | # Get the current user's username 13 | username = getpass.getuser() 14 | 15 | # Check if the current user ID is 0 (root user ID on Unix-like systems) 16 | if os.getuid() == 0: 17 | print(f"You are running this script as root. Your username is '{username}'.") 18 | else: 19 | print(f"You do not have root access. Your username is {username}.") 20 | 21 | print("The script is being run with the following python interpreter:") 22 | print(sys.executable) 23 | 24 | cwd = Path(__file__).parent 25 | workspace_data_dir = cwd.parent / "data" 26 | 27 | print(f"Copying sample submission...") 28 | 29 | shutil.copy( 30 | workspace_data_dir / "sample_submission.csv", 31 | cwd.parent / "submission" / "submission.csv", 32 | ) 33 | 34 | print(f"Sample submission copied to {cwd.parent / 'submission' / 'submission.csv'}.") 35 | -------------------------------------------------------------------------------- /automind/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | with open("README.md", "r") as f: 4 | long_description = f.read() 5 | 6 | with open("requirements.txt", "r") as f: 7 | requirements = f.read().splitlines() 8 | 9 | setup( 10 | name="automind", 11 | version="0.0.1", 12 | description="Autonomous AI for Data Science and Machine Learning", 13 | long_description=long_description, 14 | long_description_content_type="text/markdown", 15 | packages=find_packages(), 16 | package_data={ 17 | "automind": [ 18 | "../requirements.txt", 19 | "utils/config.yaml", 20 | "utils/viz_templates/*", 21 | ] 22 | }, 23 | classifiers=[ 24 | "Programming Language :: Python :: 3", 25 | "Operating System :: OS Independent", 26 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 27 | ], 28 | python_requires=">=3.10", 29 | install_requires=requirements, 30 | entry_points={ 31 | "console_scripts": [ 32 | "automind = automind.run:run", 33 | ], 34 | }, 35 | ) 36 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/learning-agency-lab-automated-essay-scoring-2/config.yaml: -------------------------------------------------------------------------------- 1 | id: learning-agency-lab-automated-essay-scoring-2 2 | name: Learning Agency Lab - Automated Essay Scoring 2.0 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 12000 8 | - position: 2 9 | value: 8000 10 | - position: 3 11 | value: 5000 12 | - position: Efficency Prize 1 13 | value: 12000 14 | - position: Efficency Prize 2 15 | value: 8000 16 | - position: Efficency Prize 3 17 | value: 5000 18 | description: mlebench/competitions/learning-agency-lab-automated-essay-scoring-2/description.md 19 | 20 | dataset: 21 | answers: learning-agency-lab-automated-essay-scoring-2/prepared/private/answers.csv 22 | sample_submission: learning-agency-lab-automated-essay-scoring-2/prepared/public/sample_submission.csv 23 | 24 | grader: 25 | name: quadratic-weighted-kappa 26 | grade_fn: mlebench.competitions.learning-agency-lab-automated-essay-scoring-2.grade:grade 27 | 28 | preparer: mlebench.competitions.learning-agency-lab-automated-essay-scoring-2.prepare:prepare 29 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/imet-2020-fgvc7/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Label artwork images with significant attributes. 4 | 5 | # Metric 6 | 7 | Micro averaged F1 score. 8 | 9 | # Submission Format 10 | 11 | ``` 12 | id,attribute_ids 13 | 00011f01965f141f5d1eea6592fa9862,0 1 2 14 | 00014abc91ed3e4bf1663fde8136fe80,0 1 2 15 | 0002e2054e303badc1a33463f6fb7973,0 1 2 16 | ``` 17 | 18 | # Dataset 19 | 20 | Multiple modalities can be expected and the camera sources are unknown. The photographs are often centered for objects, and in the case where the museum artifact is an entire room, the images are scenic in nature. 21 | 22 | Each object is annotated by a single annotator without a verification step. You should consider these annotations noisy. 23 | 24 | The filename of each image is its `id`. 25 | 26 | - **train.csv** gives the `attribute_ids` for the train images in **/train** 27 | - **/test** contains the test images. You must predict the `attribute_ids` for these images. 28 | - **sample_submission.csv** contains a submission in the correct format 29 | - **labels.csv** provides descriptions of the attributes -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/bms-molecular-translation/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Provided with images of chemicals, predict the corresponding International Chemical Identifier (InChI) text string of the image. 4 | 5 | # Metric 6 | 7 | Mean [Levenshtein distance](http://en.wikipedia.org/wiki/Levenshtein_distance) between the InChi strings you submit and the ground truth InChi values. 8 | 9 | # Submission Format 10 | 11 | For each `image_id` in the test set, you must predict the InChi string of the molecule in the corresponding image. The file should contain a header and have the following format: 12 | 13 | ``` 14 | image_id,InChI 15 | 00000d2a601c,InChI=1S/H2O/h1H2 16 | 00001f7fc849,InChI=1S/H2O/h1H2 17 | 000037687605,InChI=1S/H2O/h1H2 18 | etc. 19 | ``` 20 | 21 | # Dataset 22 | 23 | - **train/** - the training images, arranged in a 3-level folder structure by `image_id` 24 | - **test/** - the test images, arranged in the same folder structure as `train/` 25 | - **train_labels.csv** - ground truth InChi labels for the training images 26 | - **sample_submission.csv** - a sample submission file in the correct format 27 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/tweet-sentiment-extraction/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Predict the word or phrase from tweets that exemplifies the labelled sentiment. 4 | 5 | # Metric 6 | 7 | Word-level Jaccard score. 8 | 9 | # Submission Format 10 | 11 | For each ID in the test set, you must predict the string that best supports the sentiment for the tweet in question. Note that the selected text _needs_ to be **quoted** and **complete** (include punctuation, etc. - the above code splits ONLY on whitespace) to work correctly. The file should contain a header and have the following format: 12 | ``` 13 | textID,selected_text 14 | 2,"very good" 15 | 5,"I don't care" 16 | 6,"bad" 17 | 8,"it was, yes" 18 | etc. 19 | ``` 20 | 21 | # Dataset 22 | 23 | - **train.csv** - the training set 24 | - **test.csv** - the test set 25 | - **sample_submission.csv** - a sample submission file in the correct format 26 | 27 | - `textID` - unique ID for each piece of text 28 | - `text` - the text of the tweet 29 | - `sentiment` - the general sentiment of the tweet 30 | - `selected_text` - [train only] the text that supports the tweet's sentiment -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/jigsaw-toxic-comment-classification-challenge/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Given a dataset of comments from Wikipedia's talk page edits, predict the probability of each comment being toxic. 4 | 5 | # Metric 6 | 7 | Mean column-wise ROC AUC; the average of the individual AUCs of each predicted column. 8 | 9 | # Submission Format 10 | 11 | For each `id` in the test set, you must predict a probability for each of the six possible types of comment toxicity (toxic, severe_toxic, obscene, threat, insult, identity_hate). The columns must be in the same order as shown below. The file should contain a header and have the following format: 12 | 13 | ``` 14 | id,toxic,severe_toxic,obscene,threat,insult,identity_hate 15 | 00001cee341fdb12,0.5,0.5,0.5,0.5,0.5,0.5 16 | 0000247867823ef7,0.5,0.5,0.5,0.5,0.5,0.5 17 | etc. 18 | ``` 19 | 20 | # Dataset 21 | 22 | - **train.csv** - the training set, contains comments with their binary labels 23 | - **test.csv** - the test set, you must predict the toxicity probabilities for these comments. 24 | - **sample_submission.csv** - a sample submission file in the correct format -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/rsna-2022-cervical-spine-fracture-detection/config.yaml: -------------------------------------------------------------------------------- 1 | id: rsna-2022-cervical-spine-fracture-detection 2 | name: RSNA 2022 Cervical Spine Fracture Detection 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 6000 8 | - position: 2 9 | value: 5000 10 | - position: 3 11 | value: 4000 12 | - position: 4 13 | value: 3000 14 | - position: 5 15 | value: 3000 16 | - position: 6 17 | value: 3000 18 | - position: 7 19 | value: 3000 20 | - position: 8 21 | value: 3000 22 | description: mlebench/competitions/rsna-2022-cervical-spine-fracture-detection/description.md 23 | 24 | dataset: 25 | answers: rsna-2022-cervical-spine-fracture-detection/prepared/private/answers.csv 26 | sample_submission: rsna-2022-cervical-spine-fracture-detection/prepared/public/sample_submission.csv 27 | 28 | grader: 29 | name: weighted-multi-label-log-loss 30 | grade_fn: mlebench.competitions.rsna-2022-cervical-spine-fracture-detection.grade:grade 31 | 32 | preparer: mlebench.competitions.rsna-2022-cervical-spine-fracture-detection.prepare:prepare 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 InnovatingAI 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /automind/utils/additional_notes.txt: -------------------------------------------------------------------------------- 1 | ADDITIONAL NOTES 2 | ------ 3 | 4 | - **Compute**: You have access to ${HARDWARE} with the appropriate drivers installed. 5 | - **Total Runtime**: You have a maximum of ${TIME_LIMIT} hours to attempt this task. You may iteratively develop your solution within this time frame, but bear in mind that program runtime also counts towards this limit. 6 | - **Total Steps**: You have a maximum of ${STEP_LIMIT} steps to submit your solution. 7 | - You will be informed of the remaining time and steps at each step. 8 | - **Use as much of the available time as possible to refine and optimize your submission**: Your goal is to achieve the best possible score, not just to submit your first working solution. Consider strategies like validating your answer using the "Validate Submission" tool at the http://localhost:5000/validate endpoint, manually spot-checking predictions, building custom validation sets and grading functions, and comparing different algorithms. You may keep working until the time limit expires -- as long as your best submission is stored at the designated location at the end of your run, it will be picked up for grading. 9 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/google-research-identify-contrails-reduce-global-warming/config.yaml: -------------------------------------------------------------------------------- 1 | id: google-research-identify-contrails-reduce-global-warming 2 | name: Google Research - Identify Contrails to Reduce Global Warming 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 15000 8 | - position: 2 9 | value: 10000 10 | - position: 3 11 | value: 8000 12 | - position: 4 13 | value: 7000 14 | - position: 5 15 | value: 5000 16 | - position: 6 17 | value: 5000 18 | description: mlebench/competitions/google-research-identify-contrails-reduce-global-warming/description.md 19 | 20 | dataset: 21 | answers: google-research-identify-contrails-reduce-global-warming/prepared/private/answers.csv 22 | sample_submission: google-research-identify-contrails-reduce-global-warming/prepared/public/sample_submission.csv 23 | 24 | grader: 25 | name: global-dice 26 | grade_fn: mlebench.competitions.google-research-identify-contrails-reduce-global-warming.grade:grade 27 | 28 | preparer: mlebench.competitions.google-research-identify-contrails-reduce-global-warming.prepare:prepare 29 | -------------------------------------------------------------------------------- /mle-bench/agents/aide/additional_notes.txt: -------------------------------------------------------------------------------- 1 | ADDITIONAL NOTES 2 | ------ 3 | 4 | - **Compute**: You have access to ${HARDWARE} with the appropriate drivers installed. 5 | - **Total Runtime**: You have a maximum of ${TIME_LIMIT} hours to attempt this task. You may iteratively develop your solution within this time frame, but bear in mind that program runtime also counts towards this limit. 6 | - **Total Steps**: You have a maximum of ${STEP_LIMIT} steps to submit your solution. 7 | - You will be informed of the remaining time and steps at each step. 8 | - **Use as much of the available time as possible to refine and optimize your submission**: Your goal is to achieve the best possible score, not just to submit your first working solution. Consider strategies like validating your answer using the "Validate Submission" tool at the http://localhost:5000/validate endpoint, manually spot-checking predictions, building custom validation sets and grading functions, and comparing different algorithms. You may keep working until the time limit expires -- as long as your best submission is stored at the designated location at the end of your run, it will be picked up for grading. 9 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/invasive-species-monitoring/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Predict the presence of an invasive hydrangea species in images of foliage. 4 | 5 | # Metric 6 | 7 | Area under the ROC curve. 8 | 9 | # Submission Format 10 | 11 | For each image in the test set, you must predict a probability for the target variable on whether the image contains invasive species or not. The file should contain a header and have the following format: 12 | 13 | ``` 14 | name,invasive 15 | 2,0.5 16 | 5,0 17 | 6,0.2 18 | etc. 19 | ``` 20 | 21 | # Dataset 22 | 23 | The data set contains pictures taken in a forest. 24 | 25 | - **train.7z** - the training set (contains 2295 images). 26 | - **train_labels.csv** - the correct labels for the training set. 27 | - **test.7z** - the testing set (contains 1531 images), ready to be labeled by your algorithm. 28 | - **sample_submission.csv** - a sample submission file in the correct format. 29 | 30 | ## Data fields 31 | 32 | - **name** - name of the sample picture file (numbers) 33 | - **invasive** - probability of the picture containing an invasive species. A probability of 1 means the species is present. -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/facebook-recruiting-iii-keyword-extraction/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | 2 | # Task 3 | 4 | The task is to predict tags (a.k.a. keywords, topics, summaries), given only Stack Exchange question text and its title. 5 | 6 | # Metric 7 | 8 | Mean F1-Score. 9 | 10 | # Submission Format 11 | 12 | For every question in the test set, your submission file should contain two columns: Id and Tags. Id is the unique identifier of a question in test.csv. Tags should be a space-delimited list predicted tags. You should maintain the order of questions. 13 | 14 | The file should contain a header and have the following format: 15 | 16 | ``` 17 | Id,Tags 18 | 1,"c++ javaScript" 19 | 2,"php python mysql" 20 | 3,"django" 21 | etc. 22 | ``` 23 | 24 | # Dataset 25 | 26 | **Train.csv** contains 4 columns: Id,Title,Body,Tags 27 | 28 | - Id - Unique identifier for each question 29 | - Title - The question's title 30 | - Body - The body of the question 31 | - Tags - The tags associated with the question (all lowercase, should not contain tabs '\t' or ampersands '&') 32 | 33 | **Test.csv** contains the same columns but without the Tags, which you are to predict. -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/google-quest-challenge/classes.py: -------------------------------------------------------------------------------- 1 | CLASSES = [ 2 | "question_asker_intent_understanding", 3 | "question_body_critical", 4 | "question_conversational", 5 | "question_expect_short_answer", 6 | "question_fact_seeking", 7 | "question_has_commonly_accepted_answer", 8 | "question_interestingness_others", 9 | "question_interestingness_self", 10 | "question_multi_intent", 11 | "question_not_really_a_question", 12 | "question_opinion_seeking", 13 | "question_type_choice", 14 | "question_type_compare", 15 | "question_type_consequence", 16 | "question_type_definition", 17 | "question_type_entity", 18 | "question_type_instructions", 19 | "question_type_procedure", 20 | "question_type_reason_explanation", 21 | "question_type_spelling", 22 | "question_well_written", 23 | "answer_helpful", 24 | "answer_level_of_information", 25 | "answer_plausible", 26 | "answer_relevance", 27 | "answer_satisfaction", 28 | "answer_type_instructions", 29 | "answer_type_procedure", 30 | "answer_type_reason_explanation", 31 | "answer_well_written", 32 | ] 33 | -------------------------------------------------------------------------------- /mle-bench/agents/aide/src/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Weco AI Ltd 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/rsna-miccai-brain-tumor-radiogenomic-classification/config.yaml: -------------------------------------------------------------------------------- 1 | id: rsna-miccai-brain-tumor-radiogenomic-classification 2 | name: RSNA-MICCAI Brain Tumor Radiogenomic Classification 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 6000 8 | - position: 2 9 | value: 5000 10 | - position: 3 11 | value: 4000 12 | - position: 4 13 | value: 3000 14 | - position: 5 15 | value: 3000 16 | - position: 6 17 | value: 3000 18 | - position: 7 19 | value: 3000 20 | - position: 8 21 | value: 3000 22 | description: mlebench/competitions/rsna-miccai-brain-tumor-radiogenomic-classification/description.md 23 | 24 | dataset: 25 | answers: rsna-miccai-brain-tumor-radiogenomic-classification/prepared/private/test.csv 26 | sample_submission: rsna-miccai-brain-tumor-radiogenomic-classification/prepared/public/sample_submission.csv 27 | 28 | grader: 29 | name: auc-roc 30 | grade_fn: mlebench.competitions.rsna-miccai-brain-tumor-radiogenomic-classification.grade:grade 31 | 32 | preparer: mlebench.competitions.rsna-miccai-brain-tumor-radiogenomic-classification.prepare:prepare 33 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/iwildcam-2019-fgvc6/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Label images of animals with their species. 4 | 5 | # Metric 6 | 7 | Macro F1 score 8 | 9 | # Submission Format 10 | 11 | ``` 12 | Id,Predicted 13 | 58857ccf-23d2-11e8-a6a3-ec086b02610b,1 14 | 591e4006-23d2-11e8-a6a3-ec086b02610b,5 15 | ``` 16 | 17 | The `Id` column corresponds to the test image id. The `Category` is an integer value that indicates the class of the animal, or `0` to represent the absence of an animal. 18 | 19 | # Dataset 20 | 21 | The training set contains 196,157 images from 138 different locations in Southern California. 22 | 23 | The test set contains 153,730 images from 100 locations in Idaho. 24 | 25 | The task is to label each image with one of the following label ids: 26 | 27 | ``` 28 | name, id 29 | empty, 0 30 | deer, 1 31 | moose, 2 32 | squirrel, 3 33 | rodent, 4 34 | small_mammal, 5 35 | elk, 6 36 | pronghorn_antelope, 7 37 | rabbit, 8 38 | bighorn_sheep, 9 39 | fox, 10 40 | coyote, 11 41 | black_bear, 12 42 | raccoon, 13 43 | skunk, 14 44 | wolf, 15 45 | bobcat, 16 46 | cat, 17 47 | dog, 18 48 | opossum, 19 49 | bison, 20 50 | mountain_goat, 21 51 | mountain_lion, 22 52 | ``` -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/facebook-recruiting-iii-keyword-extraction/kernels.txt: -------------------------------------------------------------------------------- 1 | vikashrajluhaniwal/multi-label-classification-for-tag-predictions 2 | dwarika/stackoverflow-tag-predictor 3 | sumantindurkhya/so-tag-prediction-multilabel-bilstm 4 | aryandec25/notebook-stackoverflow 5 | mitishaagarwal/stackoverflow-tag-predictor 6 | elemento/stackoverflow-questiontagging 7 | l0new0lf/tag-classifier 8 | ezequielstarecinch/stackoverflow-tags20-classifier 9 | koredla25/predicting-tags-for-the-questions-in-satckoverflow 10 | chandanmalla/stackoverflow-multi-label-tag-prediction 11 | f13rc3/multi-label-classification-for-tag-prediction 12 | mkchaitanya1999/multilabel-classification 13 | thangchu15897/fork-of-multi-label-classification-for-tag-3352dd 14 | thienlm/fork-of-multi-label-classification-for-tag-3352dd 15 | thienlm/multi-label-classification-for-tag-predictions 16 | akashmodi7t/notebook8fafe25719 17 | vikaskumarmaurya/so-tagpredict-googlecolab 18 | anonymous55/ml-project-final 19 | canirudh/predicting-tags-for-questions-in-stack-overflow 20 | reintegrated/4-stackoverflow-training 21 | reintegrated/1-stackoverflow-eda-and-sampling 22 | ashishkkumar/kernel3fae69652f 23 | rajat2341/stackoverflow-tag-prediction 24 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/plant-pathology-2021-fgvc8/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Detect apple diseases from images. 4 | 5 | # Metric 6 | 7 | Mean F1-Score 8 | 9 | # Submission Format 10 | 11 | labels should be a space-delimited list. 12 | 13 | The file should contain a header and have the following format: 14 | 15 | ``` 16 | image, labels 17 | 85f8cb619c66b863.jpg,healthy 18 | ad8770db05586b59.jpg,healthy 19 | c7b03e718489f3ca.jpg,healthy 20 | ``` 21 | 22 | # Dataset 23 | 24 | **train.csv** - the training set metadata. 25 | 26 | - `image` - the image ID. 27 | - `labels` - the target classes, a space delimited list of all diseases found in the image. Unhealthy leaves with too many diseases to classify visually will have the `complex` class, and may also have a subset of the diseases identified. 28 | 29 | **sample_submission.csv** - A sample submission file in the correct format. 30 | 31 | - `image` 32 | - `labels` 33 | 34 | **train_images** - The training set images. 35 | 36 | **test_images** - The test set images. This competition has a hidden test set: only three images are provided here as samples while the remaining 5,000 images will be available to your notebook once it is submitted. -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/new-york-city-taxi-fare-prediction/grade.py: -------------------------------------------------------------------------------- 1 | from pandas import DataFrame 2 | from sklearn.metrics import root_mean_squared_error 3 | 4 | from mlebench.grade_helpers import InvalidSubmissionError 5 | 6 | 7 | def prepare_for_metric(submission: DataFrame, answers: DataFrame) -> dict: 8 | 9 | assert "fare_amount" in answers.columns, "Answers should have a fare_amount column" 10 | assert "key" in answers.columns, "Answers should have a key column" 11 | if "fare_amount" not in submission.columns: 12 | raise InvalidSubmissionError("Submission should have a fare_amount column") 13 | if "key" not in submission.columns: 14 | raise InvalidSubmissionError("Submission should have a key column") 15 | 16 | # Sort by 'key' to ensure alignment 17 | submission = submission.sort_values("key") 18 | answers = answers.sort_values("key") 19 | 20 | y_true = answers["fare_amount"] 21 | y_pred = submission["fare_amount"] 22 | 23 | return y_true, y_pred 24 | 25 | 26 | def grade(submission: DataFrame, answers: DataFrame) -> float: 27 | y_true, y_pred = prepare_for_metric(submission, answers) 28 | return root_mean_squared_error(y_true, y_pred) 29 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/herbarium-2021-fgvc8/kernels.txt: -------------------------------------------------------------------------------- 1 | khoongweihao/insect-augmentation-et-al 2 | ihelon/herbarium-2021-exploratory-data-analysis 3 | ateplyuk/herb2021-pytorch-starter 4 | debarshichanda/herbarium-2021-pytorch-starter 5 | yasserhessein/herbarium-2021-uing-vgg16 6 | sauravmaheshkar/herbarium-2021-pytorch-starter-weights-biases 7 | tpmeli/herbarium-starter-efficientnet-tf-keras-gpu 8 | salmanhiro/herbarium-2021-efficientnet-b0-training-starter 9 | atamazian/herb-2021-tfrecords-effnet-training 10 | drcapa/herbarium-2021-starter-eda-datagenerator 11 | twhelan/herbarium2021-creating-smaller-subsets-of-data 12 | ricardobarbosasousa/herbarium-2021-rbs-resnet 13 | yeonghyeon/step-by-step-herbarium-2021 14 | muhammadzubairkhan92/herbarium-2021-exploratory-data-analysis 15 | sauravmaheshkar/herbarium-2021-resnet18-inference 16 | atamazian/herb-2021-tfrecords-effnet-inference 17 | abhaychaudhary01/insect-augmentation-et-al 18 | sharansmenon/herbarium-pytorch 19 | ivanfeliperodriguez/tfrecords-creation-with-arbitrary-size 20 | shayantaherian/herbarium-2021 21 | anandagdhi/notebook0ac3a4bca3 22 | anandagdhi/herberium-eda 23 | siddhartamukherjee/herbarium-2021-resnet34-pytorch-gpu 24 | oricou/herbarium21-skeleton 25 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/vesuvius-challenge-ink-detection/config.yaml: -------------------------------------------------------------------------------- 1 | id: vesuvius-challenge-ink-detection 2 | name: Vesuvius Challenge - Ink Detection 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 25000 8 | - position: 2 9 | value: 20000 10 | - position: 3 11 | value: 15000 12 | - position: 4 13 | value: 1000 14 | - position: 5 15 | value: 5000 16 | - position: 6 17 | value: 5000 18 | - position: 7 19 | value: 5000 20 | - position: 8 21 | value: 5000 22 | - position: 9 23 | value: 5000 24 | - position: 10 25 | value: 5000 26 | description: mlebench/competitions/vesuvius-challenge-ink-detection/description.md 27 | 28 | dataset: 29 | answers: vesuvius-challenge-ink-detection/prepared/private/inklabels_rle.csv 30 | gold_submission: vesuvius-challenge-ink-detection/prepared/private/gold_submission.csv 31 | sample_submission: vesuvius-challenge-ink-detection/prepared/public/sample_submission.csv 32 | 33 | grader: 34 | name: f0.5-score 35 | grade_fn: mlebench.competitions.vesuvius-challenge-ink-detection.grade:grade 36 | 37 | preparer: mlebench.competitions.vesuvius-challenge-ink-detection.prepare:prepare 38 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/learning-agency-lab-automated-essay-scoring-2/description_obfuscated.md: -------------------------------------------------------------------------------- 1 | # Task 2 | 3 | Predict the score of student essays. 4 | 5 | # Metric 6 | 7 | Quadratic weighted kappa. 8 | 9 | # Submission Format 10 | 11 | For each `essay_id` in the test set, you must predict the corresponding `score` (between 1-6, see [rubric](https://storage.googleapis.com/kaggle-forum-message-attachments/2733927/20538/Rubric_%20Holistic%20Essay%20Scoring.pdf) for more details). The file should contain a header and have the following format: 12 | 13 | ``` 14 | essay_id,score 15 | 000d118,3 16 | 000fe60,3 17 | 001ab80,4 18 | ... 19 | ``` 20 | 21 | # Dataset 22 | 23 | - **train.csv** - Essays and scores to be used as training data. 24 | - `essay_id` - The unique ID of the essay 25 | - `full_text` - The full essay response 26 | - `score` - Holistic score of the essay on a 1-6 scale 27 | - **test.csv** - The essays to be used as test data. Contains the same fields as `train.csv`, aside from exclusion of `score`. 28 | - **sample_submission.csv** - A submission file in the correct format. 29 | - `essay_id` - The unique ID of the essay 30 | - `score` - The predicted holistic score of the essay on a 1-6 scale 31 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/icecube-neutrinos-in-deep-ice/config.yaml: -------------------------------------------------------------------------------- 1 | id: icecube-neutrinos-in-deep-ice 2 | name: IceCube - Neutrinos in Deep Ice 3 | competition_type: code 4 | awards_medals: true 5 | prizes: 6 | - position: 1 7 | value: 18000 8 | - position: 2 9 | value: 12000 10 | - position: 3 11 | value: 10000 12 | - position: Most Interesting Solution Writeup 1 13 | value: 1000 14 | - position: Most Interesting Solution Writeup 2 15 | value: 1000 16 | - position: Most Interesting Solution Writeup 3 17 | value: 1000 18 | - position: Most Interesting Solution Writeup 4 19 | value: 1000 20 | - position: Most Interesting Solution Writeup 5 21 | value: 1000 22 | - position: Early Sharing Prize 23 | value: 5000 24 | description: mlebench/competitions/icecube-neutrinos-in-deep-ice/description.md 25 | 26 | dataset: 27 | answers: icecube-neutrinos-in-deep-ice/prepared/private/test.csv 28 | sample_submission: icecube-neutrinos-in-deep-ice/prepared/public/sample_submission.csv 29 | 30 | grader: 31 | name: mean-angular-error 32 | grade_fn: mlebench.competitions.icecube-neutrinos-in-deep-ice.grade:grade 33 | 34 | preparer: mlebench.competitions.icecube-neutrinos-in-deep-ice.prepare:prepare 35 | -------------------------------------------------------------------------------- /mle-bench/mlebench/competitions/random-acts-of-pizza/kernels.txt: -------------------------------------------------------------------------------- 1 | alvations/basic-nlp-with-nltk 2 | jatinraina/random-acts-of-pizza-xgboost 3 | benhamner/simple-julia-benchmark 4 | ynue21/random-act-of-pizza 5 | benhamner/rmarkdown-default-text 6 | danofer/pizza-data-to-csv 7 | benhamner/wordclouds-1 8 | benhamner/exploratory-plots-with-julia-and-gadfly 9 | quantuary/random-act-of-pizza 10 | zhichaoking/pizza-data-test 11 | koushikdeb/intro-to-terminologies-of-nlp 12 | tanjinprity/ml-project-cat-and-tan 13 | dpamgautam/nlp-basic-text-data-preprocessing-1 14 | elias22/kerneld8b44bc755 15 | zahoorahmad/basic-nlp-with-nltk-fba294 16 | jihyeseo/pizza-eda 17 | rashmijrao/ml-project 18 | tonybyte/notebook4a30ab2da1 19 | mbp14mtp/raopizza 20 | rajatk95/notebookb1b8dc9bc4 21 | heyheyivan/random-pizza 22 | chqngh/random-pizza 23 | haroonc/notebookaa32534f23 24 | haroonc/notebook26e44b5093 25 | haroonc/notebookf7a104a76a 26 | ahuaay/attempt1-1 27 | justinle/notebook-02122c138523a1fb16d0 28 | sisira/random-acts-of-pizza 29 | kevinse/notebook-3e6bc92ca10925952253 30 | sidraina89/random-acts-of-pizza 31 | xiaogea/simple-julia-benchmark 32 | pritipatra/simple-julia-benchmark 33 | benhamner/default-python-script 34 | benhamner/julia-default-text 35 | benhamner/default-r-text-1 36 | --------------------------------------------------------------------------------