├── .gitignore ├── 0_create_labels_df.ipynb ├── 0_preprocess_data.ipynb ├── 1_xval_model_train.ipynb ├── 1_xval_model_train_perfect_model.ipynb ├── 2_benchmark.ipynb ├── 3_benchmark_results_complete.ipynb ├── 3_benchmark_results_truelabels.ipynb ├── 3_benchmark_results_uniform.ipynb ├── 3_benchmark_results_worst.ipynb ├── 3_benchmark_results_worst_npw.ipynb ├── 3_side_benchmark_results.ipynb ├── LICENSE ├── README.md ├── active_learning_benchmarks ├── README.md ├── cifar-10h │ ├── 0_preprocess_data.ipynb │ ├── 1_run_benchmark.ipynb │ ├── 2_plot_results.ipynb │ ├── results │ │ ├── crowdlab_accuracy_20221118124631.npy │ │ ├── crowdlab_accuracy_20221119012724.npy │ │ ├── crowdlab_accuracy_20221120032011.npy │ │ ├── crowdlab_accuracy_20221120094140.npy │ │ ├── crowdlab_accuracy_20221120220738.npy │ │ ├── crowdlab_ensemble_accuracy_20221124022113.npy │ │ ├── crowdlab_ensemble_accuracy_20221124031623.npy │ │ ├── crowdlab_ensemble_accuracy_20221125055941.npy │ │ ├── crowdlab_ensemble_accuracy_20221125073747.npy │ │ ├── crowdlab_ensemble_accuracy_20221125084702.npy │ │ ├── crowdlab_ensemble_model_accuracy_20221124022113.npy │ │ ├── crowdlab_ensemble_model_accuracy_20221124031623.npy │ │ ├── crowdlab_ensemble_model_accuracy_20221125055941.npy │ │ ├── crowdlab_ensemble_model_accuracy_20221125073747.npy │ │ ├── crowdlab_ensemble_model_accuracy_20221125084702.npy │ │ ├── crowdlab_model_accuracy_20221118124631.npy │ │ ├── crowdlab_model_accuracy_20221119012724.npy │ │ ├── crowdlab_model_accuracy_20221120032011.npy │ │ ├── crowdlab_model_accuracy_20221120094140.npy │ │ ├── crowdlab_model_accuracy_20221120220738.npy │ │ ├── disagreement_accuracy_20221124080356.npy │ │ ├── disagreement_accuracy_20221124093429.npy │ │ ├── disagreement_accuracy_20221124095748.npy │ │ ├── disagreement_accuracy_20221125164119.npy │ │ ├── disagreement_accuracy_20221125171752.npy │ │ ├── disagreement_model_accuracy_20221124080356.npy │ │ ├── disagreement_model_accuracy_20221124093429.npy │ │ ├── disagreement_model_accuracy_20221124095748.npy │ │ ├── disagreement_model_accuracy_20221125164119.npy │ │ ├── disagreement_model_accuracy_20221125171752.npy │ │ ├── entropy_accuracy_20221104235901.npy │ │ ├── entropy_accuracy_20221105030100.npy │ │ ├── entropy_accuracy_20221105055215.npy │ │ ├── entropy_accuracy_20221105085044.npy │ │ ├── entropy_accuracy_20221105115901.npy │ │ ├── entropy_model_accuracy_20221104235901.npy │ │ ├── entropy_model_accuracy_20221105030100.npy │ │ ├── entropy_model_accuracy_20221105055215.npy │ │ ├── entropy_model_accuracy_20221105085044.npy │ │ ├── entropy_model_accuracy_20221105115901.npy │ │ ├── good_random_accuracy_20221105193128.npy │ │ ├── good_random_accuracy_20221105223006.npy │ │ ├── good_random_accuracy_20221106012412.npy │ │ ├── good_random_accuracy_20221106042709.npy │ │ ├── good_random_accuracy_20221106071728.npy │ │ ├── good_random_model_accuracy_20221105193128.npy │ │ ├── good_random_model_accuracy_20221105223006.npy │ │ ├── good_random_model_accuracy_20221106012412.npy │ │ ├── good_random_model_accuracy_20221106042709.npy │ │ ├── good_random_model_accuracy_20221106071728.npy │ │ ├── random_accuracy_20221104060547.npy │ │ ├── random_accuracy_20221104094220.npy │ │ ├── random_accuracy_20221104132000.npy │ │ ├── random_accuracy_20221104160849.npy │ │ ├── random_accuracy_20221104184148.npy │ │ ├── random_model_accuracy_20221104060547.npy │ │ ├── random_model_accuracy_20221104094220.npy │ │ ├── random_model_accuracy_20221104132000.npy │ │ ├── random_model_accuracy_20221104160849.npy │ │ ├── random_model_accuracy_20221104184148.npy │ │ ├── uncertainty_accuracy_20221110081546.npy │ │ ├── uncertainty_accuracy_20221110111443.npy │ │ ├── uncertainty_accuracy_20221110141032.npy │ │ ├── uncertainty_accuracy_20221110162526.npy │ │ ├── uncertainty_accuracy_20221110182904.npy │ │ ├── uncertainty_model_accuracy_20221110081546.npy │ │ ├── uncertainty_model_accuracy_20221110111443.npy │ │ ├── uncertainty_model_accuracy_20221110141032.npy │ │ ├── uncertainty_model_accuracy_20221110162526.npy │ │ └── uncertainty_model_accuracy_20221110182904.npy │ └── utils │ │ ├── active_learning.py │ │ ├── eval.py │ │ ├── methods.py │ │ └── model_training.py ├── extra-benchmarks │ ├── single-annotator │ │ ├── 0_create_data.ipynb │ │ ├── 1_run_benchmark.ipynb │ │ ├── 2_plot_results.ipynb │ │ └── utils │ │ │ ├── active_learning.py │ │ │ └── model_training.py │ └── single-vs-multiannotator │ │ ├── 0_create_data.ipynb │ │ ├── 1_run_benchmark.ipynb │ │ ├── 2_plot_results.ipynb │ │ └── utils │ │ ├── active_learning.py │ │ └── model_training.py ├── requirements.txt ├── wall-robot-completely-labeled │ ├── 0_create_data.ipynb │ ├── 1_run_benchmark_ET.ipynb │ ├── 1_run_benchmark_MLP.ipynb │ ├── 2_plot_results_ET.ipynb │ ├── 2_plot_results_MLP.ipynb │ ├── results │ │ ├── et │ │ │ ├── active_label_cleaning_accuracy_0.npy │ │ │ ├── active_label_cleaning_accuracy_1.npy │ │ │ ├── active_label_cleaning_accuracy_2.npy │ │ │ ├── active_label_cleaning_accuracy_3.npy │ │ │ ├── active_label_cleaning_accuracy_4.npy │ │ │ ├── active_label_cleaning_model_accuracy_0.npy │ │ │ ├── active_label_cleaning_model_accuracy_1.npy │ │ │ ├── active_label_cleaning_model_accuracy_2.npy │ │ │ ├── active_label_cleaning_model_accuracy_3.npy │ │ │ ├── active_label_cleaning_model_accuracy_4.npy │ │ │ ├── crowdlab_accuracy_0.npy │ │ │ ├── crowdlab_accuracy_1.npy │ │ │ ├── crowdlab_accuracy_2.npy │ │ │ ├── crowdlab_accuracy_3.npy │ │ │ ├── crowdlab_accuracy_4.npy │ │ │ ├── crowdlab_model_accuracy_0.npy │ │ │ ├── crowdlab_model_accuracy_1.npy │ │ │ ├── crowdlab_model_accuracy_2.npy │ │ │ ├── crowdlab_model_accuracy_3.npy │ │ │ ├── crowdlab_model_accuracy_4.npy │ │ │ ├── disagreement_accuracy_0.npy │ │ │ ├── disagreement_accuracy_1.npy │ │ │ ├── disagreement_accuracy_2.npy │ │ │ ├── disagreement_accuracy_3.npy │ │ │ ├── disagreement_accuracy_4.npy │ │ │ ├── disagreement_model_accuracy_0.npy │ │ │ ├── disagreement_model_accuracy_1.npy │ │ │ ├── disagreement_model_accuracy_2.npy │ │ │ ├── disagreement_model_accuracy_3.npy │ │ │ ├── disagreement_model_accuracy_4.npy │ │ │ ├── entropy_accuracy_0.npy │ │ │ ├── entropy_accuracy_1.npy │ │ │ ├── entropy_accuracy_2.npy │ │ │ ├── entropy_accuracy_3.npy │ │ │ ├── entropy_accuracy_4.npy │ │ │ ├── entropy_model_accuracy_0.npy │ │ │ ├── entropy_model_accuracy_1.npy │ │ │ ├── entropy_model_accuracy_2.npy │ │ │ ├── entropy_model_accuracy_3.npy │ │ │ ├── entropy_model_accuracy_4.npy │ │ │ ├── good_random_accuracy_0.npy │ │ │ ├── good_random_accuracy_1.npy │ │ │ ├── good_random_accuracy_2.npy │ │ │ ├── good_random_accuracy_3.npy │ │ │ ├── good_random_accuracy_4.npy │ │ │ ├── good_random_model_accuracy_0.npy │ │ │ ├── good_random_model_accuracy_1.npy │ │ │ ├── good_random_model_accuracy_2.npy │ │ │ ├── good_random_model_accuracy_3.npy │ │ │ ├── good_random_model_accuracy_4.npy │ │ │ ├── random_accuracy_0.npy │ │ │ ├── random_accuracy_1.npy │ │ │ ├── random_accuracy_2.npy │ │ │ ├── random_accuracy_3.npy │ │ │ ├── random_accuracy_4.npy │ │ │ ├── random_model_accuracy_0.npy │ │ │ ├── random_model_accuracy_1.npy │ │ │ ├── random_model_accuracy_2.npy │ │ │ ├── random_model_accuracy_3.npy │ │ │ ├── random_model_accuracy_4.npy │ │ │ ├── uncertainty_accuracy_0.npy │ │ │ ├── uncertainty_accuracy_1.npy │ │ │ ├── uncertainty_accuracy_2.npy │ │ │ ├── uncertainty_accuracy_3.npy │ │ │ ├── uncertainty_accuracy_4.npy │ │ │ ├── uncertainty_model_accuracy_0.npy │ │ │ ├── uncertainty_model_accuracy_1.npy │ │ │ ├── uncertainty_model_accuracy_2.npy │ │ │ ├── uncertainty_model_accuracy_3.npy │ │ │ └── uncertainty_model_accuracy_4.npy │ │ └── mlp │ │ │ ├── active_label_cleaning_accuracy_0.npy │ │ │ ├── active_label_cleaning_accuracy_1.npy │ │ │ ├── active_label_cleaning_accuracy_2.npy │ │ │ ├── active_label_cleaning_accuracy_3.npy │ │ │ ├── active_label_cleaning_accuracy_4.npy │ │ │ ├── active_label_cleaning_model_accuracy_0.npy │ │ │ ├── active_label_cleaning_model_accuracy_1.npy │ │ │ ├── active_label_cleaning_model_accuracy_2.npy │ │ │ ├── active_label_cleaning_model_accuracy_3.npy │ │ │ ├── active_label_cleaning_model_accuracy_4.npy │ │ │ ├── crowdlab_accuracy_0.npy │ │ │ ├── crowdlab_accuracy_1.npy │ │ │ ├── crowdlab_accuracy_2.npy │ │ │ ├── crowdlab_accuracy_3.npy │ │ │ ├── crowdlab_accuracy_4.npy │ │ │ ├── crowdlab_ensemble_accuracy_0.npy │ │ │ ├── crowdlab_ensemble_accuracy_1.npy │ │ │ ├── crowdlab_ensemble_accuracy_2.npy │ │ │ ├── crowdlab_ensemble_accuracy_3.npy │ │ │ ├── crowdlab_ensemble_accuracy_4.npy │ │ │ ├── crowdlab_ensemble_model_accuracy_0.npy │ │ │ ├── crowdlab_ensemble_model_accuracy_1.npy │ │ │ ├── crowdlab_ensemble_model_accuracy_2.npy │ │ │ ├── crowdlab_ensemble_model_accuracy_3.npy │ │ │ ├── crowdlab_ensemble_model_accuracy_4.npy │ │ │ ├── crowdlab_model_accuracy_0.npy │ │ │ ├── crowdlab_model_accuracy_1.npy │ │ │ ├── crowdlab_model_accuracy_2.npy │ │ │ ├── crowdlab_model_accuracy_3.npy │ │ │ ├── crowdlab_model_accuracy_4.npy │ │ │ ├── disagreement_accuracy_0.npy │ │ │ ├── disagreement_accuracy_1.npy │ │ │ ├── disagreement_accuracy_2.npy │ │ │ ├── disagreement_accuracy_3.npy │ │ │ ├── disagreement_accuracy_4.npy │ │ │ ├── disagreement_model_accuracy_0.npy │ │ │ ├── disagreement_model_accuracy_1.npy │ │ │ ├── disagreement_model_accuracy_2.npy │ │ │ ├── disagreement_model_accuracy_3.npy │ │ │ ├── disagreement_model_accuracy_4.npy │ │ │ ├── entropy_accuracy_0.npy │ │ │ ├── entropy_accuracy_1.npy │ │ │ ├── entropy_accuracy_2.npy │ │ │ ├── entropy_accuracy_3.npy │ │ │ ├── entropy_accuracy_4.npy │ │ │ ├── entropy_model_accuracy_0.npy │ │ │ ├── entropy_model_accuracy_1.npy │ │ │ ├── entropy_model_accuracy_2.npy │ │ │ ├── entropy_model_accuracy_3.npy │ │ │ ├── entropy_model_accuracy_4.npy │ │ │ ├── good_random_accuracy_0.npy │ │ │ ├── good_random_accuracy_1.npy │ │ │ ├── good_random_accuracy_2.npy │ │ │ ├── good_random_accuracy_3.npy │ │ │ ├── good_random_accuracy_4.npy │ │ │ ├── good_random_model_accuracy_0.npy │ │ │ ├── good_random_model_accuracy_1.npy │ │ │ ├── good_random_model_accuracy_2.npy │ │ │ ├── good_random_model_accuracy_3.npy │ │ │ ├── good_random_model_accuracy_4.npy │ │ │ ├── random_accuracy_0.npy │ │ │ ├── random_accuracy_1.npy │ │ │ ├── random_accuracy_2.npy │ │ │ ├── random_accuracy_3.npy │ │ │ ├── random_accuracy_4.npy │ │ │ ├── random_model_accuracy_0.npy │ │ │ ├── random_model_accuracy_1.npy │ │ │ ├── random_model_accuracy_2.npy │ │ │ ├── random_model_accuracy_3.npy │ │ │ ├── random_model_accuracy_4.npy │ │ │ ├── uncertainty_accuracy_0.npy │ │ │ ├── uncertainty_accuracy_1.npy │ │ │ ├── uncertainty_accuracy_2.npy │ │ │ ├── uncertainty_accuracy_3.npy │ │ │ ├── uncertainty_accuracy_4.npy │ │ │ ├── uncertainty_model_accuracy_0.npy │ │ │ ├── uncertainty_model_accuracy_1.npy │ │ │ ├── uncertainty_model_accuracy_2.npy │ │ │ ├── uncertainty_model_accuracy_3.npy │ │ │ └── uncertainty_model_accuracy_4.npy │ └── utils │ │ ├── active_learning_utils.py │ │ ├── methods_ET.py │ │ ├── methods_MLP.py │ │ └── model_training.py └── wall-robot │ ├── 0_create_data.ipynb │ ├── 1_run_benchmark_ET.ipynb │ ├── 1_run_benchmark_MLP.ipynb │ ├── 2_plot_results_ET.ipynb │ ├── 2_plot_results_MLP.ipynb │ ├── results │ ├── et │ │ ├── crowdlab_accuracy_0.npy │ │ ├── crowdlab_accuracy_1.npy │ │ ├── crowdlab_accuracy_2.npy │ │ ├── crowdlab_accuracy_3.npy │ │ ├── crowdlab_accuracy_4.npy │ │ ├── crowdlab_model_accuracy_0.npy │ │ ├── crowdlab_model_accuracy_1.npy │ │ ├── crowdlab_model_accuracy_2.npy │ │ ├── crowdlab_model_accuracy_3.npy │ │ ├── crowdlab_model_accuracy_4.npy │ │ ├── entropy_accuracy_0.npy │ │ ├── entropy_accuracy_1.npy │ │ ├── entropy_accuracy_2.npy │ │ ├── entropy_accuracy_3.npy │ │ ├── entropy_accuracy_4.npy │ │ ├── entropy_model_accuracy_0.npy │ │ ├── entropy_model_accuracy_1.npy │ │ ├── entropy_model_accuracy_2.npy │ │ ├── entropy_model_accuracy_3.npy │ │ ├── entropy_model_accuracy_4.npy │ │ ├── good_random_accuracy_0.npy │ │ ├── good_random_accuracy_1.npy │ │ ├── good_random_accuracy_2.npy │ │ ├── good_random_accuracy_3.npy │ │ ├── good_random_accuracy_4.npy │ │ ├── good_random_model_accuracy_0.npy │ │ ├── good_random_model_accuracy_1.npy │ │ ├── good_random_model_accuracy_2.npy │ │ ├── good_random_model_accuracy_3.npy │ │ ├── good_random_model_accuracy_4.npy │ │ ├── random_accuracy_0.npy │ │ ├── random_accuracy_1.npy │ │ ├── random_accuracy_2.npy │ │ ├── random_accuracy_3.npy │ │ ├── random_accuracy_4.npy │ │ ├── random_model_accuracy_0.npy │ │ ├── random_model_accuracy_1.npy │ │ ├── random_model_accuracy_2.npy │ │ ├── random_model_accuracy_3.npy │ │ ├── random_model_accuracy_4.npy │ │ ├── uncertainty_accuracy_0.npy │ │ ├── uncertainty_accuracy_1.npy │ │ ├── uncertainty_accuracy_2.npy │ │ ├── uncertainty_accuracy_3.npy │ │ ├── uncertainty_accuracy_4.npy │ │ ├── uncertainty_model_accuracy_0.npy │ │ ├── uncertainty_model_accuracy_1.npy │ │ ├── uncertainty_model_accuracy_2.npy │ │ ├── uncertainty_model_accuracy_3.npy │ │ └── uncertainty_model_accuracy_4.npy │ └── mlp │ │ ├── crowdlab_accuracy_0.npy │ │ ├── crowdlab_accuracy_1.npy │ │ ├── crowdlab_accuracy_2.npy │ │ ├── crowdlab_accuracy_3.npy │ │ ├── crowdlab_accuracy_4.npy │ │ ├── crowdlab_ensemble_accuracy_0.npy │ │ ├── crowdlab_ensemble_accuracy_1.npy │ │ ├── crowdlab_ensemble_accuracy_2.npy │ │ ├── crowdlab_ensemble_accuracy_3.npy │ │ ├── crowdlab_ensemble_accuracy_4.npy │ │ ├── crowdlab_ensemble_model_accuracy_0.npy │ │ ├── crowdlab_ensemble_model_accuracy_1.npy │ │ ├── crowdlab_ensemble_model_accuracy_2.npy │ │ ├── crowdlab_ensemble_model_accuracy_3.npy │ │ ├── crowdlab_ensemble_model_accuracy_4.npy │ │ ├── crowdlab_model_accuracy_0.npy │ │ ├── crowdlab_model_accuracy_1.npy │ │ ├── crowdlab_model_accuracy_2.npy │ │ ├── crowdlab_model_accuracy_3.npy │ │ ├── crowdlab_model_accuracy_4.npy │ │ ├── disagreement_accuracy_0.npy │ │ ├── disagreement_accuracy_1.npy │ │ ├── disagreement_accuracy_2.npy │ │ ├── disagreement_accuracy_3.npy │ │ ├── disagreement_accuracy_4.npy │ │ ├── disagreement_model_accuracy_0.npy │ │ ├── disagreement_model_accuracy_1.npy │ │ ├── disagreement_model_accuracy_2.npy │ │ ├── disagreement_model_accuracy_3.npy │ │ ├── disagreement_model_accuracy_4.npy │ │ ├── entropy_accuracy_0.npy │ │ ├── entropy_accuracy_1.npy │ │ ├── entropy_accuracy_2.npy │ │ ├── entropy_accuracy_3.npy │ │ ├── entropy_accuracy_4.npy │ │ ├── entropy_model_accuracy_0.npy │ │ ├── entropy_model_accuracy_1.npy │ │ ├── entropy_model_accuracy_2.npy │ │ ├── entropy_model_accuracy_3.npy │ │ ├── entropy_model_accuracy_4.npy │ │ ├── good_random_accuracy_0.npy │ │ ├── good_random_accuracy_1.npy │ │ ├── good_random_accuracy_2.npy │ │ ├── good_random_accuracy_3.npy │ │ ├── good_random_accuracy_4.npy │ │ ├── good_random_model_accuracy_0.npy │ │ ├── good_random_model_accuracy_1.npy │ │ ├── good_random_model_accuracy_2.npy │ │ ├── good_random_model_accuracy_3.npy │ │ ├── good_random_model_accuracy_4.npy │ │ ├── good_random_num_added_0.npy │ │ ├── random_accuracy_0.npy │ │ ├── random_accuracy_1.npy │ │ ├── random_accuracy_2.npy │ │ ├── random_accuracy_3.npy │ │ ├── random_accuracy_4.npy │ │ ├── random_model_accuracy_0.npy │ │ ├── random_model_accuracy_1.npy │ │ ├── random_model_accuracy_2.npy │ │ ├── random_model_accuracy_3.npy │ │ ├── random_model_accuracy_4.npy │ │ ├── uncertainty_accuracy_0.npy │ │ ├── uncertainty_accuracy_1.npy │ │ ├── uncertainty_accuracy_2.npy │ │ ├── uncertainty_accuracy_3.npy │ │ ├── uncertainty_accuracy_4.npy │ │ ├── uncertainty_model_accuracy_0.npy │ │ ├── uncertainty_model_accuracy_1.npy │ │ ├── uncertainty_model_accuracy_2.npy │ │ ├── uncertainty_model_accuracy_3.npy │ │ └── uncertainty_model_accuracy_4.npy │ └── utils │ ├── active_learning_utils.py │ ├── methods_ET.py │ ├── methods_MLP.py │ └── model_training.py ├── benchmark_results ├── results.csv ├── results_complete.csv ├── results_uniform.csv └── results_worst.csv ├── cleanlab ├── .ci │ ├── README.md │ └── nblint.py ├── .coveragerc ├── .editorconfig ├── .github │ ├── ISSUE_TEMPLATE │ │ ├── 1-bug-report.md │ │ ├── 2-feature-request.md │ │ ├── 3-help.md │ │ └── 4-other.md │ └── workflows │ │ ├── ci.yml │ │ ├── gh-pages.yaml │ │ └── links.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DEVELOPMENT.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── cleanlab │ ├── __init__.py │ ├── benchmarking │ │ ├── __init__.py │ │ └── noise_generation.py │ ├── classification.py │ ├── count.py │ ├── dataset.py │ ├── experimental │ │ ├── README.md │ │ ├── __init__.py │ │ ├── cifar_cnn.py │ │ ├── coteaching.py │ │ ├── fasttext.py │ │ ├── huggingface_keras_classifier.py │ │ └── mnist_pytorch.py │ ├── filter.py │ ├── internal │ │ ├── __init__.py │ │ ├── label_quality_utils.py │ │ ├── latent_algebra.py │ │ ├── util.py │ │ └── validation.py │ ├── multiannotator.py │ ├── rank.py │ └── version.py ├── docs │ ├── .nojekyll │ ├── Makefile │ ├── README.md │ ├── make.bat │ ├── requirements.txt │ └── source │ │ ├── _static │ │ └── .gitkeep │ │ ├── _templates │ │ ├── brand.html │ │ ├── page.html │ │ ├── redirect-to-stable.html │ │ ├── versioning.html │ │ └── versioning.js │ │ ├── cleanlab │ │ ├── benchmarking │ │ │ ├── index.rst │ │ │ └── noise_generation.rst │ │ ├── classification.rst │ │ ├── count.rst │ │ ├── dataset.rst │ │ ├── experimental │ │ │ ├── cifar_cnn.rst │ │ │ ├── coteaching.rst │ │ │ ├── fasttext.rst │ │ │ ├── index.rst │ │ │ └── mnist_pytorch.rst │ │ ├── filter.rst │ │ ├── internal │ │ │ ├── index.rst │ │ │ ├── label_quality_utils.rst │ │ │ ├── latent_algebra.rst │ │ │ └── util.rst │ │ └── rank.rst │ │ ├── conf.py │ │ ├── index.rst │ │ ├── migrating │ │ └── migrate_v2.rst │ │ └── tutorials │ │ ├── .gitignore │ │ ├── audio.ipynb │ │ ├── dataset_health.ipynb │ │ ├── image.ipynb │ │ ├── indepth_overview.ipynb │ │ ├── index.rst │ │ ├── pred_probs_cross_val.rst │ │ ├── tabular.ipynb │ │ └── text.ipynb ├── pyproject.toml ├── requirements-dev.txt ├── setup.py └── tests │ ├── test_classification.py │ ├── test_dataset.py │ ├── test_filter_count.py │ ├── test_latent_algebra.py │ ├── test_model_pytorch_cnn.py │ ├── test_multiannotator.py │ ├── test_noise_generation.py │ ├── test_rank.py │ └── test_util.py ├── crowd-kit ├── .github │ ├── CODEOWNERS │ ├── ISSUE_TEMPLATE │ │ ├── bug_template.yml │ │ ├── config.yml │ │ ├── documentation_improvement_template.yml │ │ └── feature_request_template.yml │ ├── PULL_REQUEST_TEMPLATE.md │ └── workflows │ │ ├── release.yml │ │ └── tests.yml ├── .gitignore ├── AUTHORS ├── CHANGELOG.md ├── CITATION.cff ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── Pipfile ├── README.md ├── benchmarks │ ├── asv.conf.json │ ├── benchmarks │ │ ├── __init__.py │ │ ├── classification_benchmark.py │ │ ├── image_segmentation_benchmark.py │ │ ├── pairwise_benchmark.py │ │ └── text_aggregation_benchmark.py │ ├── pull_crowdkit.py │ └── run_benchmarks.py ├── crowdkit │ ├── __init__.py │ ├── aggregation │ │ ├── __init__.py │ │ ├── base │ │ │ ├── __init__.py │ │ │ └── py.typed │ │ ├── classification │ │ │ ├── __init__.py │ │ │ ├── dawid_skene.py │ │ │ ├── glad.py │ │ │ ├── gold_majority_vote.py │ │ │ ├── kos.py │ │ │ ├── m_msr.py │ │ │ ├── mace.py │ │ │ ├── majority_vote.py │ │ │ ├── py.typed │ │ │ ├── wawa.py │ │ │ └── zero_based_skill.py │ │ ├── embeddings │ │ │ ├── __init__.py │ │ │ ├── closest_to_average.py │ │ │ ├── hrrasa.py │ │ │ ├── py.typed │ │ │ └── rasa.py │ │ ├── image_segmentation │ │ │ ├── __init__.py │ │ │ ├── py.typed │ │ │ ├── segmentation_em.py │ │ │ ├── segmentation_majority_vote.py │ │ │ └── segmentation_rasa.py │ │ ├── pairwise │ │ │ ├── __init__.py │ │ │ ├── bradley_terry.py │ │ │ ├── noisy_bt.py │ │ │ └── py.typed │ │ ├── py.typed │ │ ├── texts │ │ │ ├── __init__.py │ │ │ ├── py.typed │ │ │ ├── rover.py │ │ │ ├── text_hrrasa.py │ │ │ ├── text_rasa.py │ │ │ └── text_summarization.py │ │ └── utils.py │ ├── datasets │ │ ├── __init__.py │ │ ├── _base.py │ │ ├── _loaders.py │ │ ├── load_dataset.py │ │ └── py.typed │ ├── metrics │ │ ├── __init__.py │ │ ├── data │ │ │ ├── __init__.py │ │ │ ├── _classification.py │ │ │ └── py.typed │ │ └── workers │ │ │ ├── __init__.py │ │ │ ├── accuracy_on_aggregates.py │ │ │ └── py.typed │ ├── postprocessing │ │ ├── __init__.py │ │ ├── entropy_threshold.py │ │ └── py.typed │ └── py.typed ├── docs │ ├── index.md │ ├── reference │ │ └── toc.yaml │ └── toc.yaml ├── examples │ ├── Readability-Pairwise.ipynb │ └── TlkAgg-Categorical.ipynb ├── misc │ ├── described_objects_crowdkit.py │ └── make_markdowns.py ├── pyproject.toml ├── setup.cfg ├── setup.py └── tests │ ├── __init__.py │ ├── aggregation │ ├── __init__.py │ ├── data_gold_mv.py │ ├── data_hrrasa.py │ ├── data_image.py │ ├── data_mmsr.py │ ├── data_mv.py │ ├── data_rover.py │ ├── data_wawa.py │ ├── data_zbs.py │ ├── test_bt_aggregation.py │ ├── test_classification_aggregation.py │ ├── test_classification_aggregation_edge_cases.py │ ├── test_ds_aggregation.py │ ├── test_glad_aggregation.py │ ├── test_image_aggregation.py │ ├── test_kos_aggregation.py │ ├── test_mace_aggregation.py │ ├── test_majority_vote.py │ ├── test_rover_aggregation.py │ └── test_text_aggregation.py │ ├── conftest.py │ ├── datasets │ ├── __init__.py │ └── test_datasets.py │ ├── metrics │ └── test_metrics.py │ ├── postprocessing │ └── test_entropy_threshold.py │ └── test_utils.py ├── paper.pdf ├── requirements.txt └── utils ├── benchmark_functions.py ├── cross_validation_autogluon.py └── eval.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/.gitignore -------------------------------------------------------------------------------- /0_create_labels_df.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/0_create_labels_df.ipynb -------------------------------------------------------------------------------- /0_preprocess_data.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/0_preprocess_data.ipynb -------------------------------------------------------------------------------- /1_xval_model_train.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/1_xval_model_train.ipynb -------------------------------------------------------------------------------- /1_xval_model_train_perfect_model.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/1_xval_model_train_perfect_model.ipynb -------------------------------------------------------------------------------- /2_benchmark.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/2_benchmark.ipynb -------------------------------------------------------------------------------- /3_benchmark_results_complete.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/3_benchmark_results_complete.ipynb -------------------------------------------------------------------------------- /3_benchmark_results_truelabels.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/3_benchmark_results_truelabels.ipynb -------------------------------------------------------------------------------- /3_benchmark_results_uniform.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/3_benchmark_results_uniform.ipynb -------------------------------------------------------------------------------- /3_benchmark_results_worst.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/3_benchmark_results_worst.ipynb -------------------------------------------------------------------------------- /3_benchmark_results_worst_npw.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/3_benchmark_results_worst_npw.ipynb -------------------------------------------------------------------------------- /3_side_benchmark_results.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/3_side_benchmark_results.ipynb -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/README.md -------------------------------------------------------------------------------- /active_learning_benchmarks/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/README.md -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/0_preprocess_data.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/0_preprocess_data.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/1_run_benchmark.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/1_run_benchmark.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/2_plot_results.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/2_plot_results.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_accuracy_20221118124631.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_accuracy_20221118124631.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_accuracy_20221119012724.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_accuracy_20221119012724.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_accuracy_20221120032011.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_accuracy_20221120032011.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_accuracy_20221120094140.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_accuracy_20221120094140.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_accuracy_20221120220738.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_accuracy_20221120220738.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_accuracy_20221124022113.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_accuracy_20221124022113.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_accuracy_20221124031623.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_accuracy_20221124031623.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_accuracy_20221125055941.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_accuracy_20221125055941.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_accuracy_20221125073747.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_accuracy_20221125073747.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_accuracy_20221125084702.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_accuracy_20221125084702.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_model_accuracy_20221124022113.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_model_accuracy_20221124022113.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_model_accuracy_20221124031623.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_model_accuracy_20221124031623.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_model_accuracy_20221125055941.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_model_accuracy_20221125055941.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_model_accuracy_20221125073747.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_model_accuracy_20221125073747.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_model_accuracy_20221125084702.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_ensemble_model_accuracy_20221125084702.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_model_accuracy_20221118124631.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_model_accuracy_20221118124631.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_model_accuracy_20221119012724.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_model_accuracy_20221119012724.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_model_accuracy_20221120032011.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_model_accuracy_20221120032011.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_model_accuracy_20221120094140.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_model_accuracy_20221120094140.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/crowdlab_model_accuracy_20221120220738.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/crowdlab_model_accuracy_20221120220738.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/disagreement_accuracy_20221124080356.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/disagreement_accuracy_20221124080356.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/disagreement_accuracy_20221124093429.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/disagreement_accuracy_20221124093429.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/disagreement_accuracy_20221124095748.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/disagreement_accuracy_20221124095748.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/disagreement_accuracy_20221125164119.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/disagreement_accuracy_20221125164119.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/disagreement_accuracy_20221125171752.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/disagreement_accuracy_20221125171752.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/disagreement_model_accuracy_20221124080356.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/disagreement_model_accuracy_20221124080356.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/disagreement_model_accuracy_20221124093429.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/disagreement_model_accuracy_20221124093429.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/disagreement_model_accuracy_20221124095748.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/disagreement_model_accuracy_20221124095748.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/disagreement_model_accuracy_20221125164119.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/disagreement_model_accuracy_20221125164119.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/disagreement_model_accuracy_20221125171752.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/disagreement_model_accuracy_20221125171752.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/entropy_accuracy_20221104235901.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/entropy_accuracy_20221104235901.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/entropy_accuracy_20221105030100.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/entropy_accuracy_20221105030100.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/entropy_accuracy_20221105055215.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/entropy_accuracy_20221105055215.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/entropy_accuracy_20221105085044.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/entropy_accuracy_20221105085044.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/entropy_accuracy_20221105115901.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/entropy_accuracy_20221105115901.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/entropy_model_accuracy_20221104235901.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/entropy_model_accuracy_20221104235901.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/entropy_model_accuracy_20221105030100.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/entropy_model_accuracy_20221105030100.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/entropy_model_accuracy_20221105055215.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/entropy_model_accuracy_20221105055215.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/entropy_model_accuracy_20221105085044.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/entropy_model_accuracy_20221105085044.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/entropy_model_accuracy_20221105115901.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/entropy_model_accuracy_20221105115901.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/good_random_accuracy_20221105193128.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/good_random_accuracy_20221105193128.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/good_random_accuracy_20221105223006.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/good_random_accuracy_20221105223006.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/good_random_accuracy_20221106012412.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/good_random_accuracy_20221106012412.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/good_random_accuracy_20221106042709.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/good_random_accuracy_20221106042709.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/good_random_accuracy_20221106071728.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/good_random_accuracy_20221106071728.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/good_random_model_accuracy_20221105193128.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/good_random_model_accuracy_20221105193128.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/good_random_model_accuracy_20221105223006.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/good_random_model_accuracy_20221105223006.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/good_random_model_accuracy_20221106012412.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/good_random_model_accuracy_20221106012412.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/good_random_model_accuracy_20221106042709.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/good_random_model_accuracy_20221106042709.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/good_random_model_accuracy_20221106071728.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/good_random_model_accuracy_20221106071728.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/random_accuracy_20221104060547.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/random_accuracy_20221104060547.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/random_accuracy_20221104094220.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/random_accuracy_20221104094220.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/random_accuracy_20221104132000.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/random_accuracy_20221104132000.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/random_accuracy_20221104160849.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/random_accuracy_20221104160849.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/random_accuracy_20221104184148.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/random_accuracy_20221104184148.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/random_model_accuracy_20221104060547.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/random_model_accuracy_20221104060547.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/random_model_accuracy_20221104094220.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/random_model_accuracy_20221104094220.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/random_model_accuracy_20221104132000.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/random_model_accuracy_20221104132000.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/random_model_accuracy_20221104160849.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/random_model_accuracy_20221104160849.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/random_model_accuracy_20221104184148.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/random_model_accuracy_20221104184148.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/uncertainty_accuracy_20221110081546.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/uncertainty_accuracy_20221110081546.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/uncertainty_accuracy_20221110111443.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/uncertainty_accuracy_20221110111443.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/uncertainty_accuracy_20221110141032.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/uncertainty_accuracy_20221110141032.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/uncertainty_accuracy_20221110162526.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/uncertainty_accuracy_20221110162526.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/uncertainty_accuracy_20221110182904.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/uncertainty_accuracy_20221110182904.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/uncertainty_model_accuracy_20221110081546.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/uncertainty_model_accuracy_20221110081546.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/uncertainty_model_accuracy_20221110111443.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/uncertainty_model_accuracy_20221110111443.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/uncertainty_model_accuracy_20221110141032.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/uncertainty_model_accuracy_20221110141032.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/uncertainty_model_accuracy_20221110162526.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/uncertainty_model_accuracy_20221110162526.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/results/uncertainty_model_accuracy_20221110182904.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/results/uncertainty_model_accuracy_20221110182904.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/utils/active_learning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/utils/active_learning.py -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/utils/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/utils/eval.py -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/utils/methods.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/utils/methods.py -------------------------------------------------------------------------------- /active_learning_benchmarks/cifar-10h/utils/model_training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/cifar-10h/utils/model_training.py -------------------------------------------------------------------------------- /active_learning_benchmarks/extra-benchmarks/single-annotator/0_create_data.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/extra-benchmarks/single-annotator/0_create_data.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/extra-benchmarks/single-annotator/1_run_benchmark.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/extra-benchmarks/single-annotator/1_run_benchmark.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/extra-benchmarks/single-annotator/2_plot_results.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/extra-benchmarks/single-annotator/2_plot_results.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/extra-benchmarks/single-annotator/utils/active_learning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/extra-benchmarks/single-annotator/utils/active_learning.py -------------------------------------------------------------------------------- /active_learning_benchmarks/extra-benchmarks/single-annotator/utils/model_training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/extra-benchmarks/single-annotator/utils/model_training.py -------------------------------------------------------------------------------- /active_learning_benchmarks/extra-benchmarks/single-vs-multiannotator/0_create_data.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/extra-benchmarks/single-vs-multiannotator/0_create_data.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/extra-benchmarks/single-vs-multiannotator/1_run_benchmark.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/extra-benchmarks/single-vs-multiannotator/1_run_benchmark.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/extra-benchmarks/single-vs-multiannotator/2_plot_results.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/extra-benchmarks/single-vs-multiannotator/2_plot_results.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/extra-benchmarks/single-vs-multiannotator/utils/active_learning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/extra-benchmarks/single-vs-multiannotator/utils/active_learning.py -------------------------------------------------------------------------------- /active_learning_benchmarks/extra-benchmarks/single-vs-multiannotator/utils/model_training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/extra-benchmarks/single-vs-multiannotator/utils/model_training.py -------------------------------------------------------------------------------- /active_learning_benchmarks/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/requirements.txt -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/0_create_data.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/0_create_data.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/1_run_benchmark_ET.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/1_run_benchmark_ET.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/1_run_benchmark_MLP.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/1_run_benchmark_MLP.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/2_plot_results_ET.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/2_plot_results_ET.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/2_plot_results_MLP.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/2_plot_results_MLP.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/active_label_cleaning_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/crowdlab_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/disagreement_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/entropy_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/good_random_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/random_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/et/uncertainty_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/active_label_cleaning_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_ensemble_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/crowdlab_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/disagreement_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/entropy_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/good_random_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/random_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/results/mlp/uncertainty_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/utils/active_learning_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/utils/active_learning_utils.py -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/utils/methods_ET.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/utils/methods_ET.py -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/utils/methods_MLP.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/utils/methods_MLP.py -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot-completely-labeled/utils/model_training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot-completely-labeled/utils/model_training.py -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/0_create_data.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/0_create_data.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/1_run_benchmark_ET.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/1_run_benchmark_ET.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/1_run_benchmark_MLP.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/1_run_benchmark_MLP.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/2_plot_results_ET.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/2_plot_results_ET.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/2_plot_results_MLP.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/2_plot_results_MLP.ipynb -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/crowdlab_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/crowdlab_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/crowdlab_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/crowdlab_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/crowdlab_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/crowdlab_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/crowdlab_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/crowdlab_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/crowdlab_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/crowdlab_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/crowdlab_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/crowdlab_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/crowdlab_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/crowdlab_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/crowdlab_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/crowdlab_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/crowdlab_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/crowdlab_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/crowdlab_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/crowdlab_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/entropy_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/entropy_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/entropy_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/entropy_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/entropy_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/entropy_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/entropy_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/entropy_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/entropy_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/entropy_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/entropy_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/entropy_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/entropy_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/entropy_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/entropy_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/entropy_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/entropy_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/entropy_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/entropy_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/entropy_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/good_random_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/good_random_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/good_random_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/good_random_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/good_random_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/good_random_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/good_random_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/good_random_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/good_random_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/good_random_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/good_random_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/good_random_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/good_random_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/good_random_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/good_random_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/good_random_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/good_random_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/good_random_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/good_random_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/good_random_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/random_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/random_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/random_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/random_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/random_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/random_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/random_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/random_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/random_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/random_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/random_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/random_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/random_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/random_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/random_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/random_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/random_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/random_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/random_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/random_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/uncertainty_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/uncertainty_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/uncertainty_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/uncertainty_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/uncertainty_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/uncertainty_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/uncertainty_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/uncertainty_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/uncertainty_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/uncertainty_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/uncertainty_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/uncertainty_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/uncertainty_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/uncertainty_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/uncertainty_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/uncertainty_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/uncertainty_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/uncertainty_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/et/uncertainty_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/et/uncertainty_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_ensemble_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/crowdlab_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/crowdlab_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/disagreement_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/disagreement_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/disagreement_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/disagreement_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/disagreement_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/disagreement_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/disagreement_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/disagreement_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/disagreement_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/disagreement_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/disagreement_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/disagreement_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/disagreement_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/disagreement_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/disagreement_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/disagreement_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/disagreement_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/disagreement_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/disagreement_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/disagreement_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/entropy_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/entropy_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/entropy_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/entropy_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/entropy_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/entropy_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/entropy_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/entropy_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/entropy_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/entropy_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/entropy_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/entropy_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/entropy_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/entropy_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/entropy_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/entropy_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/entropy_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/entropy_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/entropy_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/entropy_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/good_random_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/good_random_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/good_random_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/good_random_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/good_random_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/good_random_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/good_random_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/good_random_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/good_random_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/good_random_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/good_random_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/good_random_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/good_random_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/good_random_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/good_random_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/good_random_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/good_random_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/good_random_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/good_random_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/good_random_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/good_random_num_added_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/good_random_num_added_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/random_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/random_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/random_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/random_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/random_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/random_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/random_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/random_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/random_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/random_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/random_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/random_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/random_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/random_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/random_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/random_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/random_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/random_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/random_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/random_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/uncertainty_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/uncertainty_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/uncertainty_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/uncertainty_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/uncertainty_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/uncertainty_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/uncertainty_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/uncertainty_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/uncertainty_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/uncertainty_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/uncertainty_model_accuracy_0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/uncertainty_model_accuracy_0.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/uncertainty_model_accuracy_1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/uncertainty_model_accuracy_1.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/uncertainty_model_accuracy_2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/uncertainty_model_accuracy_2.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/uncertainty_model_accuracy_3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/uncertainty_model_accuracy_3.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/results/mlp/uncertainty_model_accuracy_4.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/results/mlp/uncertainty_model_accuracy_4.npy -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/utils/active_learning_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/utils/active_learning_utils.py -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/utils/methods_ET.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/utils/methods_ET.py -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/utils/methods_MLP.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/utils/methods_MLP.py -------------------------------------------------------------------------------- /active_learning_benchmarks/wall-robot/utils/model_training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/active_learning_benchmarks/wall-robot/utils/model_training.py -------------------------------------------------------------------------------- /benchmark_results/results.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/benchmark_results/results.csv -------------------------------------------------------------------------------- /benchmark_results/results_complete.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/benchmark_results/results_complete.csv -------------------------------------------------------------------------------- /benchmark_results/results_uniform.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/benchmark_results/results_uniform.csv -------------------------------------------------------------------------------- /benchmark_results/results_worst.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/benchmark_results/results_worst.csv -------------------------------------------------------------------------------- /cleanlab/.ci/README.md: -------------------------------------------------------------------------------- 1 | # .ci 2 | 3 | This directory contains support scripts for CI. 4 | -------------------------------------------------------------------------------- /cleanlab/.ci/nblint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/.ci/nblint.py -------------------------------------------------------------------------------- /cleanlab/.coveragerc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/.coveragerc -------------------------------------------------------------------------------- /cleanlab/.editorconfig: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/.editorconfig -------------------------------------------------------------------------------- /cleanlab/.github/ISSUE_TEMPLATE/1-bug-report.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/.github/ISSUE_TEMPLATE/1-bug-report.md -------------------------------------------------------------------------------- /cleanlab/.github/ISSUE_TEMPLATE/2-feature-request.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/.github/ISSUE_TEMPLATE/2-feature-request.md -------------------------------------------------------------------------------- /cleanlab/.github/ISSUE_TEMPLATE/3-help.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/.github/ISSUE_TEMPLATE/3-help.md -------------------------------------------------------------------------------- /cleanlab/.github/ISSUE_TEMPLATE/4-other.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/.github/ISSUE_TEMPLATE/4-other.md -------------------------------------------------------------------------------- /cleanlab/.github/workflows/ci.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/.github/workflows/ci.yml -------------------------------------------------------------------------------- /cleanlab/.github/workflows/gh-pages.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/.github/workflows/gh-pages.yaml -------------------------------------------------------------------------------- /cleanlab/.github/workflows/links.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/.github/workflows/links.yml -------------------------------------------------------------------------------- /cleanlab/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/.gitignore -------------------------------------------------------------------------------- /cleanlab/.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/.pre-commit-config.yaml -------------------------------------------------------------------------------- /cleanlab/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /cleanlab/CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/CONTRIBUTING.md -------------------------------------------------------------------------------- /cleanlab/DEVELOPMENT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/DEVELOPMENT.md -------------------------------------------------------------------------------- /cleanlab/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/LICENSE -------------------------------------------------------------------------------- /cleanlab/MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/MANIFEST.in -------------------------------------------------------------------------------- /cleanlab/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/README.md -------------------------------------------------------------------------------- /cleanlab/cleanlab/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/__init__.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/benchmarking/__init__.py: -------------------------------------------------------------------------------- 1 | from . import noise_generation 2 | -------------------------------------------------------------------------------- /cleanlab/cleanlab/benchmarking/noise_generation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/benchmarking/noise_generation.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/classification.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/count.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/count.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/dataset.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/experimental/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/experimental/README.md -------------------------------------------------------------------------------- /cleanlab/cleanlab/experimental/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cleanlab/cleanlab/experimental/cifar_cnn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/experimental/cifar_cnn.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/experimental/coteaching.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/experimental/coteaching.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/experimental/fasttext.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/experimental/fasttext.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/experimental/huggingface_keras_classifier.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/experimental/huggingface_keras_classifier.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/experimental/mnist_pytorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/experimental/mnist_pytorch.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/filter.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/internal/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cleanlab/cleanlab/internal/label_quality_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/internal/label_quality_utils.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/internal/latent_algebra.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/internal/latent_algebra.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/internal/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/internal/util.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/internal/validation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/internal/validation.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/multiannotator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/multiannotator.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/rank.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/rank.py -------------------------------------------------------------------------------- /cleanlab/cleanlab/version.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/cleanlab/version.py -------------------------------------------------------------------------------- /cleanlab/docs/.nojekyll: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /cleanlab/docs/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/Makefile -------------------------------------------------------------------------------- /cleanlab/docs/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/README.md -------------------------------------------------------------------------------- /cleanlab/docs/make.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/make.bat -------------------------------------------------------------------------------- /cleanlab/docs/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/requirements.txt -------------------------------------------------------------------------------- /cleanlab/docs/source/_static/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/_static/.gitkeep -------------------------------------------------------------------------------- /cleanlab/docs/source/_templates/brand.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/_templates/brand.html -------------------------------------------------------------------------------- /cleanlab/docs/source/_templates/page.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/_templates/page.html -------------------------------------------------------------------------------- /cleanlab/docs/source/_templates/redirect-to-stable.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/_templates/redirect-to-stable.html -------------------------------------------------------------------------------- /cleanlab/docs/source/_templates/versioning.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/_templates/versioning.html -------------------------------------------------------------------------------- /cleanlab/docs/source/_templates/versioning.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/_templates/versioning.js -------------------------------------------------------------------------------- /cleanlab/docs/source/cleanlab/benchmarking/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/cleanlab/benchmarking/index.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/cleanlab/benchmarking/noise_generation.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/cleanlab/benchmarking/noise_generation.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/cleanlab/classification.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/cleanlab/classification.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/cleanlab/count.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/cleanlab/count.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/cleanlab/dataset.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/cleanlab/dataset.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/cleanlab/experimental/cifar_cnn.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/cleanlab/experimental/cifar_cnn.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/cleanlab/experimental/coteaching.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/cleanlab/experimental/coteaching.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/cleanlab/experimental/fasttext.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/cleanlab/experimental/fasttext.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/cleanlab/experimental/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/cleanlab/experimental/index.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/cleanlab/experimental/mnist_pytorch.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/cleanlab/experimental/mnist_pytorch.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/cleanlab/filter.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/cleanlab/filter.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/cleanlab/internal/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/cleanlab/internal/index.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/cleanlab/internal/label_quality_utils.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/cleanlab/internal/label_quality_utils.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/cleanlab/internal/latent_algebra.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/cleanlab/internal/latent_algebra.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/cleanlab/internal/util.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/cleanlab/internal/util.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/cleanlab/rank.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/cleanlab/rank.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/conf.py -------------------------------------------------------------------------------- /cleanlab/docs/source/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/index.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/migrating/migrate_v2.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/migrating/migrate_v2.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/tutorials/.gitignore: -------------------------------------------------------------------------------- 1 | *.tar.gz 2 | pretrained_models/ 3 | spoken_digits/ 4 | -------------------------------------------------------------------------------- /cleanlab/docs/source/tutorials/audio.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/tutorials/audio.ipynb -------------------------------------------------------------------------------- /cleanlab/docs/source/tutorials/dataset_health.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/tutorials/dataset_health.ipynb -------------------------------------------------------------------------------- /cleanlab/docs/source/tutorials/image.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/tutorials/image.ipynb -------------------------------------------------------------------------------- /cleanlab/docs/source/tutorials/indepth_overview.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/tutorials/indepth_overview.ipynb -------------------------------------------------------------------------------- /cleanlab/docs/source/tutorials/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/tutorials/index.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/tutorials/pred_probs_cross_val.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/tutorials/pred_probs_cross_val.rst -------------------------------------------------------------------------------- /cleanlab/docs/source/tutorials/tabular.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/tutorials/tabular.ipynb -------------------------------------------------------------------------------- /cleanlab/docs/source/tutorials/text.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/docs/source/tutorials/text.ipynb -------------------------------------------------------------------------------- /cleanlab/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/pyproject.toml -------------------------------------------------------------------------------- /cleanlab/requirements-dev.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/requirements-dev.txt -------------------------------------------------------------------------------- /cleanlab/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/setup.py -------------------------------------------------------------------------------- /cleanlab/tests/test_classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/tests/test_classification.py -------------------------------------------------------------------------------- /cleanlab/tests/test_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/tests/test_dataset.py -------------------------------------------------------------------------------- /cleanlab/tests/test_filter_count.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/tests/test_filter_count.py -------------------------------------------------------------------------------- /cleanlab/tests/test_latent_algebra.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/tests/test_latent_algebra.py -------------------------------------------------------------------------------- /cleanlab/tests/test_model_pytorch_cnn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/tests/test_model_pytorch_cnn.py -------------------------------------------------------------------------------- /cleanlab/tests/test_multiannotator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/tests/test_multiannotator.py -------------------------------------------------------------------------------- /cleanlab/tests/test_noise_generation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/tests/test_noise_generation.py -------------------------------------------------------------------------------- /cleanlab/tests/test_rank.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/tests/test_rank.py -------------------------------------------------------------------------------- /cleanlab/tests/test_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/cleanlab/tests/test_util.py -------------------------------------------------------------------------------- /crowd-kit/.github/CODEOWNERS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/.github/CODEOWNERS -------------------------------------------------------------------------------- /crowd-kit/.github/ISSUE_TEMPLATE/bug_template.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/.github/ISSUE_TEMPLATE/bug_template.yml -------------------------------------------------------------------------------- /crowd-kit/.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/.github/ISSUE_TEMPLATE/config.yml -------------------------------------------------------------------------------- /crowd-kit/.github/ISSUE_TEMPLATE/documentation_improvement_template.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/.github/ISSUE_TEMPLATE/documentation_improvement_template.yml -------------------------------------------------------------------------------- /crowd-kit/.github/ISSUE_TEMPLATE/feature_request_template.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/.github/ISSUE_TEMPLATE/feature_request_template.yml -------------------------------------------------------------------------------- /crowd-kit/.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/.github/PULL_REQUEST_TEMPLATE.md -------------------------------------------------------------------------------- /crowd-kit/.github/workflows/release.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/.github/workflows/release.yml -------------------------------------------------------------------------------- /crowd-kit/.github/workflows/tests.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/.github/workflows/tests.yml -------------------------------------------------------------------------------- /crowd-kit/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/.gitignore -------------------------------------------------------------------------------- /crowd-kit/AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/AUTHORS -------------------------------------------------------------------------------- /crowd-kit/CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/CHANGELOG.md -------------------------------------------------------------------------------- /crowd-kit/CITATION.cff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/CITATION.cff -------------------------------------------------------------------------------- /crowd-kit/CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/CONTRIBUTING.md -------------------------------------------------------------------------------- /crowd-kit/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/LICENSE -------------------------------------------------------------------------------- /crowd-kit/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE AUTHORS CONTRIBUTING.md 2 | -------------------------------------------------------------------------------- /crowd-kit/Pipfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/Pipfile -------------------------------------------------------------------------------- /crowd-kit/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/README.md -------------------------------------------------------------------------------- /crowd-kit/benchmarks/asv.conf.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/benchmarks/asv.conf.json -------------------------------------------------------------------------------- /crowd-kit/benchmarks/benchmarks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /crowd-kit/benchmarks/benchmarks/classification_benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/benchmarks/benchmarks/classification_benchmark.py -------------------------------------------------------------------------------- /crowd-kit/benchmarks/benchmarks/image_segmentation_benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/benchmarks/benchmarks/image_segmentation_benchmark.py -------------------------------------------------------------------------------- /crowd-kit/benchmarks/benchmarks/pairwise_benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/benchmarks/benchmarks/pairwise_benchmark.py -------------------------------------------------------------------------------- /crowd-kit/benchmarks/benchmarks/text_aggregation_benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/benchmarks/benchmarks/text_aggregation_benchmark.py -------------------------------------------------------------------------------- /crowd-kit/benchmarks/pull_crowdkit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/benchmarks/pull_crowdkit.py -------------------------------------------------------------------------------- /crowd-kit/benchmarks/run_benchmarks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/benchmarks/run_benchmarks.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/__init__.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/__init__.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/base/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/base/__init__.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/base/py.typed: -------------------------------------------------------------------------------- 1 | inline 2 | -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/classification/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/classification/__init__.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/classification/dawid_skene.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/classification/dawid_skene.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/classification/glad.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/classification/glad.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/classification/gold_majority_vote.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/classification/gold_majority_vote.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/classification/kos.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/classification/kos.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/classification/m_msr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/classification/m_msr.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/classification/mace.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/classification/mace.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/classification/majority_vote.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/classification/majority_vote.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/classification/py.typed: -------------------------------------------------------------------------------- 1 | inline 2 | -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/classification/wawa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/classification/wawa.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/classification/zero_based_skill.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/classification/zero_based_skill.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/embeddings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/embeddings/__init__.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/embeddings/closest_to_average.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/embeddings/closest_to_average.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/embeddings/hrrasa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/embeddings/hrrasa.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/embeddings/py.typed: -------------------------------------------------------------------------------- 1 | inline 2 | -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/embeddings/rasa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/embeddings/rasa.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/image_segmentation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/image_segmentation/__init__.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/image_segmentation/py.typed: -------------------------------------------------------------------------------- 1 | inline 2 | -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/image_segmentation/segmentation_em.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/image_segmentation/segmentation_em.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/image_segmentation/segmentation_majority_vote.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/image_segmentation/segmentation_majority_vote.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/image_segmentation/segmentation_rasa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/image_segmentation/segmentation_rasa.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/pairwise/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/pairwise/__init__.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/pairwise/bradley_terry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/pairwise/bradley_terry.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/pairwise/noisy_bt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/pairwise/noisy_bt.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/pairwise/py.typed: -------------------------------------------------------------------------------- 1 | inline 2 | -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/py.typed: -------------------------------------------------------------------------------- 1 | inline 2 | -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/texts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/texts/__init__.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/texts/py.typed: -------------------------------------------------------------------------------- 1 | inline 2 | -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/texts/rover.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/texts/rover.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/texts/text_hrrasa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/texts/text_hrrasa.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/texts/text_rasa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/texts/text_rasa.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/texts/text_summarization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/texts/text_summarization.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/aggregation/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/aggregation/utils.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/datasets/__init__.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/datasets/_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/datasets/_base.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/datasets/_loaders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/datasets/_loaders.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/datasets/load_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/datasets/load_dataset.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/datasets/py.typed: -------------------------------------------------------------------------------- 1 | inline 2 | -------------------------------------------------------------------------------- /crowd-kit/crowdkit/metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/metrics/__init__.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/metrics/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/metrics/data/__init__.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/metrics/data/_classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/metrics/data/_classification.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/metrics/data/py.typed: -------------------------------------------------------------------------------- 1 | inline 2 | -------------------------------------------------------------------------------- /crowd-kit/crowdkit/metrics/workers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/metrics/workers/__init__.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/metrics/workers/accuracy_on_aggregates.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/metrics/workers/accuracy_on_aggregates.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/metrics/workers/py.typed: -------------------------------------------------------------------------------- 1 | inline 2 | -------------------------------------------------------------------------------- /crowd-kit/crowdkit/postprocessing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/postprocessing/__init__.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/postprocessing/entropy_threshold.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/crowdkit/postprocessing/entropy_threshold.py -------------------------------------------------------------------------------- /crowd-kit/crowdkit/postprocessing/py.typed: -------------------------------------------------------------------------------- 1 | inline 2 | -------------------------------------------------------------------------------- /crowd-kit/crowdkit/py.typed: -------------------------------------------------------------------------------- 1 | inline 2 | -------------------------------------------------------------------------------- /crowd-kit/docs/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/docs/index.md -------------------------------------------------------------------------------- /crowd-kit/docs/reference/toc.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/docs/reference/toc.yaml -------------------------------------------------------------------------------- /crowd-kit/docs/toc.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/docs/toc.yaml -------------------------------------------------------------------------------- /crowd-kit/examples/Readability-Pairwise.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/examples/Readability-Pairwise.ipynb -------------------------------------------------------------------------------- /crowd-kit/examples/TlkAgg-Categorical.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/examples/TlkAgg-Categorical.ipynb -------------------------------------------------------------------------------- /crowd-kit/misc/described_objects_crowdkit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/misc/described_objects_crowdkit.py -------------------------------------------------------------------------------- /crowd-kit/misc/make_markdowns.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/misc/make_markdowns.py -------------------------------------------------------------------------------- /crowd-kit/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/pyproject.toml -------------------------------------------------------------------------------- /crowd-kit/setup.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/setup.cfg -------------------------------------------------------------------------------- /crowd-kit/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/setup.py -------------------------------------------------------------------------------- /crowd-kit/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/data_gold_mv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/data_gold_mv.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/data_hrrasa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/data_hrrasa.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/data_image.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/data_image.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/data_mmsr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/data_mmsr.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/data_mv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/data_mv.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/data_rover.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/data_rover.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/data_wawa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/data_wawa.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/data_zbs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/data_zbs.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/test_bt_aggregation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/test_bt_aggregation.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/test_classification_aggregation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/test_classification_aggregation.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/test_classification_aggregation_edge_cases.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/test_classification_aggregation_edge_cases.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/test_ds_aggregation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/test_ds_aggregation.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/test_glad_aggregation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/test_glad_aggregation.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/test_image_aggregation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/test_image_aggregation.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/test_kos_aggregation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/test_kos_aggregation.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/test_mace_aggregation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/test_mace_aggregation.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/test_majority_vote.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/test_majority_vote.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/test_rover_aggregation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/test_rover_aggregation.py -------------------------------------------------------------------------------- /crowd-kit/tests/aggregation/test_text_aggregation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/aggregation/test_text_aggregation.py -------------------------------------------------------------------------------- /crowd-kit/tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/conftest.py -------------------------------------------------------------------------------- /crowd-kit/tests/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /crowd-kit/tests/datasets/test_datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/datasets/test_datasets.py -------------------------------------------------------------------------------- /crowd-kit/tests/metrics/test_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/metrics/test_metrics.py -------------------------------------------------------------------------------- /crowd-kit/tests/postprocessing/test_entropy_threshold.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/postprocessing/test_entropy_threshold.py -------------------------------------------------------------------------------- /crowd-kit/tests/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/crowd-kit/tests/test_utils.py -------------------------------------------------------------------------------- /paper.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/paper.pdf -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/requirements.txt -------------------------------------------------------------------------------- /utils/benchmark_functions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/utils/benchmark_functions.py -------------------------------------------------------------------------------- /utils/cross_validation_autogluon.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/utils/cross_validation_autogluon.py -------------------------------------------------------------------------------- /utils/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cleanlab/multiannotator-benchmarks/HEAD/utils/eval.py --------------------------------------------------------------------------------