├── .DS_Store ├── .dockerignore ├── .gitignore ├── .gitmodules ├── Dockerfile.rf100.benchmark ├── Dockerfile.rf100.download ├── GLIP-benchmark ├── glip_Swin_T_O365_GoldG.yaml ├── override_config.py └── train.sh ├── LICENSE.md ├── README.md ├── datasets_links.txt ├── datasets_links_640.txt ├── doc └── images │ ├── comparison-yolos-bar.png │ ├── comparison-yolos-box.png │ └── grid │ ├── 4-fold-defect.jpg │ ├── abdomen-mri.jpg │ ├── acl-x-ray.jpg │ ├── activity-diagrams-qdobr.jpg │ ├── aerial-cows.jpg │ ├── aerial-pool.jpg │ ├── aerial-spheres.jpg │ ├── animals-ij5d2.jpg │ ├── apex-videogame.jpg │ ├── apples-fvpl5.jpg │ ├── aquarium-qlnqy.jpg │ ├── asbestos.jpg │ ├── avatar-recognition-nuexe.jpg │ ├── axial-mri.jpg │ ├── bacteria-ptywi.jpg │ ├── bccd-ouzjz.jpg │ ├── bees-jt5in.jpg │ ├── bone-fracture-7fylg.jpg │ ├── brain-tumor-m2pbp.jpg │ ├── cable-damage.jpg │ ├── cables-nl42k.jpg │ ├── cavity-rs0uf.jpg │ ├── cell-towers.jpg │ ├── cells-uyemf.jpg │ ├── chess-pieces-mjzgj.jpg │ ├── circuit-elements.jpg │ ├── circuit-voltages.jpg │ ├── cloud-types.jpg │ ├── coins-1apki.jpg │ ├── construction-safety-gsnvb.jpg │ ├── coral-lwptl.jpg │ ├── corrosion-bi3q3.jpg │ ├── cotton-20xz5.jpg │ ├── cotton-plant-disease.jpg │ ├── csgo-videogame.jpg │ ├── currency-v4f8j.jpg │ ├── digits-t2eg6.jpg │ ├── document-parts.jpg │ ├── excavators-czvg9.jpg │ ├── farcry6-videogame.jpg │ ├── fish-market-ggjso.jpg │ ├── flir-camera-objects.jpg │ ├── furniture-ngpea.jpg │ ├── gauge-u2lwv.jpg │ ├── grass-weeds.jpg │ ├── gynecology-mri.jpg │ ├── halo-infinite-angel-videogame.jpg │ ├── hand-gestures-jps7z.jpg │ ├── insects-mytwu.jpg │ ├── leaf-disease-nsdsr.jpg │ ├── lettuce-pallets.jpg │ ├── liver-disease.jpg │ ├── marbles.jpg │ ├── mask-wearing-608pr.jpg │ ├── mitosis-gjs3g.jpg │ ├── number-ops.jpg │ ├── paper-parts.jpg │ ├── paragraphs-co84b.jpg │ ├── parasites-1s07h.jpg │ ├── peanuts-sd4kf.jpg │ ├── peixos-fish.jpg │ ├── people-in-paintings.jpg │ ├── pests-2xlvx.jpg │ ├── phages.jpg │ ├── pills-sxdht.jpg │ ├── poker-cards-cxcvz.jpg │ ├── printed-circuit-board.jpg │ ├── radio-signal.jpg │ ├── road-signs-6ih4y.jpg │ ├── road-traffic.jpg │ ├── robomasters-285km.jpg │ ├── secondary-chains.jpg │ ├── sedimentary-features-9eosf.jpg │ ├── shark-teeth-5atku.jpg │ ├── sign-language-sokdr.jpg │ ├── signatures-xc8up.jpg │ ├── smoke-uvylj.jpg │ ├── soccer-players-5fuqs.jpg │ ├── soda-bottles.jpg │ ├── solar-panels-taxvb.jpg │ ├── stomata-cells.jpg │ ├── street-work.jpg │ ├── tabular-data-wf9uh.jpg │ ├── team-fight-tactics.jpg │ ├── thermal-cheetah-my4dp.jpg │ ├── thermal-dogs-and-people-x6ejw.jpg │ ├── trail-camera.jpg │ ├── truck-movement.jpg │ ├── tweeter-posts.jpg │ ├── tweeter-profile.jpg │ ├── underwater-objects-5v7p8.jpg │ ├── underwater-pipes-4ng4t.jpg │ ├── uno-deck.jpg │ ├── valentines-chocolate.jpg │ ├── vehicles-q0x2v.jpg │ ├── wall-damage.jpg │ ├── washroom-rf1fa.jpg │ ├── weed-crop-aerial.jpg │ ├── wine-labels.jpg │ └── x-ray-rheumatology.jpg ├── metadata ├── bbox_stats.csv ├── categories.csv ├── datasets_stats.csv ├── labeling_hours.csv ├── labels_names.json ├── original_datasets.csv └── results_glip.csv ├── notebooks ├── analysis.ipynb ├── kmeans.ipynb ├── roboflow-100-pytorch.ipynb └── zero_shot_analysis.ipynb ├── results.csv ├── scripts ├── convert_table_from_old_flo_paper_to_csv.py ├── create_rf100-coco-eval.sh ├── download_dataset.py ├── download_datasets.sh ├── get_labels_names.py ├── image │ ├── cluster.py │ ├── encode.py │ ├── kmeans.py │ └── pca.py ├── make_dataset_table_with_images.py ├── make_montages.sh ├── make_results_from_evals.py ├── make_results_table.py ├── parse_categories.py ├── parse_dataset_link.py ├── reduce_dims.py └── split_for_webapp.py ├── yolov5-benchmark ├── final_eval.txt ├── parse_eval.py └── train.sh ├── yolov7-benchmark ├── final_eval.txt ├── parse_eval.py ├── requirements.txt └── train.sh └── yolov8-benchmark ├── final_eval.txt └── train.sh /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/.DS_Store -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | rf100/ 2 | runs/ 3 | runs_first_run/ 4 | **/*.pth 5 | **/*.pt -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | montages/ 3 | paper/*= 4 | *.zip 5 | *.pk 6 | *.pkl 7 | temp/ 8 | rf100/ 9 | runs/ 10 | yolov5-benchmark/attributes.txt 11 | yolov5-benchmark/loc.txt 12 | venv/ 13 | .env 14 | attributes.txt 15 | loc.txt 16 | yolos-benchmark/detr/ 17 | mAP_results.txt 18 | yolos-benchmark/yolos_res.txt 19 | *.pt 20 | # Byte-compiled / optimized / DLL files 21 | __pycache__/ 22 | *.py[cod] 23 | *$py.class 24 | 25 | # C extensions 26 | *.so 27 | 28 | # Distribution / packaging 29 | .Python 30 | build/ 31 | develop-eggs/ 32 | dist/ 33 | downloads/ 34 | eggs/ 35 | .eggs/ 36 | lib/ 37 | lib64/ 38 | parts/ 39 | sdist/ 40 | var/ 41 | wheels/ 42 | share/python-wheels/ 43 | *.egg-info/ 44 | .installed.cfg 45 | *.egg 46 | MANIFEST 47 | 48 | # PyInstaller 49 | # Usually these files are written by a python script from a template 50 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 51 | *.manifest 52 | *.spec 53 | 54 | # Installer logs 55 | pip-log.txt 56 | pip-delete-this-directory.txt 57 | 58 | # Unit test / coverage reports 59 | htmlcov/ 60 | .tox/ 61 | .nox/ 62 | .coverage 63 | .coverage.* 64 | .cache 65 | nosetests.xml 66 | coverage.xml 67 | *.cover 68 | *.py,cover 69 | .hypothesis/ 70 | .pytest_cache/ 71 | cover/ 72 | 73 | # Translations 74 | *.mo 75 | *.pot 76 | 77 | # Django stuff: 78 | *.log 79 | local_settings.py 80 | db.sqlite3 81 | db.sqlite3-journal 82 | 83 | # Flask stuff: 84 | instance/ 85 | .webassets-cache 86 | 87 | # Scrapy stuff: 88 | .scrapy 89 | 90 | # Sphinx documentation 91 | docs/_build/ 92 | 93 | # PyBuilder 94 | .pybuilder/ 95 | target/ 96 | 97 | # Jupyter Notebook 98 | .ipynb_checkpoints 99 | 100 | # IPython 101 | profile_default/ 102 | ipython_config.py 103 | 104 | # pyenv 105 | # For a library or package, you might want to ignore these files since the code is 106 | # intended to run in multiple environments; otherwise, check them in: 107 | # .python-version 108 | 109 | # pipenv 110 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 111 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 112 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 113 | # install all needed dependencies. 114 | #Pipfile.lock 115 | 116 | # poetry 117 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 118 | # This is especially recommended for binary packages to ensure reproducibility, and is more 119 | # commonly ignored for libraries. 120 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 121 | #poetry.lock 122 | 123 | # pdm 124 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 125 | #pdm.lock 126 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 127 | # in version control. 128 | # https://pdm.fming.dev/#use-with-ide 129 | .pdm.toml 130 | 131 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 132 | __pypackages__/ 133 | 134 | # Celery stuff 135 | celerybeat-schedule 136 | celerybeat.pid 137 | 138 | # SageMath parsed files 139 | *.sage.py 140 | 141 | # Environments 142 | .env 143 | .venv 144 | env/ 145 | venv/ 146 | ENV/ 147 | env.bak/ 148 | venv.bak/ 149 | 150 | # Spyder project settings 151 | .spyderproject 152 | .spyproject 153 | 154 | # Rope project settings 155 | .ropeproject 156 | 157 | # mkdocs documentation 158 | /site 159 | 160 | # mypy 161 | .mypy_cache/ 162 | .dmypy.json 163 | dmypy.json 164 | 165 | # Pyre type checker 166 | .pyre/ 167 | 168 | # pytype static type analyzer 169 | .pytype/ 170 | 171 | # Cython debug symbols 172 | cython_debug/ 173 | 174 | # PyCharm 175 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 176 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 177 | # and can be added to the global gitignore or merged into this file. For a more nuclear 178 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 179 | #.idea/ -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "GLIP-benchmark/GLIP"] 2 | path = GLIP-benchmark/GLIP 3 | url = https://github.com/microsoft/GLIP 4 | [submodule "yolov7-benchmark/yolov7"] 5 | path = yolov7-benchmark/yolov7 6 | url = https://github.com/WongKinYiu/yolov7 7 | [submodule "yolov5-benchmark/yolov5"] 8 | path = yolov5-benchmark/yolov5 9 | url = https://github.com/ultralytics/yolov5 -------------------------------------------------------------------------------- /Dockerfile.rf100.benchmark: -------------------------------------------------------------------------------- 1 | FROM nvcr.io/nvidia/pytorch:22.08-py3 2 | RUN apt update && \ 3 | apt install -y zip htop screen libgl1-mesa-glx 4 | RUN pip install --no-cache-dir seaborn thop 5 | RUN pip install --no-cache-dir roboflow 6 | # https://stackoverflow.com/questions/72706073/attributeerror-partially-initialized-module-cv2-has-no-attribute-gapi-wip-gs 7 | RUN pip install --no-cache-dir opencv-contrib-python==4.5.5.62 yacs transformers timm einops shapely timm yacs tensorboardX ftfy prettytable pymongo albumentations 8 | COPY . . 9 | RUN cd /workspace/GLIP-benchmark/GLIP && python setup.py build develop --user 10 | ENTRYPOINT [ "/bin/bash" ] 11 | -------------------------------------------------------------------------------- /Dockerfile.rf100.download: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim 2 | RUN pip install roboflow 3 | WORKDIR /workspace 4 | COPY ./scripts /workspace/scripts/ 5 | COPY ./datasets_links_640.txt /workspace/datasets_links_640.txt 6 | ENTRYPOINT [ "./scripts/download_datasets.sh" ] -------------------------------------------------------------------------------- /GLIP-benchmark/glip_Swin_T_O365_GoldG.yaml: -------------------------------------------------------------------------------- 1 | MODEL: 2 | META_ARCHITECTURE: "GeneralizedVLRCNN" 3 | WEIGHT: "swin_tiny_patch4_window7_224.pth" 4 | RPN_ONLY: True 5 | RPN_ARCHITECTURE: "VLDYHEAD" 6 | 7 | BACKBONE: 8 | CONV_BODY: "SWINT-FPN-RETINANET" 9 | OUT_CHANNELS: 256 10 | FREEZE_CONV_BODY_AT: -1 11 | 12 | LANGUAGE_BACKBONE: 13 | FREEZE: False 14 | MODEL_TYPE: "bert-base-uncased" # "roberta-base", "clip" 15 | MASK_SPECIAL: False 16 | 17 | RPN: 18 | USE_FPN: True 19 | ANCHOR_SIZES: (64, 128, 256, 512, 1024) 20 | ANCHOR_STRIDE: (8, 16, 32, 64, 128) 21 | ASPECT_RATIOS: (1.0,) 22 | SCALES_PER_OCTAVE: 1 23 | 24 | DYHEAD: 25 | CHANNELS: 256 26 | NUM_CONVS: 6 27 | USE_GN: True 28 | USE_DYRELU: True 29 | USE_DFCONV: True 30 | USE_DYFUSE: True 31 | TOPK: 9 # topk for selecting candidate positive samples from each level 32 | SCORE_AGG: "MEAN" 33 | LOG_SCALE: 0.0 34 | 35 | FUSE_CONFIG: 36 | EARLY_FUSE_ON: True 37 | TYPE: "MHA-B" 38 | USE_CLASSIFICATION_LOSS: False 39 | USE_TOKEN_LOSS: False 40 | USE_CONTRASTIVE_ALIGN_LOSS: False 41 | CONTRASTIVE_HIDDEN_DIM: 64 42 | USE_DOT_PRODUCT_TOKEN_LOSS: True 43 | USE_FUSED_FEATURES_DOT_PRODUCT: True 44 | USE_LAYER_SCALE: True 45 | CLAMP_MIN_FOR_UNDERFLOW: True 46 | CLAMP_MAX_FOR_OVERFLOW: True 47 | CLAMP_BERTATTN_MIN_FOR_UNDERFLOW: True 48 | CLAMP_BERTATTN_MAX_FOR_OVERFLOW: True 49 | CLAMP_DOT_PRODUCT: True 50 | 51 | USE_CHECKPOINT: True 52 | 53 | TEST: 54 | DURING_TRAINING: False 55 | IMS_PER_BATCH: 64 56 | 57 | # use for grounding model 58 | DATASETS: 59 | TRAIN: ("object365_dt_train", "mixed_train_no_coco", "flickr30k_train", ) 60 | TEST: ("coco_2017_val", ) 61 | DISABLE_SHUFFLE: False 62 | ADD_DET_PROMPT: False 63 | RANDOM_SAMPLE_NEG: 85 64 | CONTROL_PROB: (0.0, 0.0, 0.5, 0.0) 65 | 66 | SEPARATION_TOKENS: ". " 67 | 68 | INPUT: 69 | PIXEL_MEAN: [ 103.530, 116.280, 123.675 ] 70 | PIXEL_STD: [ 57.375, 57.120, 58.395 ] 71 | MIN_SIZE_TRAIN: 800 72 | MAX_SIZE_TRAIN: 1333 73 | MIN_SIZE_TEST: 800 74 | MAX_SIZE_TEST: 1333 75 | 76 | AUGMENT: 77 | MULT_MIN_SIZE_TRAIN: (480,560,640,720,800) 78 | 79 | DATALOADER: 80 | SIZE_DIVISIBILITY: 32 81 | 82 | SOLVER: 83 | OPTIMIZER: ADAMW 84 | BASE_LR: 0.0001 85 | LANG_LR: 0.00001 86 | WEIGHT_DECAY: 0.0001 87 | STEPS: (0.67, 0.89) 88 | MAX_EPOCH: 30 89 | IMS_PER_BATCH: 64 90 | WARMUP_ITERS: 2000 91 | WARMUP_FACTOR: 0.001 92 | USE_AMP: True 93 | MODEL_EMA: 0.999 94 | FIND_UNUSED_PARAMETERS: False 95 | 96 | CLIP_GRADIENTS: 97 | ENABLED: True 98 | CLIP_TYPE: "full_model" 99 | CLIP_VALUE: 1.0 100 | NORM_TYPE: 2.0 -------------------------------------------------------------------------------- /GLIP-benchmark/override_config.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | import json 3 | import argparse 4 | 5 | # This file contains helper functions to setup the GLIP repo for Evaluation 6 | 7 | 8 | def write_yaml(data): 9 | filename = "dataset.yaml" 10 | 11 | file_data = dict( 12 | DATALOADER=dict(ASPECT_RATIO_GROUPING=False, SIZE_DIVISIBILITY=32), 13 | DATASETS=dict( 14 | GENERAL_COPY=16, 15 | OVERRIDE_CATEGORY=data, 16 | REGISTER=dict( 17 | test=dict( 18 | ann_file="odinw/dataset/test/_annotations.coco.json", 19 | img_dir="odinw/dataset/test", 20 | ), 21 | train=dict( 22 | ann_file="odinw/dataset/train/_annotations.coco.json", 23 | img_dir="odinw/dataset/train", 24 | ), 25 | val=dict( 26 | ann_file="odinw/dataset/valid/_annotations.coco.json", 27 | img_dir="odinw/dataset/valid", 28 | ), 29 | ), 30 | TEST='("val",)', 31 | TRAIN='("train",)', 32 | ), 33 | INPUT=dict( 34 | MAX_SIZE_TEST=1333, 35 | MAX_SIZE_TRAIN=1333, 36 | MIN_SIZE_TEST=800, 37 | MIN_SIZE_TRAIN=800, 38 | ), 39 | MODEL=dict( 40 | ATSS=dict(NUM_CLASSES=8), 41 | DYHEAD=dict(NUM_CLASSES=8), 42 | FCOS=dict(NUM_CLASSES=8), 43 | ROI_BOX_HEAD=dict(NUM_CLASSES=8), 44 | ), 45 | SOLVER=dict(CHECKPOINT_PERIOD=100, MAX_EPOCH=12, WARMUP_ITERS=0), 46 | TEST=dict(IMS_PER_BATCH=8), 47 | ) 48 | 49 | with open(filename, "w") as outfile: 50 | yaml.dump(file_data, outfile, default_flow_style=False) 51 | 52 | 53 | def delete_first_entry(): 54 | print("Deleting first JSON entry...") 55 | folders = ["dataset/test/", "dataset/train/", "dataset/valid/"] 56 | 57 | for folder in folders: 58 | with open(folder + "_annotations.coco.json") as f: 59 | data = json.load(f) 60 | 61 | for entry in list(data["categories"]): 62 | if entry["id"] == 0: 63 | data["categories"].remove(entry) 64 | 65 | with open(folder + "_annotations.coco.json", "w") as f: 66 | json.dump(data, f) 67 | 68 | 69 | def gen_data(): 70 | f = open("dataset/test/_annotations.coco.json") 71 | data = json.load(f) 72 | 73 | data_list = [] 74 | g = open("../custom_prompts.json") 75 | custom_data = json.load(g) 76 | 77 | for i in data["categories"]: 78 | # data_list.append(i) <-- use this line for normal evaluation 79 | dataset_name = i["supercategory"] 80 | break 81 | 82 | for dataset in custom_data: 83 | if dataset["dataset_name"] == dataset_name: 84 | data_list = dataset["classes"] 85 | 86 | if len(data_list) == 0: 87 | data_list = data["categories"] 88 | 89 | return data_list 90 | 91 | 92 | def main(): 93 | 94 | delete_first_entry() # remove the zeroth entry in JSON file 95 | data = gen_data() # generate the dictionary data to pass to yaml file 96 | write_yaml(str(data)) # generate yaml file for evaluation 97 | 98 | 99 | if __name__ == "__main__": 100 | main() 101 | -------------------------------------------------------------------------------- /GLIP-benchmark/train.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | dir=$(pwd)/runs/glip 5 | echo $dir 6 | datasets=$dir/rf100 7 | 8 | if [ ! -d $datasets ] ; then 9 | $(pwd)/scripts/download_datasets.sh -l $datasets -f coco 10 | fi 11 | 12 | if [ ! -f "$dir/final_eval.txt" ] ; then 13 | touch "$dir/final_eval.txt" 14 | fi 15 | 16 | 17 | cd $(pwd)/GLIP-benchmark/ 18 | 19 | if [ ! -f "$dir/glip_tiny_model_o365_goldg_cc_sbu.pth" ] ; then 20 | wget -P $dir https://github.com/microsoft/GLIP/blob/main/configs/pretrain/glip_Swin_T_O365_GoldG.yaml 21 | fi 22 | 23 | cd GLIP/ 24 | 25 | python setup.py build develop --user 26 | 27 | mkdir -p ./DATASET/coco/annotations 28 | 29 | if [ -L "./DATASET/coco/val2017/test" ] ; then 30 | rm ./DATASET/coco/val2017/test 31 | rm ./DATASET/coco/annotations/instances_val2017.json 32 | fi 33 | 34 | for dataset in $(ls $datasets) 35 | do 36 | dataset=$datasets/$dataset 37 | echo "Training on $dataset" 38 | if [ ! -d "$dataset/results" ] ; 39 | then 40 | ln -s $dataset/test ./DATASET/coco/val2017 41 | ln -s $dataset/test/_annotations.coco.json ./DATASET/coco/annotations/instances_val2017.json 42 | 43 | PYTHONPATH="" 44 | # interesting, we need to append this module to the PYTHONPATH 45 | export PYTHONPATH="${PYTHONPATH}:${PWD}/" && python tools/test_grounding_net.py --config-file ../glip_Swin_T_O365_GoldG.yaml --weight $dir/glip_tiny_model_o365_goldg_cc_sbu.pth \ 46 | TEST.IMS_PER_BATCH 1 \ 47 | MODEL.DYHEAD.SCORE_AGG "MEAN" \ 48 | TEST.EVAL_TASK detection \ 49 | MODEL.DYHEAD.FUSE_CONFIG.MLM_LOSS False \ 50 | OUTPUT_DIR $dataset/results 51 | rm ./DATASET/coco/val2017 52 | rm ./DATASET/coco/annotations/instances_val2017.json 53 | fi 54 | done 55 | 56 | echo "Done evaluating all the datasets with GLIP!" 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Roboflow 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /datasets_links.txt: -------------------------------------------------------------------------------- 1 | https://app.roboflow.com/roboflow-100/tweeter-profile/2 2 | https://app.roboflow.com/roboflow-100/gauge-u2lwv/3 3 | https://app.roboflow.com/roboflow-100/road-traffic/4 4 | https://app.roboflow.com/roboflow-100/wall-damage/2 5 | https://app.roboflow.com/roboflow-100/fish-market-ggjso/6 6 | https://app.roboflow.com/roboflow-100/soda-bottles/4 7 | https://app.roboflow.com/roboflow-100/flir-camera-objects/2 8 | https://app.roboflow.com/roboflow-100/stomata-cells/2 9 | https://app.roboflow.com/roboflow-100/leaf-disease-nsdsr/2 10 | https://app.roboflow.com/roboflow-100/bees-jt5in/2 11 | https://app.roboflow.com/roboflow-100/team-fight-tactics/3 12 | https://app.roboflow.com/roboflow-100/phages/2 13 | https://app.roboflow.com/roboflow-100/robomasters-285km/2 14 | https://app.roboflow.com/roboflow-100/lettuce-pallets/2 15 | https://app.roboflow.com/roboflow-100/trail-camera/2 16 | https://app.roboflow.com/roboflow-100/sedimentary-features-9eosf/3 17 | https://app.roboflow.com/roboflow-100/liver-disease/2 18 | https://app.roboflow.com/roboflow-100/cell-towers/2 19 | https://app.roboflow.com/roboflow-100/shark-teeth-5atku/2 20 | https://app.roboflow.com/roboflow-100/currency-v4f8j/2 21 | https://app.roboflow.com/roboflow-100/asbestos/2 22 | https://app.roboflow.com/roboflow-100/insects-mytwu/2 23 | https://app.roboflow.com/roboflow-100/cotton-20xz5/2 24 | https://app.roboflow.com/roboflow-100/uno-deck/2 25 | https://app.roboflow.com/roboflow-100/grass-weeds/2 26 | https://app.roboflow.com/roboflow-100/circuit-voltages/2 27 | https://app.roboflow.com/roboflow-100/people-in-paintings/2 28 | https://app.roboflow.com/roboflow-100/apples-fvpl5/2 29 | https://app.roboflow.com/roboflow-100/number-ops/2 30 | https://app.roboflow.com/roboflow-100/cable-damage/2 31 | https://app.roboflow.com/roboflow-100/furniture-ngpea/2 32 | https://app.roboflow.com/roboflow-100/poker-cards-cxcvz/2 33 | https://app.roboflow.com/roboflow-100/pills-sxdht/2 34 | https://app.roboflow.com/roboflow-100/bone-fracture-7fylg/2 35 | https://app.roboflow.com/roboflow-100/marbles/3 36 | https://app.roboflow.com/roboflow-100/cavity-rs0uf/3 37 | https://app.roboflow.com/roboflow-100/pests-2xlvx/2 38 | https://app.roboflow.com/roboflow-100/printed-circuit-board/4 39 | https://app.roboflow.com/roboflow-100/peanuts-sd4kf/2 40 | https://app.roboflow.com/roboflow-100/vehicles-q0x2v/2 41 | https://app.roboflow.com/roboflow-100/digits-t2eg6/2 42 | https://app.roboflow.com/roboflow-100/wine-labels/2 43 | https://app.roboflow.com/roboflow-100/truck-movement/4 44 | https://app.roboflow.com/roboflow-100/coral-lwptl/2 45 | https://app.roboflow.com/roboflow-100/brain-tumor-m2pbp/2 46 | https://app.roboflow.com/roboflow-100/cotton-plant-disease/2 47 | https://app.roboflow.com/roboflow-100/bacteria-ptywi/2 48 | https://app.roboflow.com/roboflow-100/4-fold-defect/2 49 | https://app.roboflow.com/roboflow-100/cells-uyemf/2 50 | https://app.roboflow.com/roboflow-100/gynecology-mri/2 51 | https://app.roboflow.com/roboflow-100/axial-mri/2 52 | https://app.roboflow.com/roboflow-100/abdomen-mri/2 53 | https://app.roboflow.com/roboflow-100/acl-x-ray/2 54 | https://app.roboflow.com/roboflow-100/radio-signal/2 55 | https://app.roboflow.com/roboflow-100/x-ray-rheumatology/2 56 | https://app.roboflow.com/roboflow-100/parasites-1s07h/2 57 | https://app.roboflow.com/roboflow-100/aerial-cows/2 58 | https://app.roboflow.com/roboflow-100/aerial-spheres/2 59 | https://app.roboflow.com/roboflow-100/secondary-chains/2 60 | https://app.roboflow.com/roboflow-100/aerial-pool/4 61 | https://app.roboflow.com/roboflow-100/underwater-objects-5v7p8/2 62 | https://app.roboflow.com/roboflow-100/peixos-fish/4 63 | https://app.roboflow.com/roboflow-100/underwater-pipes-4ng4t/2 64 | https://app.roboflow.com/roboflow-100/signatures-xc8up/2 65 | https://app.roboflow.com/roboflow-100/activity-diagrams-qdobr/2 66 | https://app.roboflow.com/roboflow-100/document-parts/2 67 | https://app.roboflow.com/roboflow-100/tweeter-posts/2 68 | https://app.roboflow.com/roboflow-100/avatar-recognition-nuexe/2 69 | https://app.roboflow.com/roboflow-100/csgo-videogame/2 70 | https://app.roboflow.com/roboflow-100/farcry6-videogame/2 71 | https://app.roboflow.com/roboflow-100/apex-videogame/2 72 | https://app.roboflow.com/roboflow-100/cables-nl42k/2 73 | https://app.roboflow.com/roboflow-100/circuit-elements/4 74 | https://app.roboflow.com/roboflow-100/washroom-rf1fa/2 75 | https://app.roboflow.com/roboflow-100/construction-safety-gsnvb/2 76 | https://app.roboflow.com/roboflow-100/street-work/4 77 | https://app.roboflow.com/roboflow-100/excavators-czvg9/2 78 | https://app.roboflow.com/roboflow-100/corrosion-bi3q3/2 79 | https://app.roboflow.com/roboflow-100/solar-panels-taxvb/2 80 | https://app.roboflow.com/roboflow-100/animals-ij5d2/2 81 | https://app.roboflow.com/roboflow-100/valentines-chocolate/4 82 | https://app.roboflow.com/roboflow-100/sign-language-sokdr/2 83 | https://app.roboflow.com/roboflow-100/halo-infinite-angel-videogame/2 84 | https://app.roboflow.com/roboflow-100/aquarium-qlnqy/2 85 | https://app.roboflow.com/roboflow-100/thermal-cheetah-my4dp/2 86 | https://app.roboflow.com/roboflow-100/chess-pieces-mjzgj/2 87 | https://app.roboflow.com/roboflow-100/bccd-ouzjz/2 88 | https://app.roboflow.com/roboflow-100/mask-wearing-608pr/2 89 | https://app.roboflow.com/roboflow-100/thermal-dogs-and-people-x6ejw/2 90 | https://app.roboflow.com/roboflow-100/weed-crop-aerial/2 91 | https://app.roboflow.com/roboflow-100/mitosis-gjs3g/2 92 | https://app.roboflow.com/roboflow-100/smoke-uvylj/2 93 | https://app.roboflow.com/roboflow-100/road-signs-6ih4y/2 94 | https://app.roboflow.com/roboflow-100/soccer-players-5fuqs/2 95 | https://app.roboflow.com/roboflow-100/hand-gestures-jps7z/2 96 | https://app.roboflow.com/roboflow-100/paper-parts/4 97 | https://app.roboflow.com/roboflow-100/cloud-types/2 98 | https://app.roboflow.com/roboflow-100/tabular-data-wf9uh/2 99 | https://app.roboflow.com/roboflow-100/paragraphs-co84b/2 100 | https://app.roboflow.com/roboflow-100/coins-1apki/2 -------------------------------------------------------------------------------- /datasets_links_640.txt: -------------------------------------------------------------------------------- 1 | https://app.roboflow.com/roboflow-100/tweeter-profile/1 2 | https://app.roboflow.com/roboflow-100/gauge-u2lwv/4 3 | https://app.roboflow.com/roboflow-100/road-traffic/3 4 | https://app.roboflow.com/roboflow-100/wall-damage/1 5 | https://app.roboflow.com/roboflow-100/fish-market-ggjso/5 6 | https://app.roboflow.com/roboflow-100/soda-bottles/3 7 | https://app.roboflow.com/roboflow-100/flir-camera-objects/1 8 | https://app.roboflow.com/roboflow-100/stomata-cells/1 9 | https://app.roboflow.com/roboflow-100/leaf-disease-nsdsr/1 10 | https://app.roboflow.com/roboflow-100/bees-jt5in/1 11 | https://app.roboflow.com/roboflow-100/team-fight-tactics/1 12 | https://app.roboflow.com/roboflow-100/phages/1 13 | https://app.roboflow.com/roboflow-100/robomasters-285km/1 14 | https://app.roboflow.com/roboflow-100/lettuce-pallets/1 15 | https://app.roboflow.com/roboflow-100/trail-camera/1 16 | https://app.roboflow.com/roboflow-100/sedimentary-features-9eosf/4 17 | https://app.roboflow.com/roboflow-100/liver-disease/1 18 | https://app.roboflow.com/roboflow-100/cell-towers/1 19 | https://app.roboflow.com/roboflow-100/shark-teeth-5atku/1 20 | https://app.roboflow.com/roboflow-100/currency-v4f8j/1 21 | https://app.roboflow.com/roboflow-100/asbestos/1 22 | https://app.roboflow.com/roboflow-100/insects-mytwu/1 23 | https://app.roboflow.com/roboflow-100/cotton-20xz5/1 24 | https://app.roboflow.com/roboflow-100/uno-deck/1 25 | https://app.roboflow.com/roboflow-100/grass-weeds/1 26 | https://app.roboflow.com/roboflow-100/circuit-voltages/1 27 | https://app.roboflow.com/roboflow-100/people-in-paintings/1 28 | https://app.roboflow.com/roboflow-100/apples-fvpl5/1 29 | https://app.roboflow.com/roboflow-100/number-ops/1 30 | https://app.roboflow.com/roboflow-100/cable-damage/1 31 | https://app.roboflow.com/roboflow-100/furniture-ngpea/1 32 | https://app.roboflow.com/roboflow-100/poker-cards-cxcvz/1 33 | https://app.roboflow.com/roboflow-100/pills-sxdht/1 34 | https://app.roboflow.com/roboflow-100/bone-fracture-7fylg/1 35 | https://app.roboflow.com/roboflow-100/marbles/1 36 | https://app.roboflow.com/roboflow-100/cavity-rs0uf/1 37 | https://app.roboflow.com/roboflow-100/pests-2xlvx/1 38 | https://app.roboflow.com/roboflow-100/printed-circuit-board/3 39 | https://app.roboflow.com/roboflow-100/peanuts-sd4kf/1 40 | https://app.roboflow.com/roboflow-100/vehicles-q0x2v/1 41 | https://app.roboflow.com/roboflow-100/digits-t2eg6/1 42 | https://app.roboflow.com/roboflow-100/wine-labels/1 43 | https://app.roboflow.com/roboflow-100/truck-movement/3 44 | https://app.roboflow.com/roboflow-100/coral-lwptl/1 45 | https://app.roboflow.com/roboflow-100/brain-tumor-m2pbp/1 46 | https://app.roboflow.com/roboflow-100/cotton-plant-disease/1 47 | https://app.roboflow.com/roboflow-100/bacteria-ptywi/1 48 | https://app.roboflow.com/roboflow-100/4-fold-defect/1 49 | https://app.roboflow.com/roboflow-100/cells-uyemf/1 50 | https://app.roboflow.com/roboflow-100/gynecology-mri/1 51 | https://app.roboflow.com/roboflow-100/axial-mri/1 52 | https://app.roboflow.com/roboflow-100/abdomen-mri/1 53 | https://app.roboflow.com/roboflow-100/acl-x-ray/1 54 | https://app.roboflow.com/roboflow-100/radio-signal/1 55 | https://app.roboflow.com/roboflow-100/x-ray-rheumatology/1 56 | https://app.roboflow.com/roboflow-100/parasites-1s07h/1 57 | https://app.roboflow.com/roboflow-100/aerial-cows/1 58 | https://app.roboflow.com/roboflow-100/aerial-spheres/1 59 | https://app.roboflow.com/roboflow-100/secondary-chains/1 60 | https://app.roboflow.com/roboflow-100/aerial-pool/3 61 | https://app.roboflow.com/roboflow-100/underwater-objects-5v7p8/1 62 | https://app.roboflow.com/roboflow-100/peixos-fish/3 63 | https://app.roboflow.com/roboflow-100/underwater-pipes-4ng4t/1 64 | https://app.roboflow.com/roboflow-100/signatures-xc8up/1 65 | https://app.roboflow.com/roboflow-100/activity-diagrams-qdobr/1 66 | https://app.roboflow.com/roboflow-100/document-parts/1 67 | https://app.roboflow.com/roboflow-100/tweeter-posts/1 68 | https://app.roboflow.com/roboflow-100/avatar-recognition-nuexe/1 69 | https://app.roboflow.com/roboflow-100/csgo-videogame/1 70 | https://app.roboflow.com/roboflow-100/farcry6-videogame/1 71 | https://app.roboflow.com/roboflow-100/apex-videogame/1 72 | https://app.roboflow.com/roboflow-100/cables-nl42k/1 73 | https://app.roboflow.com/roboflow-100/circuit-elements/3 74 | https://app.roboflow.com/roboflow-100/washroom-rf1fa/1 75 | https://app.roboflow.com/roboflow-100/construction-safety-gsnvb/1 76 | https://app.roboflow.com/roboflow-100/street-work/3 77 | https://app.roboflow.com/roboflow-100/excavators-czvg9/1 78 | https://app.roboflow.com/roboflow-100/corrosion-bi3q3/1 79 | https://app.roboflow.com/roboflow-100/solar-panels-taxvb/1 80 | https://app.roboflow.com/roboflow-100/animals-ij5d2/1 81 | https://app.roboflow.com/roboflow-100/valentines-chocolate/3 82 | https://app.roboflow.com/roboflow-100/sign-language-sokdr/1 83 | https://app.roboflow.com/roboflow-100/halo-infinite-angel-videogame/1 84 | https://app.roboflow.com/roboflow-100/aquarium-qlnqy/1 85 | https://app.roboflow.com/roboflow-100/thermal-cheetah-my4dp/1 86 | https://app.roboflow.com/roboflow-100/chess-pieces-mjzgj/1 87 | https://app.roboflow.com/roboflow-100/bccd-ouzjz/1 88 | https://app.roboflow.com/roboflow-100/mask-wearing-608pr/1 89 | https://app.roboflow.com/roboflow-100/thermal-dogs-and-people-x6ejw/1 90 | https://app.roboflow.com/roboflow-100/weed-crop-aerial/1 91 | https://app.roboflow.com/roboflow-100/mitosis-gjs3g/1 92 | https://app.roboflow.com/roboflow-100/smoke-uvylj/1 93 | https://app.roboflow.com/roboflow-100/road-signs-6ih4y/1 94 | https://app.roboflow.com/roboflow-100/soccer-players-5fuqs/1 95 | https://app.roboflow.com/roboflow-100/hand-gestures-jps7z/1 96 | https://app.roboflow.com/roboflow-100/paper-parts/3 97 | https://app.roboflow.com/roboflow-100/cloud-types/1 98 | https://app.roboflow.com/roboflow-100/tabular-data-wf9uh/1 99 | https://app.roboflow.com/roboflow-100/paragraphs-co84b/1 100 | https://app.roboflow.com/roboflow-100/coins-1apki/1 -------------------------------------------------------------------------------- /doc/images/comparison-yolos-bar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/comparison-yolos-bar.png -------------------------------------------------------------------------------- /doc/images/comparison-yolos-box.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/comparison-yolos-box.png -------------------------------------------------------------------------------- /doc/images/grid/4-fold-defect.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/4-fold-defect.jpg -------------------------------------------------------------------------------- /doc/images/grid/abdomen-mri.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/abdomen-mri.jpg -------------------------------------------------------------------------------- /doc/images/grid/acl-x-ray.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/acl-x-ray.jpg -------------------------------------------------------------------------------- /doc/images/grid/activity-diagrams-qdobr.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/activity-diagrams-qdobr.jpg -------------------------------------------------------------------------------- /doc/images/grid/aerial-cows.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/aerial-cows.jpg -------------------------------------------------------------------------------- /doc/images/grid/aerial-pool.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/aerial-pool.jpg -------------------------------------------------------------------------------- /doc/images/grid/aerial-spheres.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/aerial-spheres.jpg -------------------------------------------------------------------------------- /doc/images/grid/animals-ij5d2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/animals-ij5d2.jpg -------------------------------------------------------------------------------- /doc/images/grid/apex-videogame.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/apex-videogame.jpg -------------------------------------------------------------------------------- /doc/images/grid/apples-fvpl5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/apples-fvpl5.jpg -------------------------------------------------------------------------------- /doc/images/grid/aquarium-qlnqy.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/aquarium-qlnqy.jpg -------------------------------------------------------------------------------- /doc/images/grid/asbestos.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/asbestos.jpg -------------------------------------------------------------------------------- /doc/images/grid/avatar-recognition-nuexe.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/avatar-recognition-nuexe.jpg -------------------------------------------------------------------------------- /doc/images/grid/axial-mri.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/axial-mri.jpg -------------------------------------------------------------------------------- /doc/images/grid/bacteria-ptywi.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/bacteria-ptywi.jpg -------------------------------------------------------------------------------- /doc/images/grid/bccd-ouzjz.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/bccd-ouzjz.jpg -------------------------------------------------------------------------------- /doc/images/grid/bees-jt5in.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/bees-jt5in.jpg -------------------------------------------------------------------------------- /doc/images/grid/bone-fracture-7fylg.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/bone-fracture-7fylg.jpg -------------------------------------------------------------------------------- /doc/images/grid/brain-tumor-m2pbp.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/brain-tumor-m2pbp.jpg -------------------------------------------------------------------------------- /doc/images/grid/cable-damage.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/cable-damage.jpg -------------------------------------------------------------------------------- /doc/images/grid/cables-nl42k.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/cables-nl42k.jpg -------------------------------------------------------------------------------- /doc/images/grid/cavity-rs0uf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/cavity-rs0uf.jpg -------------------------------------------------------------------------------- /doc/images/grid/cell-towers.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/cell-towers.jpg -------------------------------------------------------------------------------- /doc/images/grid/cells-uyemf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/cells-uyemf.jpg -------------------------------------------------------------------------------- /doc/images/grid/chess-pieces-mjzgj.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/chess-pieces-mjzgj.jpg -------------------------------------------------------------------------------- /doc/images/grid/circuit-elements.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/circuit-elements.jpg -------------------------------------------------------------------------------- /doc/images/grid/circuit-voltages.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/circuit-voltages.jpg -------------------------------------------------------------------------------- /doc/images/grid/cloud-types.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/cloud-types.jpg -------------------------------------------------------------------------------- /doc/images/grid/coins-1apki.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/coins-1apki.jpg -------------------------------------------------------------------------------- /doc/images/grid/construction-safety-gsnvb.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/construction-safety-gsnvb.jpg -------------------------------------------------------------------------------- /doc/images/grid/coral-lwptl.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/coral-lwptl.jpg -------------------------------------------------------------------------------- /doc/images/grid/corrosion-bi3q3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/corrosion-bi3q3.jpg -------------------------------------------------------------------------------- /doc/images/grid/cotton-20xz5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/cotton-20xz5.jpg -------------------------------------------------------------------------------- /doc/images/grid/cotton-plant-disease.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/cotton-plant-disease.jpg -------------------------------------------------------------------------------- /doc/images/grid/csgo-videogame.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/csgo-videogame.jpg -------------------------------------------------------------------------------- /doc/images/grid/currency-v4f8j.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/currency-v4f8j.jpg -------------------------------------------------------------------------------- /doc/images/grid/digits-t2eg6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/digits-t2eg6.jpg -------------------------------------------------------------------------------- /doc/images/grid/document-parts.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/document-parts.jpg -------------------------------------------------------------------------------- /doc/images/grid/excavators-czvg9.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/excavators-czvg9.jpg -------------------------------------------------------------------------------- /doc/images/grid/farcry6-videogame.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/farcry6-videogame.jpg -------------------------------------------------------------------------------- /doc/images/grid/fish-market-ggjso.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/fish-market-ggjso.jpg -------------------------------------------------------------------------------- /doc/images/grid/flir-camera-objects.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/flir-camera-objects.jpg -------------------------------------------------------------------------------- /doc/images/grid/furniture-ngpea.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/furniture-ngpea.jpg -------------------------------------------------------------------------------- /doc/images/grid/gauge-u2lwv.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/gauge-u2lwv.jpg -------------------------------------------------------------------------------- /doc/images/grid/grass-weeds.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/grass-weeds.jpg -------------------------------------------------------------------------------- /doc/images/grid/gynecology-mri.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/gynecology-mri.jpg -------------------------------------------------------------------------------- /doc/images/grid/halo-infinite-angel-videogame.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/halo-infinite-angel-videogame.jpg -------------------------------------------------------------------------------- /doc/images/grid/hand-gestures-jps7z.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/hand-gestures-jps7z.jpg -------------------------------------------------------------------------------- /doc/images/grid/insects-mytwu.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/insects-mytwu.jpg -------------------------------------------------------------------------------- /doc/images/grid/leaf-disease-nsdsr.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/leaf-disease-nsdsr.jpg -------------------------------------------------------------------------------- /doc/images/grid/lettuce-pallets.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/lettuce-pallets.jpg -------------------------------------------------------------------------------- /doc/images/grid/liver-disease.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/liver-disease.jpg -------------------------------------------------------------------------------- /doc/images/grid/marbles.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/marbles.jpg -------------------------------------------------------------------------------- /doc/images/grid/mask-wearing-608pr.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/mask-wearing-608pr.jpg -------------------------------------------------------------------------------- /doc/images/grid/mitosis-gjs3g.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/mitosis-gjs3g.jpg -------------------------------------------------------------------------------- /doc/images/grid/number-ops.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/number-ops.jpg -------------------------------------------------------------------------------- /doc/images/grid/paper-parts.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/paper-parts.jpg -------------------------------------------------------------------------------- /doc/images/grid/paragraphs-co84b.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/paragraphs-co84b.jpg -------------------------------------------------------------------------------- /doc/images/grid/parasites-1s07h.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/parasites-1s07h.jpg -------------------------------------------------------------------------------- /doc/images/grid/peanuts-sd4kf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/peanuts-sd4kf.jpg -------------------------------------------------------------------------------- /doc/images/grid/peixos-fish.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/peixos-fish.jpg -------------------------------------------------------------------------------- /doc/images/grid/people-in-paintings.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/people-in-paintings.jpg -------------------------------------------------------------------------------- /doc/images/grid/pests-2xlvx.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/pests-2xlvx.jpg -------------------------------------------------------------------------------- /doc/images/grid/phages.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/phages.jpg -------------------------------------------------------------------------------- /doc/images/grid/pills-sxdht.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/pills-sxdht.jpg -------------------------------------------------------------------------------- /doc/images/grid/poker-cards-cxcvz.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/poker-cards-cxcvz.jpg -------------------------------------------------------------------------------- /doc/images/grid/printed-circuit-board.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/printed-circuit-board.jpg -------------------------------------------------------------------------------- /doc/images/grid/radio-signal.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/radio-signal.jpg -------------------------------------------------------------------------------- /doc/images/grid/road-signs-6ih4y.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/road-signs-6ih4y.jpg -------------------------------------------------------------------------------- /doc/images/grid/road-traffic.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/road-traffic.jpg -------------------------------------------------------------------------------- /doc/images/grid/robomasters-285km.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/robomasters-285km.jpg -------------------------------------------------------------------------------- /doc/images/grid/secondary-chains.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/secondary-chains.jpg -------------------------------------------------------------------------------- /doc/images/grid/sedimentary-features-9eosf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/sedimentary-features-9eosf.jpg -------------------------------------------------------------------------------- /doc/images/grid/shark-teeth-5atku.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/shark-teeth-5atku.jpg -------------------------------------------------------------------------------- /doc/images/grid/sign-language-sokdr.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/sign-language-sokdr.jpg -------------------------------------------------------------------------------- /doc/images/grid/signatures-xc8up.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/signatures-xc8up.jpg -------------------------------------------------------------------------------- /doc/images/grid/smoke-uvylj.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/smoke-uvylj.jpg -------------------------------------------------------------------------------- /doc/images/grid/soccer-players-5fuqs.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/soccer-players-5fuqs.jpg -------------------------------------------------------------------------------- /doc/images/grid/soda-bottles.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/soda-bottles.jpg -------------------------------------------------------------------------------- /doc/images/grid/solar-panels-taxvb.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/solar-panels-taxvb.jpg -------------------------------------------------------------------------------- /doc/images/grid/stomata-cells.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/stomata-cells.jpg -------------------------------------------------------------------------------- /doc/images/grid/street-work.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/street-work.jpg -------------------------------------------------------------------------------- /doc/images/grid/tabular-data-wf9uh.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/tabular-data-wf9uh.jpg -------------------------------------------------------------------------------- /doc/images/grid/team-fight-tactics.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/team-fight-tactics.jpg -------------------------------------------------------------------------------- /doc/images/grid/thermal-cheetah-my4dp.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/thermal-cheetah-my4dp.jpg -------------------------------------------------------------------------------- /doc/images/grid/thermal-dogs-and-people-x6ejw.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/thermal-dogs-and-people-x6ejw.jpg -------------------------------------------------------------------------------- /doc/images/grid/trail-camera.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/trail-camera.jpg -------------------------------------------------------------------------------- /doc/images/grid/truck-movement.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/truck-movement.jpg -------------------------------------------------------------------------------- /doc/images/grid/tweeter-posts.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/tweeter-posts.jpg -------------------------------------------------------------------------------- /doc/images/grid/tweeter-profile.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/tweeter-profile.jpg -------------------------------------------------------------------------------- /doc/images/grid/underwater-objects-5v7p8.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/underwater-objects-5v7p8.jpg -------------------------------------------------------------------------------- /doc/images/grid/underwater-pipes-4ng4t.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/underwater-pipes-4ng4t.jpg -------------------------------------------------------------------------------- /doc/images/grid/uno-deck.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/uno-deck.jpg -------------------------------------------------------------------------------- /doc/images/grid/valentines-chocolate.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/valentines-chocolate.jpg -------------------------------------------------------------------------------- /doc/images/grid/vehicles-q0x2v.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/vehicles-q0x2v.jpg -------------------------------------------------------------------------------- /doc/images/grid/wall-damage.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/wall-damage.jpg -------------------------------------------------------------------------------- /doc/images/grid/washroom-rf1fa.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/washroom-rf1fa.jpg -------------------------------------------------------------------------------- /doc/images/grid/weed-crop-aerial.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/weed-crop-aerial.jpg -------------------------------------------------------------------------------- /doc/images/grid/wine-labels.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/wine-labels.jpg -------------------------------------------------------------------------------- /doc/images/grid/x-ray-rheumatology.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboflow/roboflow-100-benchmark/45226420a600bf7fd6eaecb140cd73f5b379f7c3/doc/images/grid/x-ray-rheumatology.jpg -------------------------------------------------------------------------------- /metadata/bbox_stats.csv: -------------------------------------------------------------------------------- 1 | area_mean,area_std,num_small,num_medium,num_large 2 | 0.054357342422008514,0.030122648924589157,82.0,11.0,0.0 3 | 0.26623955368995667,0.16934259235858917,19.0,11.0,11.0 4 | 0.35584744811058044,0.21767005324363708,3.0,7.0,9.0 5 | 0.0030717775225639343,0.005782486405223608,100.0,0.0,0.0 6 | 0.3612661063671112,0.22255025804042816,30.0,14.0,11.0 7 | 0.009229364804923534,0.0033523228485137224,28.0,0.0,0.0 8 | 0.13373510539531708,0.08654186874628067,110.0,87.0,25.0 9 | 0.09290185570716858,0.13463422656059265,61.0,12.0,9.0 10 | 0.09684932231903076,0.16856230795383453,70.0,7.0,5.0 11 | 0.20443764328956604,0.21899673342704773,77.0,24.0,11.0 12 | 0.05806122347712517,0.05842362344264984,251.0,44.0,14.0 13 | 0.0005208333604969084,0.001295924186706543,36.0,0.0,0.0 14 | 0.08443094044923782,0.1240839809179306,11.0,3.0,0.0 15 | 0.24976608157157898,0.28780943155288696,194.0,79.0,59.0 16 | 0.03238772973418236,0.011246869340538979,216.0,0.0,0.0 17 | 0.03059276007115841,0.031107094138860703,96.0,7.0,0.0 18 | 0.11400854587554932,0.13177397847175598,398.0,113.0,52.0 19 | 0.005142728332430124,0.0013112083543092012,367.0,0.0,0.0 20 | 0.00394018180668354,0.007802590262144804,453.0,1.0,0.0 21 | 0.005418410059064627,0.0006790984189137816,42.0,0.0,0.0 22 | 0.0028046206571161747,0.009252841584384441,44.0,0.0,0.0 23 | 0.02310498058795929,0.03168782591819763,54.0,1.0,0.0 24 | 0.01054210402071476,0.010606996715068817,38.0,0.0,0.0 25 | 0.2013472318649292,0.18554741144180298,171.0,39.0,43.0 26 | 0.0028106898535043,0.0018492909148335457,29.0,0.0,0.0 27 | 0.04103148356080055,0.03310590982437134,40.0,5.0,0.0 28 | 0.11106543242931366,0.043324992060661316,57.0,66.0,5.0 29 | 0.24537314474582672,0.10483761131763458,51.0,163.0,227.0 30 | 0.40245407819747925,0.22090654075145721,4.0,16.0,17.0 31 | 0.2269040048122406,0.20650224387645721,6.0,6.0,2.0 32 | 0.3960130512714386,0.11402531713247299,0.0,2.0,14.0 33 | 0.0198796596378088,0.025236302986741066,134.0,0.0,0.0 34 | 0.5280275940895081,0.26964500546455383,7.0,12.0,5.0 35 | 0.3555130958557129,0.44449716806411743,346.0,0.0,6.0 36 | 0.1524413675069809,0.15501300990581512,15.0,8.0,4.0 37 | 0.22549471259117126,0.27539706230163574,44.0,6.0,2.0 38 | 0.010462401434779167,0.005113694351166487,15.0,0.0,0.0 39 | 0.004904378205537796,0.0016120315995067358,899.0,0.0,0.0 40 | 0.029611090198159218,0.018795959651470184,242.0,2.0,0.0 41 | 0.020128784701228142,0.029466798529028893,23.0,2.0,0.0 42 | 0.27454665303230286,0.14099113643169403,6.0,17.0,23.0 43 | 0.01664428785443306,0.003718553576618433,18.0,0.0,0.0 44 | 0.30547067523002625,0.13489313423633575,61.0,311.0,514.0 45 | 0.02334369346499443,0.025967368856072426,135.0,14.0,1.0 46 | 0.19458287954330444,0.12498975545167923,7.0,17.0,6.0 47 | 0.004844557028263807,0.00574044743552804,709.0,0.0,0.0 48 | 0.007729736156761646,0.005730015691369772,20.0,0.0,0.0 49 | 0.23021860420703888,0.29688024520874023,35.0,16.0,12.0 50 | 0.15047220885753632,0.10856442898511887,78.0,19.0,22.0 51 | 0.009001914411783218,0.007606551516801119,95.0,0.0,0.0 52 | 0.017661329358816147,0.01999436318874359,388.0,19.0,6.0 53 | 0.22489598393440247,0.2561335265636444,5.0,4.0,3.0 54 | 0.024333156645298004,0.03773677349090576,188.0,14.0,3.0 55 | 0.03898724168539047,0.03868435323238373,28.0,2.0,0.0 56 | 0.04881041496992111,0.1311672329902649,52.0,1.0,4.0 57 | 0.005656721536070108,0.0023153931833803654,110.0,0.0,0.0 58 | 0.027952300384640694,0.04443540796637535,249.0,14.0,10.0 59 | 0.5938368439674377,0.20541322231292725,0.0,1.0,0.0 60 | 0.0788995698094368,0.04109147563576698,45.0,14.0,0.0 61 | 0.13533616065979004,0.0805160254240036,40.0,26.0,20.0 62 | 0.01486705057322979,0.028045713901519775,44.0,1.0,0.0 63 | 0.02635451965034008,0.021868616342544556,33.0,0.0,0.0 64 | 0.22322823107242584,0.22555823624134064,570.0,119.0,79.0 65 | 0.11423677951097488,0.2845735549926758,168.0,8.0,4.0 66 | 0.11531835794448853,0.10967831313610077,349.0,202.0,41.0 67 | 0.18601374328136444,0.1648966521024704,255.0,221.0,119.0 68 | 0.0800371840596199,0.11551466584205627,49.0,8.0,3.0 69 | 0.004442375618964434,0.005547626875340939,118.0,0.0,0.0 70 | 0.009585818275809288,0.010507742874324322,739.0,5.0,0.0 71 | 0.007886047475039959,0.011615199968218803,70.0,0.0,0.0 72 | 0.009998969733715057,0.01144008431583643,93.0,0.0,0.0 73 | 0.05397418886423111,0.07198095321655273,13.0,2.0,1.0 74 | 0.0006580307963304222,6.606380338780582e-05,51.0,0.0,0.0 75 | 0.004908294416964054,0.0018631535349413753,16.0,0.0,0.0 76 | 0.0033367401920259,0.014353371225297451,117.0,1.0,0.0 77 | 0.0002914767828769982,0.0002747478138189763,178.0,0.0,0.0 78 | 0.42475828528404236,0.27009862661361694,41.0,94.0,90.0 79 | 0.03870256990194321,0.014728624373674393,209.0,0.0,0.0 80 | 0.04154537618160248,0.020983219146728516,35.0,1.0,0.0 81 | 0.1778300255537033,0.18558365106582642,103.0,47.0,20.0 82 | 0.0009326171712018549,0.00029692956013605,2.0,0.0,0.0 83 | 0.0010339355794712901,0.0005077195237390697,33.0,0.0,0.0 84 | 0.00016886393132153898,4.769439692609012e-05,9.0,0.0,0.0 85 | 0.003867683932185173,0.004892387427389622,102.0,0.0,0.0 86 | 0.004292378202080727,0.0018755252240225673,30.0,0.0,0.0 87 | 0.007211581338196993,0.003961037378758192,101.0,0.0,0.0 88 | 0.05934333801269531,0.036344025284051895,380.0,18.0,2.0 89 | 0.0024013672955334187,0.00125244224909693,132.0,0.0,0.0 90 | 0.2611139416694641,0.31157976388931274,9.0,0.0,3.0 91 | 0.0021655273158103228,0.0007675010710954666,10.0,0.0,0.0 92 | 0.008020907640457153,0.002998563228175044,278.0,0.0,0.0 93 | 0.0709468275308609,0.08291144669055939,9.0,4.0,1.0 94 | 0.007085914723575115,0.0026640526484698057,16.0,0.0,0.0 95 | 0.08051616698503494,0.014058250002563,271.0,35.0,0.0 96 | 0.0800485834479332,0.0480751171708107,176.0,59.0,3.0 97 | 0.01930963061749935,0.0137125039473176,39.0,0.0,0.0 98 | 0.0061462982557713985,0.005057338625192642,253.0,0.0,0.0 99 | 0.023044854402542114,0.02167961746454239,963.0,12.0,0.0 100 | 0.014259954914450645,0.030328284949064255,41.0,2.0,0.0 101 | 0.013855285942554474,0.018007680773735046,1281.0,16.0,1.0 102 | -------------------------------------------------------------------------------- /metadata/categories.csv: -------------------------------------------------------------------------------- 1 | dataset,category 2 | hand-gestures-jps7z,real world 3 | smoke-uvylj,real world 4 | wall-damage,real world 5 | corrosion-bi3q3,real world 6 | excavators-czvg9,real world 7 | chess-pieces-mjzgj,real world 8 | road-signs-6ih4y,real world 9 | street-work,real world 10 | construction-safety-gsnvb,real world 11 | road-traffic,real world 12 | washroom-rf1fa,real world 13 | circuit-elements,real world 14 | mask-wearing-608pr,real world 15 | cables-nl42k,real world 16 | soda-bottles,real world 17 | truck-movement,real world 18 | wine-labels,real world 19 | digits-t2eg6,real world 20 | vehicles-q0x2v,real world 21 | peanuts-sd4kf,real world 22 | printed-circuit-board,real world 23 | pests-2xlvx,real world 24 | cavity-rs0uf,real world 25 | leaf-disease-nsdsr,real world 26 | marbles,real world 27 | pills-sxdht,real world 28 | poker-cards-cxcvz,real world 29 | number-ops,real world 30 | insects-mytwu,real world 31 | cotton-20xz5,real world 32 | furniture-ngpea,real world 33 | cable-damage,real world 34 | animals-ij5d2,real world 35 | coins-1apki,real world 36 | apples-fvpl5,real world 37 | people-in-paintings,real world 38 | circuit-voltages,real world 39 | uno-deck,real world 40 | grass-weeds,real world 41 | gauge-u2lwv,real world 42 | sign-language-sokdr,real world 43 | valentines-chocolate,real world 44 | fish-market-ggjso,real world 45 | lettuce-pallets,real world 46 | shark-teeth-5atku,real world 47 | bees-jt5in,real world 48 | sedimentary-features-9eosf,real world 49 | currency-v4f8j,real world 50 | trail-camera,real world 51 | cell-towers,real world 52 | apex-videogame,videogames 53 | farcry6-videogame,videogames 54 | csgo-videogame,videogames 55 | avatar-recognition-nuexe,videogames 56 | halo-infinite-angel-videogame,videogames 57 | team-fight-tactics,videogames 58 | robomasters-285km,videogames 59 | tweeter-posts,documents 60 | tweeter-profile,documents 61 | document-parts,documents 62 | activity-diagrams-qdobr,documents 63 | signatures-xc8up,documents 64 | paper-parts,documents 65 | tabular-data-wf9uh,documents 66 | paragraphs-co84b,documents 67 | underwater-pipes-4ng4t,underwater 68 | aquarium-qlnqy,underwater 69 | peixos-fish,underwater 70 | underwater-objects-5v7p8,underwater 71 | coral-lwptl,underwater 72 | aerial-pool,aerial 73 | secondary-chains,aerial 74 | aerial-spheres,aerial 75 | soccer-players-5fuqs,aerial 76 | weed-crop-aerial,aerial 77 | aerial-cows,aerial 78 | cloud-types,aerial 79 | stomata-cells,microscopic 80 | bccd-ouzjz,microscopic 81 | parasites-1s07h,microscopic 82 | cells-uyemf,microscopic 83 | 4-fold-defect,microscopic 84 | bacteria-ptywi,microscopic 85 | cotton-plant-disease,microscopic 86 | mitosis-gjs3g,microscopic 87 | phages,microscopic 88 | liver-disease,microscopic 89 | asbestos,microscopic 90 | thermal-dogs-and-people-x6ejw,electromagnetic 91 | solar-panels-taxvb,electromagnetic 92 | radio-signal,electromagnetic 93 | thermal-cheetah-my4dp,electromagnetic 94 | x-ray-rheumatology,electromagnetic 95 | acl-x-ray,electromagnetic 96 | abdomen-mri,electromagnetic 97 | axial-mri,electromagnetic 98 | gynecology-mri,electromagnetic 99 | brain-tumor-m2pbp,electromagnetic 100 | bone-fracture-7fylg,electromagnetic 101 | flir-camera-objects,electromagnetic 102 | -------------------------------------------------------------------------------- /metadata/datasets_stats.csv: -------------------------------------------------------------------------------- 1 | dataset,category,train,test,valid,size,num_classes,yolov5,yolov7,glip,num_datasets,yolov8 2 | 4-fold-defect,microscopic,503,33,134,670,1.0,0.97,0.938,0.0,1,0.964 3 | abdomen-mri,electromagnetic,1887,238,479,2604,1.0,0.965,0.958,0.021,1,0.962 4 | acl-x-ray,electromagnetic,2141,306,612,3059,1.0,0.995,0.998,0.0,1,0.995 5 | activity-diagrams-qdobr,documents,259,45,74,378,19.0,0.427,0.509,0.002,1,0.705 6 | aerial-cows,aerial,1084,299,340,1723,1.0,0.854,0.568,0.056,1,0.826 7 | aerial-pool,aerial,673,96,177,946,7.0,0.513,0.791,0.013,1,0.744 8 | aerial-spheres,aerial,318,51,104,473,6.0,0.993,0.539,0.0,1,0.993 9 | animals-ij5d2,real world,700,100,200,1000,10.0,0.761,0.342,0.249,1,0.835 10 | apex-videogame,videogames,2583,415,691,3689,2.0,0.839,0.875,NULL,1,0.865 11 | apples-fvpl5,real world,489,30,178,697,2.0,0.779,0.791,NULL,1,0.785 12 | aquarium-qlnqy,underwater,448,63,127,638,7.0,0.79,0.822,0.195,1,0.781 13 | asbestos,microscopic,932,133,266,1331,4.0,0.596,0.611,0.007,1,0.631 14 | avatar-recognition-nuexe,videogames,225,30,59,314,1.0,0.889,0.943,0.367,1,0.877 15 | axial-mri,electromagnetic,253,39,79,371,2.0,0.638,0.549,0.039,1,0.637 16 | bacteria-ptywi,microscopic,30,10,10,50,1.0,0.162,0.000961,0.0,1,0.57 17 | bccd-ouzjz,microscopic,255,36,73,364,3.0,0.912,0.922,0.191,1,0.927 18 | bees-jt5in,real world,5640,836,1604,8080,1.0,0.891,0.68,0.009,1,0.901 19 | bone-fracture-7fylg,electromagnetic,326,44,88,458,4.0,0.0851,0.0896,0.0,1,NULL 20 | brain-tumor-m2pbp,electromagnetic,6930,990,1980,9900,3.0,0.768,0.809,0.003,1,NULL 21 | cable-damage,real world,919,134,265,1318,2.0,0.91,0.574,0.006,1,NULL 22 | cables-nl42k,real world,4816,794,1220,6830,11.0,0.688,0.722,0.01,1,0.76 23 | cavity-rs0uf,real world,287,38,93,418,2.0,0.782,0.799,0.029,1,0.765 24 | cell-towers,real world,705,101,202,1008,2.0,0.939,0.942,0.053,1,0.947 25 | cells-uyemf,microscopic,16,2,4,22,1.0,0.249,0.0845,0.005,1,0.861 26 | chess-pieces-mjzgj,real world,202,29,58,289,13.0,0.977,0.83,0.017,1,0.988 27 | circuit-elements,real world,672,36,64,772,46.0,0.0627,NULL,0.001,1,NULL 28 | circuit-voltages,real world,92,15,25,132,6.0,0.797,0.257,0.009,1,0.894 29 | cloud-types,aerial,3528,504,1008,5040,4.0,0.271,0.306,0.006,1,0.336 30 | coins-1apki,real world,6121,699,1599,8419,4.0,0.932,0.977,0.175,1,0.966 31 | construction-safety-gsnvb,real world,997,90,119,1206,5.0,0.915,0.915,0.259,1,0.874 32 | coral-lwptl,underwater,427,74,93,594,14.0,0.174,0.218,0.001,1,0.199 33 | corrosion-bi3q3,real world,840,105,304,1249,3.0,0.768,0.764,0.003,1,0.735 34 | cotton-20xz5,real world,367,20,19,406,4.0,0.569,0.591,0.157,1,0.513 35 | cotton-plant-disease,microscopic,724,102,198,1024,1.0,0.204,0.0518,0.0,1,0.176 36 | csgo-videogame,videogames,1774,207,446,2427,2.0,0.974,0.964,0.184,1,0.978 37 | currency-v4f8j,real world,576,82,155,813,10.0,0.583,0.514,0.099,1,0.877 38 | digits-t2eg6,real world,2912,367,824,4103,10.0,0.989,0.989,0.003,1,0.991 39 | document-parts,documents,906,150,318,1374,2.0,0.677,0.666,0.033,1,0.67 40 | excavators-czvg9,real world,2244,144,267,2655,3.0,0.946,0.895,0.274,1,0.922 41 | farcry6-videogame,videogames,82,14,24,120,11.0,0.619,0.216,0.248,1,0.709 42 | fish-market-ggjso,real world,14180,1202,3116,18498,21.0,0.92,0.988,0.013,1,0.993 43 | flir-camera-objects,electromagnetic,9306,1452,2854,13612,4.0,0.796,0.824,0.073,1,0.782 44 | furniture-ngpea,real world,454,74,161,689,3.0,0.983,0.968,0.836,1,0.979 45 | gauge-u2lwv,real world,158,25,52,235,2.0,0.642,0.668,0.217,1,0.661 46 | grass-weeds,real world,1661,245,580,2486,1.0,0.781,0.781,0.106,1,0.784 47 | gynecology-mri,electromagnetic,2122,253,526,2901,3.0,0.323,0.171,0.0,1,0.288 48 | halo-infinite-angel-videogame,videogames,462,71,136,669,4.0,0.921,0.924,0.173,1,0.935 49 | hand-gestures-jps7z,real world,642,94,178,914,14.0,0.995,0.995,NULL,1,0.994 50 | insects-mytwu,real world,696,100,199,995,10.0,0.89,0.858,0.024,1,0.882 51 | leaf-disease-nsdsr,real world,1589,296,616,2501,3.0,0.531,0.56,0.027,1,0.573 52 | lettuce-pallets,real world,1060,151,299,1510,5.0,0.945,0.966,0.031,1,0.956 53 | liver-disease,microscopic,2782,400,794,3976,4.0,0.592,0.583,0.0,1,0.576 54 | marbles,real world,54,32,19,105,2.0,0.992,0.473,0.03,1,0.986 55 | mask-wearing-608pr,real world,105,15,29,149,2.0,0.788,0.513,0.008,1,0.898 56 | mitosis-gjs3g,microscopic,213,30,61,304,1.0,0.931,0.739,0.001,1,0.962 57 | number-ops,real world,4869,623,1636,7128,15.0,0.99,0.992,0.055,1,0.992 58 | paper-parts,documents,8472,1209,2359,12040,46.0,0.59,0.796,0.007,1,0.78 59 | paragraphs-co84b,documents,4209,633,1221,6063,7.0,0.626,0.61,0.0,1,0.653 60 | parasites-1s07h,microscopic,1484,215,411,2110,8.0,0.848,0.889,0.036,1,0.861 61 | peanuts-sd4kf,real world,268,42,77,387,2.0,0.995,0.997,0.358,1,0.995 62 | peixos-fish,underwater,821,118,261,1200,12.0,0.148,0.821,0.004,1,0.809 63 | people-in-paintings,real world,634,81,194,909,1.0,0.575,0.678,0.168,1,0.561 64 | pests-2xlvx,real world,509,55,153,717,28.0,0.136,0.029,0.004,1,0.209 65 | phages,microscopic,1155,103,164,1422,2.0,0.854,0.842,0.002,1,0.839 66 | pills-sxdht,real world,316,45,90,451,8.0,0.869,0.867,0.194,1,0.987 67 | poker-cards-cxcvz,real world,964,128,193,1285,53.0,0.886,0.251,-0.0,1,0.995 68 | printed-circuit-board,real world,548,44,80,672,34.0,0.0907,NULL,0.0,1,NULL 69 | radio-signal,electromagnetic,1954,278,566,2798,2.0,0.673,0.653,0.066,1,0.695 70 | road-signs-6ih4y,real world,1376,229,488,2093,21.0,0.963,0.944,0.036,1,0.973 71 | road-traffic,real world,494,133,187,814,12.0,0.597,0.847,NULL,1,0.798 72 | robomasters-285km,videogames,1945,278,556,2779,9.0,0.816,0.772,0.003,1,0.814 73 | secondary-chains,aerial,103,16,43,162,1.0,0.341,0.312,0.0,1,0.315 74 | sedimentary-features-9eosf,real world,156,21,45,222,5.0,0.327,0.244,0.0,1,0.455 75 | shark-teeth-5atku,real world,191,36,53,280,4.0,0.948,0.863,0.025,1,0.991 76 | sign-language-sokdr,real world,504,72,144,720,26.0,0.87,0.255,0.006,1,0.966 77 | signatures-xc8up,documents,257,37,74,368,1.0,0.961,0.932,0.082,1,0.941 78 | smoke-uvylj,real world,522,76,148,746,1.0,0.959,0.962,0.431,1,0.975 79 | soccer-players-5fuqs,aerial,114,16,33,163,3.0,0.66,0.399,0.065,1,0.946 80 | soda-bottles,real world,1547,216,486,2249,6.0,0.964,NULL,0.098,1,0.96 81 | solar-panels-taxvb,electromagnetic,112,19,30,161,5.0,0.413,0.261,0.0,1,0.761 82 | stomata-cells,microscopic,1482,209,414,2105,2.0,0.84,0.847,0.012,1,0.855 83 | street-work,real world,611,87,175,873,11.0,0.478,0.708,0.148,1,0.693 84 | tabular-data-wf9uh,documents,3251,206,409,3866,12.0,0.752,0.782,0.018,1,0.78 85 | team-fight-tactics,videogames,1162,112,307,1581,59.0,0.961,0.88,0.0,1,0.973 86 | thermal-cheetah-my4dp,electromagnetic,90,14,25,129,2.0,0.931,0.513,0.028,1,0.945 87 | thermal-dogs-and-people-x6ejw,electromagnetic,142,20,41,203,2.0,0.967,0.957,0.47,1,0.976 88 | trail-camera,real world,941,131,239,1311,2.0,0.966,0.969,0.512,1,0.966 89 | truck-movement,real world,740,107,215,1062,7.0,0.786,0.846,0.007,1,0.858 90 | tweeter-posts,documents,87,9,21,117,2.0,0.708,0.495,0.005,1,0.866 91 | tweeter-profile,documents,425,61,121,607,1.0,0.988,0.99,0.002,1,0.987 92 | underwater-objects-5v7p8,underwater,5320,760,1520,7600,5.0,0.693,0.453,0.005,1,0.728 93 | underwater-pipes-4ng4t,underwater,5617,779,1575,7971,1.0,0.995,0.998,0.733,1,0.995 94 | uno-deck,real world,6295,899,1798,8992,15.0,0.993,0.994,0.013,1,0.993 95 | valentines-chocolate,real world,68,6,13,87,22.0,0.11,0.0593,0.013,1,0.978 96 | vehicles-q0x2v,real world,2634,458,966,4058,NULL,0.454,0.464,0.029,1,0.472 97 | wall-damage,real world,325,40,96,461,3.0,0.5,0.434,NULL,1,0.571 98 | washroom-rf1fa,real world,1885,318,775,2978,10.0,0.619,0.634,0.146,1,0.62 99 | weed-crop-aerial,aerial,823,118,235,1176,2.0,0.82,0.615,0.027,1,0.791 100 | wine-labels,real world,3172,630,841,4643,12.0,0.569,0.632,0.045,1,0.642 101 | x-ray-rheumatology,electromagnetic,135,16,34,185,12.0,0.722,0.506,0.0,1,0.858 -------------------------------------------------------------------------------- /metadata/labeling_hours.csv: -------------------------------------------------------------------------------- 1 | dataset,labelling hours 2 | smoke-uvylj,7 3 | damage-bah7a,0 4 | corrosion-bi3q3,186 5 | excavators-czvg9,0 6 | chess-pieces-mjzgj,0 7 | road-signs-6ih4y,0 8 | street-work,2 9 | construction-safety-gsnvb,505 10 | vehicles-80lt0,816 11 | washroom-rf1fa,449 12 | circuit-elements,311 13 | mask-wearing-608pr,0 14 | cables-nl42k,0 15 | soda-bottles,243 16 | truck-movement,282 17 | wine-labels,249 18 | digits-t2eg6,144 19 | vehicles-q0x2v,1121 20 | peanuts-sd4kf,212 21 | printed-circuit-board,311 22 | pests-2xlvx,188 23 | cavity-rs0uf,165 24 | leaf-disease-nsdsr,143 25 | marbles,133 26 | pills-sxdht,0 27 | poker-cards-cxcvz,0 28 | number-ops,28 29 | insects-mytwu,0 30 | cotton-20xz5,17 31 | furniture-ngpea,0 32 | cable-damage,2 33 | animals-ij5d2,12 34 | coins-1apki,0 35 | apples-fvpl,0 36 | people-in-paintings,5 37 | circuit-voltages,11 38 | uno-deck,0 39 | grass-weeds,105 40 | gauge-u2lwv,141 41 | sign-language-sokdr,0 42 | valentines-chocolate,4 43 | fish-market-ggjso,252 44 | lettuce-pallets,168 45 | shark-teeth-5atku,154 46 | bees-jt5in,163 47 | sedimentary-features-9eosf,31 48 | currency-v4f8j,1 49 | trail-camera,4 50 | cell-towers,25 51 | farcry6-videogame,0 52 | csgo-videogame,0 53 | avatar-recognition-nuexe,3 54 | halo-infinite-angel-videogame,4 55 | team-fight-tactics,88 56 | robomasters-285km,27 57 | underwater-pipes-4ng4t,316 58 | aquarium-qlnqy,0 59 | peixos-fish,0 60 | underwater-objects-5v7p8,0 61 | coral-lwptl,165 62 | stomata-cells,0 63 | bccd-ouzjz,0 64 | parasites-1s07h,0 65 | cells-uyemf,210 66 | 4-fold-defect,279 67 | bacteria-ptywi,472 68 | cotton-plant-disease,259 69 | mitosis-gjs3g,0 70 | phages,74 71 | liver-disease,31 72 | asbestos,126 73 | aerial-pool,421 74 | secondary-chains,201 75 | aerial-spheres,177 76 | soccer-players-5fuqs,0 77 | weed-crop-aerial,0 78 | aerial-cows,179 79 | cloud-types,0 80 | thermal-dogs-and-people-x6ejw,0 81 | solar-panels-taxvb,175 82 | radio-signal,0 83 | thermal-cheetah-my4dp,0 84 | x-ray-rheumatology,16 85 | acl-x-ray,0 86 | abdomen-mri,0 87 | axial-mri,0 88 | gynecology-mri,7 89 | brain-tumor-m2pbp,0 90 | bone-fracture-7fylg,0 91 | flir-camera-objects,17 92 | tweeter-posts,0 93 | tweeter-profile,0 94 | document-parts,192 95 | activity-diagrams-qdobr,192 96 | signatures-xc8up,0 97 | paper-parts,211 98 | tabular-data-wf9uh,271 99 | paragraphs-co84b,228 100 | -------------------------------------------------------------------------------- /metadata/original_datasets.csv: -------------------------------------------------------------------------------- 1 | dataset,category,original 2 | hand-gestures-jps7z,real world,https://universe.roboflow.com/hand-gestures-recognition/hand-gestures-dataset 3 | smoke-uvylj,real world,https://universe.roboflow.com/sigma-pub/smoke-detection-sigma 4 | wall-damage,real world,https://universe.roboflow.com/sina-uyen0/damage_level_detection 5 | corrosion-bi3q3,real world,https://universe.roboflow.com/khaingwintz-gmail-com/dataset--2-pathein-train-plus-v-3-update-mm 6 | excavators-czvg9,real world,https://universe.roboflow.com/mohamed-sabek-6zmr6/excavators-cwlh0 7 | chess-pieces-mjzgj,real world,https://universe.roboflow.com/joseph-nelson/chess-pieces-new 8 | road-signs-6ih4y,real world,https://universe.roboflow.com/project-sign-detection/traffic-sign-cdfml 9 | street-work,real world,https://universe.roboflow.com/cone/capacetes-e-cones 10 | construction-safety-gsnvb,real world,https://universe.roboflow.com/computer-vision/worker-safety 11 | road-traffic,real world,https://universe.roboflow.com/due/detection-dlzhy 12 | washroom-rf1fa,real world,https://universe.roboflow.com/imagetaggingworkspace/washroom-image-tagging 13 | circuit-elements,real world,https://universe.roboflow.com/new-workspace-rzrja/pcb-2.0 14 | mask-wearing-608pr,real world,https://universe.roboflow.com/joseph-nelson/mask-wearing 15 | cables-nl42k,real world,https://universe.roboflow.com/annotationericsson/annotation-2.0 16 | soda-bottles,real world,https://universe.roboflow.com/food7/test1-iajnv 17 | truck-movement,real world,https://universe.roboflow.com/psi-dhxqe/psi-rossville-pano 18 | wine-labels,real world,https://universe.roboflow.com/wine-label/wine-label-detection 19 | digits-t2eg6,real world,https://universe.roboflow.com/dmrs/number-1gmaw 20 | vehicles-q0x2v,real world,https://universe.roboflow.com/7-class/11-11-2021-09.41 21 | peanuts-sd4kf,real world,https://universe.roboflow.com/molds-onbk3/peanuts-mckge/ 22 | printed-circuit-board,real world,https://universe.roboflow.com/new-workspace-rzrja/pcb-2.0 23 | pests-2xlvx,real world,https://universe.roboflow.com/gugugu/pests-f8kkr 24 | cavity-rs0uf,real world,https://universe.roboflow.com/duong-duc-cuong/cavity-n3ioq 25 | leaf-disease-nsdsr,real world,https://universe.roboflow.com/puri/puri4-ygapu 26 | marbles,real world,https://universe.roboflow.com/zhe-fan/marble-images 27 | pills-sxdht,real world,https://universe.roboflow.com/mohamed-attia-e2mor/pill-detection-llp4r 28 | poker-cards-cxcvz,real world,https://universe.roboflow.com/roboflow-100/poker-cards-cxcvz 29 | number-ops,real world,https://universe.roboflow.com/mnist-bvalq/mnist-icrul 30 | insects-mytwu,real world,https://universe.roboflow.com/nirmani/yolo-custome-925 31 | cotton-20xz5,real world,https://universe.roboflow.com/cotton-nqp2x/bt-cotton 32 | furniture-ngpea,real world,https://universe.roboflow.com/minoj-selvaraj/furniture-sfocl 33 | cable-damage,real world,https://universe.roboflow.com/st-hedgehog-yusupov-gmail-com/kanaaat 34 | animals-ij5d2,real world,https://universe.roboflow.com/dane-sprsiter/barnyard 35 | coins-1apki,real world,https://universe.roboflow.com/labelimg/label_coin 36 | apples-fvpl5,real world,https://universe.roboflow.com/arfiani-nur-sayidah-9lizr/apple-sorting-2bfhk 37 | people-in-paintings,real world,https://universe.roboflow.com/raya-al/french-paintings-dataset-d2vbe 38 | circuit-voltages,real world,https://universe.roboflow.com/vanitchaporn/circuit-gexit 39 | uno-deck,real world, https://universe.roboflow.com/joseph-nelson/uno-cards 40 | grass-weeds,real world,https://universe.roboflow.com/jan-douwe/testbl 41 | gauge-u2lwv,real world,https://universe.roboflow.com/evankim9903-gmail-com/gauge_detection 42 | sign-language-sokdr,real world,https://universe.roboflow.com/david-lee-d0rhs/american-sign-language-letters 43 | valentines-chocolate,real world,https://universe.roboflow.com/chocolates/valentines-chocolates 44 | fish-market-ggjso,real world,https://universe.roboflow.com/commolybroken/dataset-z2vab 45 | lettuce-pallets,real world,https://universe.roboflow.com/lettucedetector 46 | shark-teeth-5atku,real world,https://universe.roboflow.com/sharks/shark-taxonomy 47 | bees-jt5in,real world,https://universe.roboflow.com/jjb-object-detection-projects/bee-detection-pry0w 48 | sedimentary-features-9eosf,real world,https://universe.roboflow.com/sedimentary-structures/sedimentary-features-rmadz 49 | currency-v4f8j,real world,https://universe.roboflow.com/alex-hyams-cosqx/cash-counter/ 50 | trail-camera,real world,https://universe.roboflow.com/my-game-pics/my-game-pics 51 | cell-towers,real world,https://universe.roboflow.com/yuyang-li/tower_jointv1 52 | apex-videogame,videogames,https://universe.roboflow.com/apex-esoic/apexyolov4 53 | farcry6-videogame,videogames,https://universe.roboflow.com/kais-al-hajjih/farcry6-hackathon 54 | csgo-videogame,videogames,https://universe.roboflow.com/new-workspace-rp0z0/csgo-train-yolo-v5 55 | avatar-recognition-nuexe,videogames,https://universe.roboflow.com/new-workspace-0pohs/avatar-recognition-rfw8d 56 | halo-infinite-angel-videogame,videogames,https://universe.roboflow.com/graham-doerksen/halo-infinite-angel-aim 57 | team-fight-tactics,videogames,https://universe.roboflow.com/lamaitw/lama-itw/ 58 | robomasters-285km,videogames,https://universe.roboflow.com/etp5501-gmail-com/robomasters-colored 59 | tweeter-posts,documents,https://universe.roboflow.com/tweeter/tweeter 60 | tweeter-profile,documents,https://universe.roboflow.com/wojciech-blachowski/tweets 61 | document-parts,documents,https://universe.roboflow.com/new-workspace-vf0ib/calpers 62 | activity-diagrams-qdobr,documents,https://universe.roboflow.com/public1/activity-diagrams-s7sxv 63 | signatures-xc8up,documents,https://universe.roboflow.com/signature-detection/signaturesdetectiob 64 | paper-parts,documents,https://universe.roboflow.com/object-detection-alan-devera/object-detection-ycqjb 65 | tabular-data-wf9uh,documents,https://universe.roboflow.com/rik-biswas/tabular-data-dh4ek 66 | paragraphs-co84b,documents,https://universe.roboflow.com/new-workspace-4vus5/singlemcq 67 | underwater-pipes-4ng4t,underwater,https://universe.roboflow.com/underwaterpipes/underwater_pipes_orginal_pictures 68 | aquarium-qlnqy,underwater,https://universe.roboflow.com/brad-dwyer/aquarium-combined 69 | peixos-fish,underwater,https://universe.roboflow.com/nasca37/peixos3 70 | underwater-objects-5v7p8,underwater,https://universe.roboflow.com/workspace-txxpz/underwater-detection 71 | coral-lwptl,underwater,https://universe.roboflow.com/nikita-manolis-je2ii/coral-growth-form 72 | aerial-pool,aerial,https://universe.roboflow.com/a-s/uwh 73 | secondary-chains,aerial,https://universe.roboflow.com/cc_moon/secondaries 74 | aerial-spheres,aerial,https://universe.roboflow.com/mevil-crasta/annotating-spheres---11-04 75 | soccer-players-5fuqs,aerial,https://universe.roboflow.com/ilyes-talbi-ptwsp/futbol-players 76 | weed-crop-aerial,aerial,https://universe.roboflow.com/new-workspace-csmgu/weedcrop-waifl 77 | aerial-cows,aerial,https://universe.roboflow.com/omarkapur-berkeley-edu/livestalk 78 | cloud-types,aerial,https://universe.roboflow.com/research-project/shallow-cloud 79 | stomata-cells,microscopic,https://universe.roboflow.com/new-workspace-fditd/stomata02 80 | bccd-ouzjz,microscopic,https://universe.roboflow.com/team-roboflow/blood-cell-detection-1ekwu 81 | parasites-1s07h,microscopic,https://universe.roboflow.com/graduao/sistema-para-analise-de-ovos-de-parasitas-em-amostra-de-agua-e-sangue 82 | cells-uyemf,microscopic,https://universe.roboflow.com/new-workspace-86q1t/t03-proyecto-celula-dataset-ampliado 83 | 4-fold-defect,microscopic,https://universe.roboflow.com/kat-laura/defect-detection-gil85 84 | bacteria-ptywi,microscopic,https://universe.roboflow.com/terada-shoma/gram-positive-bacteria 85 | cotton-plant-disease,microscopic,https://universe.roboflow.com/quandong-qian/desease-cotton-plant 86 | mitosis-gjs3g,microscopic,https://universe.roboflow.com/20029-tkmce-ac-in/mitosis-dwute 87 | phages,microscopic,https://universe.roboflow.com/danish2562022-gmail-com/microglia_rgb/ 88 | liver-disease,microscopic,https://universe.roboflow.com/liver-t5yvf/liver-diseases 89 | asbestos,microscopic,https://universe.roboflow.com/ahmad-rabiee/asbest91 90 | thermal-dogs-and-people-x6ejw,electromagnetic,https://universe.roboflow.com/joseph-nelson/thermal-dogs-and-people 91 | solar-panels-taxvb,electromagnetic,https://universe.roboflow.com/new-workspace-rt1da/solarpaneldetectmodel 92 | radio-signal,electromagnetic,https://universe.roboflow.com/danil/ 93 | thermal-cheetah-my4dp,electromagnetic,https://universe.roboflow.com/brad-dwyer/thermal-cheetah 94 | x-ray-rheumatology,electromagnetic,https://universe.roboflow.com/publictestsite/xray-rheumatology-images-public 95 | acl-x-ray,electromagnetic,https://universe.roboflow.com/objectdetection-9lu9z/detectron2-acl 96 | abdomen-mri,electromagnetic,https://universe.roboflow.com/xinweihe/circle-3train 97 | axial-mri,electromagnetic,https://universe.roboflow.com/tfg-2nmge/axial-dataset 98 | gynecology-mri,electromagnetic,https://universe.roboflow.com/yuanyuanpei7/5-8w 99 | brain-tumor-m2pbp,electromagnetic,https://universe.roboflow.com/yousef-ghanem-jzj4y/brain-tumor-detection-fpf1f 100 | bone-fracture-7fylg,electromagnetic,https://universe.roboflow.com/science-research/science-research-2022:-bone-fracture-detection 101 | flir-camera-objects,electromagnetic,https://universe.roboflow.com/thermal-imaging-0hwfw/flir-data-set 102 | -------------------------------------------------------------------------------- /metadata/results_glip.csv: -------------------------------------------------------------------------------- 1 | dataset,glip 2 | smoke-uvylj,0.431 3 | damage-bah7a,0.093 4 | corrosion-bi3q3,0.003 5 | excavators-czvg9,0.274 6 | chess-pieces-mjzgj,0.017 7 | road-signs-6ih4y,0.036 8 | street-work,0.148 9 | construction-safety-gsnvb,0.259 10 | vehicles-80lt0,0.147 11 | washroom-rf1fa,0.146 12 | circuit-elements,0.001 13 | mask-wearing-608pr,0.008 14 | cables-nl42k,0.01 15 | soda-bottles,0.098 16 | truck-movement,0.007 17 | wine-labels,0.045 18 | digits-t2eg6,0.003 19 | vehicles-q0x2v,0.029 20 | peanuts-sd4kf,0.358 21 | printed-circuit-board,0.0 22 | pests-2xlvx,0.004 23 | cavity-rs0uf,0.029 24 | leaf-disease-nsdsr,0.027 25 | marbles,0.03 26 | pills-sxdht,0.194 27 | poker-cards-cxcvz,-1.0 28 | number-ops,0.055 29 | insects-mytwu,0.024 30 | cotton-20xz5,0.157 31 | furniture-ngpea,0.836 32 | cable-damage,0.006 33 | animals-ij5d2,0.249 34 | coins-1apki,0.175 35 | apples-fvpl,0.271 36 | people-in-paintings,0.168 37 | circuit-voltages,0.009 38 | uno-deck,0.013 39 | grass-weeds,0.106 40 | gauge-u2lwv,0.217 41 | sign-language-sokdr,0.006 42 | valentines-chocolate,0.013 43 | fish-market-ggjso,0.013 44 | lettuce-pallets,0.031 45 | shark-teeth-5atku,0.025 46 | bees-jt5in,0.009 47 | sedimentary-features-9eosf,0.0 48 | currency-v4f8j,0.099 49 | trail-camera,0.512 50 | cell-towers,0.053 51 | farcry6-1videogame,0.248 52 | csgo-1videogame,0.184 53 | avatar-1recognition-1nuexe,0.367 54 | halo-1infinite-1angel-1videogame,0.173 55 | team-1fight-1tactics,0.0 56 | robomasters-1285km,0.003 57 | underwater-1pipes-14ng4t,0.733 58 | aquarium-1qlnqy,0.195 59 | peixos-1fish,0.004 60 | underwater-1objects-15v7p8,0.005 61 | coral-1lwptl,0.001 62 | stomata-1cells,0.012 63 | bccd-1ouzjz,0.191 64 | parasites-11s07h,0.036 65 | cells-1uyemf,0.005 66 | 4-1fold-1defect,0.0 67 | bacteria-1ptywi,0.0 68 | cotton-1plant-1disease,0.0 69 | mitosis-1gjs3g,0.001 70 | phages,0.002 71 | liver-1disease,0.0 72 | asbestos,0.007 73 | aerial-1pool,0.013 74 | secondary-1chains,0.0 75 | aerial-1spheres,0.0 76 | soccer-1players-15fuqs,0.065 77 | weed-1crop-1aerial,0.027 78 | aerial-1cows,0.056 79 | cloud-1types,0.006 80 | thermal-1dogs-1and-1people-1x6ejw,0.47 81 | solar-1panels-1taxvb,0.0 82 | radio-1signal,0.066 83 | thermal-1cheetah-1my4dp,0.028 84 | x-1ray-1rheumatology,0.0 85 | acl-1x-1ray,0.0 86 | abdomen-1mri,0.021 87 | axial-1mri,0.039 88 | gynecology-1mri,0.0 89 | brain-1tumor-1m2pbp,0.003 90 | bone-1fracture-17fylg,0.0 91 | flir-1camera-1objects,0.073 92 | tweeter-1posts,0.005 93 | tweeter-1profile,0.002 94 | document-1parts,0.033 95 | activity-1diagrams-1qdobr,0.002 96 | signatures-1xc8up,0.082 97 | paper-1parts,0.007 98 | tabular-1data-1wf9uh,0.018 99 | paragraphs-1co84b,0.0 100 | -------------------------------------------------------------------------------- /notebooks/analysis.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "9f917105", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "%matplotlib notebook\n", 11 | "%load_ext autoreload" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "id": "113abe90", 17 | "metadata": {}, 18 | "source": [ 19 | "# Analysis\n", 20 | "\n", 21 | "Notebook containing code to create our plots" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "id": "fa2c22b9", 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "import pandas as pd\n", 32 | "from tqdm.autonotebook import tqdm\n", 33 | "import seaborn as sns\n", 34 | "import matplotlib.pyplot as plt\n", 35 | "from pathlib import Path\n", 36 | "import pickle\n", 37 | "from typing import Dict\n", 38 | "\n", 39 | "\n", 40 | "plt.style.use(['science', 'notebook'])" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "id": "a86ca053", 46 | "metadata": {}, 47 | "source": [ 48 | "## Preambula\n", 49 | "We are going to run some computations, to save time let's create a decorator that stores and read from disk" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": null, 55 | "id": "36e31589", 56 | "metadata": {}, 57 | "outputs": [], 58 | "source": [ 59 | "def maybe_load_from_disk(location: Path):\n", 60 | " def decorator(func):\n", 61 | " def _inner(*args, **kwargs):\n", 62 | " if location.exists():\n", 63 | " print(f\"[INFO] loading from {location}\")\n", 64 | " with open(location, \"rb\") as f:\n", 65 | " return pickle.load(f)\n", 66 | " res = func(*args, **kwargs)\n", 67 | " with open(location, \"wb\") as f:\n", 68 | " print(f\"[INFO] saving to {location}\")\n", 69 | " pickle.dump(res, f)\n", 70 | " return res\n", 71 | " return _inner\n", 72 | " return decorator\n", 73 | " " 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": null, 79 | "id": "0a4c26a9", 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "df = pd.read_csv(\"../metadata/categories.csv\", index_col=0)\n", 84 | "\n", 85 | "df.head()" 86 | ] 87 | }, 88 | { 89 | "cell_type": "markdown", 90 | "id": "62926f8a", 91 | "metadata": {}, 92 | "source": [ 93 | "# Sizes\n", 94 | "\n", 95 | "Find out total dataset sizes, we have `rf100` download at `rf100`. We can use the index to iterate and get the size of each folder" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": null, 101 | "id": "838827a7", 102 | "metadata": {}, 103 | "outputs": [], 104 | "source": [ 105 | "from pathlib import Path\n", 106 | "from functools import reduce\n", 107 | "from collections import defaultdict\n", 108 | "\n", 109 | "RF100_ROOT = Path('../rf100')\n", 110 | "\n", 111 | "def count_num_files(dataset: str):\n", 112 | " dataset_path = RF100_ROOT / dataset\n", 113 | " sub_dirs = [\"train\", \"valid\", \"test\"]\n", 114 | " num_files = defaultdict(int)\n", 115 | " for sub_dir in sub_dirs:\n", 116 | " sub_dir_path = dataset_path / sub_dir / 'images'\n", 117 | " num_files[sub_dir] += sum([1 if curr.is_file() else 0 for curr in sub_dir_path.iterdir()])\n", 118 | " \n", 119 | " return pd.Series(num_files)" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "id": "7953231f", 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [ 129 | "# @maybe_load_from_disk(Path('../temp/df.pkl'))\n", 130 | "def apply_num_files(df):\n", 131 | " df[[\"train\", \"test\", \"valid\"]] = df.apply(lambda row: count_num_files(row.name), axis=1)[[\"train\", \"test\", \"valid\"]]\n", 132 | " df[\"size\"] = df[\"train\"] + df[\"test\"] + df[\"valid\"]\n", 133 | " return df\n", 134 | "\n", 135 | "df = apply_num_files(df)" 136 | ] 137 | }, 138 | { 139 | "cell_type": "markdown", 140 | "id": "c91f21a0", 141 | "metadata": {}, 142 | "source": [ 143 | "We now want to add the number of classes for each dataset, obtained before hand" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": null, 149 | "id": "7b3c54dd", 150 | "metadata": {}, 151 | "outputs": [], 152 | "source": [ 153 | "import json\n", 154 | "# obtained by running `export ROBOFLOW_API_KEY=.... && python ./scripts/get_labels_names.py` \n", 155 | "with open(\"../metadata/labels_names.json\", 'r') as f: \n", 156 | " labels_names = json.load(f)" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": null, 162 | "id": "c7dc3585", 163 | "metadata": {}, 164 | "outputs": [], 165 | "source": [ 166 | "df" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": null, 172 | "id": "55ec6b88", 173 | "metadata": {}, 174 | "outputs": [], 175 | "source": [ 176 | "def get_num_classes_per_dataset(labels_names: Dict) -> pd.DataFrame:\n", 177 | " records = []\n", 178 | " for item in labels_names:\n", 179 | " num_classes = len(item[\"classes\"])\n", 180 | " records.append({\n", 181 | " \"dataset\" : item[\"name\"],\n", 182 | " \"num_classes\": num_classes\n", 183 | " })\n", 184 | " return pd.DataFrame.from_records(records).set_index(\"dataset\")\n", 185 | "\n", 186 | "df = df.join(get_num_classes_per_dataset(labels_names))" 187 | ] 188 | }, 189 | { 190 | "cell_type": "markdown", 191 | "id": "4dca41aa", 192 | "metadata": {}, 193 | "source": [ 194 | "Finally, we also want to add the yolov5/7 - glip results" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": null, 200 | "id": "605e4c47", 201 | "metadata": {}, 202 | "outputs": [], 203 | "source": [ 204 | "results = pd.read_csv(\"../results.csv\", index_col=0)\n", 205 | "df = df.join(results)" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": null, 211 | "id": "6f4e585f", 212 | "metadata": {}, 213 | "outputs": [], 214 | "source": [ 215 | "df" 216 | ] 217 | }, 218 | { 219 | "cell_type": "markdown", 220 | "id": "f07434e6", 221 | "metadata": {}, 222 | "source": [ 223 | "Let's see how many of them there are for each category" 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": null, 229 | "id": "2ec16012", 230 | "metadata": {}, 231 | "outputs": [], 232 | "source": [ 233 | "df" 234 | ] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "execution_count": null, 239 | "id": "a8d794a4", 240 | "metadata": {}, 241 | "outputs": [], 242 | "source": [ 243 | "df[\"num_datasets\"] = 1\n", 244 | "aggretations = {\n", 245 | " \"train\" : \"sum\", \"test\" : \"sum\", \"valid\" : \"sum\", \"size\" : \"sum\", \"num_classes\" : \"sum\",\n", 246 | " \"yolov5\": \"mean\", \"yolov7\": \"mean\",\n", 247 | " \"num_datasets\" : \"sum\" \n", 248 | " }\n", 249 | "grouped_df = df.groupby(\"category\").agg(aggretations).reset_index()\n", 250 | "grouped_df = grouped_df.sort_values(\"size\")\n", 251 | "grouped_df[\"perc\"] = grouped_df[\"size\"] / grouped_df[\"size\"].sum()\n", 252 | "grouped_df" 253 | ] 254 | }, 255 | { 256 | "cell_type": "markdown", 257 | "id": "83da60ea", 258 | "metadata": {}, 259 | "source": [ 260 | "Now, we want to use the order of the categories to sort our original dataframe, till will make it easier to visualize them" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": null, 266 | "id": "360925ae", 267 | "metadata": {}, 268 | "outputs": [], 269 | "source": [ 270 | "df_with_ordered_categories = pd.DataFrame(index=grouped_df.index, data={\"category\": grouped_df.category})\n", 271 | "df = df_with_ordered_categories.merge(df.reset_index(\"dataset\"), on=\"category\", how=\"inner\")\n", 272 | "df = df.set_index(\"dataset\")\n", 273 | "df" 274 | ] 275 | }, 276 | { 277 | "cell_type": "markdown", 278 | "id": "f809a972", 279 | "metadata": {}, 280 | "source": [ 281 | "Let's store it to disk" 282 | ] 283 | }, 284 | { 285 | "cell_type": "code", 286 | "execution_count": null, 287 | "id": "54988bbd", 288 | "metadata": {}, 289 | "outputs": [], 290 | "source": [ 291 | "df.to_csv(\"../metadata/datasets_stats.csv\")" 292 | ] 293 | }, 294 | { 295 | "cell_type": "markdown", 296 | "id": "d0543c26", 297 | "metadata": {}, 298 | "source": [ 299 | "## Bounding boxes stats\n", 300 | "\n", 301 | "Cool, so we may also want to plot/show the mean size of bboxes for each dataset" 302 | ] 303 | }, 304 | { 305 | "cell_type": "markdown", 306 | "id": "7654efd1", 307 | "metadata": {}, 308 | "source": [ 309 | "Let's create something to read all the annotations. We can take advantage of PyTorch Dataloader to use multiple cores and make the computation go brum brum" 310 | ] 311 | }, 312 | { 313 | "cell_type": "code", 314 | "execution_count": null, 315 | "id": "50c1f2ce", 316 | "metadata": {}, 317 | "outputs": [], 318 | "source": [ 319 | "from torch.utils.data import Dataset, DataLoader\n", 320 | "import torch\n", 321 | "\n", 322 | "IGNORE = -1\n", 323 | "# all images are resized to 640\n", 324 | "size = (640, 640)\n", 325 | "\n", 326 | "class AnnotationDataset(Dataset):\n", 327 | " def __init__(self, root: Path, fmt: str = \"txt\"):\n", 328 | " super().__init__()\n", 329 | " self.annotations_paths = list(root.glob(f\"**/*.{fmt}\"))\n", 330 | " \n", 331 | " def maybe_convert_polygon_to_bbox(self, line: str):\n", 332 | " splitted = line.split(\" \")\n", 333 | " label, rest = splitted[0], splitted[1:]\n", 334 | " label = torch.as_tensor(int(label))\n", 335 | " is_bbox = len(rest) == 4\n", 336 | " if is_bbox:\n", 337 | " return label, torch.as_tensor([float(el) for el in rest])\n", 338 | " else:\n", 339 | " # must be a polygon\n", 340 | " poly = torch.as_tensor([float(el) for el in rest])\n", 341 | " poly = poly.view(-1, 2)\n", 342 | " xmax, ymax = torch.max(poly, dim=0).values\n", 343 | " xmin, ymin = torch.min(poly, dim=0).values\n", 344 | " width, heigh = xmax - xmin, ymax - ymin\n", 345 | " xcenter, ycenter = xmin + width / 2, ymin + heigh / 2\n", 346 | " return label, torch.stack([xcenter, ycenter, width, heigh])\n", 347 | " \n", 348 | " def __getitem__(self, idx: int):\n", 349 | " with self.annotations_paths[idx].open('r') as f:\n", 350 | " for line in f.readlines():\n", 351 | " label, bbox = self.maybe_convert_polygon_to_bbox(line)\n", 352 | " return label, bbox \n", 353 | " return torch.tensor(IGNORE), torch.as_tensor([IGNORE, IGNORE, IGNORE, IGNORE], dtype=torch.float32)\n", 354 | " \n", 355 | " def __len__(self):\n", 356 | " return len(self.annotations_paths)" 357 | ] 358 | }, 359 | { 360 | "cell_type": "markdown", 361 | "id": "d6f71fe7", 362 | "metadata": {}, 363 | "source": [ 364 | "Let's try it out" 365 | ] 366 | }, 367 | { 368 | "cell_type": "code", 369 | "execution_count": null, 370 | "id": "d33de9be", 371 | "metadata": {}, 372 | "outputs": [], 373 | "source": [ 374 | "ds = AnnotationDataset(RF100_ROOT / df.index[0] / 'test' / 'labels')\n", 375 | "ds[0]" 376 | ] 377 | }, 378 | { 379 | "cell_type": "markdown", 380 | "id": "8dc11209", 381 | "metadata": {}, 382 | "source": [ 383 | "gg. Now we can use a torch `DataLoader` to speed up stuff. Let's define a couple of functions to help us out" 384 | ] 385 | }, 386 | { 387 | "cell_type": "code", 388 | "execution_count": null, 389 | "id": "e4c70374", 390 | "metadata": {}, 391 | "outputs": [], 392 | "source": [ 393 | "def get_areas_and_labels(dataset: str, split: str =\"test\"):\n", 394 | " ds = AnnotationDataset(RF100_ROOT / dataset / split / 'labels')\n", 395 | " dl = DataLoader(ds, \n", 396 | "# num_workers=1, \n", 397 | " batch_size=128)\n", 398 | "\n", 399 | " all_areas = None\n", 400 | " all_labels = None\n", 401 | " for (labels, bboxes) in dl:\n", 402 | " bboxes = bboxes[labels != IGNORE] \n", 403 | " # area = w * h\n", 404 | " areas = bboxes[:,2] * bboxes[:,3]\n", 405 | " all_areas = torch.cat((all_areas, areas)) if all_areas is not None else areas\n", 406 | " all_labels = torch.cat((all_labels, labels)) if all_labels is not None else labels\n", 407 | "\n", 408 | " return all_areas, all_labels\n", 409 | "\n", 410 | "\n", 411 | "def compute_stats(areas: torch.Tensor):\n", 412 | " # let's compute the number of small, medium and large bbox\n", 413 | " bins = torch.histc(areas, bins=3, min=0, max=0.3)\n", 414 | " return areas.mean(), areas.std(), *bins" 415 | ] 416 | }, 417 | { 418 | "cell_type": "code", 419 | "execution_count": null, 420 | "id": "1bb8c0b9", 421 | "metadata": {}, 422 | "outputs": [], 423 | "source": [ 424 | "@maybe_load_from_disk(Path(\"../temp/bbox.pkl\"))\n", 425 | "def create_bbox_df(df):\n", 426 | " records = []\n", 427 | " dataset_bar = tqdm(df.index)\n", 428 | " for dataset in dataset_bar:\n", 429 | " dataset_bar.set_postfix_str(dataset)\n", 430 | " split_bar = tqdm([\"train\", \"test\", \"valid\"], leave=False)\n", 431 | " for split in split_bar:\n", 432 | " split_bar.set_postfix_str(split)\n", 433 | " areas, labels = get_areas_and_labels(dataset, split)\n", 434 | " vals = compute_stats(areas)\n", 435 | " vals = [val.float().item() for val in vals]\n", 436 | " area_mean, area_std, num_small, num_medium, num_large = vals\n", 437 | " labels = labels[labels != IGNORE]\n", 438 | " records.append(dict(\n", 439 | " num_classes=labels.unique().numpy().shape[0],\n", 440 | " labels=labels.unique().numpy(),\n", 441 | " areas=areas.numpy(),\n", 442 | " area_mean=area_mean, \n", 443 | " area_std=area_std, \n", 444 | " num_small=num_small, \n", 445 | " num_medium=num_medium, \n", 446 | " num_large=num_large,\n", 447 | " split=split,\n", 448 | " dataset=dataset,\n", 449 | " )\n", 450 | " )\n", 451 | "\n", 452 | " return pd.DataFrame.from_records(records)" 453 | ] 454 | }, 455 | { 456 | "cell_type": "code", 457 | "execution_count": null, 458 | "id": "a40c34a5", 459 | "metadata": {}, 460 | "outputs": [], 461 | "source": [ 462 | "bbox_df = create_bbox_df(df)\n", 463 | "bbox_df" 464 | ] 465 | }, 466 | { 467 | "cell_type": "code", 468 | "execution_count": null, 469 | "id": "3957a3c9", 470 | "metadata": {}, 471 | "outputs": [], 472 | "source": [ 473 | "train_df = bbox_df[bbox_df[\"split\"] == \"train\"].reset_index(drop=True)\n", 474 | "valid_df = bbox_df[bbox_df[\"split\"] == \"valid\"].reset_index(drop=True)\n", 475 | "test_df = bbox_df[bbox_df[\"split\"] == \"test\"].reset_index(drop=True)" 476 | ] 477 | }, 478 | { 479 | "cell_type": "code", 480 | "execution_count": null, 481 | "id": "d7612dea", 482 | "metadata": {}, 483 | "outputs": [], 484 | "source": [ 485 | "train_df" 486 | ] 487 | }, 488 | { 489 | "cell_type": "markdown", 490 | "id": "32b95209", 491 | "metadata": {}, 492 | "source": [ 493 | "check if we have all the labels in all splits" 494 | ] 495 | }, 496 | { 497 | "cell_type": "code", 498 | "execution_count": null, 499 | "id": "916c32b0", 500 | "metadata": {}, 501 | "outputs": [], 502 | "source": [ 503 | "import numpy as np \n", 504 | "\n", 505 | "all_missing_labels = []\n", 506 | "all_is_correct = []\n", 507 | "for idx, (train_labels, valid_labels, test_labels) in tqdm(\n", 508 | " zip(train_df.index,\n", 509 | " zip(train_df[\"labels\"].values, valid_df[\"labels\"].values, test_df[\"labels\"].values\n", 510 | " )\n", 511 | " ),\n", 512 | " total=len(train_df)):\n", 513 | " # see https://numpy.org/doc/stable/reference/generated/numpy.setdiff1d.html\n", 514 | " missing_from_valid = np.setdiff1d(valid_labels, train_labels)\n", 515 | " missing_from_test = np.setdiff1d(test_labels, train_labels)\n", 516 | " missing_labels = np.array([])\n", 517 | " \n", 518 | " if missing_from_valid.shape[0] > 0:\n", 519 | " missing_labels = np.concatenate((missing_labels, missing_from_valid))\n", 520 | " if missing_from_test.shape[0] > 0:\n", 521 | " missing_labels = np.concatenate((missing_labels, missing_from_test))\n", 522 | " \n", 523 | " is_correct = missing_labels.shape[0] == 0\n", 524 | " all_missing_labels.append(missing_labels)\n", 525 | " all_is_correct.append(is_correct)\n", 526 | "\n", 527 | "train_df[\"missing_labels\"] = all_missing_labels\n", 528 | "train_df[\"is_correct\"] = all_is_correct\n" 529 | ] 530 | }, 531 | { 532 | "cell_type": "code", 533 | "execution_count": null, 534 | "id": "65602f35", 535 | "metadata": {}, 536 | "outputs": [], 537 | "source": [ 538 | "train_df" 539 | ] 540 | }, 541 | { 542 | "cell_type": "code", 543 | "execution_count": null, 544 | "id": "cf70e426", 545 | "metadata": {}, 546 | "outputs": [], 547 | "source": [ 548 | "wrong_df = train_df[~train_df[\"is_correct\"]]" 549 | ] 550 | }, 551 | { 552 | "cell_type": "code", 553 | "execution_count": null, 554 | "id": "1192f4e2", 555 | "metadata": {}, 556 | "outputs": [], 557 | "source": [ 558 | "wrong_df[[\"dataset\", \"missing_labels\"]].to_csv(\"missing.csv\")" 559 | ] 560 | }, 561 | { 562 | "cell_type": "markdown", 563 | "id": "cd0f48c0", 564 | "metadata": {}, 565 | "source": [ 566 | "Let's add all the prev informations" 567 | ] 568 | }, 569 | { 570 | "cell_type": "markdown", 571 | "id": "72fb6bab", 572 | "metadata": {}, 573 | "source": [ 574 | "merging areas" 575 | ] 576 | }, 577 | { 578 | "cell_type": "code", 579 | "execution_count": null, 580 | "id": "cd43bd1b", 581 | "metadata": {}, 582 | "outputs": [], 583 | "source": [ 584 | "records = {}\n", 585 | "\n", 586 | "for idx, train_area, valid_area, test_area in zip(train_df[\"dataset\"], \n", 587 | " train_df[\"areas\"].values, \n", 588 | " valid_df[\"areas\"].values,\n", 589 | " test_df[\"areas\"].values):\n", 590 | " records[idx] = np.concatenate([train_area, valid_area, test_area])\n", 591 | " \n", 592 | "\n", 593 | "areas_series = pd.Series(records)\n", 594 | "areas_series" 595 | ] 596 | }, 597 | { 598 | "cell_type": "code", 599 | "execution_count": null, 600 | "id": "c79cf5ba", 601 | "metadata": {}, 602 | "outputs": [], 603 | "source": [ 604 | "bbox_df_grouped[\"areas\"] = areas_series\n", 605 | "bbox_df_grouped.head(2)" 606 | ] 607 | }, 608 | { 609 | "cell_type": "code", 610 | "execution_count": null, 611 | "id": "2a7fcc76", 612 | "metadata": {}, 613 | "outputs": [], 614 | "source": [ 615 | "df_with_bbox = df.join(bbox_df_grouped, how=\"inner\")" 616 | ] 617 | }, 618 | { 619 | "cell_type": "code", 620 | "execution_count": null, 621 | "id": "376d3b5c", 622 | "metadata": {}, 623 | "outputs": [], 624 | "source": [ 625 | "df_with_bbox" 626 | ] 627 | }, 628 | { 629 | "cell_type": "code", 630 | "execution_count": null, 631 | "id": "6836851a", 632 | "metadata": {}, 633 | "outputs": [], 634 | "source": [ 635 | "fig = plt.figure(figsize=(25, 40))\n", 636 | "plot = sns.boxplot(data=df_with_bbox[\"areas\"], orient='h')\n", 637 | "plt.xlabel(\"bbox\")\n", 638 | "plt.ylabel(\"dataset\")\n", 639 | "plot.set_yticklabels(df.index)\n", 640 | "plt.gcf().savefig(\"plot_all_train.png\")" 641 | ] 642 | }, 643 | { 644 | "cell_type": "code", 645 | "execution_count": null, 646 | "id": "e582f4bd", 647 | "metadata": {}, 648 | "outputs": [], 649 | "source": [ 650 | "plt.gcf().savefig(\"plot.png\")" 651 | ] 652 | }, 653 | { 654 | "cell_type": "markdown", 655 | "id": "09e3f3cc", 656 | "metadata": {}, 657 | "source": [ 658 | "### Clip Embeddings\n", 659 | "\n", 660 | "I have sampled 512 points per dataset and encoded them with CLIP. Let's load them, avg them and plot in 2D (after PCA). Let's do it! " 661 | ] 662 | }, 663 | { 664 | "cell_type": "code", 665 | "execution_count": null, 666 | "id": "1a0acf53", 667 | "metadata": {}, 668 | "outputs": [], 669 | "source": [ 670 | "import clip\n", 671 | "from typing import Callable\n", 672 | "from PIL import Image\n", 673 | "\n", 674 | "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", 675 | "RF100_ROOT = Path(\"./rf100/\")\n", 676 | "\n", 677 | "class ImageDataset(Dataset):\n", 678 | " def __init__(self, root: Path, fmt: str = \"jpg\", transform: Callable = None):\n", 679 | " super().__init__()\n", 680 | " self.images_path = list(root.glob(f\"**/*.{fmt}\"))\n", 681 | " self.transform = transform or ToTensor()\n", 682 | "\n", 683 | " def __getitem__(self, idx: int):\n", 684 | " image = Image.open(self.images_path[idx]).convert(\"RGB\")\n", 685 | " return self.transform(image), idx, str(self.images_path[idx])\n", 686 | "\n", 687 | " def __len__(self):\n", 688 | " return len(self.images_path)\n", 689 | "\n", 690 | "def pca(x, k, center=True):\n", 691 | " if center:\n", 692 | " m = x.mean(0, keepdim=True)\n", 693 | " s = x.std(0, unbiased=False, keepdim=True)\n", 694 | " x -= m\n", 695 | " x /= s\n", 696 | " # why pca related to svd? https://www.cs.cmu.edu/~elaw/papers/pca.pdf chap VI\n", 697 | " U, S, V = torch.linalg.svd(x) \n", 698 | " reduced = torch.mm(x, V[:k].T)\n", 699 | "\n", 700 | " return reduced\n", 701 | "\n", 702 | "@maybe_load_from_disk(Path(\"./embeddings_means.pkl\"))\n", 703 | "def get_embeddings(df):\n", 704 | " MAX_BATCHES = 2\n", 705 | " model, preprocess = clip.load(\"ViT-B/32\", device=device, jit=True)\n", 706 | " records = []\n", 707 | " for dataset in tqdm(df.index):\n", 708 | " ds = ImageDataset(RF100_ROOT / dataset / \"train/images\", transform=preprocess)\n", 709 | " dl = DataLoader(\n", 710 | " ds, batch_size=256, num_workers=8, pin_memory=True, shuffle=True\n", 711 | " ) # we shuffle and we sample MAX_BATCHES batches per dataset\n", 712 | " i = 0\n", 713 | " means = None\n", 714 | " for (x, _, _) in dl:\n", 715 | " with torch.no_grad():\n", 716 | " x = x.to(\"cuda\")\n", 717 | " x = model.encode_image(x)\n", 718 | " means = torch.vstack((means, x.mean(0))) if means is not None else x.mean(0)\n", 719 | " i += 1\n", 720 | " if i >= MAX_BATCHES: break\n", 721 | " if len(means.shape) == 1: means = means.unsqueeze(0)\n", 722 | " \n", 723 | " records.append(dict(dataset=dataset, clip_mean=means.mean(0).squeeze().cpu().numpy()))\n", 724 | " \n", 725 | " return pd.DataFrame.from_records(records, index=df.index)\n", 726 | "\n", 727 | "\n" 728 | ] 729 | }, 730 | { 731 | "cell_type": "code", 732 | "execution_count": null, 733 | "id": "34aeb7b8", 734 | "metadata": {}, 735 | "outputs": [], 736 | "source": [ 737 | "embed_df = get_embeddings(df)\n", 738 | "embed_df.head(2)" 739 | ] 740 | }, 741 | { 742 | "cell_type": "code", 743 | "execution_count": null, 744 | "id": "2b45d642", 745 | "metadata": {}, 746 | "outputs": [], 747 | "source": [ 748 | "clip_means = torch.stack([torch.from_numpy(el) for el in embed_df.clip_mean.tolist()])" 749 | ] 750 | }, 751 | { 752 | "cell_type": "code", 753 | "execution_count": null, 754 | "id": "9614baa4", 755 | "metadata": {}, 756 | "outputs": [], 757 | "source": [ 758 | "clip_means_reduced = pca(clip_means.float(), k=2)\n", 759 | "clip_means_reduced.shape" 760 | ] 761 | }, 762 | { 763 | "cell_type": "code", 764 | "execution_count": null, 765 | "id": "f94b4279", 766 | "metadata": {}, 767 | "outputs": [], 768 | "source": [ 769 | "fig = plt.figure(figsize=(10, 8))\n", 770 | "ax = sns.scatterplot(x=clip_means_reduced[:,0], y=clip_means_reduced[:,1], size=df[\"size\"], sizes=(0, 500), hue=df.category, alpha=.66, legend=\"brief\")\n", 771 | "sns.move_legend(ax, bbox_to_anchor=(1.02, 1), loc='upper left')" 772 | ] 773 | }, 774 | { 775 | "cell_type": "code", 776 | "execution_count": null, 777 | "id": "34c0850a", 778 | "metadata": {}, 779 | "outputs": [], 780 | "source": [ 781 | "plt.savefig(\"embedds.png\",)" 782 | ] 783 | }, 784 | { 785 | "cell_type": "markdown", 786 | "id": "5b350266", 787 | "metadata": {}, 788 | "source": [ 789 | "## Plotting" 790 | ] 791 | }, 792 | { 793 | "cell_type": "code", 794 | "execution_count": null, 795 | "id": "0e82fb55", 796 | "metadata": {}, 797 | "outputs": [], 798 | "source": [ 799 | "df_with_bbox" 800 | ] 801 | }, 802 | { 803 | "cell_type": "code", 804 | "execution_count": null, 805 | "id": "5a2b42f3", 806 | "metadata": {}, 807 | "outputs": [], 808 | "source": [ 809 | "plt.savefig(\"datasets.png\")" 810 | ] 811 | }, 812 | { 813 | "cell_type": "code", 814 | "execution_count": null, 815 | "id": "287b1b55", 816 | "metadata": {}, 817 | "outputs": [], 818 | "source": [ 819 | "fig, axs = plt.subplots(2, 2, figsize=(20,16))\n", 820 | "import matplotlib as mpl\n", 821 | "mpl.rcParams['axes.titlepad'] = 16\n", 822 | "# fig.suptitle('Datasets Categories')\n", 823 | "axs[0,0].set_title(\"Total datasets size/category\")\n", 824 | "ax = sns.barplot(data=grouped_df, x=\"size\", y=\"category\", linewidth=2, edgecolor=\".2\", ax=axs[0,0])\n", 825 | "ax.set_ylabel('category')\n", 826 | "ax.set_xlabel('size')\n", 827 | "secax = ax.secondary_xaxis('top', functions=(lambda x: x / df_with_bbox[\"size\"].sum(), lambda x:x))\n", 828 | "secax.set_xlabel('size (%)')\n", 829 | "ax.minorticks_on()\n", 830 | "secax.minorticks_on()\n", 831 | "\n", 832 | "axs[0,1].set_title(\"Mean datasets size/category\")\n", 833 | "ax = sns.boxplot(data=df_with_bbox, x=\"size\", y=\"category\", ax=axs[0,1])\n", 834 | "ax.set_xlabel('size')\n", 835 | "ax.get_yaxis().set_visible(False)\n", 836 | "secax = ax.secondary_xaxis('top', functions=(lambda x: x / df_with_bbox[\"size\"].sum(), lambda x:x))\n", 837 | "secax.set_xlabel('size (%)')\n", 838 | "ax.minorticks_on()\n", 839 | "secax.minorticks_on()\n", 840 | "\n", 841 | "axs[1,0].set_title(\"Mean bbox area\")\n", 842 | "ax = sns.boxplot(data=df_with_bbox, x=\"area_mean\", y=\"category\", ax=axs[1,0])\n", 843 | "ax.set_xlabel(\"bbox\")\n", 844 | "\n", 845 | "axs[1,1].set_title(\"Mean num_classes\")\n", 846 | "ax = sns.boxplot(data=df_with_bbox, x=\"num_classes\", y=\"category\", ax=axs[1,1])\n", 847 | "ax.set_xlabel(\"labels\")\n", 848 | "ax.get_yaxis().set_visible(False)\n" 849 | ] 850 | }, 851 | { 852 | "cell_type": "code", 853 | "execution_count": null, 854 | "id": "08cc6098", 855 | "metadata": {}, 856 | "outputs": [], 857 | "source": [ 858 | "plt.savefig(\"datasets_stats.png\")" 859 | ] 860 | } 861 | ], 862 | "metadata": { 863 | "kernelspec": { 864 | "display_name": "Python 3 (ipykernel)", 865 | "language": "python", 866 | "name": "python3" 867 | }, 868 | "language_info": { 869 | "codemirror_mode": { 870 | "name": "ipython", 871 | "version": 3 872 | }, 873 | "file_extension": ".py", 874 | "mimetype": "text/x-python", 875 | "name": "python", 876 | "nbconvert_exporter": "python", 877 | "pygments_lexer": "ipython3", 878 | "version": "3.9.13" 879 | }, 880 | "vscode": { 881 | "interpreter": { 882 | "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" 883 | } 884 | } 885 | }, 886 | "nbformat": 4, 887 | "nbformat_minor": 5 888 | } 889 | -------------------------------------------------------------------------------- /notebooks/kmeans.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "384929b3", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "%matplotlib notebook" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "id": "80ed0138", 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "import sys\n", 21 | "sys.path.append(\"../\")" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "id": "90893ff0", 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "import numpy as np\n", 32 | "from pickle import load, dump\n", 33 | "from pathlib import Path\n", 34 | "from tqdm import tqdm\n", 35 | "import torch\n", 36 | "from torchvision.utils import make_grid\n", 37 | "import torchvision.transforms as T\n", 38 | "from torchvision.transforms.functional import to_pil_image\n", 39 | "from PIL import Image\n", 40 | "from multiprocessing import Pool\n", 41 | "from scripts.image.kmeans import kmeans\n", 42 | "import pandas as pd\n", 43 | "import matplotlib.pyplot as plt\n", 44 | "import pickle" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "id": "afc25fa4", 50 | "metadata": {}, 51 | "source": [ 52 | "Let's open and get the data from the pickles file, we are using the embedddings **after tsne**. You can obtain them by running the `/scripts/reduce_dims.py` script.\n", 53 | "\n", 54 | "```\n", 55 | "python ./scripts/reduce_dims -i ./ -i . -k 2 --method tsne\n", 56 | "```" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "id": "a10b4c86", 63 | "metadata": {}, 64 | "outputs": [], 65 | "source": [ 66 | "DATA_PATH = \"../reduced-tsne-k=2.pk\"" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": null, 72 | "id": "617cdad8", 73 | "metadata": {}, 74 | "outputs": [], 75 | "source": [ 76 | "with open(DATA_PATH, \"rb\") as f:\n", 77 | " data = pickle.load(f)\n", 78 | "\n", 79 | "data" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "id": "de0cde35", 85 | "metadata": {}, 86 | "source": [ 87 | "Let's also add the category to each element" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "id": "a2603f1c", 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "def add_categories_to_data(data, categories):\n", 98 | " data['categories'] = []\n", 99 | " for image_paths in tqdm(data['image_paths']):\n", 100 | " # e.g.rf100/chess-pieces-mjzgj/train/images/foo.jpg'\n", 101 | " dataset_name = Path(image_paths).parts[1]\n", 102 | " category = categories.loc[dataset_name].category\n", 103 | " data['categories'].append(category)\n", 104 | " \n", 105 | " return data" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "id": "c1b624b2", 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "categories = pd.read_csv(\"../metadata/categories.csv\", index_col=0)\n", 116 | "data = add_categories_to_data(data, categories)" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "id": "29b6db3b", 123 | "metadata": {}, 124 | "outputs": [], 125 | "source": [ 126 | "np.unique(data['categories'])" 127 | ] 128 | }, 129 | { 130 | "cell_type": "markdown", 131 | "id": "9216ce3d", 132 | "metadata": {}, 133 | "source": [ 134 | "sweet, now we have all the data we need. Let's do some clustering" 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "id": "5c01c80c", 140 | "metadata": {}, 141 | "source": [ 142 | "## Clustering\n", 143 | "Let's define some transformations" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": null, 149 | "id": "764a475a", 150 | "metadata": {}, 151 | "outputs": [], 152 | "source": [ 153 | "import torchvision.transforms.functional as F\n", 154 | "\n", 155 | "def read_image_and_transform(image_path, size=(224, 224)):\n", 156 | " image_path = \"../\" + image_path\n", 157 | " img = Image.open(image_path).convert(\"RGB\")\n", 158 | " img = F.resize(img, size)\n", 159 | " img = F.to_tensor(img)\n", 160 | " return img" 161 | ] 162 | }, 163 | { 164 | "cell_type": "markdown", 165 | "id": "30d76b81", 166 | "metadata": {}, 167 | "source": [ 168 | "### Cluster per category\n", 169 | "\n", 170 | "We first want to get the 100 most representative images per category, thus we will first filter per category and run kmeans with 100" 171 | ] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "execution_count": null, 176 | "id": "3dff0180", 177 | "metadata": {}, 178 | "outputs": [], 179 | "source": [ 180 | "def reored_by_left_top(x):\n", 181 | " left, top = x[:,0].min(), x[:,1].max()\n", 182 | " to_compare = torch.tensor([left, top])\n", 183 | " indexes = ((to_compare - x) ** 2).sum(dim=-1).argsort()\n", 184 | " return indexes" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": null, 190 | "id": "7958dec4", 191 | "metadata": {}, 192 | "outputs": [], 193 | "source": [ 194 | "plt.rcParams[\"savefig.bbox\"] = 'tight'\n", 195 | "\n", 196 | "def show(imgs):\n", 197 | " if not isinstance(imgs, list):\n", 198 | " imgs = [imgs]\n", 199 | " fig, axs = plt.subplots(ncols=len(imgs), squeeze=False)\n", 200 | " for i, img in enumerate(imgs):\n", 201 | " img = img.detach()\n", 202 | " img = F.to_pil_image(img)\n", 203 | " axs[0, i].imshow(np.asarray(img))\n", 204 | " axs[0, i].set(xticklabels=[], yticklabels=[], xticks=[], yticks=[])\n", 205 | " return fig" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": null, 211 | "id": "4bbc9d66", 212 | "metadata": {}, 213 | "outputs": [], 214 | "source": [ 215 | "def cluter_per_category(category=\"real world\", num_clusters=100):\n", 216 | " filtered_indexes = np.array(data[\"categories\"]) == category\n", 217 | " # let's use the pca ones\n", 218 | " filtered_x = data['x'][filtered_indexes]\n", 219 | " filtered_image_paths = np.array(data['image_paths'])[filtered_indexes]\n", 220 | " # do kmeans\n", 221 | " means, bins = kmeans(torch.from_numpy(filtered_x), num_clusters=num_clusters, num_iters=100)\n", 222 | "# means = reored_by_left_top(means)\n", 223 | " # compute distance between means and all points\n", 224 | " diffs = (means[:,None,:] - filtered_x[None,...])\n", 225 | " diffs = (diffs**2).sum(dim=-1)\n", 226 | " indexes = diffs.argmin(axis=1)\n", 227 | " # create the grid\n", 228 | " image_paths = filtered_image_paths[indexes]\n", 229 | " indexes = reored_by_left_top(filtered_x[indexes])\n", 230 | " image_paths = image_paths[indexes]\n", 231 | "\n", 232 | " image = show(\n", 233 | " make_grid(\n", 234 | " list(map(read_image_and_transform, image_paths)),\n", 235 | " nrow=25\n", 236 | " )\n", 237 | " )\n", 238 | " return image" 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": null, 244 | "id": "cfc059a3", 245 | "metadata": {}, 246 | "outputs": [], 247 | "source": [ 248 | "dst = Path(\"../paper/images/grid/\")\n", 249 | "\n", 250 | "for category in categories.category.unique():\n", 251 | " num_clusters = 50\n", 252 | " if category == \"real world\":\n", 253 | " num_clusters = 200\n", 254 | " cluter_per_category(category, num_clusters).savefig(dst / f\"{category}.png\", dpi=800, bbox_inches='tight')" 255 | ] 256 | }, 257 | { 258 | "cell_type": "markdown", 259 | "id": "97c9cd7f", 260 | "metadata": {}, 261 | "source": [ 262 | "## Grid Image" 263 | ] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": null, 268 | "id": "40b3fea9", 269 | "metadata": {}, 270 | "outputs": [], 271 | "source": [ 272 | "def make_cluster_grid(num_clusters, nrow):\n", 273 | " # let's use the pca ones\n", 274 | " x = data['x']\n", 275 | " image_paths = np.array(data['image_paths'])\n", 276 | " # do kmeans\n", 277 | " means, bins = kmeans(torch.from_numpy(x), num_clusters=num_clusters, num_iters=50)\n", 278 | " diffs = (means[:,None,:] - x[None,...])\n", 279 | " diffs = (diffs**2).sum(dim=-1)\n", 280 | " indexes = diffs.argmin(axis=1)\n", 281 | " # create the grid\n", 282 | " image_paths = image_paths[indexes]\n", 283 | " indexes = reored_by_left_top(x[indexes])\n", 284 | " image_paths = image_paths[indexes]\n", 285 | " # create the grid\n", 286 | " image = show(\n", 287 | " make_grid(\n", 288 | " list(map(lambda x: read_image_and_transform(x, size=(128,128)), image_paths)),\n", 289 | " nrow=nrow\n", 290 | " )\n", 291 | " )\n", 292 | " return image" 293 | ] 294 | }, 295 | { 296 | "cell_type": "code", 297 | "execution_count": null, 298 | "id": "4bcae7d4", 299 | "metadata": {}, 300 | "outputs": [], 301 | "source": [ 302 | "num_clusters = 40 * 60 \n", 303 | "\n", 304 | "make_cluster_grid(num_clusters, nrow=60).savefig(dst / f\"rf100-40x60.png\", dpi=800, bbox_inches='tight')" 305 | ] 306 | }, 307 | { 308 | "cell_type": "code", 309 | "execution_count": null, 310 | "id": "8d2a5423", 311 | "metadata": {}, 312 | "outputs": [], 313 | "source": [ 314 | "make_cluster_grid(num_clusters, nrow=8).savefig(dst / f\"rf100-8x8.png\", dpi=800, bbox_inches='tight')" 315 | ] 316 | }, 317 | { 318 | "cell_type": "code", 319 | "execution_count": null, 320 | "id": "a33aa47d", 321 | "metadata": {}, 322 | "outputs": [], 323 | "source": [] 324 | } 325 | ], 326 | "metadata": { 327 | "kernelspec": { 328 | "display_name": "Python 3 (ipykernel)", 329 | "language": "python", 330 | "name": "python3" 331 | }, 332 | "language_info": { 333 | "codemirror_mode": { 334 | "name": "ipython", 335 | "version": 3 336 | }, 337 | "file_extension": ".py", 338 | "mimetype": "text/x-python", 339 | "name": "python", 340 | "nbconvert_exporter": "python", 341 | "pygments_lexer": "ipython3", 342 | "version": "3.9.13" 343 | }, 344 | "vscode": { 345 | "interpreter": { 346 | "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" 347 | } 348 | } 349 | }, 350 | "nbformat": 4, 351 | "nbformat_minor": 5 352 | } 353 | -------------------------------------------------------------------------------- /notebooks/roboflow-100-pytorch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "b471e7ce", 6 | "metadata": {}, 7 | "source": [ 8 | "# RoboFlow 100 - Visualize images with PyTorch\n", 9 | "\n", 10 | "Let's see how to parse and visualize images inside RF100 with PyTorch. Let's start by importing a bunch of stuff" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 8, 16 | "id": "3ead1e30", 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "from pathlib import Path\n", 21 | "from ipywidgets import interact\n", 22 | "from torch.utils.data import Dataset, DataLoader\n", 23 | "import torchvision.transforms as T\n", 24 | "from typing import Callable, Tuple, Optional, List, Dict\n", 25 | "import torch\n", 26 | "from PIL import Image\n", 27 | "from dataclasses import dataclass" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "id": "1113bef0", 33 | "metadata": {}, 34 | "source": [ 35 | "## Downloading RF100\n", 36 | "\n", 37 | "We will assume you are `RF100` saved on your disk, we provided a detailed guide on the official [README](https://github.com/roboflow-ai/roboflow-100-benchmark/blob/main/README.md) with instruction for dowloading it.\n", 38 | "\n", 39 | "**We assume the dataset was downloaded in yolov5/7 format** using the `-f yolov5` flag" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "id": "03f09607", 45 | "metadata": {}, 46 | "source": [ 47 | "## Dataset\n", 48 | "\n", 49 | "We need to know where `RF100` is stored, feel free to change `ROOT`.\n" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 9, 55 | "id": "cd85ea27", 56 | "metadata": {}, 57 | "outputs": [], 58 | "source": [ 59 | "ROOT = Path('../rf100/') # <- change me :)\n", 60 | "datasets = sorted(list(ROOT.iterdir()), key=lambda x: x.stem)" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "id": "d2d6ba27", 66 | "metadata": {}, 67 | "source": [ 68 | "Next, we need the classic [PyTorch `Dataset`](https://pytorch.org/tutorials/beginner/basics/data_tutorial.html)." 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 10, 74 | "id": "28f824fc", 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "class ImageDataset(Dataset):\n", 79 | " def __init__(self, root: Path, split: str = \"train\", transform: Callable = None):\n", 80 | " super().__init__()\n", 81 | " # we have three splits, \"train\", \"valid\" and \"test\"\n", 82 | " self.src = root / split\n", 83 | " # images and labels are linked by the same name\n", 84 | " self.names = list(map(lambda x: x.stem, (self.src / \"labels\").glob(\"*.txt\")))\n", 85 | " self.transform = transform\n", 86 | "\n", 87 | " def get_image(self, image_path: Path) -> Image.Image:\n", 88 | " \"\"\"\n", 89 | " This function opens the image and returns it\n", 90 | " \"\"\"\n", 91 | " image = Image.open(image_path).convert(\"RGB\")\n", 92 | " return image\n", 93 | "\n", 94 | " def get_labels(self, labels_path: Path) -> Optional[torch.Tensor]:\n", 95 | " \"\"\"\n", 96 | " This function reads the label txt file in yolo format, each line of the file looks like\n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " The coordinates are with respect to the image's width and height, so between 0 - 1\n", 101 | " \n", 102 | " We parse the labels with the following steps:\n", 103 | " 1) read line by line\n", 104 | " 2) for each line, get the label id, x_center, y_center, width and height\n", 105 | " 3) convert to a tensor\n", 106 | " 4) add to the previous one by stacking them vertically\n", 107 | " \n", 108 | " The return tensor has shape `batch_size, 5`\n", 109 | " \"\"\"\n", 110 | " labels = None\n", 111 | " with labels_path.open(\"r\") as f:\n", 112 | " for line in f.readlines():\n", 113 | " parsed_line = [float(e) for e in line.strip().split(\" \")]\n", 114 | " if len(parsed_line) != 5:\n", 115 | " continue\n", 116 | " c, x, y, w, h = [float(e) for e in line.strip().split(\" \")]\n", 117 | " label = torch.as_tensor([[c, x, y, w, h]])\n", 118 | " if labels is None:\n", 119 | " labels = label\n", 120 | " else:\n", 121 | " labels = torch.vstack([labels, label])\n", 122 | " return labels\n", 123 | "\n", 124 | " def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:\n", 125 | " \"\"\"\n", 126 | " Parse image and associated labels are return them as a tensor.\n", 127 | " \"\"\"\n", 128 | " image_path = self.src / \"images\" / f\"{self.names[idx]}.jpg\"\n", 129 | " labels_path = self.src / \"labels\" / f\"{self.names[idx]}.txt\"\n", 130 | " image = self.get_image(image_path)\n", 131 | " labels = self.get_labels(labels_path)\n", 132 | " bboxes = None\n", 133 | " if labels is not None:\n", 134 | " bboxes = labels[..., 1:]\n", 135 | " labels = labels[..., 0].to(torch.int)\n", 136 | " image, bboxes = self.transform(image, bboxes)\n", 137 | " else:\n", 138 | " # if we don't have labels, let's use this values\n", 139 | " labels = torch.tensor([-1.0])\n", 140 | " bboxes = torch.as_tensor([[0.0, 0.0, 0.0, 0.0]])\n", 141 | " image, bboxes = self.transform(image, bboxes)\n", 142 | " image = T.functional.to_tensor(image)\n", 143 | " return {\"image\": image, \"bboxes\": bboxes, \"labels\": labels}\n", 144 | "\n", 145 | " def __len__(self):\n", 146 | " return len(self.names)\n" 147 | ] 148 | }, 149 | { 150 | "cell_type": "markdown", 151 | "id": "05f393fd", 152 | "metadata": {}, 153 | "source": [ 154 | "### Transformations\n", 155 | "\n", 156 | "Since we will probably need to resize the image, we can create a custom transformation that works on `image` and `bboxes` as well! [Latest version of `torchvision`](https://pytorch.org/blog/extending-torchvisions-transforms-to-object-detection-segmentation-and-video-tasks/) supports both images and bboxes, feel free to use it instead! " 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": 11, 162 | "id": "a8e7f8cd", 163 | "metadata": {}, 164 | "outputs": [], 165 | "source": [ 166 | "@dataclass\n", 167 | "class Resize:\n", 168 | " size: Tuple[int, int]\n", 169 | " \n", 170 | " def __call__(self, image: Image.Image, bboxes: Optional[torch.Tensor]) -> Tuple[Image.Image, torch.Tensor]:\n", 171 | " # in PIL they are inverted LoL\n", 172 | " w, h = image.size\n", 173 | " if bboxes is not None:\n", 174 | " bboxes *= torch.as_tensor([w, h, w, h])\n", 175 | " image = T.functional.resize(image, self.size)\n", 176 | " new_w, new_h = image.size\n", 177 | " if bboxes is not None:\n", 178 | " # map to new sizes\n", 179 | " bboxes /= torch.as_tensor([w / new_w , h / new_h, w / new_w, h / new_h])\n", 180 | " # to 0 - 1\n", 181 | " bboxes /= torch.as_tensor([new_w, new_h, new_w, new_h])\n", 182 | " return image, bboxes" 183 | ] 184 | }, 185 | { 186 | "cell_type": "markdown", 187 | "id": "f8f52bb4", 188 | "metadata": {}, 189 | "source": [ 190 | "## Visualisation\n", 191 | "Let's create `AnnotatedImage` to help us draw the bboxes on the image, since `torchvision.utils.draw_bounding_boxes` uses `xyxy` format, we will use `torchvision.ops.boxes.box_convert` to convert yolo style annotation (`x_center`, `y_center`, `width`, `height`) to `xyxy`." 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": 12, 197 | "id": "9d1d6e72", 198 | "metadata": {}, 199 | "outputs": [], 200 | "source": [ 201 | "from torchvision.utils import draw_bounding_boxes\n", 202 | "from torchvision.ops.boxes import box_convert\n", 203 | "\n", 204 | "class AnnotatedImage:\n", 205 | " def __init__(self, image: torch.Tensor, bboxes: torch.Tensor, labels: torch.Tensor):\n", 206 | " self.image = image\n", 207 | " self.bboxes = bboxes\n", 208 | " self.labels = labels\n", 209 | " \n", 210 | " def draw(self):\n", 211 | " c, h, w = self.image.shape\n", 212 | " bboxes = self.bboxes * torch.as_tensor([w, h, w, h])\n", 213 | " return draw_bounding_boxes(\n", 214 | " (self.image * 255).to(torch.uint8), \n", 215 | " bboxes, \n", 216 | " width=3,\n", 217 | " colors=[\"yellow\"] * len(self.labels),\n", 218 | " labels=[str(i.item()) for i in self.labels])\n", 219 | " \n", 220 | " " 221 | ] 222 | }, 223 | { 224 | "cell_type": "markdown", 225 | "id": "6d1e1736", 226 | "metadata": {}, 227 | "source": [ 228 | "Finally, we can use jupyter `ipywidgets` module to visualize the images and the labels" 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "execution_count": 15, 234 | "id": "579f0244", 235 | "metadata": { 236 | "scrolled": false 237 | }, 238 | "outputs": [ 239 | { 240 | "data": { 241 | "application/vnd.jupyter.widget-view+json": { 242 | "model_id": "c7e5f633c7d84f8b9b3e3019f3fd1cb6", 243 | "version_major": 2, 244 | "version_minor": 0 245 | }, 246 | "text/plain": [ 247 | "interactive(children=(Dropdown(description='dataset_path', options=('4-fold-defect', 'abdomen-mri', 'acl-x-ray…" 248 | ] 249 | }, 250 | "metadata": {}, 251 | "output_type": "display_data" 252 | } 253 | ], 254 | "source": [ 255 | "import matplotlib.pyplot as plt\n", 256 | "from torchvision.ops.boxes import box_convert\n", 257 | "\n", 258 | "from ipywidgets import interactive,Dropdown, IntSlider\n", 259 | "\n", 260 | "dataset_path_dropdown = Dropdown(options=map(lambda x: x.stem, datasets))\n", 261 | "dataset_path_split = Dropdown(options=[\"train\", \"valid\", \"test\"])\n", 262 | "image_idx_slider = IntSlider(min=0, max=1)\n", 263 | "\n", 264 | "@interact(dataset_path=dataset_path_dropdown, split=dataset_path_split, image_idx=image_idx_slider)\n", 265 | "def visualize(dataset_path: Path, split: str = \"train\", image_idx: int = 0):\n", 266 | " ds = ImageDataset(ROOT / dataset_path, split=split, transform=Resize((640, 640)))\n", 267 | " image_idx_slider.max = len(ds)\n", 268 | " # let's be sure we are within range\n", 269 | " image_idx = min(image_idx_slider.value, len(ds) - 1)\n", 270 | " data = ds[image_idx]\n", 271 | " bboxes = box_convert(data[\"bboxes\"], in_fmt=\"cxcywh\", out_fmt=\"xyxy\")\n", 272 | " img = AnnotatedImage(data[\"image\"], bboxes, data[\"labels\"]).draw()\n", 273 | " return Image.fromarray(img.permute(1,2,0).numpy())\n" 274 | ] 275 | }, 276 | { 277 | "cell_type": "markdown", 278 | "id": "162a9360", 279 | "metadata": {}, 280 | "source": [ 281 | "Et voilà 🥳" 282 | ] 283 | } 284 | ], 285 | "metadata": { 286 | "kernelspec": { 287 | "display_name": "Python 3 (ipykernel)", 288 | "language": "python", 289 | "name": "python3" 290 | }, 291 | "language_info": { 292 | "codemirror_mode": { 293 | "name": "ipython", 294 | "version": 3 295 | }, 296 | "file_extension": ".py", 297 | "mimetype": "text/x-python", 298 | "name": "python", 299 | "nbconvert_exporter": "python", 300 | "pygments_lexer": "ipython3", 301 | "version": "3.9.12" 302 | } 303 | }, 304 | "nbformat": 4, 305 | "nbformat_minor": 5 306 | } 307 | -------------------------------------------------------------------------------- /results.csv: -------------------------------------------------------------------------------- 1 | dataset,yolov5,yolov7,glip 2 | 4-fold-defect,0.97,0.938,0.0 3 | abdomen-mri,0.965,0.958,0.021 4 | acl-x-ray,0.995,0.998,0.0 5 | activity-diagrams-qdobr,0.427,0.509,0.002 6 | aerial-cows,0.854,0.568,0.056 7 | aerial-pool,0.513,0.791,0.013 8 | aerial-spheres,0.993,0.539,0.0 9 | animals-ij5d2,0.761,0.342,0.249 10 | apex-videogame,0.839,0.875, 11 | apples-fvpl5,0.779,0.791, 12 | aquarium-qlnqy,0.79,0.822,0.195 13 | asbestos,0.596,0.611,0.007 14 | avatar-recognition-nuexe,0.889,0.943,0.367 15 | axial-mri,0.638,0.549,0.039 16 | bacteria-ptywi,0.162,0.000961,0.0 17 | bccd-ouzjz,0.912,0.922,0.191 18 | bees-jt5in,0.891,0.68,0.009 19 | bone-fracture-7fylg,0.0851,0.0896,0.0 20 | brain-tumor-m2pbp,0.768,0.809,0.003 21 | cable-damage,0.91,0.574,0.006 22 | cables-nl42k,0.688,0.722,0.01 23 | cavity-rs0uf,0.782,0.799,0.029 24 | cell-towers,0.939,0.942,0.053 25 | cells-uyemf,0.249,0.0845,0.005 26 | chess-pieces-mjzgj,0.977,0.83,0.017 27 | circuit-elements,0.0627,,0.001 28 | circuit-voltages,0.797,0.257,0.009 29 | cloud-types,0.271,0.306,0.006 30 | coins-1apki,0.932,0.977,0.175 31 | construction-safety-gsnvb,0.915,0.915,0.259 32 | coral-lwptl,0.174,0.218,0.001 33 | corrosion-bi3q3,0.768,0.764,0.003 34 | cotton-20xz5,0.569,0.591,0.157 35 | cotton-plant-disease,0.204,0.0518,0.0 36 | csgo-videogame,0.974,0.964,0.184 37 | currency-v4f8j,0.583,0.514,0.099 38 | digits-t2eg6,0.989,0.989,0.003 39 | document-parts,0.677,0.666,0.033 40 | excavators-czvg9,0.946,0.895,0.274 41 | farcry6-videogame,0.619,0.216,0.248 42 | fish-market-ggjso,0.92,0.988,0.013 43 | flir-camera-objects,0.796,0.824,0.073 44 | furniture-ngpea,0.983,0.968,0.836 45 | gauge-u2lwv,0.642,0.668,0.217 46 | grass-weeds,0.781,0.781,0.106 47 | gynecology-mri,0.323,0.171,0.0 48 | halo-infinite-angel-videogame,0.921,0.924,0.173 49 | hand-gestures-jps7z,0.995,0.995, 50 | insects-mytwu,0.89,0.858,0.024 51 | leaf-disease-nsdsr,0.531,0.56,0.027 52 | lettuce-pallets,0.945,0.966,0.031 53 | liver-disease,0.592,0.583,0.0 54 | marbles,0.992,0.473,0.03 55 | mask-wearing-608pr,0.788,0.513,0.008 56 | mitosis-gjs3g,0.931,0.739,0.001 57 | number-ops,0.99,0.992,0.055 58 | paper-parts,0.59,0.796,0.007 59 | paragraphs-co84b,0.626,0.61,0.0 60 | parasites-1s07h,0.848,0.889,0.036 61 | peanuts-sd4kf,0.995,0.997,0.358 62 | peixos-fish,0.148,0.821,0.004 63 | people-in-paintings,0.575,0.678,0.168 64 | pests-2xlvx,0.136,0.029,0.004 65 | phages,0.854,0.842,0.002 66 | pills-sxdht,0.869,0.867,0.194 67 | poker-cards-cxcvz,0.886,0.251,-0.0 68 | printed-circuit-board,0.0907,,0.0 69 | radio-signal,0.673,0.653,0.066 70 | road-signs-6ih4y,0.963,0.944,0.036 71 | road-traffic,0.597,0.847, 72 | robomasters-285km,0.816,0.772,0.003 73 | secondary-chains,0.341,0.312,0.0 74 | sedimentary-features-9eosf,0.327,0.244,0.0 75 | shark-teeth-5atku,0.948,0.863,0.025 76 | sign-language-sokdr,0.87,0.255,0.006 77 | signatures-xc8up,0.961,0.932,0.082 78 | smoke-uvylj,0.959,0.962,0.431 79 | soccer-players-5fuqs,0.66,0.399,0.065 80 | soda-bottles,0.964,,0.098 81 | solar-panels-taxvb,0.413,0.261,0.0 82 | stomata-cells,0.84,0.847,0.012 83 | street-work,0.478,0.708,0.148 84 | tabular-data-wf9uh,0.752,0.782,0.018 85 | team-fight-tactics,0.961,0.88,0.0 86 | thermal-cheetah-my4dp,0.931,0.513,0.028 87 | thermal-dogs-and-people-x6ejw,0.967,0.957,0.47 88 | trail-camera,0.966,0.969,0.512 89 | truck-movement,0.786,0.846,0.007 90 | tweeter-posts,0.708,0.495,0.005 91 | tweeter-profile,0.988,0.99,0.002 92 | underwater-objects-5v7p8,0.693,0.453,0.005 93 | underwater-pipes-4ng4t,0.995,0.998,0.733 94 | uno-deck,0.993,0.994,0.013 95 | valentines-chocolate,0.11,0.0593,0.013 96 | vehicles-q0x2v,0.454,0.464,0.029 97 | wall-damage,0.5,0.434, 98 | washroom-rf1fa,0.619,0.634,0.146 99 | weed-crop-aerial,0.82,0.615,0.027 100 | wine-labels,0.569,0.632,0.045 101 | x-ray-rheumatology,0.722,0.506,0.0 102 | -------------------------------------------------------------------------------- /scripts/convert_table_from_old_flo_paper_to_csv.py: -------------------------------------------------------------------------------- 1 | """ 2 | Poor man script to convert the table from an old version of the paper. This was the only way since we didn't have access to the original data source from the table 3 | """ 4 | from argparse import ArgumentParser 5 | from pathlib import Path 6 | 7 | import pandas as pd 8 | 9 | parser = ArgumentParser() 10 | parser.add_argument("-f", "--file", type=Path) 11 | 12 | args = parser.parse_args() 13 | filepath = args.file 14 | filepath = "./file.csv" 15 | 16 | src_df = pd.read_csv( 17 | filepath, 18 | header=None, 19 | names=[ 20 | "name", 21 | "classes", 22 | "labelling hours", 23 | "train", 24 | "valid", 25 | "test", 26 | "yolov5", 27 | "yolov7", 28 | "glip", 29 | "url", 30 | ], 31 | ) 32 | 33 | src_df["dataset"] = src_df["url"].apply(lambda x: Path(x).parent.stem) 34 | glip_df = src_df[["dataset", "glip"]].set_index("dataset") 35 | src_df = src_df[["dataset", "labelling hours"]].set_index("dataset") 36 | 37 | 38 | src_df.to_csv("./metadata/labeling_hours.csv") 39 | glip_df.to_csv("./metadata/results_glip.csv") 40 | -------------------------------------------------------------------------------- /scripts/create_rf100-coco-eval.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | root=$1 3 | dest=$(pwd)/rf100-coco-eval 4 | 5 | mkdir -p $dest 6 | 7 | for dataset in $(ls $root); do 8 | echo "$dataset" 9 | mkdir -p $dest/$dataset 10 | cp $root/$dataset/test/_annotations.coco.json $dest/$dataset/ 11 | done 12 | 13 | echo "zipping ..." 14 | zip -r rf100-coco-eval.zip rf100-coco-eval/ -------------------------------------------------------------------------------- /scripts/download_dataset.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | from os import environ 3 | from pathlib import Path 4 | 5 | from roboflow import Roboflow 6 | 7 | 8 | def main(): 9 | # construct the argument parser and parse the arguments 10 | parser = ArgumentParser() 11 | 12 | parser.add_argument( 13 | "-p", 14 | "--project", 15 | required=True, 16 | type=str, 17 | help="The project ID of the dataset found in the dataset URL.", 18 | ) 19 | parser.add_argument( 20 | "-v", 21 | "--version", 22 | required=True, 23 | type=int, 24 | help="The version the dataset you want to use", 25 | ) 26 | parser.add_argument( 27 | "-f", 28 | "--model_format", 29 | required=False, 30 | type=str, 31 | default="coco", 32 | help="The format of the export you want to use (i.e. coco or yolov5)", 33 | ) 34 | 35 | parser.add_argument( 36 | "-l", 37 | "--location", 38 | required=False, 39 | type=str, 40 | default="./rf100", 41 | help="Where to store the dataset", 42 | ) 43 | # parses command line arguments 44 | args = vars(parser.parse_args()) 45 | 46 | try: 47 | api_key = environ["ROBOFLOW_API_KEY"] 48 | except KeyError: 49 | raise KeyError( 50 | "You must export your Roboflow api key, to obtain one see https://docs.roboflow.com/rest-api." 51 | ) 52 | # create location if it doesn't exist 53 | out_dir = Path(args["location"]) / args["project"] 54 | # out_dir.mkdir(parents=True, exist_ok=True) 55 | print(f'Storing {args["project"] } in {out_dir} for {args["model_format"]}') 56 | # get and download the dataset 57 | rf = Roboflow(api_key=api_key) 58 | project = rf.workspace("roboflow-100").project(args["project"]) 59 | project.version(args["version"]).download( 60 | args["model_format"], location=str(out_dir) 61 | ) 62 | print("Done!") 63 | 64 | 65 | if __name__ == "__main__": 66 | main() 67 | -------------------------------------------------------------------------------- /scripts/download_datasets.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | input="$(pwd)/datasets_links_640.txt" 4 | 5 | while getopts f:l: flag 6 | do 7 | case "${flag}" in 8 | f) format=${OPTARG};; 9 | l) location=${OPTARG};; 10 | esac 11 | done 12 | # default values 13 | format=${format:-coco} 14 | location=${location:-$(pwd)/rf100} 15 | 16 | echo "Starting downloading RF100..." 17 | 18 | for link in $(cat $input) 19 | do 20 | attributes=$(python3 $(pwd)/scripts/parse_dataset_link.py -l $link) 21 | 22 | project=$(echo $attributes | cut -d' ' -f 3) 23 | version=$(echo $attributes | cut -d' ' -f 4) 24 | if [ ! -d "$location/$project" ] ; 25 | then 26 | python3 $(pwd)/scripts/download_dataset.py -p $project -v $version -l $location -f $format 27 | fi 28 | done 29 | 30 | echo "Done!" -------------------------------------------------------------------------------- /scripts/get_labels_names.py: -------------------------------------------------------------------------------- 1 | import os 2 | from concurrent.futures import ThreadPoolExecutor 3 | from pprint import pprint 4 | 5 | import pandas as pd 6 | import requests 7 | from tqdm import tqdm 8 | 9 | ROBOFLOW_API_KEY = os.environ["ROBOFLOW_API_KEY"] 10 | API_URL = "https://api.roboflow.com/roboflow-100" 11 | 12 | df = pd.read_csv("./metadata/categories.csv", index_col=0) 13 | datasets = df.index 14 | 15 | 16 | def get_labels(dataset): 17 | res = requests.get(f"{API_URL}/{dataset}", params={"api_key": ROBOFLOW_API_KEY}) 18 | classes = res.json()["project"]["classes"] 19 | return { 20 | "name": dataset, 21 | "classes": classes, 22 | "category": df.loc[dataset]["category"], 23 | } 24 | 25 | 26 | with ThreadPoolExecutor() as executor: 27 | res = list(tqdm(executor.map(get_labels, datasets))) 28 | pprint(res) 29 | 30 | # let's 31 | -------------------------------------------------------------------------------- /scripts/image/cluster.py: -------------------------------------------------------------------------------- 1 | from multiprocessing import Pool 2 | from pathlib import Path 3 | from pickle import dump, load 4 | 5 | import numpy as np 6 | import pandas as pd 7 | import torch 8 | import torchvision.transforms as T 9 | from kmeans import kmeans 10 | from PIL import Image 11 | from torchvision.transforms.functional import to_pil_image 12 | from torchvision.utils import make_grid 13 | from tqdm import tqdm 14 | 15 | NUM_CLUSTERS = 32 * 56 16 | 17 | 18 | def get_data(root: Path): 19 | for encode_path in tqdm(list(root.glob("*.pk"))): 20 | with open(encode_path, "rb") as f: 21 | batch = load(f) 22 | batch["indxs"] = batch["indxs"].numpy() 23 | if data is None: 24 | data = batch 25 | else: 26 | data["x"] = np.concatenate([data["x"], batch["x"]]) 27 | data["indxs"] = np.concatenate([data["indxs"], batch["indxs"]]) 28 | data["image_paths"] += batch["image_paths"] 29 | 30 | return data 31 | 32 | 33 | def pca(x, k, center=True): 34 | if center: 35 | m = x.mean(0, keepdim=True) 36 | s = x.std(0, unbiased=False, keepdim=True) 37 | x -= m 38 | x /= s 39 | # why pca related to svd? https://www.cs.cmu.edu/~elaw/papers/pca.pdf chap VI 40 | U, S, V = torch.linalg.svd(x) 41 | reduced = torch.mm(x, V[:k].T) 42 | 43 | return reduced 44 | 45 | 46 | transform = T.Compose([T.Resize((128, 128)), T.ToTensor()]) 47 | 48 | 49 | def read_image_and_transform(image_path): 50 | img = Image.open(image_path).convert("RGB") 51 | img = transform(img) 52 | return img 53 | 54 | 55 | def make_image_grid(closest): 56 | x = list( 57 | tqdm( 58 | map(read_image_and_transform, [data["image_paths"][i] for i in closest]), 59 | total=len(closest), 60 | ) 61 | ) 62 | grid = make_grid(x, nrow=56) 63 | 64 | to_pil_image(grid).save("./rf100-grid=32x56.png") 65 | 66 | 67 | data = get_data(Path("./encoded/")) 68 | x = torch.from_numpy(data["x"]) 69 | x = pca(x, k=32) 70 | means, bins = kmeans(x, num_clusters=NUM_CLUSTERS, num_iters=25) 71 | 72 | # save to disk 73 | kmeans_outfile_name = f"../temp/torch-kmeans_num-clusters={NUM_CLUSTERS}-pca=32.pk" 74 | with open(kmeans_outfile_name, "wb") as f: 75 | dump(means, f) 76 | 77 | # from disk 78 | with open(kmeans_outfile_name, "rb") as f: 79 | means = load(f) 80 | means = means.numpy() 81 | 82 | print(f"Loaded means, shape={means.shape}") 83 | 84 | means = np.sort(means, axis=0) 85 | x = np.sort(x, axis=0) 86 | 87 | closest, distances = vq(means[0], x) 88 | print(closest) 89 | print(closest.shape) 90 | print("Computed distances") 91 | 92 | make_image_grid(closest) 93 | -------------------------------------------------------------------------------- /scripts/image/encode.py: -------------------------------------------------------------------------------- 1 | # read images from rf100 2 | # encode them with clip 3 | 4 | from pathlib import Path 5 | from pickle import dump 6 | from typing import Callable, Optional 7 | 8 | import clip 9 | import numpy as np 10 | import torch 11 | import torch as nn 12 | from PIL import Image 13 | from torch.utils.data import DataLoader, Dataset 14 | from torchvision.transforms import ToTensor 15 | from tqdm import tqdm 16 | 17 | device = "cuda" if torch.cuda.is_available() else "cpu" 18 | 19 | model, preprocess = clip.load("ViT-L/14", device=device, jit=True) 20 | 21 | 22 | class ImageDataset(Dataset): 23 | def __init__(self, root: Path, fmt: str = "jpg", transform: Callable = None): 24 | super().__init__() 25 | self.images_path = list(root.glob(f"**/*.{fmt}")) 26 | self.transform = transform or ToTensor() 27 | 28 | def __getitem__(self, idx: int): 29 | image = Image.open(self.images_path[idx]).convert("RGB") 30 | return self.transform(image), idx, str(self.images_path[idx]) 31 | 32 | def __len__(self): 33 | return len(self.images_path) 34 | 35 | 36 | if __name__ == "__main__": 37 | 38 | from argparse import ArgumentParser 39 | 40 | parser = ArgumentParser() 41 | 42 | parser.add_argument( 43 | "--batches", 44 | type=int, 45 | default=4, 46 | help="max amout of batches per datasets, default to 4.", 47 | ) 48 | parser.add_argument("-i", type=Path, default=Path("./rf100/"), help="RF100 dir.") 49 | parser.add_argument( 50 | "-o", type=Path, default=Path("./temp/rf100/embeddings"), help="output dir." 51 | ) 52 | 53 | args = parser.parse_args() 54 | 55 | MAX_BATCHES, input_dir, output_dir = args.batches, args.i, args.o 56 | 57 | output_dir: Path = output_dir 58 | output_dir.mkdir(exist_ok=True, parents=True) 59 | 60 | for dataset_path in tqdm(list(input_dir.glob("*/"))): 61 | ds = ImageDataset(dataset_path / "train/images", transform=preprocess) 62 | dl = DataLoader( 63 | ds, batch_size=128, num_workers=8, pin_memory=True, shuffle=True 64 | ) # we shuffle and we sample 2 batches per dataset 65 | i = 0 66 | for (x, indxs, images_paths) in dl: 67 | with torch.no_grad(): 68 | x = x.to(device) 69 | x = model.encode_image(x) 70 | x = x.cpu().numpy() 71 | encoded_file_name = f"{dataset_path.stem}_{i}.pk" 72 | 73 | with open(Path(output_dir) / encoded_file_name, "wb") as f: 74 | dump( 75 | { 76 | "x": x, 77 | "indxs": indxs, 78 | "image_paths": images_paths, 79 | }, 80 | f, 81 | ) 82 | 83 | print(f"Stored to {encoded_file_name}") 84 | 85 | i += 1 86 | if i >= MAX_BATCHES: 87 | break 88 | -------------------------------------------------------------------------------- /scripts/image/kmeans.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from einops import rearrange, repeat 4 | from tqdm import tqdm 5 | 6 | 7 | def l2norm(t): 8 | return F.normalize(t, p=2, dim=-1) 9 | 10 | 11 | def sample_vectors(samples, num): 12 | num_samples, device = samples.shape[0], samples.device 13 | 14 | if num_samples >= num: 15 | indices = torch.randperm(num_samples, device=device)[:num] 16 | else: 17 | indices = torch.randint(0, num_samples, (num,), device=device) 18 | 19 | return samples[indices] 20 | 21 | 22 | def kmeans(samples, num_clusters, num_iters=10, use_cosine_sim=False): 23 | dim, dtype, device = samples.shape[-1], samples.dtype, samples.device 24 | 25 | means = sample_vectors(samples, num_clusters) 26 | 27 | for i in tqdm(range(num_iters)): 28 | if use_cosine_sim: 29 | dists = samples @ means.t() 30 | else: 31 | diffs = rearrange(samples, "n d -> n () d") - rearrange( 32 | means, "c d -> () c d" 33 | ) 34 | dists = -(diffs**2).sum(dim=-1) 35 | 36 | buckets = dists.max(dim=-1).indices 37 | bins = torch.bincount(buckets, minlength=num_clusters) 38 | zero_mask = bins == 0 39 | bins_min_clamped = bins.masked_fill(zero_mask, 1) 40 | 41 | new_means = buckets.new_zeros(num_clusters, dim, dtype=dtype) 42 | new_means.scatter_add_(0, repeat(buckets, "n -> n d", d=dim), samples) 43 | new_means = new_means / bins_min_clamped[..., None] 44 | 45 | if use_cosine_sim: 46 | new_means = l2norm(new_means) 47 | 48 | means = torch.where(zero_mask[..., None], means, new_means) 49 | 50 | return means, bins 51 | -------------------------------------------------------------------------------- /scripts/image/pca.py: -------------------------------------------------------------------------------- 1 | """ 2 | PyTorch PCA implementation 3 | """ 4 | 5 | import torch 6 | 7 | 8 | def pca(x: torch.Tensor, k: int, center=True): 9 | if center: 10 | m = x.mean(0, keepdim=True) 11 | s = x.std(0, unbiased=False, keepdim=True) 12 | x -= m 13 | x /= s 14 | # why pca related to svd? https://www.cs.cmu.edu/~elaw/papers/pca.pdf chap VI 15 | U, S, V = torch.linalg.svd(x) 16 | reduced = torch.mm(x, V[:k].T) 17 | 18 | return reduced 19 | -------------------------------------------------------------------------------- /scripts/make_dataset_table_with_images.py: -------------------------------------------------------------------------------- 1 | """Code to generate the markdown table in the README.md 2 | """ 3 | import random 4 | from pathlib import Path 5 | from typing import Tuple 6 | 7 | import pandas as pd 8 | import torch 9 | from PIL import Image 10 | from torchvision.transforms.functional import pil_to_tensor, to_pil_image 11 | from torchvision.utils import make_grid 12 | 13 | ROOT = Path("./rf100") 14 | 15 | 16 | def read_and_transform(image_path: Path) -> torch.Tensor: 17 | image = Image.open(image_path).convert("RGB").resize((128, 128)) 18 | return pil_to_tensor(image) 19 | 20 | 21 | def make_grid_from_dataset(dataset_dir: Path, n=5) -> Tuple[str, Image.Image]: 22 | # get random 5 images 23 | images_paths = list((dataset_dir / "train" / "images").glob("*")) 24 | sampled_images_paths = random.choices(images_paths, k=n) 25 | tensors = [read_and_transform(image_path) for image_path in sampled_images_paths] 26 | grid = make_grid(tensors, nrow=n) 27 | return dataset_dir.stem, to_pil_image(grid) 28 | 29 | 30 | def save(dataset_name: str, grid: Image.Image) -> Tuple[str, str]: 31 | grid_path = f"doc/images/grid/{dataset_name}.jpg" 32 | grid.save(grid_path, quality=70) 33 | return dataset_name, grid_path 34 | 35 | 36 | def make_grids(root: Path): 37 | out = list(map(make_grid_from_dataset, root.iterdir())) 38 | out = list(map(lambda x: save(*x), out)) 39 | return out 40 | 41 | 42 | def make_table(root: Path) -> str: 43 | out = make_grids(root) 44 | dataset_names, grid_paths = zip(*out) 45 | category_df = pd.read_csv("metadata/categories.csv", index_col=0) 46 | df = pd.DataFrame( 47 | data=dict(dataset=dataset_names, samples=grid_paths), 48 | columns=["dataset", "samples"], 49 | ) 50 | df.samples = df.samples.apply(lambda x: f"![alt]({x})") 51 | df = df.set_index("dataset", drop=True) 52 | df = category_df.join(df) 53 | # del df["category"] 54 | df = df.reset_index() 55 | df.dataset = df.dataset.apply( 56 | lambda x: f"[{x}](https://universe.roboflow.com/roboflow-100/{x})" 57 | ) 58 | df = df.set_index("dataset", drop=True) 59 | return df.to_markdown() 60 | 61 | 62 | print(make_table(ROOT)) 63 | -------------------------------------------------------------------------------- /scripts/make_montages.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | dir=$(pwd) 5 | echo $dir 6 | root=$dir/montages 7 | 8 | for split_id in $(ls $root | sort -V) 9 | do 10 | echo "Montage for $split_id" 11 | montage_dir=$root/$split_id/ 12 | ls $root/$split_id/images/* | sort -V > $root/$split_id/images_to_montage.txt 13 | montage `cat $root/$split_id/images_to_montage.txt` -geometry +0+0 -background none -tile 32x $root/$split_id/2048-img-atlas.jpg 14 | rm -r $root/$split_id/images/ 15 | done 16 | 17 | echo "Done!" 18 | -------------------------------------------------------------------------------- /scripts/make_results_from_evals.py: -------------------------------------------------------------------------------- 1 | """ 2 | This script merge the model's results in one single csv fie 3 | """ 4 | import pandas as pd 5 | 6 | yolov5_df = pd.read_csv( 7 | "./yolov5-benchmark/final_eval.txt", 8 | sep=" ", 9 | index_col=0, 10 | header=None, 11 | names=["dataset", "yolov5"], 12 | ) 13 | yolov7_df = pd.read_csv( 14 | "./yolov7-benchmark/final_eval.txt", 15 | sep=" ", 16 | index_col=0, 17 | header=None, 18 | names=["dataset", "yolov7"], 19 | ) 20 | 21 | df = yolov5_df.join(yolov7_df) 22 | # let's add glip as well! 23 | glip_df = pd.read_csv("./metadata/results_glip.csv", index_col=0) 24 | df = df.join(glip_df) 25 | df.to_csv("./results.csv") 26 | -------------------------------------------------------------------------------- /scripts/make_results_table.py: -------------------------------------------------------------------------------- 1 | """ 2 | This script create the result table in the paper. 3 | 4 | 1) get all links from `dataset_links.txt` 5 | 2) get all the stats from RF100 (name, number of images in train/val/test) + labeling hours 6 | 3) merge the stats with the result dataframe (created by `merge_final_eval.py`) 7 | 4) print the latex code for the table 8 | """ 9 | from pathlib import Path 10 | import numpy as np 11 | 12 | import pandas as pd 13 | 14 | RF100_BASE_URL = "https://app.roboflow.com/roboflow-100" 15 | 16 | 17 | def get_rf100_version_and_links(root: Path) -> pd.DataFrame: 18 | filepath = root / "datasets_links_640.txt" 19 | records = [] 20 | with filepath.open("r") as f: 21 | for line in f.readlines(): 22 | link = Path(line.strip()) 23 | name, version = link.parent.stem, link.stem 24 | records.append( 25 | { 26 | "dataset": name, 27 | "version": version, 28 | "link": f"{RF100_BASE_URL}/{name}/{version}", 29 | } 30 | ) 31 | return pd.DataFrame.from_records(records).set_index("dataset") 32 | 33 | 34 | def get_rf100_results( 35 | rf100_version_and_links: pd.DataFrame, 36 | rf100_stats: pd.DataFrame, 37 | rf100_labeling_info: pd.DataFrame, 38 | ) -> pd.DataFrame: 39 | 40 | df = rf100_stats.join(rf100_version_and_links).join(rf100_labeling_info) 41 | df = df.fillna(-1) 42 | df = df.astype({"labelling hours": int, "num_classes": int}) 43 | return df 44 | 45 | 46 | rf100_version_and_links = get_rf100_version_and_links(Path(".")) 47 | rf100_stats = pd.read_csv("./metadata/datasets_stats.csv", index_col=0) 48 | rf100_train_results = pd.read_csv("./results.csv", index_col=0) 49 | # let's also add labeling hours on the fly 50 | rf100_labeling_info = pd.read_csv("./metadata/labeling_hours.csv", index_col=0) 51 | results = get_rf100_results(rf100_version_and_links, rf100_stats, rf100_labeling_info) 52 | results = results.reset_index() 53 | # formatting for latex 54 | format_name = lambda x: " ".join(x.split("-")[:2]) 55 | add_link = lambda link, name: "\href{" + link + "}{" + name + "}" 56 | results["name"] = results.apply( 57 | lambda x: add_link(x["link"], format_name(x["dataset"])), axis=1 58 | ) 59 | del results["version"] 60 | del results["link"] 61 | results = results.set_index("name", drop=True) 62 | del results["dataset"] 63 | del results["num_datasets"] 64 | del results["size"] 65 | labelling_hours_col = results.pop("labelling hours") 66 | results.insert(4, "labelling hours", labelling_hours_col) 67 | print(results.head(5)) 68 | # https://pandas.pydata.org/docs/reference/api/pandas.io.formats.style.Styler.to_latex.html 69 | table_style = results.style 70 | table_style.clear() 71 | table_style.table_styles = [] 72 | table_style.caption = None 73 | table_style = table_style.format( 74 | { 75 | "train": "{}", 76 | "val": "{}", 77 | "test": "{}", 78 | "yolov5": "{:.3f}", 79 | "yolov7": "{:.3f}", 80 | "glip": "{:.3f}", 81 | } 82 | ) 83 | 84 | 85 | def highlight_max(s: pd.Series): 86 | max_idx = np.argmax([s["yolov5"], s["yolov7"], s["glip"]]) 87 | is_max = [False for _ in range(s.shape[0])] 88 | is_max[6 + max_idx] = True 89 | return ["font-weight: bold" if cell else "" for cell in is_max] 90 | 91 | 92 | table_style = table_style.apply(highlight_max, axis=1) 93 | print( 94 | table_style.to_latex( 95 | "table.tex", 96 | hrules=True, 97 | clines="all;data", 98 | position_float="centering", 99 | convert_css=True, 100 | ) 101 | ) 102 | -------------------------------------------------------------------------------- /scripts/parse_categories.py: -------------------------------------------------------------------------------- 1 | """ 2 | This script takes a file text (lost in time) and outputs a correctly formated csv file with category -> dataset 3 | """ 4 | import re 5 | from argparse import ArgumentParser 6 | from pathlib import Path 7 | 8 | import pandas as pd 9 | 10 | parser = ArgumentParser() 11 | parser.add_argument("-i", "--input", type=Path) 12 | parser.add_argument("-o", "--output", type=Path) 13 | 14 | args = parser.parse_args() 15 | 16 | records = [] 17 | 18 | with open(args.input, "r") as f: 19 | for line in f.readlines(): 20 | splitted = line.strip().split(" ") 21 | is_category = len(splitted) <= 2 22 | if is_category: 23 | current_category = " ".join(splitted).lower() 24 | else: 25 | dataset = f"{splitted[0].lower()} {splitted[1].lower()}" 26 | dataset = re.findall(r"\D+", dataset)[0].strip() 27 | dataset = dataset.replace(" ", "-") 28 | print(f"[{current_category}] - {dataset}") 29 | records.append({"category": current_category, "dataset": dataset}) 30 | 31 | df = pd.DataFrame.from_records(records) 32 | df.to_csv(args.output, index=True) 33 | -------------------------------------------------------------------------------- /scripts/parse_dataset_link.py: -------------------------------------------------------------------------------- 1 | import re 2 | from argparse import ArgumentParser 3 | 4 | 5 | def main(): 6 | parser = ArgumentParser( 7 | description="A handy script that will decompose and print from a roboflow dataset link it's workspace, project and version" 8 | ) 9 | parser.add_argument( 10 | "-l", "--link", required=True, help="A link to a roboflow dataset" 11 | ) 12 | args = vars(parser.parse_args()) 13 | # first one gonna be protocol, e.g. http 14 | _, url, workspace, project, version = re.split("/+", args["link"]) 15 | print(url, workspace, project, version) 16 | 17 | 18 | if __name__ == "__main__": 19 | main() 20 | -------------------------------------------------------------------------------- /scripts/reduce_dims.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from pickle import dump, load 3 | from typing import Dict, Iterator, List, Tuple 4 | 5 | import numpy as np 6 | import torch 7 | from image.pca import pca 8 | from sklearn.manifold import TSNE 9 | from tqdm import tqdm 10 | 11 | 12 | def get_data(root: Path) -> dict: 13 | data = None 14 | for path in tqdm(list(root.glob("*.pk"))): 15 | with path.open("rb") as f: 16 | batch = load(f) 17 | batch["indxs"] = batch["indxs"].numpy() 18 | if data is None: 19 | data = batch 20 | else: 21 | data["x"] = np.concatenate([data["x"], batch["x"]]) 22 | data["indxs"] = np.concatenate([data["indxs"], batch["indxs"]]) 23 | data["image_paths"] += batch["image_paths"] 24 | return data 25 | 26 | 27 | def encode_with_pca(data: dict, k: int) -> dict: 28 | x = torch.from_numpy(data["x"]).float() 29 | x = pca(x, k=k) 30 | data["x"] = x.cpu().numpy() 31 | return data 32 | 33 | 34 | def encode_with(data: dict, method: str, k: int) -> dict: 35 | if method == "pca": 36 | data = encode_with_pca(data, k) 37 | elif method == "tsne": 38 | data = encode_with_pca(data, 50) 39 | model = TSNE(n_components=k, random_state=0, init="pca") 40 | np.set_printoptions(suppress=True) 41 | data["x"] = model.fit_transform(data["x"]) 42 | else: 43 | raise ValueError(f"{method} not known.") 44 | return data 45 | 46 | 47 | def reduce_dims(input_dir: Path, output_dir: Path, method: str, k: int): 48 | print("Loading data ...") 49 | data = get_data(input_dir) 50 | print(f"Encoding with method={method} k={k}") 51 | data = encode_with(data, method, k) 52 | print(f"data shape = {data['x'].shape}") 53 | with (output_dir / f"reduced-{method}-k={k}.pk").open("wb") as f: 54 | dump(data, f) 55 | 56 | 57 | if __name__ == "__main__": 58 | from argparse import ArgumentParser 59 | 60 | parser = ArgumentParser() 61 | 62 | parser.add_argument( 63 | "-i", 64 | type=Path, 65 | default=Path("./rf100-temp/ "), 66 | help="Where the embeddings pickle file are.", 67 | ) 68 | parser.add_argument("-o", type=Path, default=Path("."), help="output dir.") 69 | parser.add_argument( 70 | "-k", type=int, default=3, help="number of dimensions, defaults to 3." 71 | ) 72 | parser.add_argument( 73 | "-tmethod", type=str, default="pca", help="type of method, for now only PCA" 74 | ) 75 | 76 | args = parser.parse_args() 77 | 78 | input_dir, output_dir, k, method = args.i, args.o, args.k, args.method 79 | 80 | output_dir.mkdir(exist_ok=True, parents=True) 81 | 82 | reduce_dims(input_dir, output_dir, method, k) 83 | -------------------------------------------------------------------------------- /scripts/split_for_webapp.py: -------------------------------------------------------------------------------- 1 | """ 2 | This script will split the reduced-* pickle file into folders with the right dimension and metadata for the collage. 3 | """ 4 | 5 | import json 6 | from concurrent.futures import ThreadPoolExecutor 7 | from pathlib import Path 8 | from pickle import dump, load 9 | from typing import Iterable, List, Tuple 10 | 11 | import numpy as np 12 | from PIL import Image 13 | from tqdm import tqdm 14 | 15 | """ 16 | 1) read the data 17 | 2) divide the data in chuck, each chunk is the amount of images we can fit 18 | a) read images with a thread pool 19 | b) resize all the images with PIL 20 | c) save all the images with a thread pool 21 | d) create a json with { x, y, z } 22 | c) the json is named from 0 ... (number of chunks) to link json <-> montage 23 | """ 24 | 25 | 26 | def get_data(input_path: Path) -> dict: 27 | with input_path.open("rb") as f: 28 | data = load(f) 29 | 30 | data["image_paths"] = np.array(data["image_paths"]) 31 | return data 32 | 33 | 34 | def make_json_and_image_for_split( 35 | split_id: int, 36 | x: np.array, 37 | image_paths: np.array, 38 | size: Tuple, 39 | output_dir: Path, 40 | points_file_prefix: str = "", 41 | ): 42 | # split_output_dir = output_dir / str(split_id) 43 | split_output_dir = output_dir 44 | split_output_dir_images = split_output_dir / "images" 45 | 46 | images_ids = range(len(image_paths)) 47 | images_ids_to_names = {} 48 | # resize images 49 | should_store_images = not split_output_dir_images.exists() 50 | if should_store_images: 51 | split_output_dir_images.mkdir(exist_ok=True, parents=True) 52 | # open all the images 53 | images: List[Image.Image] = list( 54 | map(lambda x: Image.open(x).convert("RGB"), image_paths) 55 | ) 56 | images: List[Image.Image] = list(map(lambda x: x.resize(size), images)) 57 | # store them to disk 58 | for i, image_id in enumerate(images_ids): 59 | # add image filename to the dict 60 | images_ids_to_names[image_id] = Path(image_paths[i]).stem 61 | if should_store_images: 62 | image = images[image_id] 63 | image.save( 64 | split_output_dir_images / f"{str(image_id)}.jpeg", 65 | format="JPEG", 66 | quality=70, 67 | ) 68 | # create the json 69 | points_json = [] 70 | for points, image_id in zip(x, images_ids): 71 | if points.shape[0] == 3: 72 | x, y, z = points.tolist() 73 | elif points.shape[0] == 2: 74 | x, y = points.tolist() 75 | z = 1 76 | else: 77 | raise ValueError( 78 | f"points have shape={points.shape}, only 2D and 3D are supported." 79 | ) 80 | points_json.append({"x": x, "y": y, "z": z, "image_id": image_id}) 81 | 82 | with (split_output_dir / f"{points_file_prefix}-points.json").open("w") as f: 83 | json.dump(points_json, f) 84 | 85 | with (split_output_dir / "images_ids_to_names.json").open("w") as f: 86 | json.dump(images_ids_to_names, f) 87 | 88 | def get_data_chunks( 89 | data: dict, chunk_size: int 90 | ) -> Iterable[Tuple[int, dict, List[Path]]]: 91 | num_splits = max(1, data["x"].shape[0] // chunk_size) 92 | for i in range(num_splits): 93 | x = data["x"][i * chunk_size : (i + 1) * chunk_size] 94 | image_paths = data["image_paths"][i * chunk_size : (i + 1) * chunk_size] 95 | yield i, x, image_paths 96 | 97 | 98 | def make_split( 99 | data: dict, output_dir: Path, montage_size: str, image_size: str, *args, **kwargs 100 | ): 101 | """ 102 | Takes `data`, writes to disk all the images and the embeddings in json format 103 | """ 104 | montage_width, montage_height = [int(el) for el in montage_size.split("x")] 105 | image_width, image_height = [int(el) for el in image_size.split("x")] 106 | 107 | chunk_size = (montage_width * montage_height) // (image_width * image_height) 108 | 109 | with ThreadPoolExecutor() as executor: 110 | list( 111 | tqdm( 112 | executor.map( 113 | lambda x: make_json_and_image_for_split( 114 | *x, (image_width, image_height), output_dir, *args, **kwargs 115 | ), 116 | get_data_chunks(data, chunk_size), 117 | ), 118 | total=data["x"].shape[0] // chunk_size, 119 | ) 120 | ) 121 | 122 | 123 | def make_splits_by_dataset( 124 | data: dict, output_dir: Path, montage_size: str, image_size: str, **kwargs 125 | ) -> List[dict]: 126 | # better way? 127 | datasets = np.array( 128 | [Path(x).parent.parent.parent.stem for x in data["image_paths"]] 129 | ) 130 | buckets = {} 131 | for dataset in np.unique(datasets): 132 | indices = datasets == dataset 133 | buckets[str(dataset)] = { 134 | "x": data["x"][indices], 135 | "image_paths": data["image_paths"][indices], 136 | } 137 | 138 | for key, bucket in tqdm(buckets.items()): 139 | make_split(bucket, output_dir / key, montage_size, image_size, **kwargs) 140 | 141 | 142 | if __name__ == "__main__": 143 | # # from argparse import ArgumentParser 144 | 145 | # # parser = ArgumentParser() 146 | 147 | # # parser.add_argument("-i", type=Path, help="Input pkl file/") 148 | # # parser.add_argument( 149 | # # "-o", type=Path, help="output dir." 150 | # # ) 151 | # # parser.add_argument( 152 | # # "--montage-size", type=str, default="2048x2048", help="size of the montage in pixels." 153 | # # ) 154 | # # parser.add_argument( 155 | # # "--image-size", type=str, default="64x64", help="new size for the images in pixels." 156 | # # ) 157 | 158 | # # args = parser.parse_args() 159 | 160 | # # input_path, output_dir, k, method = args.i, args.o, args.k, args.method 161 | input_path = Path("./reduced-tsne-k=3.pk") 162 | input_path, output_dir, montage_size, image_size = ( 163 | Path("./reduced-tsne-k=3.pk"), 164 | Path("./montages"), 165 | "2048x2048", 166 | "64x64", 167 | ) 168 | output_dir.mkdir(exist_ok=True, parents=True) 169 | 170 | make_splits_by_dataset( 171 | get_data(input_path), 172 | output_dir, 173 | montage_size, 174 | image_size, 175 | points_file_prefix=input_path.stem, 176 | ) 177 | 178 | # make_split(input_path, output_dir, montage_size, image_size, points_file_prefix=input_path.stem) 179 | -------------------------------------------------------------------------------- /yolov5-benchmark/final_eval.txt: -------------------------------------------------------------------------------- 1 | 4-fold-defect 0.97 2 | abdomen-mri 0.965 3 | acl-x-ray 0.995 4 | activity-diagrams-qdobr 0.427 5 | aerial-cows 0.854 6 | aerial-pool 0.513 7 | aerial-spheres 0.993 8 | animals-ij5d2 0.761 9 | apex-videogame 0.839 10 | apples-fvpl5 0.779 11 | aquarium-qlnqy 0.79 12 | asbestos 0.596 13 | avatar-recognition-nuexe 0.889 14 | axial-mri 0.638 15 | bacteria-ptywi 0.162 16 | bccd-ouzjz 0.912 17 | bees-jt5in 0.891 18 | bone-fracture-7fylg 0.0851 19 | brain-tumor-m2pbp 0.768 20 | cable-damage 0.91 21 | cables-nl42k 0.688 22 | cavity-rs0uf 0.782 23 | cell-towers 0.939 24 | cells-uyemf 0.249 25 | chess-pieces-mjzgj 0.977 26 | circuit-elements 0.0627 27 | circuit-voltages 0.797 28 | cloud-types 0.271 29 | coins-1apki 0.932 30 | construction-safety-gsnvb 0.915 31 | coral-lwptl 0.174 32 | corrosion-bi3q3 0.768 33 | cotton-20xz5 0.569 34 | cotton-plant-disease 0.204 35 | csgo-videogame 0.974 36 | currency-v4f8j 0.583 37 | digits-t2eg6 0.989 38 | document-parts 0.677 39 | excavators-czvg9 0.946 40 | farcry6-videogame 0.619 41 | fish-market-ggjso 0.92 42 | flir-camera-objects 0.796 43 | furniture-ngpea 0.983 44 | gauge-u2lwv 0.642 45 | grass-weeds 0.781 46 | gynecology-mri 0.323 47 | halo-infinite-angel-videogame 0.921 48 | hand-gestures-jps7z 0.995 49 | insects-mytwu 0.89 50 | leaf-disease-nsdsr 0.531 51 | lettuce-pallets 0.945 52 | liver-disease 0.592 53 | marbles 0.992 54 | mask-wearing-608pr 0.788 55 | mitosis-gjs3g 0.931 56 | number-ops 0.99 57 | paper-parts 0.59 58 | paragraphs-co84b 0.626 59 | parasites-1s07h 0.848 60 | peanuts-sd4kf 0.995 61 | peixos-fish 0.148 62 | people-in-paintings 0.575 63 | pests-2xlvx 0.136 64 | phages 0.854 65 | pills-sxdht 0.869 66 | poker-cards-cxcvz 0.886 67 | printed-circuit-board 0.0907 68 | radio-signal 0.673 69 | road-signs-6ih4y 0.963 70 | road-traffic 0.597 71 | robomasters-285km 0.816 72 | secondary-chains 0.341 73 | sedimentary-features-9eosf 0.327 74 | shark-teeth-5atku 0.948 75 | sign-language-sokdr 0.87 76 | signatures-xc8up 0.961 77 | smoke-uvylj 0.959 78 | soccer-players-5fuqs 0.66 79 | soda-bottles 0.964 80 | solar-panels-taxvb 0.413 81 | stomata-cells 0.84 82 | street-work 0.478 83 | tabular-data-wf9uh 0.752 84 | team-fight-tactics 0.961 85 | thermal-cheetah-my4dp 0.931 86 | thermal-dogs-and-people-x6ejw 0.967 87 | trail-camera 0.966 88 | truck-movement 0.786 89 | tweeter-posts 0.708 90 | tweeter-profile 0.988 91 | underwater-objects-5v7p8 0.693 92 | underwater-pipes-4ng4t 0.995 93 | uno-deck 0.993 94 | valentines-chocolate 0.11 95 | vehicles-q0x2v 0.454 96 | wall-damage 0.5 97 | washroom-rf1fa 0.619 98 | weed-crop-aerial 0.82 99 | wine-labels 0.569 100 | x-ray-rheumatology 0.722 101 | -------------------------------------------------------------------------------- /yolov5-benchmark/parse_eval.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | import yaml 4 | 5 | # construct the argument parser and parse the arguments 6 | ap = argparse.ArgumentParser() 7 | 8 | 9 | ap.add_argument("-i", "--input", required=True, help="Input file to parse") 10 | ap.add_argument( 11 | "-l", "--location", required=True, help="Dataset location (where data.yaml is)" 12 | ) 13 | ap.add_argument("-o", "--output", required=True, help="Output file to write") 14 | 15 | args = vars(ap.parse_args()) 16 | location = args["location"] 17 | 18 | 19 | def should_remove_line(line, stop_words): 20 | return any([word in line for word in stop_words]) 21 | 22 | 23 | with open(location + "/data.yaml", "r") as stream: 24 | class_names = yaml.safe_load(stream)["names"] 25 | 26 | with open(args["input"], "r") as f: 27 | lines = f.readlines() 28 | eval_lines = [] 29 | for line in lines: 30 | entries = line.split(" ") 31 | entries = list(filter(lambda val: val != "", entries)) 32 | entries = [e.strip("\n") for e in entries] 33 | start_class = False 34 | for e in entries: 35 | if e == "all": 36 | if "(AP)" not in entries: 37 | if "(AR)" not in entries: 38 | # parse all 39 | eval = {} 40 | eval["class"] = entries[0] 41 | eval["images"] = entries[1] 42 | eval["targets"] = entries[2] 43 | eval["precision"] = entries[3] 44 | eval["recall"] = entries[4] 45 | eval["map50"] = entries[5] 46 | eval["map95"] = entries[6] 47 | eval_lines.append(eval) 48 | 49 | if e in class_names: 50 | eval = {} 51 | eval["class"] = entries[0] 52 | eval["images"] = entries[1] 53 | eval["targets"] = entries[2] 54 | eval["precision"] = entries[3] 55 | eval["recall"] = entries[4] 56 | eval["map50"] = entries[5] 57 | eval["map95"] = entries[6] 58 | eval_lines.append(eval) 59 | 60 | print("-- This is the result:") 61 | print(eval_lines) 62 | 63 | # get the mAP50 value 64 | if len(eval_lines) > 1: 65 | print("There's more dicts") 66 | for lst in eval_lines: 67 | if lst["class"] == "all": 68 | map_val = lst["map50"] 69 | 70 | else: 71 | print("There's only one dict res") 72 | map_val = [res["map50"] for res in eval_lines][0] 73 | 74 | res = location, ": ", map_val 75 | 76 | 77 | with open(args["output"], "a") as f: 78 | f.write("".join(res)) 79 | f.write("\n") 80 | -------------------------------------------------------------------------------- /yolov5-benchmark/train.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | dir=$(pwd)/runs/yolo-v5 5 | echo $dir 6 | datasets=$dir/rf100 7 | 8 | if [ ! -d $datasets ] ; then 9 | $(pwd)/scripts/download_datasets.sh -l $datasets -f yolov5 10 | fi 11 | 12 | if [ ! -f "$dir/final_eval.txt" ] ; then 13 | touch "$dir/final_eval.txt" 14 | fi 15 | 16 | 17 | cd $(pwd)/yolov5-benchmark/ 18 | 19 | if [ ! -f "yolov5s.pt" ] ; then 20 | wget https://github.com/ultralytics/yolov5/releases/download/v6.2/yolov5s.pt 21 | fi 22 | 23 | cd yolov5/ 24 | # for AttributeError: partially initialized module ‘cv2’ has no attribute ‘gapi_wip_gst_GStreamerPipeline’ (most likely due to a circular import) 25 | pip install opencv-python==4.5.5.64 26 | 27 | for dataset in $(ls $datasets) 28 | do 29 | dataset=$datasets/$dataset 30 | echo "Training on $dataset" 31 | if [ ! -d "$dataset/results" ] ; 32 | then 33 | python train.py --img 640 --batch 16 --epochs 1 --name $dataset/results --data $dataset/data.yaml --weights ./yolov5s.pt 34 | if [ ! -d $dataset/test ] 35 | then 36 | echo "[ERROR] $dataset don't have a test set" 37 | continue 38 | fi 39 | sed -i "s/valid/test/" $dataset/data.yaml 40 | python val.py --data $dataset/data.yaml --img 640 --batch 16 --weights $dataset/results/weights/best.pt --name $dataset --exist-ok --verbose |& tee $dataset/val_eval.txt 41 | sed -i "s/test/valid/" $dataset/data.yaml 42 | python ../parse_eval.py -i $dataset/val_eval.txt -l $dataset -o $dir/final_eval.txt 43 | fi 44 | done 45 | 46 | echo "Done training all the datasets with YOLOv5!" 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /yolov7-benchmark/final_eval.txt: -------------------------------------------------------------------------------- 1 | 4-fold-defect 0.938 2 | apples-fvpl5 0.791 3 | asbestos 0.611 4 | bacteria-ptywi 0.000961 5 | bees-jt5in 0.68 6 | bone-fracture-7fylg 0.0896 7 | brain-tumor-m2pbp 0.809 8 | cable-damage 0.574 9 | cavity-rs0uf 0.799 10 | cell-towers 0.942 11 | cells-uyemf 0.0845 12 | circuit-voltages 0.257 13 | coral-lwptl 0.218 14 | cotton-20xz5 0.591 15 | cotton-plant-disease 0.0518 16 | currency-v4f8j 0.514 17 | digits-t2eg6 0.989 18 | fish-market-ggjso 0.988 19 | flir-camera-objects 0.824 20 | furniture-ngpea 0.968 21 | gauge-u2lwv 0.668 22 | grass-weeds 0.781 23 | gynecology-mri 0.171 24 | insects-mytwu 0.858 25 | leaf-disease-nsdsr 0.56 26 | lettuce-pallets 0.966 27 | liver-disease 0.583 28 | marbles 0.473 29 | number-ops 0.992 30 | peanuts-sd4kf 0.997 31 | people-in-paintings 0.678 32 | pests-2xlvx 0.029 33 | phages 0.842 34 | pills-sxdht 0.867 35 | poker-cards-cxcvz 0.251 36 | circuit-voltages 0.386 37 | road-traffic 0.847 38 | robomasters-285km 0.772 39 | sedimentary-features-9eosf 0.244 40 | shark-teeth-5atku 0.863 41 | shark-teeth-5atku 0.863 42 | stomata-cells 0.847 43 | team-fight-tactics 0.88 44 | trail-camera 0.969 45 | truck-movement 0.846 46 | tweeter-profile 0.99 47 | uno-deck 0.994 48 | vehicles-q0x2v 0.464 49 | wall-damage 0.434 50 | wine-labels 0.632 51 | abdomen-mri 0.958 52 | acl-x-ray 0.998 53 | activity-diagrams-qdobr 0.509 54 | aerial-cows 0.568 55 | aerial-pool 0.791 56 | aerial-spheres 0.539 57 | animals-ij5d2 0.342 58 | apex-videogame 0.875 59 | aquarium-qlnqy 0.822 60 | avatar-recognition-nuexe 0.943 61 | axial-mri 0.549 62 | bccd-ouzjz 0.922 63 | cables-nl42k 0.722 64 | chess-pieces-mjzgj 0.83 65 | cloud-types 0.306 66 | coins-1apki 0.977 67 | construction-safety-gsnvb 0.915 68 | corrosion-bi3q3 0.764 69 | csgo-videogame 0.964 70 | document-parts 0.666 71 | excavators-czvg9 0.895 72 | farcry6-videogame 0.216 73 | halo-infinite-angel-videogame 0.924 74 | hand-gestures-jps7z 0.995 75 | mask-wearing-608pr 0.513 76 | mitosis-gjs3g 0.739 77 | paper-parts 0.796 78 | paragraphs-co84b 0.61 79 | parasites-1s07h 0.889 80 | peixos-fish 0.821 81 | radio-signal 0.653 82 | road-signs-6ih4y 0.944 83 | secondary-chains 0.312 84 | sign-language-sokdr 0.255 85 | signatures-xc8up 0.932 86 | smoke-uvylj 0.962 87 | soccer-players-5fuqs 0.399 88 | solar-panels-taxvb 0.261 89 | street-work 0.708 90 | tabular-data-wf9uh 0.782 91 | thermal-cheetah-my4dp 0.513 92 | thermal-dogs-and-people-x6ejw 0.957 93 | tweeter-posts 0.495 94 | underwater-objects-5v7p8 0.453 95 | underwater-pipes-4ng4t 0.998 96 | valentines-chocolate 0.0593 97 | washroom-rf1fa 0.634 98 | weed-crop-aerial 0.615 99 | x-ray-rheumatology 0.506 100 | -------------------------------------------------------------------------------- /yolov7-benchmark/parse_eval.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | import yaml 4 | 5 | # construct the argument parser and parse the arguments 6 | ap = argparse.ArgumentParser() 7 | 8 | ap.add_argument("-i", "--input", required=True, help="Input file to parse") 9 | ap.add_argument( 10 | "-l", "--location", required=True, help="Dataset location (where data.yaml is)" 11 | ) 12 | ap.add_argument("-o", "--output", required=True, help="Output file to write") 13 | 14 | args = vars(ap.parse_args()) 15 | location = args["location"] 16 | 17 | 18 | def should_remove_line(line, stop_words): 19 | return any([word in line for word in stop_words]) 20 | 21 | 22 | with open(location + "/data.yaml", "r") as stream: 23 | class_names = yaml.safe_load(stream)["names"] 24 | 25 | with open(args["input"], "r") as f: 26 | lines = f.readlines() 27 | eval_lines = [] 28 | for line in lines: 29 | entries = line.split(" ") 30 | entries = list(filter(lambda val: val != "", entries)) 31 | entries = [e.strip("\n") for e in entries] 32 | start_class = False 33 | for e in entries: 34 | if e == "all": 35 | if "(AP)" not in entries: 36 | if "(AR)" not in entries: 37 | # parse all 38 | eval = {} 39 | eval["class"] = entries[0] 40 | eval["images"] = entries[1] 41 | eval["targets"] = entries[2] 42 | eval["precision"] = entries[3] 43 | eval["recall"] = entries[4] 44 | eval["map50"] = entries[5] 45 | eval["map95"] = entries[6] 46 | eval_lines.append(eval) 47 | 48 | if e in class_names: 49 | eval = {} 50 | eval["class"] = entries[0] 51 | eval["images"] = entries[1] 52 | eval["targets"] = entries[2] 53 | eval["precision"] = entries[3] 54 | eval["recall"] = entries[4] 55 | eval["map50"] = entries[5] 56 | eval["map95"] = entries[6] 57 | eval_lines.append(eval) 58 | 59 | print("-- This is the result:") 60 | print(eval_lines) 61 | 62 | # get the mAP50 value 63 | if len(eval_lines) > 1: 64 | print("There's more dicts") 65 | for lst in eval_lines: 66 | if lst["class"] == "all": 67 | map_val = lst["map50"] 68 | 69 | else: 70 | print("There's only one dict res") 71 | map_val = [res["map50"] for res in eval_lines][0] 72 | 73 | res = location, ": ", map_val 74 | 75 | 76 | with open(args["output"], "a") as f: 77 | f.write("".join(res)) 78 | f.write("\n") 79 | -------------------------------------------------------------------------------- /yolov7-benchmark/requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==1.2.0 2 | asttokens==2.0.7 3 | backcall==0.2.0 4 | cachetools==5.2.0 5 | certifi==2024.7.4 6 | chardet==4.0.0 7 | charset-normalizer==2.1.0 8 | cycler==0.10.0 9 | decorator==5.1.1 10 | executing==0.9.1 11 | fonttools==4.43.0 12 | glob2==0.7 13 | google-auth==2.10.0 14 | google-auth-oauthlib==0.4.6 15 | grpcio==1.53.2 16 | idna==3.7 17 | importlib-metadata==4.12.0 18 | ipython==8.10.0 19 | jedi==0.18.1 20 | kiwisolver==1.3.1 21 | Markdown==3.4.1 22 | MarkupSafe==2.1.1 23 | matplotlib==3.5.2 24 | matplotlib-inline==0.1.3 25 | numpy==1.23.1 26 | oauthlib==3.2.2 27 | opencv-python==4.8.1.78 28 | opencv-python-headless==4.8.1.78 29 | packaging==21.3 30 | pandas==1.4.3 31 | parso==0.8.3 32 | pexpect==4.8.0 33 | pickleshare==0.7.5 34 | Pillow==10.3.0 35 | prompt-toolkit==3.0.30 36 | protobuf==4.21.6 37 | psutil==5.9.1 38 | ptyprocess==0.7.0 39 | pure-eval==0.2.2 40 | pyasn1==0.4.8 41 | pyasn1-modules==0.2.8 42 | Pygments==2.12.0 43 | pyparsing==2.4.7 44 | python-dateutil==2.8.2 45 | python-dotenv==0.20.0 46 | pytz==2022.1 47 | PyYAML==6.0 48 | requests==2.32.2 49 | requests-oauthlib==1.3.1 50 | requests-toolbelt==0.9.1 51 | roboflow==0.2.14 52 | rsa==4.9 53 | scipy==1.9.0 54 | seaborn==0.11.2 55 | six==1.16.0 56 | stack-data==0.3.0 57 | tensorboard==2.9.1 58 | tensorboard-data-server==0.6.1 59 | tensorboard-plugin-wit==1.8.1 60 | thop==0.1.1.post2207130030 61 | torch==2.2.0 62 | torchvision==0.13.1 63 | tqdm==4.66.3 64 | traitlets==5.3.0 65 | typing-extensions==4.3.0 66 | urllib3==1.26.19 67 | wcwidth==0.2.5 68 | Werkzeug==3.0.1 69 | wget==3.2 70 | zipp==3.19.1 -------------------------------------------------------------------------------- /yolov7-benchmark/train.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | dir=$(pwd)/runs/yolo-v7 5 | echo $dir 6 | datasets=$dir/rf100 7 | 8 | if [ ! -d $datasets ] ; then 9 | $(pwd)/scripts/download_datasets.sh -l $datasets -f yolov5 10 | fi 11 | 12 | if [ ! -f "$dir/final_eval.txt" ] ; then 13 | touch "$dir/final_eval.txt" 14 | fi 15 | 16 | 17 | cd $(pwd)/yolov7-benchmark/ 18 | 19 | cd yolov7/ 20 | 21 | if [ ! -f "yolov7_training.pt" ] ; then 22 | wget https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7_training.pt 23 | fi 24 | 25 | for dataset in $(ls $datasets) 26 | do 27 | dataset=$datasets/$dataset 28 | echo "Training on $dataset" 29 | if [ ! -d "$dataset/results" ] ; 30 | then 31 | python train.py --img 640 --batch 16 --epochs 100 --name $dataset/results --data $dataset/data.yaml --weights 'yolov7_training.pt' |& tee $dataset/logs.txt 32 | python test.py --data $dataset/data.yaml --img 640 --batch 16 --weights $dataset/results/weights/best.pt --name $dataset --exist-ok |& tee $dataset/val_eval.txt 33 | python ../parse_eval.py -i $dataset/val_eval.txt -l $dataset -o $dir/final_eval.txt 34 | fi 35 | done 36 | 37 | echo "Done training all the datasets with YOLOv7!" 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /yolov8-benchmark/final_eval.txt: -------------------------------------------------------------------------------- 1 | 4-fold-defect 0.964 2 | abdomen-mri 0.962 3 | acl-x-ray 0.995 4 | activity-diagrams-qdobr 0.705 5 | aerial-cows 0.826 6 | aerial-pool 0.744 7 | aerial-spheres 0.993 8 | animals-ij5d2 0.835 9 | apex-videogame 0.865 10 | apples-fvpl5 0.785 11 | aquarium-qlnqy 0.781 12 | asbestos 0.631 13 | avatar-recognition-nuexe 0.877 14 | axial-mri 0.637 15 | bacteria-ptywi 0.57 16 | bccd-ouzjz 0.927 17 | bees-jt5in 0.901 18 | cables-nl42k 0.76 19 | cavity-rs0uf 0.765 20 | cell-towers 0.947 21 | cells-uyemf 0.861 22 | chess-pieces-mjzgj 0.988 23 | circuit-voltages 0.894 24 | cloud-types 0.336 25 | coins-1apki 0.966 26 | construction-safety-gsnvb 0.874 27 | coral-lwptl 0.199 28 | corrosion-bi3q3 0.735 29 | cotton-20xz5 0.513 30 | cotton-plant-disease 0.176 31 | csgo-videogame 0.978 32 | currency-v4f8j 0.877 33 | digits-t2eg6 0.991 34 | document-parts 0.67 35 | excavators-czvg9 0.922 36 | farcry6-videogame 0.709 37 | fish-market-ggjso 0.993 38 | flir-camera-objects 0.782 39 | furniture-ngpea 0.979 40 | gauge-u2lwv 0.661 41 | grass-weeds 0.784 42 | gynecology-mri 0.288 43 | halo-infinite-angel-videogame 0.935 44 | hand-gestures-jps7z 0.994 45 | insects-mytwu 0.882 46 | leaf-disease-nsdsr 0.573 47 | lettuce-pallets 0.956 48 | liver-disease 0.576 49 | marbles 0.986 50 | mask-wearing-608pr 0.898 51 | mitosis-gjs3g 0.962 52 | number-ops 0.992 53 | paper-parts 0.78 54 | paragraphs-co84b 0.653 55 | parasites-1s07h 0.861 56 | peanuts-sd4kf 0.995 57 | peixos-fish 0.809 58 | people-in-paintings 0.561 59 | pests-2xlvx 0.209 60 | phages 0.839 61 | pills-sxdht 0.987 62 | poker-cards-cxcvz 0.995 63 | radio-signal 0.695 64 | road-signs-6ih4y 0.973 65 | road-traffic 0.798 66 | robomasters-285km 0.814 67 | secondary-chains 0.315 68 | sedimentary-features-9eosf 0.455 69 | shark-teeth-5atku 0.991 70 | sign-language-sokdr 0.966 71 | signatures-xc8up 0.941 72 | smoke-uvylj 0.975 73 | soccer-players-5fuqs 0.946 74 | soda-bottles 0.96 75 | solar-panels-taxvb 0.761 76 | stomata-cells 0.855 77 | street-work 0.693 78 | tabular-data-wf9uh 0.78 79 | team-fight-tactics 0.973 80 | thermal-cheetah-my4dp 0.945 81 | thermal-dogs-and-people-x6ejw 0.976 82 | trail-camera 0.966 83 | truck-movement 0.858 84 | tweeter-posts 0.866 85 | tweeter-profile 0.987 86 | underwater-objects-5v7p8 0.728 87 | underwater-pipes-4ng4t 0.995 88 | uno-deck 0.993 89 | valentines-chocolate 0.978 90 | vehicles-q0x2v 0.472 91 | wall-damage 0.571 92 | washroom-rf1fa 0.62 93 | weed-crop-aerial 0.791 94 | wine-labels 0.642 95 | x-ray-rheumatology 0.858 -------------------------------------------------------------------------------- /yolov8-benchmark/train.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | dir=$(pwd)/runs/yolo-v8 5 | echo $dir 6 | datasets=$dir/rf100 7 | 8 | if [ ! -d $datasets ] ; then 9 | $(pwd)/scripts/download_datasets.sh -l $datasets -f yolov5 10 | fi 11 | 12 | if [ ! -f "$dir/final_eval.txt" ] ; then 13 | touch "$dir/final_eval.txt" 14 | fi 15 | 16 | 17 | cd $(pwd)/yolov8-benchmark/ 18 | 19 | # fo rhttps://stackoverflow.com/questions/4011705/python-the-imagingft-c-module-is-not-installed 20 | apt-get install -y libfreetype6-dev 21 | # setting up yolov8 - specific version, 20.01.2023 22 | pip install git+https://github.com/ultralytics/ultralytics.git@fix_shape_mismatch 23 | # for AttributeError: partially initialized module ‘cv2’ has no attribute ‘gapi_wip_gst_GStreamerPipeline’ (most likely due to a circular import) 24 | pip install --force --no-cache-dir opencv-contrib-python==4.5.5.62 25 | wget https://github.com/ultralytics/assets/releases/download/v0.0.0/yolov8s.pt 26 | 27 | 28 | 29 | for dataset in $(ls $datasets) 30 | do 31 | dataset=$datasets/$dataset 32 | echo "Training on $dataset" 33 | if [ ! -d "$dataset/results" ] ; 34 | then 35 | yolo detect train data=$dataset/data.yaml model=yolov8s.pt epochs=1 batch=16 36 | 37 | yolo detect val data=$dataset/data.yaml model=/workspace/yolov8-benchmark/runs/detect/train/weights/best.pt 38 | # yolov8 doesn't have a param for outdir LoL and save it in /runs without any reason 39 | cp -r /workspace/yolov8-benchmark/runs/detect/ $dataset/results 40 | rm -rf /workspace/yolov8-benchmark/runs/detect/ 41 | # python train.py --img 640 --batch 16 --epochs 100 --name $dataset/results --data $dataset/data.yaml --weights ./yolov5s.pt 42 | # python val.py --data $dataset/data.yaml --img 640 --batch 16 --weights $dataset/results/weights/best.pt --name $dataset --exist-ok --verbose |& tee $dataset/val_eval.txt 43 | # python ../parse_eval.py -i $dataset/val_eval.txt -l $dataset -o $dir/final_eval.txt 44 | fi 45 | done 46 | 47 | echo "Done training all the datasets with YOLOv5!" 48 | 49 | 50 | 51 | 52 | --------------------------------------------------------------------------------