├── src └── detectron2_al │ ├── __init__.py │ ├── engine │ ├── __init__.py │ └── al_engine.py │ ├── modeling │ ├── __init__.py │ ├── utils.py │ ├── rcnn.py │ └── roi_heads.py │ ├── dataset │ ├── __init__.py │ ├── dataset_mapper.py │ ├── utils.py │ ├── object_fusion.py │ └── al_dataset.py │ ├── configs │ ├── __init__.py │ ├── config.py │ └── defaults.py │ ├── scheduling_utils.py │ └── scoring_utils.py ├── configs ├── coco │ ├── faster_rcnn_R_50_FPN │ │ ├── B-base.yaml │ │ ├── B-add-rounds.yaml │ │ ├── B-add-buget.yaml │ │ ├── O-random-object-scoring.yaml │ │ ├── B-reduce-budget.yaml │ │ ├── B-change-object-scoring.yaml │ │ ├── I-change-image-score-agg.yaml │ │ ├── O-perturbation-object-scoring.yaml │ │ ├── O-change-pred-th.yaml │ │ └── O-use-exp-decay.yaml │ └── faster_rcnn_R_50_FPN.yaml ├── tk │ ├── faster_rcnn_R_50_FPN │ │ ├── B-base.yaml │ │ ├── B-add-rounds.yaml │ │ ├── B-add-buget.yaml │ │ ├── B-reduce-budget.yaml │ │ ├── O-random-object-scoring.yaml │ │ ├── I-change-image-score-agg.yaml │ │ ├── B-change-object-scoring.yaml │ │ ├── O-change-pred-th.yaml │ │ └── O-use-exp-decay.yaml │ └── faster_rcnn_R_50_FPN.yaml ├── HJDataset │ ├── faster_rcnn_R_50_FPN │ │ ├── B-base.yaml │ │ ├── B-add-rounds.yaml │ │ ├── B-add-buget.yaml │ │ ├── O-random-object-scoring.yaml │ │ ├── B-reduce-budget.yaml │ │ ├── B-change-object-scoring.yaml │ │ ├── I-change-image-score-agg.yaml │ │ ├── I-random-image-scoring.yaml │ │ ├── O-perturbation-object-scoring.yaml │ │ ├── O-change-correction-a.yaml │ │ ├── O-base-v2.yaml │ │ ├── O-change-correction-b.yaml │ │ ├── O-perturbation-object-scoring-v2.yaml │ │ ├── O-perturbation-object-scoring-v3.yaml │ │ ├── O-perturbation-object-scoring-v4.yaml │ │ ├── O-perturbation-object-scoring-v5.yaml │ │ ├── O-change-correction-c.yaml │ │ ├── O-perturbation-object-scoring-v4-c.yaml │ │ ├── O-use-exp-decay.yaml │ │ ├── O-change-correction-with-perturbation-a.yaml │ │ ├── O-change-correction-with-perturbation-b.yaml │ │ ├── O-perturbation-object-scoring-v4-b.yaml │ │ ├── O-perturbation-object-scoring-v5-b.yaml │ │ ├── O-perturbation-object-scoring-v6.yaml │ │ ├── O-exp-decay+perturbation.yaml │ │ ├── O-perturbation-object-scoring-v6-b.yaml │ │ ├── O-change-correction-with-perturbation-c.yaml │ │ ├── O-perturbation-object-scoring-v6-c.yaml │ │ └── O-perturbation-object-scoring-v6-d.yaml │ └── faster_rcnn_R_50_FPN.yaml ├── mamort │ ├── faster_rcnn_R_50_FPN │ │ ├── B-base.yaml │ │ ├── B-add-rounds.yaml │ │ ├── B-add-buget.yaml │ │ ├── B-reduce-budget.yaml │ │ ├── O-random-object-scoring.yaml │ │ ├── I-change-image-score-agg.yaml │ │ ├── I-random-image-scoring.yaml │ │ ├── B-change-object-scoring.yaml │ │ ├── O-perturbation-object-scoring.yaml │ │ ├── O-change-pred-th.yaml │ │ ├── O-use-exp-decay.yaml │ │ └── O-exp-decay+perturbation.yaml │ └── faster_rcnn_R_50_FPN.yaml ├── prima │ ├── faster_rcnn_R_50_FPN │ │ ├── B-base.yaml │ │ ├── B-add-rounds.yaml │ │ ├── B-add-buget.yaml │ │ ├── O-random-object-scoring.yaml │ │ ├── B-reduce-budget.yaml │ │ ├── B-change-object-scoring.yaml │ │ ├── I-change-image-score-agg.yaml │ │ ├── I-random-image-scoring.yaml │ │ ├── O-perturbation-object-scoring.yaml │ │ ├── O-change-correction-a.yaml │ │ ├── O-change-correction-b.yaml │ │ ├── O-change-pred-th.yaml │ │ ├── O-perturbation-object-scoring-v2.yaml │ │ ├── O-perturbation-object-scoring-v3.yaml │ │ ├── O-perturbation-object-scoring-v4.yaml │ │ ├── O-perturbation-object-scoring-v5.yaml │ │ ├── O-change-correction-c.yaml │ │ ├── O-perturbation-object-scoring-v4-c.yaml │ │ ├── O-use-exp-decay.yaml │ │ ├── O-change-correction-with-perturbation-a.yaml │ │ ├── O-change-correction-with-perturbation-b.yaml │ │ ├── O-perturbation-object-scoring-v4-b.yaml │ │ ├── O-perturbation-object-scoring-v5-b.yaml │ │ ├── O-perturbation-object-scoring-v6.yaml │ │ ├── O-perturbation-object-scoring-v6-b.yaml │ │ ├── O-change-correction-with-perturbation-c.yaml │ │ ├── O-exp-decay+perturbation.yaml │ │ ├── O-perturbation-object-scoring-v6-c.yaml │ │ └── O-perturbation-object-scoring-v6-d.yaml │ └── faster_rcnn_R_50_FPN.yaml ├── publaynet │ ├── faster_rcnn_R_50_FPN │ │ ├── B-base.yaml │ │ ├── B-add-rounds.yaml │ │ ├── B-add-buget.yaml │ │ ├── O-random-object-scoring.yaml │ │ ├── B-reduce-budget.yaml │ │ ├── B-change-object-scoring.yaml │ │ ├── I-change-image-score-agg.yaml │ │ ├── I-random-image-scoring.yaml │ │ ├── O-perturbation-object-scoring.yaml │ │ ├── O-change-correction-a.yaml │ │ ├── O-base-v2.yaml │ │ ├── O-change-correction-b.yaml │ │ ├── O-perturbation-object-scoring-v2.yaml │ │ ├── O-perturbation-object-scoring-v3.yaml │ │ ├── O-perturbation-object-scoring-v4.yaml │ │ ├── O-perturbation-object-scoring-v5.yaml │ │ ├── O-change-correction-c.yaml │ │ ├── O-perturbation-object-scoring-v4-c.yaml │ │ ├── O-use-exp-decay.yaml │ │ ├── O-change-correction-with-perturbation-a.yaml │ │ ├── O-change-correction-with-perturbation-b.yaml │ │ ├── O-perturbation-object-scoring-v4-b.yaml │ │ ├── O-perturbation-object-scoring-v5-b.yaml │ │ ├── O-perturbation-object-scoring-v6.yaml │ │ ├── O-exp-decay+perturbation.yaml │ │ ├── O-perturbation-object-scoring-v6-b.yaml │ │ ├── O-change-correction-with-perturbation-c.yaml │ │ ├── O-perturbation-object-scoring-v6-c.yaml │ │ └── O-perturbation-object-scoring-v6-d.yaml │ └── faster_rcnn_R_50_FPN.yaml └── base │ ├── faster_rcnn_R_101_FPN.yaml │ └── faster_rcnn_R_50_FPN.yaml ├── requirements.txt ├── tools ├── RobotoCondensed-Regular.ttf ├── process_HJDataset.py ├── cococv.py ├── cocosplit.py ├── train_al_model.py ├── convert_prima_to_coco.py └── show_results.py ├── .gitmodules ├── scripts ├── create_dataset_cv.sh ├── train_prima.sh ├── train_mamort.sh ├── train_coco.sh ├── train_HJDataset.sh ├── train_tk.sh ├── train_publaynet.sh ├── create_HJ.sh ├── train_prima_cv.sh ├── train_base.sh ├── train_base_cv.sh └── create_dataset.sh ├── annotating ├── config.xml └── backend_model.py ├── README.md └── .gitignore /src/detectron2_al/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/detectron2_al/engine/__init__.py: -------------------------------------------------------------------------------- 1 | from .al_engine import * -------------------------------------------------------------------------------- /configs/coco/faster_rcnn_R_50_FPN/B-base.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" -------------------------------------------------------------------------------- /configs/tk/faster_rcnn_R_50_FPN/B-base.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/B-base.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" -------------------------------------------------------------------------------- /configs/mamort/faster_rcnn_R_50_FPN/B-base.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/B-base.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/B-base.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" -------------------------------------------------------------------------------- /src/detectron2_al/modeling/__init__.py: -------------------------------------------------------------------------------- 1 | from .rcnn import * 2 | from .roi_heads import * -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | layoutparser==0.1.1 2 | dataclasses 3 | funcy 4 | lxml 5 | imagesize 6 | beautifulsoup4 7 | SciencePlots -------------------------------------------------------------------------------- /tools/RobotoCondensed-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lolipopshock/Detectron2_AL/HEAD/tools/RobotoCondensed-Regular.ttf -------------------------------------------------------------------------------- /configs/coco/faster_rcnn_R_50_FPN/B-add-rounds.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | TRAINING: 4 | ROUNDS: 12 -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/B-add-rounds.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | TRAINING: 4 | ROUNDS: 12 -------------------------------------------------------------------------------- /configs/tk/faster_rcnn_R_50_FPN/B-add-rounds.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | TRAINING: 4 | ROUNDS: 12 -------------------------------------------------------------------------------- /src/detectron2_al/dataset/__init__.py: -------------------------------------------------------------------------------- 1 | from .al_dataset import * 2 | from .dataset_mapper import * 3 | from .object_fusion import * -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/B-add-rounds.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | TRAINING: 4 | ROUNDS: 16 -------------------------------------------------------------------------------- /configs/coco/faster_rcnn_R_50_FPN/B-add-buget.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 3000 -------------------------------------------------------------------------------- /configs/coco/faster_rcnn_R_50_FPN/O-random-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: random -------------------------------------------------------------------------------- /configs/mamort/faster_rcnn_R_50_FPN/B-add-rounds.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | TRAINING: 4 | ROUNDS: 12 -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/B-add-buget.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 320 -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-random-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: random -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/B-add-rounds.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | TRAINING: 4 | ROUNDS: 16 -------------------------------------------------------------------------------- /configs/tk/faster_rcnn_R_50_FPN/B-add-buget.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 320 -------------------------------------------------------------------------------- /configs/tk/faster_rcnn_R_50_FPN/B-reduce-budget.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 80 -------------------------------------------------------------------------------- /configs/tk/faster_rcnn_R_50_FPN/O-random-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: random -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/B-add-buget.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 4000 -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-random-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: random -------------------------------------------------------------------------------- /configs/coco/faster_rcnn_R_50_FPN/B-reduce-budget.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 1500 -------------------------------------------------------------------------------- /configs/mamort/faster_rcnn_R_50_FPN/B-add-buget.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 320 -------------------------------------------------------------------------------- /configs/mamort/faster_rcnn_R_50_FPN/B-reduce-budget.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 80 -------------------------------------------------------------------------------- /configs/mamort/faster_rcnn_R_50_FPN/O-random-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: random -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/B-reduce-budget.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 80 -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/B-add-buget.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 4000 -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-random-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: random -------------------------------------------------------------------------------- /configs/tk/faster_rcnn_R_50_FPN/I-change-image-score-agg.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | IMAGE_SCORE_AGGREGATION: max -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/B-reduce-budget.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 1000 -------------------------------------------------------------------------------- /configs/coco/faster_rcnn_R_50_FPN/B-change-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: least_confidence -------------------------------------------------------------------------------- /configs/coco/faster_rcnn_R_50_FPN/I-change-image-score-agg.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | IMAGE_SCORE_AGGREGATION: max -------------------------------------------------------------------------------- /configs/mamort/faster_rcnn_R_50_FPN/I-change-image-score-agg.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | IMAGE_SCORE_AGGREGATION: max -------------------------------------------------------------------------------- /configs/mamort/faster_rcnn_R_50_FPN/I-random-image-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | IMAGE_SCORE_AGGREGATION: random -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/B-change-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: least_confidence -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/I-change-image-score-agg.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | IMAGE_SCORE_AGGREGATION: max -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/I-random-image-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | IMAGE_SCORE_AGGREGATION: random -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/B-reduce-budget.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 1000 -------------------------------------------------------------------------------- /configs/tk/faster_rcnn_R_50_FPN/B-change-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: least_confidence -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/B-change-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: least_confidence -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/I-change-image-score-agg.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | IMAGE_SCORE_AGGREGATION: max -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/I-random-image-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | IMAGE_SCORE_AGGREGATION: random -------------------------------------------------------------------------------- /configs/coco/faster_rcnn_R_50_FPN/O-perturbation-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation -------------------------------------------------------------------------------- /configs/mamort/faster_rcnn_R_50_FPN/B-change-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: least_confidence -------------------------------------------------------------------------------- /configs/mamort/faster_rcnn_R_50_FPN/O-perturbation-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-perturbation-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/B-change-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: least_confidence -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/I-change-image-score-agg.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | IMAGE_SCORE_AGGREGATION: max -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/I-random-image-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | IMAGE_SCORE_AGGREGATION: random -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-perturbation-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-perturbation-object-scoring.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-change-correction-a.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | REMOVE_DUPLICATES: false -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-change-correction-a.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | REMOVE_DUPLICATES: false -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-change-correction-b.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | RECOVER_MISSING_OBJECTS: false -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-change-correction-a.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | REMOVE_DUPLICATES: false -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-base-v2.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | DECAY: geometry 5 | LAST_RATIO: 0.50 -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-change-correction-b.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | RECOVER_MISSING_OBJECTS: false -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-base-v2.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | DECAY: geometry 5 | LAST_RATIO: 0.50 -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-change-correction-b.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | RECOVER_MISSING_OBJECTS: false -------------------------------------------------------------------------------- /configs/tk/faster_rcnn_R_50_FPN/O-change-pred-th.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | MODEL: 3 | ROI_HEADS: 4 | NMS_THRESH_TEST: 0.25 5 | SCORE_THRESH_TEST: 0.25 -------------------------------------------------------------------------------- /configs/coco/faster_rcnn_R_50_FPN/O-change-pred-th.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | MODEL: 3 | ROI_HEADS: 4 | NMS_THRESH_TEST: 0.25 5 | SCORE_THRESH_TEST: 0.25 -------------------------------------------------------------------------------- /configs/mamort/faster_rcnn_R_50_FPN/O-change-pred-th.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | MODEL: 3 | ROI_HEADS: 4 | NMS_THRESH_TEST: 0.25 5 | SCORE_THRESH_TEST: 0.25 -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-change-pred-th.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | MODEL: 3 | ROI_HEADS: 4 | NMS_THRESH_TEST: 0.25 5 | SCORE_THRESH_TEST: 0.25 -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v2.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 2 -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v3.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 3 -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v4.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v5.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 5 -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v2.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 2 -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v3.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 3 -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v4.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v5.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 5 -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v2.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 2 -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v3.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 3 -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v4.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v5.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 5 -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-change-correction-c.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | REMOVE_DUPLICATES: false 5 | RECOVER_MISSING_OBJECTS: false -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-change-correction-c.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | REMOVE_DUPLICATES: false 5 | RECOVER_MISSING_OBJECTS: false -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-change-correction-c.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | REMOVE_DUPLICATES: false 5 | RECOVER_MISSING_OBJECTS: false -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v4-c.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 6 | LAMBDA: 1.5 -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v4-c.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 6 | LAMBDA: 1.5 -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v4-c.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 6 | LAMBDA: 1.5 -------------------------------------------------------------------------------- /configs/tk/faster_rcnn_R_50_FPN/O-use-exp-decay.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | DECAY: geometry 5 | SELECTION_RAIO_DECAY: geometry 6 | TRAINING: 7 | EPOCHS_PER_ROUND_DECAY: geometry -------------------------------------------------------------------------------- /configs/coco/faster_rcnn_R_50_FPN/O-use-exp-decay.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | DECAY: geometry 5 | SELECTION_RAIO_DECAY: geometry 6 | TRAINING: 7 | EPOCHS_PER_ROUND_DECAY: geometry -------------------------------------------------------------------------------- /configs/mamort/faster_rcnn_R_50_FPN/O-use-exp-decay.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | DECAY: geometry 5 | SELECTION_RAIO_DECAY: geometry 6 | TRAINING: 7 | EPOCHS_PER_ROUND_DECAY: geometry -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-use-exp-decay.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | DECAY: geometry 5 | SELECTION_RAIO_DECAY: geometry 6 | TRAINING: 7 | EPOCHS_PER_ROUND_DECAY: geometry -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-use-exp-decay.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | DECAY: geometry 5 | SELECTION_RAIO_DECAY: geometry 6 | TRAINING: 7 | EPOCHS_PER_ROUND_DECAY: geometry -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-change-correction-with-perturbation-a.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 6 | OBJECT_FUSION: 7 | REMOVE_DUPLICATES: false -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-use-exp-decay.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | DECAY: geometry 5 | SELECTION_RAIO_DECAY: geometry 6 | TRAINING: 7 | EPOCHS_PER_ROUND_DECAY: geometry -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-change-correction-with-perturbation-a.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 6 | OBJECT_FUSION: 7 | REMOVE_DUPLICATES: false -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-change-correction-with-perturbation-a.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 6 | OBJECT_FUSION: 7 | REMOVE_DUPLICATES: false -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-change-correction-with-perturbation-b.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 6 | OBJECT_FUSION: 7 | RECOVER_MISSING_OBJECTS: false -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-change-correction-with-perturbation-b.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 6 | OBJECT_FUSION: 7 | RECOVER_MISSING_OBJECTS: false -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v4-b.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 6 | ALPHAS: [0.10] 7 | BETAS: [0.05] 8 | RANDOM: true -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v5-b.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 5 6 | ALPHAS: [0.10] 7 | BETAS: [0.05] 8 | RANDOM: true -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v6.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | OBJECT_FUSION: 5 | DECAY: geometry 6 | LAST_RATIO: 0.50 7 | PERTURBATION: 8 | VERSION: 4 -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-change-correction-with-perturbation-b.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 6 | OBJECT_FUSION: 7 | RECOVER_MISSING_OBJECTS: false -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v4-b.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 6 | ALPHAS: [0.10] 7 | BETAS: [0.05] 8 | RANDOM: true -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v5-b.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 5 6 | ALPHAS: [0.10] 7 | BETAS: [0.05] 8 | RANDOM: true -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v6.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | OBJECT_FUSION: 5 | DECAY: geometry 6 | LAST_RATIO: 0.50 7 | PERTURBATION: 8 | VERSION: 4 -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v4-b.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 6 | ALPHAS: [0.10] 7 | BETAS: [0.05] 8 | RANDOM: true -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v5-b.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 5 6 | ALPHAS: [0.10] 7 | BETAS: [0.05] 8 | RANDOM: true -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v6.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | OBJECT_FUSION: 5 | DECAY: geometry 6 | LAST_RATIO: 0.50 7 | PERTURBATION: 8 | VERSION: 4 -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "src/label-studio"] 2 | path = src/label-studio 3 | url = git@github.com:lolipopshock/label-studio.git 4 | [submodule "src/label-studio-frontend"] 5 | path = src/label-studio-frontend 6 | url = git@github.com:lolipopshock/label-studio-frontend.git 7 | -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v6-b.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | OBJECT_FUSION: 5 | DECAY: geometry 6 | LAST_RATIO: 0.50 7 | PERTURBATION: 8 | VERSION: 4 9 | RANDOM: true -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-exp-decay+perturbation.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | DECAY: geometry 5 | SELECTION_RAIO_DECAY: geometry 6 | TRAINING: 7 | EPOCHS_PER_ROUND_DECAY: geometry 8 | OBJECT_SCORING: perturbation -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v6-b.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | OBJECT_FUSION: 5 | DECAY: geometry 6 | LAST_RATIO: 0.50 7 | PERTURBATION: 8 | VERSION: 4 9 | RANDOM: true -------------------------------------------------------------------------------- /configs/mamort/faster_rcnn_R_50_FPN/O-exp-decay+perturbation.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | DECAY: geometry 5 | SELECTION_RAIO_DECAY: geometry 6 | TRAINING: 7 | EPOCHS_PER_ROUND_DECAY: geometry 8 | OBJECT_SCORING: perturbation -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-change-correction-with-perturbation-c.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 6 | OBJECT_FUSION: 7 | REMOVE_DUPLICATES: false 8 | RECOVER_MISSING_OBJECTS: false -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-exp-decay+perturbation.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | DECAY: geometry 5 | SELECTION_RAIO_DECAY: geometry 6 | TRAINING: 7 | EPOCHS_PER_ROUND_DECAY: geometry 8 | OBJECT_SCORING: perturbation -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-exp-decay+perturbation.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_FUSION: 4 | DECAY: geometry 5 | SELECTION_RAIO_DECAY: geometry 6 | TRAINING: 7 | EPOCHS_PER_ROUND_DECAY: geometry 8 | OBJECT_SCORING: perturbation -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v6-b.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | OBJECT_FUSION: 5 | DECAY: geometry 6 | LAST_RATIO: 0.50 7 | PERTURBATION: 8 | VERSION: 4 9 | RANDOM: true -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-change-correction-with-perturbation-c.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 6 | OBJECT_FUSION: 7 | REMOVE_DUPLICATES: false 8 | RECOVER_MISSING_OBJECTS: false -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-change-correction-with-perturbation-c.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | PERTURBATION: 5 | VERSION: 4 6 | OBJECT_FUSION: 7 | REMOVE_DUPLICATES: false 8 | RECOVER_MISSING_OBJECTS: false -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v6-c.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | OBJECT_FUSION: 5 | DECAY: geometry 6 | LAST_RATIO: 0.50 7 | PERTURBATION: 8 | VERSION: 4 9 | ALPHAS: [0.04, 0.08] 10 | BETAS: [0.04, 0.08] -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v6-d.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | OBJECT_FUSION: 5 | DECAY: geometry 6 | LAST_RATIO: 0.50 7 | PERTURBATION: 8 | VERSION: 4 9 | ALPHAS: [0.04, 0.08] 10 | BETAS: [0.03, 0.06] -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v6-c.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | OBJECT_FUSION: 5 | DECAY: geometry 6 | LAST_RATIO: 0.50 7 | PERTURBATION: 8 | VERSION: 4 9 | ALPHAS: [0.04, 0.08] 10 | BETAS: [0.04, 0.08] -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v6-d.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | OBJECT_FUSION: 5 | DECAY: geometry 6 | LAST_RATIO: 0.50 7 | PERTURBATION: 8 | VERSION: 4 9 | ALPHAS: [0.04, 0.08] 10 | BETAS: [0.03, 0.06] -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v6-c.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | OBJECT_FUSION: 5 | DECAY: geometry 6 | LAST_RATIO: 0.50 7 | PERTURBATION: 8 | VERSION: 4 9 | ALPHAS: [0.04, 0.08] 10 | BETAS: [0.04, 0.08] -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN/O-perturbation-object-scoring-v6-d.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | OBJECT_SCORING: perturbation 4 | OBJECT_FUSION: 5 | DECAY: geometry 6 | LAST_RATIO: 0.50 7 | PERTURBATION: 8 | VERSION: 4 9 | ALPHAS: [0.04, 0.08] 10 | BETAS: [0.03, 0.06] -------------------------------------------------------------------------------- /src/detectron2_al/configs/__init__.py: -------------------------------------------------------------------------------- 1 | from detectron2.config.compat import downgrade_config, upgrade_config 2 | from .config import CfgNode, get_cfg, global_cfg, set_global_cfg 3 | 4 | __all__ = [ 5 | "CfgNode", 6 | "get_cfg", 7 | "global_cfg", 8 | "set_global_cfg", 9 | "downgrade_config", 10 | "upgrade_config", 11 | ] -------------------------------------------------------------------------------- /scripts/create_dataset_cv.sh: -------------------------------------------------------------------------------- 1 | cd ../tools/ 2 | 3 | python cococv.py \ 4 | --annotation_path ../data/prima/annotations/all.json \ 5 | --save_path ../data/prima/annotations/cv \ 6 | --folds 5 \ 7 | --having-annotations 8 | 9 | echo "The prima dataset has splitted into training and testing set for 5 folds!" 10 | echo "======================================" -------------------------------------------------------------------------------- /annotating/config.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /configs/coco/faster_rcnn_R_50_FPN.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../base/faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 2000 5 | OBJECT_FUSION: 6 | BUDGET_ETA: 0.4 7 | INITIAL_RATIO: 0.90 8 | LAST_RATIO: 0.40 9 | REMOVE_DUPLICATES_TH: 0.25 10 | TRAINING: 11 | EPOCHS_PER_ROUND_INITIAL: 20 12 | EPOCHS_PER_ROUND_LAST: 10 13 | ROUNDS: 10 14 | MODEL: 15 | ROI_HEADS: 16 | SCORE_THRESH_TEST: 0.5 17 | SOLVER: 18 | IMS_PER_BATCH: 6 19 | TEST: 20 | DETECTIONS_PER_IMAGE: 100 -------------------------------------------------------------------------------- /configs/mamort/faster_rcnn_R_50_FPN.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../base/faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 1200 5 | OBJECT_FUSION: 6 | BUDGET_ETA: 0.4 7 | INITIAL_RATIO: 0.90 8 | LAST_RATIO: 0.40 9 | REMOVE_DUPLICATES_TH: 0.25 10 | TRAINING: 11 | EPOCHS_PER_ROUND_INITIAL: 20 12 | EPOCHS_PER_ROUND_LAST: 10 13 | ROUNDS: 10 14 | MODEL: 15 | ROI_HEADS: 16 | SCORE_THRESH_TEST: 0.5 17 | SOLVER: 18 | IMS_PER_BATCH: 6 19 | TEST: 20 | DETECTIONS_PER_IMAGE: 100 -------------------------------------------------------------------------------- /configs/tk/faster_rcnn_R_50_FPN.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../base/faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 160 5 | OBJECT_FUSION: 6 | BUDGET_ETA: 0.4 7 | INITIAL_RATIO: 0.80 8 | LAST_RATIO: 0.30 9 | REMOVE_DUPLICATES_TH: 0.25 10 | TRAINING: 11 | EPOCHS_PER_ROUND_INITIAL: 320 12 | EPOCHS_PER_ROUND_LAST: 150 13 | ROUNDS: 8 14 | MODEL: 15 | ROI_HEADS: 16 | NUM_CLASSES: 5 17 | SCORE_THRESH_TEST: 0.5 18 | SOLVER: 19 | IMS_PER_BATCH: 8 20 | TEST: 21 | DETECTIONS_PER_IMAGE: 150 -------------------------------------------------------------------------------- /configs/HJDataset/faster_rcnn_R_50_FPN.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../base/faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 1000 5 | OBJECT_FUSION: 6 | BUDGET_ETA: 0.4 7 | INITIAL_RATIO: 0.90 8 | LAST_RATIO: 0.40 9 | REMOVE_DUPLICATES_TH: 0.25 10 | TRAINING: 11 | EPOCHS_PER_ROUND_INITIAL: 30 12 | EPOCHS_PER_ROUND_LAST: 10 13 | ROUNDS: 6 14 | MODEL: 15 | ROI_HEADS: 16 | NUM_CLASSES: 5 17 | SCORE_THRESH_TEST: 0.5 18 | SOLVER: 19 | IMS_PER_BATCH: 6 20 | TEST: 21 | DETECTIONS_PER_IMAGE: 100 -------------------------------------------------------------------------------- /configs/publaynet/faster_rcnn_R_50_FPN.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../base/faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 2000 5 | OBJECT_FUSION: 6 | BUDGET_ETA: 0.4 7 | INITIAL_RATIO: 0.90 8 | LAST_RATIO: 0.40 9 | REMOVE_DUPLICATES_TH: 0.25 10 | TRAINING: 11 | EPOCHS_PER_ROUND_INITIAL: 20 12 | EPOCHS_PER_ROUND_LAST: 10 13 | ROUNDS: 10 14 | MODEL: 15 | ROI_HEADS: 16 | NUM_CLASSES: 5 17 | SCORE_THRESH_TEST: 0.5 18 | SOLVER: 19 | IMS_PER_BATCH: 6 20 | TEST: 21 | DETECTIONS_PER_IMAGE: 100 -------------------------------------------------------------------------------- /src/detectron2_al/modeling/utils.py: -------------------------------------------------------------------------------- 1 | def one_vs_two_scoring(probs): 2 | """Compute the one_vs_two_scores for the input probabilities 3 | 4 | 5 | Args: 6 | probs (torch.Tensor): NxC tensor 7 | 8 | Returns: 9 | scores (torch.Tensor): N tensor 10 | the one_vs_two_scores 11 | """ 12 | 13 | N, C = probs.shape 14 | assert C>=2, "the number of classes must be more than 1" 15 | 16 | sorted_probs, _ = probs.sort(dim=-1, descending=True) 17 | 18 | return (1 - (sorted_probs[:, 0] - sorted_probs[:, 1])) 19 | -------------------------------------------------------------------------------- /configs/prima/faster_rcnn_R_50_FPN.yaml: -------------------------------------------------------------------------------- 1 | _BASE_: "../base/faster_rcnn_R_50_FPN.yaml" 2 | AL: 3 | DATASET: 4 | IMAGE_BUDGET: 240 5 | OBJECT_FUSION: 6 | BUDGET_ETA: 0.4 7 | INITIAL_RATIO: 0.90 8 | LAST_RATIO: 0.75 9 | REMOVE_DUPLICATES_TH: 0.25 10 | TRAINING: 11 | EPOCHS_PER_ROUND_INITIAL: 200 12 | EPOCHS_PER_ROUND_LAST: 120 13 | ROUNDS: 4 14 | MODEL: 15 | ROI_HEADS: 16 | NUM_CLASSES: 5 17 | SCORE_THRESH_TEST: 0.5 18 | NMS_THRESH_TEST: 0.5 19 | SOLVER: 20 | IMS_PER_BATCH: 6 21 | TEST: 22 | DETECTIONS_PER_IMAGE: 100 23 | -------------------------------------------------------------------------------- /scripts/train_prima.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | config="$1" 4 | gpu_num="$2" 5 | shift 2 6 | 7 | base_command="python train_al_model.py \ 8 | --num-gpus $gpu_num \ 9 | --dataset_name prima \ 10 | --json_annotation_train ../data/prima/annotations/train.json \ 11 | --image_path_train ../data/prima/raw/Images \ 12 | --json_annotation_val ../data/prima/annotations/val.json \ 13 | --image_path_val ../data/prima/raw/Images \ 14 | --config-file $config" 15 | 16 | bash ./train_base.sh -c "$config" -b "$base_command" -n prima "$@" -------------------------------------------------------------------------------- /scripts/train_mamort.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | config="$1" 4 | gpu_num="$2" 5 | shift 2 6 | 7 | base_command="python train_al_model.py \ 8 | --num-gpus $gpu_num \ 9 | --dataset_name mamort \ 10 | --json_annotation_train ../data/mamort/annotations/train.json \ 11 | --image_path_train ../data/mamort/raw/images \ 12 | --json_annotation_val ../data/mamort/annotations/val.json \ 13 | --image_path_val ../data/mamort/raw/images \ 14 | --config-file $config" 15 | 16 | bash ./train_base.sh -c "$config" -b "$base_command" -n mamort "$@" -------------------------------------------------------------------------------- /scripts/train_coco.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | config="$1" 4 | gpu_num="$2" 5 | shift 2 6 | 7 | base_command="python train_al_model.py \ 8 | --num-gpus $gpu_num \ 9 | --dataset_name coco-sml \ 10 | --json_annotation_train ../data/coco/annotations/train-sml.json \ 11 | --image_path_train ../data/coco/raw/val2017 \ 12 | --json_annotation_val ../data/coco/annotations/val-sml.json \ 13 | --image_path_val ../data/coco/raw/val2017 \ 14 | --config-file $config" 15 | 16 | bash ./train_base.sh -c "$config" -b "$base_command" -n coco-sml "$@" -------------------------------------------------------------------------------- /scripts/train_HJDataset.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | config="$1" 4 | gpu_num="$2" 5 | shift 2 6 | 7 | base_command="python train_al_model.py \ 8 | --num-gpus $gpu_num \ 9 | --dataset_name HJDataset \ 10 | --json_annotation_train ../data/HJDataset/annotations/train.json \ 11 | --image_path_train ../data/HJDataset/raw/train \ 12 | --json_annotation_val ../data/HJDataset/annotations/val.json \ 13 | --image_path_val ../data/HJDataset/raw/val \ 14 | --config-file $config" 15 | 16 | bash ./train_base.sh -c "$config" -b "$base_command" -n HJDataset "$@" -------------------------------------------------------------------------------- /scripts/train_tk.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | config="$1" 4 | gpu_num="$2" 5 | shift 2 6 | 7 | base_command="python train_al_model.py \ 8 | --num-gpus $gpu_num\ 9 | --dataset_name tkdata-v10 \ 10 | --json_annotation_train ../data/tk1957-v10/annotations/train.json \ 11 | --image_path_train ../data/tk1957-v10/images \ 12 | --json_annotation_val ../data/tk1957-v10/annotations/val.json \ 13 | --image_path_val ../data/tk1957-v10/images \ 14 | --config-file $config" 15 | 16 | bash ./train_base.sh -c "$config" -b "$base_command" -n tk "$@" -------------------------------------------------------------------------------- /scripts/train_publaynet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | config="$1" 4 | gpu_num="$2" 5 | shift 2 6 | 7 | base_command="python train_al_model.py \ 8 | --num-gpus $gpu_num \ 9 | --dataset_name publaynet-sml \ 10 | --json_annotation_train ../data/publaynet/annotations/train-sml.json \ 11 | --image_path_train ../data/publaynet/raw/val \ 12 | --json_annotation_val ../data/publaynet/annotations/val-sml.json \ 13 | --image_path_val ../data/publaynet/raw/val \ 14 | --config-file $config" 15 | 16 | bash ./train_base.sh -c "$config" -b "$base_command" -n publaynet-sml "$@" -------------------------------------------------------------------------------- /scripts/create_HJ.sh: -------------------------------------------------------------------------------- 1 | cd ../tools/ 2 | 3 | mkdir -p ../data/HJDataset/annotations/ 4 | 5 | python process_HJDataset.py \ 6 | --origpath ../data/HJDataset/raw/annotations/main_train.json \ 7 | --savepath ../data/HJDataset/annotations/train.json \ 8 | --selected_categories 4 5 6 7 9 | 10 | echo "[1/2] The train set for HJDataset has been processed!" 11 | echo "======================================" 12 | 13 | python process_HJDataset.py \ 14 | --origpath ../data/HJDataset/raw/annotations/main_val.json \ 15 | --savepath ../data/HJDataset/annotations/val.json \ 16 | --selected_categories 4 5 6 7 17 | echo "[2/2] The val set for HJDataset has been processed!" 18 | echo "======================================" -------------------------------------------------------------------------------- /scripts/train_prima_cv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | config="$1" 4 | gpu_num="$2" 5 | fold="$3" 6 | shift 3 7 | 8 | base_command="python train_al_model.py \ 9 | --num-gpus $gpu_num \ 10 | --dataset_name prima \ 11 | --json_annotation_train ../data/prima/annotations/cv/$fold/train.json \ 12 | --image_path_train ../data/prima/raw/Images \ 13 | --json_annotation_val ../data/prima/annotations/cv/$fold/val.json \ 14 | --image_path_val ../data/prima/raw/Images \ 15 | --config-file $config" 16 | 17 | bash ./train_base_cv.sh -c "$config" -b "$base_command" -n prima -f $fold "$@" -------------------------------------------------------------------------------- /tools/process_HJDataset.py: -------------------------------------------------------------------------------- 1 | import json 2 | import argparse 3 | 4 | 5 | parser = argparse.ArgumentParser() 6 | 7 | parser.add_argument('--origpath', type=str, default='') 8 | parser.add_argument('--savepath', type=str, default='') 9 | parser.add_argument('--selected_categories', type=int, nargs='+', default=[]) 10 | 11 | def load_json(filename): 12 | with open(filename, 'r') as fp: 13 | return json.load(fp) 14 | 15 | def write_json(res, filename): 16 | with open(filename, 'w') as fp: 17 | json.dump(res, fp) 18 | 19 | if __name__ == "__main__": 20 | args = parser.parse_args() 21 | 22 | coco = load_json(args.origpath) 23 | cats = args.selected_categories 24 | annos = [ele for ele in coco['annotations'] if ele['category_id'] in cats] 25 | coco['annotations'] = annos 26 | 27 | write_json(coco, args.savepath) -------------------------------------------------------------------------------- /src/detectron2_al/scheduling_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class DefaultSchedular(object): 5 | 6 | def __init__(self, start, end, steps, mode): 7 | 8 | if mode == 'linear': 9 | # at = a0 + (t-1) * d 10 | self._val = np.linspace(start, end, steps) 11 | elif mode == 'geometry': 12 | # at = a0 * q ** (t-1) 13 | q = (end/start)**(1/(steps-1)) 14 | self._val = np.array([start*q**i for i in range(steps)]) 15 | else: 16 | raise NotImplementedError 17 | 18 | def __getitem__(self, idx): 19 | 20 | return self._val[idx] 21 | 22 | 23 | class IntegerSchedular(DefaultSchedular): 24 | 25 | def __init__(self, start, end, steps, mode): 26 | 27 | super().__init__(start, end, steps, mode) 28 | 29 | self._val = self._val.astype('int') -------------------------------------------------------------------------------- /src/detectron2_al/configs/config.py: -------------------------------------------------------------------------------- 1 | from detectron2.config.config import CfgNode 2 | 3 | global_cfg = CfgNode() 4 | 5 | def get_cfg() -> CfgNode: 6 | """ 7 | Get a copy of the default config. 8 | 9 | Returns: 10 | a detectron2 CfgNode instance. 11 | """ 12 | from .defaults import _C 13 | 14 | return _C.clone() 15 | 16 | 17 | def set_global_cfg(cfg: CfgNode) -> None: 18 | """ 19 | Let the global config point to the given cfg. 20 | 21 | Assume that the given "cfg" has the key "KEY", after calling 22 | `set_global_cfg(cfg)`, the key can be accessed by: 23 | 24 | .. code-block:: python 25 | 26 | from detectron2.config import global_cfg 27 | print(global_cfg.KEY) 28 | 29 | By using a hacky global config, you can access these configs anywhere, 30 | without having to pass the config object or the values deep into the code. 31 | This is a hacky feature introduced for quick prototyping / research exploration. 32 | """ 33 | global global_cfg 34 | global_cfg.clear() 35 | global_cfg.update(cfg) 36 | -------------------------------------------------------------------------------- /scripts/train_base.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd ../tools 4 | 5 | ## Handle Input 6 | print_usage() { 7 | printf "use -c to specify the config file. \n" 8 | printf "use -b to specify the base command. \n" 9 | printf "use -n to specify the dataset name. \n" 10 | } 11 | 12 | config='' 13 | base_command='' 14 | dataset_name='' 15 | while getopts 'c:b:n:h' flag; do 16 | case "${flag}" in 17 | c) config="${OPTARG}" ;; 18 | b) base_command="${OPTARG}" ;; 19 | n) dataset_name="${OPTARG}" ;; 20 | h) print_usage 21 | exit 1 ;; 22 | esac 23 | done 24 | shift $((OPTIND-1)) 25 | extra_config=$@ 26 | 27 | exp_name=`basename "$config"` 28 | exp_name="${exp_name%.*}" 29 | 30 | exp_setting=`dirname "$config"` 31 | exp_setting=`basename "$exp_setting"` 32 | 33 | function run { 34 | extra_command="AL.MODE $1 \ 35 | OUTPUT_DIR ../outputs/$dataset_name/$1/$exp_setting/${exp_name:2}" 36 | 37 | eval "$base_command $extra_command $extra_config" 38 | } 39 | 40 | if [[ "$config" == *"B-"* ]]; then 41 | run object 42 | run image 43 | elif [[ "$config" == *"I-"* ]]; then 44 | run image 45 | elif [[ "$config" == *"O-"* ]]; then 46 | run object 47 | else 48 | echo "Unknow config type" 49 | fi 50 | -------------------------------------------------------------------------------- /scripts/train_base_cv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd ../tools 4 | 5 | ## Handle Input 6 | print_usage() { 7 | printf "use -c to specify the config file. \n" 8 | printf "use -b to specify the base command. \n" 9 | printf "use -n to specify the dataset name. \n" 10 | printf "use -f to specify the fold number. \n" 11 | } 12 | 13 | config='' 14 | base_command='' 15 | dataset_name='' 16 | fold='' 17 | while getopts 'c:b:n:f:h' flag; do 18 | case "${flag}" in 19 | c) config="${OPTARG}" ;; 20 | b) base_command="${OPTARG}" ;; 21 | n) dataset_name="${OPTARG}" ;; 22 | f) fold="${OPTARG}" ;; 23 | h) print_usage 24 | exit 1 ;; 25 | esac 26 | done 27 | shift $((OPTIND-1)) 28 | extra_config=$@ 29 | 30 | exp_name=`basename "$config"` 31 | exp_name="${exp_name%.*}" 32 | 33 | exp_setting=`dirname "$config"` 34 | exp_setting=`basename "$exp_setting"` 35 | 36 | function run { 37 | extra_command="AL.MODE $1 \ 38 | OUTPUT_DIR ../outputs-cv/$dataset_name/$1/$exp_setting/${exp_name:2}/$fold" 39 | 40 | eval "$base_command $extra_command $extra_config" 41 | } 42 | 43 | if [[ "$config" == *"B-"* ]]; then 44 | run object 45 | run image 46 | elif [[ "$config" == *"I-"* ]]; then 47 | run image 48 | elif [[ "$config" == *"O-"* ]]; then 49 | run object 50 | else 51 | echo "Unknow config type" 52 | fi 53 | -------------------------------------------------------------------------------- /src/detectron2_al/dataset/dataset_mapper.py: -------------------------------------------------------------------------------- 1 | from detectron2.data import DatasetMapper 2 | from detectron2.data.detection_utils import T, logging 3 | 4 | __all__ = ['DatasetMapper'] 5 | 6 | def build_transform_gen_al(cfg, is_train): 7 | # Almost the same as detectron2.data.detection_utils.build_transform_gen 8 | # Yet there is no horizontal flip for the input image 9 | if is_train: 10 | min_size = cfg.INPUT.MIN_SIZE_TRAIN 11 | max_size = cfg.INPUT.MAX_SIZE_TRAIN 12 | sample_style = cfg.INPUT.MIN_SIZE_TRAIN_SAMPLING 13 | else: 14 | min_size = cfg.INPUT.MIN_SIZE_TEST 15 | max_size = cfg.INPUT.MAX_SIZE_TEST 16 | sample_style = "choice" 17 | if sample_style == "range": 18 | assert len(min_size) == 2, "more than 2 ({}) min_size(s) are provided for ranges".format( 19 | len(min_size) 20 | ) 21 | 22 | logger = logging.getLogger(__name__) 23 | tfm_gens = [] 24 | tfm_gens.append(T.ResizeShortestEdge(min_size, max_size, sample_style)) 25 | return tfm_gens 26 | 27 | 28 | class DatasetMapperAL(DatasetMapper): 29 | 30 | def __init__(self, cfg, is_train=True): 31 | super(DatasetMapperAL, self).__init__(cfg, is_train) 32 | # Only substitute the tfm_gens with the al version, 33 | # where there should not be any horizontal flips 34 | self.tfm_gens = build_transform_gen_al(cfg, is_train) 35 | self.crop_gen = None # Enforce no crop_generator -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Detectron2 for Active Learning in Object Detection 2 | 3 | 4 | ## Usage 5 | 6 | 1. Clone the repository with all the submodules: 7 | ```bash 8 | git clone --recurse-submodules git@github.com:lolipopshock/Detectron2_AL.git 9 | ``` 10 | 2. Install dependencies: 11 | 1. Installing object detection environment with according to your environment 12 | - The tested version of pytorch is 1.4.0 with CUDA 10 13 | - And you **must** install Detectron2 with version 0.1.1. Newer versions has different APIs. 14 | ```bash 15 | pip install detectron2==0.1.1 \ 16 | -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu100/torch1.4/index.html 17 | ``` 18 | 2. Installing other necessary dependencies: 19 | ```bash 20 | pip install -r requirements.txt 21 | ``` 22 | 3. Installing UI components 23 | ```bash 24 | cd src/label-studio 25 | pip install -e . 26 | ``` 27 | 3. Setting up the label-studio server and modeling backend 28 | 1. Initialize the labeling server (If your image folder is `./data`) 29 | ```bash 30 | label-studio init labeling/tk-labeling \ 31 | --input-path=./data \ 32 | --input-format=image-dir \ 33 | --allow-serving-local-files --force \ 34 | --label-config=extra/config.xml \ 35 | --ml-backends http://localhost:9090 36 | ``` 37 | And you can start the server via 38 | ``` 39 | label-studio start labeling/tk-labeling 40 | ``` 41 | 1. Initialize the model backend server 42 | ```bash 43 | label-studio-ml init labeling/backend_model --script extra/backend_model.py 44 | ``` 45 | And similarly, you can start the backend server by 46 | ```bash 47 | label-studio-ml start labeling/backend_model 48 | # There's a relative import of the libraries 49 | # So you have to run this command in the project project 50 | # root path to avoid import errors 51 | ``` 52 | 4. Start using active learning for annotation -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # folrder 2 | data 3 | data/ 4 | credential 5 | credential/ 6 | model 7 | model/ 8 | outputs 9 | outputs*/ 10 | labeling/ 11 | labeling*/ 12 | *archived/ 13 | 14 | # Mac Finder Configurations 15 | .DS_Store 16 | 17 | # IDEA configurations 18 | .idea/ 19 | 20 | # IPython checkpoints 21 | .ipynb_checkpoints/ 22 | log 23 | 24 | # Visual Studio Code 25 | .vscode/ 26 | 27 | # Byte-compiled / optimized / DLL files 28 | __pycache__/ 29 | *.py[cod] 30 | *$py.class 31 | 32 | # C extensions 33 | *.so 34 | 35 | # Distribution / packaging 36 | .Python 37 | build/ 38 | develop-eggs/ 39 | dist/ 40 | downloads/ 41 | eggs/ 42 | .eggs/ 43 | lib64/ 44 | parts/ 45 | sdist/ 46 | var/ 47 | wheels/ 48 | *.egg-info/ 49 | .installed.cfg 50 | *.egg 51 | MANIFEST 52 | 53 | # PyInstaller 54 | # Usually these files are written by a python script from a template 55 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 56 | *.manifest 57 | *.spec 58 | 59 | # Installer logs 60 | pip-log.txt 61 | pip-delete-this-directory.txt 62 | 63 | # Unit test / coverage reports 64 | htmlcov/ 65 | .tox/ 66 | .coverage 67 | .coverage.* 68 | .cache 69 | nosetests.xml 70 | coverage.xml 71 | *.cover 72 | .hypothesis/ 73 | .pytest_cache/ 74 | 75 | # Translations 76 | *.mo 77 | *.pot 78 | 79 | # Django stuff: 80 | *.log 81 | local_settings.py 82 | db.sqlite3 83 | 84 | # Flask stuff: 85 | instance/ 86 | .webassets-cache 87 | 88 | # Scrapy stuff: 89 | .scrapy 90 | 91 | # Sphinx documentation 92 | docs/_build/ 93 | 94 | # PyBuilder 95 | target/ 96 | 97 | # Jupyter Notebook 98 | .ipynb_checkpoints 99 | 100 | # IPython 101 | profile_default/ 102 | ipython_config.py 103 | 104 | # pyenv 105 | .python-version 106 | 107 | # celery beat schedule file 108 | celerybeat-schedule 109 | 110 | # SageMath parsed files 111 | *.sage.py 112 | 113 | # Environments 114 | .env 115 | .venv 116 | env/ 117 | venv/ 118 | ENV/ 119 | env.bak/ 120 | venv.bak/ 121 | 122 | # Spyder project settings 123 | .spyderproject 124 | .spyproject 125 | 126 | # Rope project settings 127 | .ropeproject 128 | 129 | # mkdocs documentation 130 | /site 131 | 132 | # mypy 133 | .mypy_cache/ 134 | .dmypy.json 135 | dmypy.json 136 | -------------------------------------------------------------------------------- /scripts/create_dataset.sh: -------------------------------------------------------------------------------- 1 | cd ../tools/ 2 | 3 | python convert_prima_to_coco.py \ 4 | --prima_datapath ../data/prima/raw \ 5 | --anno_savepath ../data/prima/annotations/all.json \ 6 | --drop_categories 4 5 6 7 | echo "[1/4] The prima dataset has been converted to the COCO format!" 8 | echo "======================================" 9 | 10 | python cocosplit.py \ 11 | --annotation_path ../data/prima/annotations/all.json \ 12 | --split_ratio 0.8 \ 13 | --train ../data/prima/annotations/train.json \ 14 | --test ../data/prima/annotations/val.json \ 15 | --having-annotations 16 | echo "[2/4] The prima dataset has splitted into training and testing set!" 17 | echo "======================================" 18 | 19 | python cocosplit.py \ 20 | --annotation_path ../data/publaynet/raw/annotations/val.json \ 21 | --split_ratio 0.8 \ 22 | --train ../data/publaynet/annotations/train-sml.json \ 23 | --test ../data/publaynet/annotations/val-sml.json \ 24 | --having-annotations 25 | echo "[3/4] The publaynet dataset has splitted into training and testing set!" 26 | echo "======================================" 27 | 28 | python cocosplit.py \ 29 | --annotation_path ../data/coco/raw/annotations/instances_val2017.json \ 30 | --split_ratio 0.8 \ 31 | --train ../data/coco/annotations/train-sml.json \ 32 | --test ../data/coco/annotations/val-sml.json \ 33 | --having-annotations 34 | # Used for adding a dummy background class in the original train and val files 35 | python cocosplit.py \ 36 | --annotation_path ../data/coco/raw/annotations/instances_val2017.json \ 37 | --split_ratio 1 \ 38 | --train ../data/coco/annotations/val.json \ 39 | --test ../data/coco/annotations/tmp.json \ 40 | --having-annotations 41 | 42 | python cocosplit.py \ 43 | --annotation_path ../data/coco/raw/annotations/instances_train2017.json \ 44 | --split_ratio 1 \ 45 | --train ../data/coco/annotations/train.json \ 46 | --test ../data/coco/annotations/tmp.json \ 47 | --having-annotations 48 | echo "[4/4] The coco dataset has splitted into training and testing set!" 49 | echo "======================================" -------------------------------------------------------------------------------- /src/detectron2_al/modeling/rcnn.py: -------------------------------------------------------------------------------- 1 | from detectron2.modeling.meta_arch.build import META_ARCH_REGISTRY 2 | from detectron2.modeling.meta_arch.rcnn import ProposalNetwork, GeneralizedRCNN 3 | import torch 4 | 5 | __all__ = ['ActiveLearningRCNN'] 6 | 7 | 8 | @META_ARCH_REGISTRY.register() 9 | class ActiveLearningRCNN(GeneralizedRCNN): 10 | 11 | def _estimate_feature_proposal(self, batched_inputs): 12 | 13 | with torch.no_grad(): 14 | images = self.preprocess_image(batched_inputs) 15 | features = self.backbone(images.tensor) 16 | if self.proposal_generator: 17 | rpn_proposals, _ = self.proposal_generator(images, features, None) 18 | else: 19 | assert "proposals" in batched_inputs[0] 20 | rpn_proposals = [x["proposals"].to(self.device) for x in batched_inputs] 21 | 22 | return features, rpn_proposals 23 | 24 | def generate_object_al_scores(self, batched_inputs, do_postprocess=False): 25 | 26 | with torch.no_grad(): 27 | features, rpn_proposals = self._estimate_feature_proposal(batched_inputs) 28 | 29 | return self.roi_heads.generate_object_scores(features, rpn_proposals) 30 | 31 | def generate_image_al_scores(self, batched_inputs): 32 | """ 33 | Returns: List[Tuple] 34 | A list of (image_score, image_id) tuple 35 | """ 36 | with torch.no_grad(): 37 | features, rpn_proposals = self._estimate_feature_proposal(batched_inputs) 38 | 39 | image_scores = self.roi_heads.generate_image_scores(features, rpn_proposals) 40 | 41 | return [(score, gt['image_id']) for score, gt in zip(image_scores, batched_inputs)] 42 | 43 | def forward_al(self, batched_inputs, do_postprocess=True): 44 | 45 | """ 46 | Run inference on the given inputs with active learning. 47 | 48 | Returns: 49 | same as in :meth:`forward`. 50 | """ 51 | assert not self.training 52 | 53 | images = self.preprocess_image(batched_inputs) 54 | 55 | with torch.no_grad(): 56 | features, rpn_proposals = self._estimate_feature_proposal(batched_inputs) 57 | 58 | detection_results = self.roi_heads.generate_object_scores(features, rpn_proposals) 59 | 60 | if do_postprocess: 61 | detection_results = GeneralizedRCNN._postprocess(detection_results, batched_inputs, images.image_sizes) 62 | 63 | return detection_results -------------------------------------------------------------------------------- /tools/cococv.py: -------------------------------------------------------------------------------- 1 | # Modified based on https://github.com/akarazniewicz/cocosplit/blob/master/cocosplit.py 2 | 3 | import json 4 | import argparse 5 | import funcy 6 | import random 7 | import os 8 | 9 | parser = argparse.ArgumentParser(description='Splits COCO annotations file into training and val sets.') 10 | parser.add_argument('--annotation_path', type=str, help='Path to COCO annotations file.') 11 | parser.add_argument('--save_path', type=str, help='Where to store COCO generated annotations.') 12 | parser.add_argument('--folds', type=int, required=True, help="The number of folds.") 13 | parser.add_argument('--having-annotations', dest='having_annotations', action='store_true', 14 | help='Ignore all images without annotations. Keep only these with at least one annotation') 15 | 16 | def save_coco(file, info, licenses, images, annotations, categories): 17 | with open(file, 'wt', encoding='UTF-8') as coco: 18 | json.dump({ 'info': info, 'licenses': licenses, 'images': images, 19 | 'annotations': annotations, 'categories': categories}, coco) 20 | 21 | def filter_annotations(annotations, images): 22 | image_ids = funcy.lmap(lambda i: int(i['id']), images) 23 | return funcy.lfilter(lambda a: int(a['image_id']) in image_ids, annotations) 24 | 25 | def main(annotation_path, 26 | save_path, 27 | folds, 28 | having_annotations, 29 | random_state=None): 30 | 31 | random.seed(random_state) 32 | 33 | with open(annotation_path, 'rt', encoding='UTF-8') as annotations: 34 | coco = json.load(annotations) 35 | info = coco.get('info', '') 36 | licenses = coco.get('licenses', '') 37 | images = coco['images'] 38 | annotations = coco['annotations'] 39 | categories = coco['categories'] 40 | 41 | if having_annotations: 42 | images_with_annotations = funcy.lmap(lambda a: int(a['image_id']), annotations) 43 | images = funcy.lremove(lambda i: i['id'] not in images_with_annotations, images) 44 | 45 | num_images = len(images) 46 | fold_size = num_images // folds 47 | 48 | image_indices = list(range(num_images)) 49 | random.shuffle(image_indices) 50 | 51 | for fold in range(folds): 52 | 53 | val_indices = list(range(fold_size*(fold), fold_size*(fold+1))) 54 | 55 | train = [images[idx] for idx in image_indices if idx not in val_indices] 56 | val = [images[idx] for idx in image_indices if idx in val_indices] 57 | 58 | os.makedirs(f'{save_path}/{fold}') 59 | train_save_path = f'{save_path}/{fold}/train.json' 60 | val_save_path = f'{save_path}/{fold}/val.json' 61 | 62 | save_coco(train_save_path, info, licenses, train, filter_annotations(annotations, train), categories) 63 | save_coco(val_save_path, info, licenses, val, filter_annotations(annotations, val), categories) 64 | 65 | print("[Fold {}] Saved {} entries in {} and {} in {}".format(fold, len(train), train_save_path, len(val), val_save_path)) 66 | 67 | 68 | if __name__ == "__main__": 69 | args = parser.parse_args() 70 | 71 | main(args.annotation_path, 72 | args.save_path, 73 | args.folds, 74 | args.having_annotations, 75 | random_state=42) -------------------------------------------------------------------------------- /tools/cocosplit.py: -------------------------------------------------------------------------------- 1 | # Modified based on https://github.com/akarazniewicz/cocosplit/blob/master/cocosplit.py 2 | 3 | import json 4 | import argparse 5 | import funcy 6 | from sklearn.model_selection import train_test_split 7 | 8 | parser = argparse.ArgumentParser(description='Splits COCO annotations file into training and test sets.') 9 | parser.add_argument('--annotation_path', type=str, help='Path to COCO annotations file.') 10 | parser.add_argument('--train', type=str, help='Where to store COCO training annotations') 11 | parser.add_argument('--test', type=str, help='Where to store COCO test annotations') 12 | parser.add_argument('--split_ratio', type=float, required=True, help="A percentage of a split; a number in (0, 1)") 13 | parser.add_argument('--having-annotations', dest='having_annotations', action='store_true', 14 | help='Ignore all images without annotations. Keep only these with at least one annotation') 15 | 16 | def save_coco(file, info, licenses, images, annotations, categories): 17 | with open(file, 'wt', encoding='UTF-8') as coco: 18 | json.dump({ 'info': info, 'licenses': licenses, 'images': images, 19 | 'annotations': annotations, 'categories': categories}, coco) 20 | 21 | def filter_annotations(annotations, images): 22 | image_ids = funcy.lmap(lambda i: int(i['id']), images) 23 | return funcy.lfilter(lambda a: int(a['image_id']) in image_ids, annotations) 24 | 25 | def main(annotation_path, 26 | split_ratio, 27 | having_annotations, 28 | train_save_path, 29 | test_save_path, 30 | random_state=None): 31 | 32 | with open(annotation_path, 'rt', encoding='UTF-8') as annotations: 33 | coco = json.load(annotations) 34 | info = coco.get('info', '') 35 | licenses = coco.get('licenses', '') 36 | images = coco['images'] 37 | annotations = coco['annotations'] 38 | categories = coco['categories'] 39 | 40 | if not any([ele['id']==0 for ele in categories]): 41 | categories.insert(0, {'supercategory': '', 'id': 0, 'name': '__background__'}) 42 | 43 | if split_ratio == 1: 44 | 45 | save_coco(train_save_path, info, licenses, images, annotations, categories) 46 | print("Saved {} entries in {}.".format(len(images), train_save_path)) 47 | return None 48 | 49 | number_of_images = len(images) 50 | 51 | images_with_annotations = funcy.lmap(lambda a: int(a['image_id']), annotations) 52 | 53 | if having_annotations: 54 | images = funcy.lremove(lambda i: i['id'] not in images_with_annotations, images) 55 | 56 | x, y = train_test_split(images, train_size=split_ratio, random_state=random_state) 57 | 58 | save_coco(train_save_path, info, licenses, x, filter_annotations(annotations, x), categories) 59 | save_coco(test_save_path, info, licenses, y, filter_annotations(annotations, y), categories) 60 | 61 | print("Saved {} entries in {} and {} in {}".format(len(x), train_save_path, len(y), test_save_path)) 62 | 63 | 64 | if __name__ == "__main__": 65 | args = parser.parse_args() 66 | 67 | main(args.annotation_path, 68 | args.split_ratio, 69 | args.having_annotations, 70 | args.train, 71 | args.test, 72 | random_state=42) -------------------------------------------------------------------------------- /tools/train_al_model.py: -------------------------------------------------------------------------------- 1 | """ 2 | The script is based on https://github.com/facebookresearch/detectron2/blob/master/tools/train_net.py. 3 | """ 4 | 5 | import logging 6 | import os 7 | import json 8 | from collections import OrderedDict 9 | import torch 10 | import sys 11 | import pandas as pd 12 | 13 | import detectron2.utils.comm as comm 14 | from detectron2.checkpoint import DetectionCheckpointer 15 | from detectron2.data import MetadataCatalog, DatasetCatalog 16 | from detectron2.data.datasets import register_coco_instances 17 | from detectron2.engine import default_argument_parser, default_setup, launch 18 | from detectron2.evaluation import verify_results 19 | 20 | sys.path.append('../src') 21 | from detectron2_al.configs import get_cfg 22 | from detectron2_al.engine import build_al_trainer 23 | from detectron2_al.modeling import * 24 | 25 | def setup(args): 26 | """ 27 | Create configs and perform basic setups. 28 | """ 29 | 30 | # Add the val dataset to detectron2 directory 31 | dataset_name = args.dataset_name 32 | register_coco_instances(f"{dataset_name}-val", {}, 33 | args.json_annotation_val, 34 | args.image_path_val) 35 | 36 | # Initialize the configurations 37 | cfg = get_cfg() 38 | cfg.merge_from_file(args.config_file) 39 | cfg.merge_from_list(args.opts) 40 | 41 | # Ensure it uses appropriate names and architecture 42 | cfg.MODEL.ROI_HEADS.NAME = 'ROIHeadsAL' 43 | cfg.MODEL.META_ARCHITECTURE = 'ActiveLearningRCNN' 44 | 45 | # Extra configurations for the active learning model 46 | # for better object selection 47 | # cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST = 0.25 48 | # cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.0 49 | 50 | # Initializethe dataset information 51 | cfg.AL.DATASET.NAME = args.dataset_name 52 | cfg.AL.DATASET.IMG_ROOT = args.image_path_train 53 | cfg.AL.DATASET.ANNO_PATH = args.json_annotation_train 54 | cfg.DATASETS.TEST = (f"{args.dataset_name}-val",) 55 | 56 | with open(args.json_annotation_train, 'r') as fp: 57 | anno_file = json.load(fp) 58 | cfg.MODEL.ROI_HEADS.NUM_CLASSES = len(anno_file["categories"]) 59 | del anno_file 60 | 61 | cfg.freeze() 62 | default_setup(cfg, args) 63 | return cfg 64 | 65 | def main(args): 66 | cfg = setup(args) 67 | 68 | trainer = build_al_trainer(cfg) 69 | 70 | # if args.eval_only: 71 | # model = Trainer.build_model(cfg) 72 | # DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( 73 | # cfg.MODEL.WEIGHTS, resume=args.resume 74 | # ) 75 | # res = Trainer.test(cfg, model) 76 | 77 | # if cfg.TEST.AUG.ENABLED: 78 | # res.update(Trainer.test_with_TTA(cfg, model)) 79 | # if comm.is_main_process(): 80 | # verify_results(cfg, res) 81 | 82 | # # Save the evaluation results 83 | # pd.DataFrame(res).to_csv(f'{cfg.OUTPUT_DIR}/eval.csv') 84 | # return res 85 | 86 | return trainer.train_al() 87 | 88 | if __name__ == "__main__": 89 | parser = default_argument_parser() 90 | 91 | # Extra Configurations for dataset names and paths 92 | parser.add_argument("--dataset_name", default="", help="The Dataset Name") 93 | parser.add_argument("--json_annotation_train", default="", metavar="FILE", help="The path to the training set JSON annotation") 94 | parser.add_argument("--image_path_train", default="", metavar="FILE", help="The path to the training set image folder") 95 | parser.add_argument("--json_annotation_val", default="", metavar="FILE", help="The path to the validation set JSON annotation") 96 | parser.add_argument("--image_path_val", default="", metavar="FILE", help="The path to the validation set image folder") 97 | 98 | args = parser.parse_args() 99 | print("Command Line Args:", args) 100 | 101 | launch( 102 | main, 103 | args.num_gpus, 104 | num_machines=args.num_machines, 105 | machine_rank=args.machine_rank, 106 | dist_url=args.dist_url, 107 | args=(args,), 108 | ) 109 | -------------------------------------------------------------------------------- /src/detectron2_al/dataset/utils.py: -------------------------------------------------------------------------------- 1 | import bisect 2 | import copy 3 | import itertools 4 | import logging 5 | import numpy as np 6 | import operator 7 | import pickle 8 | import torch.utils.data 9 | from fvcore.common.file_io import PathManager 10 | from tabulate import tabulate 11 | from termcolor import colored 12 | 13 | from detectron2.data import samplers 14 | from detectron2.data.catalog import DatasetCatalog, MetadataCatalog 15 | from detectron2.data.common import AspectRatioGroupedDataset, DatasetFromList, MapDataset 16 | from detectron2.data.dataset_mapper import DatasetMapper 17 | from detectron2.data.detection_utils import check_metadata_consistency 18 | 19 | from detectron2.structures import BoxMode 20 | from detectron2.utils.comm import get_world_size 21 | from detectron2.utils.env import seed_all_rng 22 | from detectron2.utils.logger import log_first_n 23 | 24 | from detectron2.data.build import ( 25 | get_detection_dataset_dicts, 26 | load_proposals_into_dataset, 27 | print_instances_class_histogram, 28 | trivial_batch_collator, 29 | worker_init_reset_seed, 30 | ) 31 | 32 | 33 | def build_detection_train_loader_drop_ids(cfg, drop_image_ids, mapper=None): 34 | """ 35 | A rewrite for the detectron2.data.build.build_detection_train_loader 36 | function, as it supports drop images of certian_ids specified by 37 | drop_image_ids. 38 | 39 | Returns: 40 | an infinite iterator of training data 41 | """ 42 | num_workers = get_world_size() 43 | images_per_batch = cfg.SOLVER.IMS_PER_BATCH 44 | assert ( 45 | images_per_batch % num_workers == 0 46 | ), "SOLVER.IMS_PER_BATCH ({}) must be divisible by the number of workers ({}).".format( 47 | images_per_batch, num_workers 48 | ) 49 | assert ( 50 | images_per_batch >= num_workers 51 | ), "SOLVER.IMS_PER_BATCH ({}) must be larger than the number of workers ({}).".format( 52 | images_per_batch, num_workers 53 | ) 54 | images_per_worker = images_per_batch // num_workers 55 | 56 | dataset_dicts = get_detection_dataset_dicts( 57 | cfg.DATASETS.TRAIN, 58 | filter_empty=cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS, 59 | min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE 60 | if cfg.MODEL.KEYPOINT_ON 61 | else 0, 62 | proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN if cfg.MODEL.LOAD_PROPOSALS else None, 63 | ) 64 | 65 | dataset = DatasetFromList([dd for dd in dataset_dicts if dd['image_id'] not in drop_image_ids], copy=False) 66 | 67 | if mapper is None: 68 | mapper = DatasetMapper(cfg, True) 69 | dataset = MapDataset(dataset, mapper) 70 | 71 | sampler_name = cfg.DATALOADER.SAMPLER_TRAIN 72 | logger = logging.getLogger(__name__) 73 | logger.info("Using training sampler {}".format(sampler_name)) 74 | if sampler_name == "TrainingSampler": 75 | sampler = samplers.TrainingSampler(len(dataset)) 76 | elif sampler_name == "RepeatFactorTrainingSampler": 77 | sampler = samplers.RepeatFactorTrainingSampler( 78 | dataset_dicts, cfg.DATALOADER.REPEAT_THRESHOLD 79 | ) 80 | else: 81 | raise ValueError("Unknown training sampler: {}".format(sampler_name)) 82 | 83 | if cfg.DATALOADER.ASPECT_RATIO_GROUPING: 84 | data_loader = torch.utils.data.DataLoader( 85 | dataset, 86 | sampler=sampler, 87 | num_workers=cfg.DATALOADER.NUM_WORKERS, 88 | batch_sampler=None, 89 | collate_fn=operator.itemgetter(0), # don't batch, but yield individual elements 90 | worker_init_fn=worker_init_reset_seed, 91 | ) # yield individual mapped dict 92 | data_loader = AspectRatioGroupedDataset(data_loader, images_per_worker) 93 | else: 94 | batch_sampler = torch.utils.data.sampler.BatchSampler( 95 | sampler, images_per_worker, drop_last=True 96 | ) 97 | # drop_last so the batch always have the same size 98 | data_loader = torch.utils.data.DataLoader( 99 | dataset, 100 | num_workers=cfg.DATALOADER.NUM_WORKERS, 101 | batch_sampler=batch_sampler, 102 | collate_fn=trivial_batch_collator, 103 | worker_init_fn=worker_init_reset_seed, 104 | ) 105 | 106 | return data_loader 107 | -------------------------------------------------------------------------------- /src/detectron2_al/scoring_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from typing import List 3 | 4 | def dice_coefficient(area1, area2, inter_area): 5 | return 2*inter_area / (area1 + area2) 6 | 7 | def iou(area1, area2, inter_area): 8 | return inter_area / (area1 + area2 - inter_area) 9 | 10 | def overlap_coefficient(area1, area2, inter_area): 11 | return inter_area / torch.min(area1, area2) 12 | 13 | def _elementwise_scoring_template(boxes1, boxes2, formula): 14 | 15 | area1, area2 = boxes1.area(), boxes2.area() 16 | inter_area = elementwise_intersect_area(boxes1, boxes2) 17 | 18 | scores = torch.where( 19 | inter_area > 0, 20 | formula(area1, area2, inter_area), 21 | torch.zeros(1, dtype=inter_area.dtype, device=inter_area.device), 22 | ) 23 | return scores 24 | 25 | def _pairwise_scoring_template(boxes1, boxes2, formula): 26 | 27 | area1, area2 = boxes1.area(), boxes2.area() 28 | inter_area = pairwise_intersect_area(boxes1, boxes2) 29 | 30 | scores = torch.where( 31 | inter_area > 0, 32 | formula(area1[:, None], area2, inter_area), 33 | torch.zeros(1, dtype=inter_area.dtype, device=inter_area.device), 34 | ) 35 | return scores 36 | 37 | def elementwise_intersect_area(boxes1, boxes2): 38 | 39 | # Modified based on 40 | # https://detectron2.readthedocs.io/_modules/detectron2/structures/boxes.html#pairwise_iou 41 | boxes1, boxes2 = boxes1.tensor, boxes2.tensor 42 | 43 | width_height = \ 44 | torch.min(boxes1[:, 2:], boxes2[:, 2:]) - \ 45 | torch.max(boxes1[:, :2], boxes2[:, :2]) 46 | # [N,M,2] 47 | 48 | width_height.clamp_(min=0) # [N,2] 49 | inter = width_height.prod(dim=-1) # [N] 50 | del width_height 51 | return inter 52 | 53 | def pairwise_intersect_area(boxes1, boxes2): 54 | 55 | # Modified based on 56 | # https://detectron2.readthedocs.io/_modules/detectron2/structures/boxes.html#pairwise_iou 57 | boxes1, boxes2 = boxes1.tensor, boxes2.tensor 58 | 59 | width_height = \ 60 | torch.min(boxes1[:, None, 2:], boxes2[:, 2:]) - \ 61 | torch.max(boxes1[:, None, :2], boxes2[:, :2]) 62 | # [N,M,2] 63 | 64 | width_height.clamp_(min=0) # [N,M,2] 65 | inter = width_height.prod(dim=2) # [N,M] 66 | del width_height 67 | return inter 68 | 69 | def elementwise_iou(boxes1, boxes2): 70 | 71 | return _elementwise_scoring_template(boxes1, boxes2, iou) 72 | 73 | def elementwise_dice_coefficient(boxes1, boxes2): 74 | 75 | return _elementwise_scoring_template(boxes1, boxes2, dice_coefficient) 76 | 77 | def elementwise_overlap_coefficient(boxes1, boxes2): 78 | 79 | return _elementwise_scoring_template(boxes1, boxes2, overlap_coefficient) 80 | 81 | def pairwise_iou(boxes1, boxes2): 82 | 83 | return _pairwise_scoring_template(boxes1, boxes2, iou) 84 | 85 | def pairwise_dice_coefficient(boxes1, boxes2): 86 | 87 | return _pairwise_scoring_template(boxes1, boxes2, dice_coefficient) 88 | 89 | def pairwise_overlap_coefficient(boxes1, boxes2): 90 | 91 | return _pairwise_scoring_template(boxes1, boxes2, overlap_coefficient) 92 | 93 | 94 | 95 | 96 | # def select_top(gt, overlappings): 97 | # max_idx = np.argmax(overlappings) 98 | # if overlappings[max_idx]<=0.0: 99 | # return [] 100 | # else: 101 | # return [gt[max_idx]] 102 | 103 | # def select_nonzero(gt, overlappings): 104 | # return [box for (box, overlap) in zip(gt, overlappings) if overlap>0.0] 105 | 106 | 107 | 108 | def select_top(overlapping_scores) -> List[List[int]]: 109 | """ 110 | Obtain the gt box ids of the top overlapping score 111 | for each prediction box. Drop the gt box the if all 112 | the overlapping scores are smaller than zero for a 113 | prediction box. 114 | """ 115 | 116 | max_scores = overlapping_scores.max(dim=-1) 117 | return [[idx.item()] if val>0 else [] \ 118 | for idx, val in zip(max_scores.indices, max_scores.values)] 119 | 120 | def select_above(overlapping_scores, threshold=0) -> List[List[int]]: 121 | """ 122 | Obtain the gt box ids of the overlapping score more than 123 | some threshold for each prediction box. 124 | """ 125 | 126 | return_indices = [[] for _ in range(overlapping_scores.shape[0])] 127 | for pred_idx, gt_idx in zip(*torch.where(overlapping_scores>threshold)): 128 | return_indices[pred_idx].append(gt_idx.item()) 129 | 130 | return return_indices 131 | 132 | def select_nonzero(overlapping_scores) -> List[List[int]]: 133 | """ 134 | Obtain the gt box ids of the overlapping score more than 135 | 0 for each prediction box. 136 | """ 137 | 138 | return select_above(overlapping_scores, 0) 139 | -------------------------------------------------------------------------------- /src/detectron2_al/configs/defaults.py: -------------------------------------------------------------------------------- 1 | from detectron2.config.defaults import _C 2 | from detectron2.config.config import CfgNode as CN 3 | 4 | # ---------------------------------------------------------------------------- # 5 | # Additional Configuration for Active Learning 6 | # ---------------------------------------------------------------------------- # 7 | 8 | _C.AL = CN() 9 | _C.AL.MODE = 'object' # {'image', 'object'} 10 | # Perform active learning on whether image-level or object-level 11 | _C.AL.OBJECT_SCORING = '1vs2' # {'1vs2, 'least_confidence', 'random', 'perturbation'} 12 | # The method to compute the individual object scores 13 | _C.AL.IMAGE_SCORE_AGGREGATION = 'avg' # {'avg', 'max', 'sum'} 14 | # The method to aggregate the individual object scores to the whole image score 15 | 16 | _C.AL.PERTURBATION = CN() 17 | # Configurations for perturbation scoring method 18 | _C.AL.PERTURBATION.VERSION = 1 # 1 to 5 19 | # 1 - CE SCORING 20 | # 2 - KL SCORING 21 | # 3 - IOU SCORING 22 | # 4 - IOU + CE 23 | # 5 - IOU + KL * 3 24 | _C.AL.PERTURBATION.ALPHAS = [0.08, 0.12] 25 | # Horizontal translation ratio 26 | _C.AL.PERTURBATION.BETAS = [0.04, 0.16] 27 | # Vertical translation ratio 28 | _C.AL.PERTURBATION.RANDOM = False 29 | # whether generate random perturbation at different iterations 30 | _C.AL.PERTURBATION.LAMBDA = 1. 31 | # the weighting constant 32 | 33 | _C.AL.DATASET = CN() 34 | # Specifies the configs for creating new datasets 35 | # It will also combines configs from DATASETS and DATALOADER 36 | # when creating the DynamicDataset for training. 37 | _C.AL.DATASET.NAME = '' 38 | _C.AL.DATASET.IMG_ROOT = '' 39 | _C.AL.DATASET.ANNO_PATH = '' 40 | # Extra meta information for the dataset, only supports COCO Dataset 41 | # This is somehow ugly, and shall be updated in the future. 42 | _C.AL.DATASET.CACHE_DIR = 'al_datasets' 43 | # The created dataset will be saved in during the active learning 44 | # training process 45 | _C.AL.DATASET.NAME_PREFIX = 'r' 46 | # The created dataset will be named as '{NAME}-{NAME_PREFIX}{idx}' in the 47 | # Detectron2 system 48 | _C.AL.DATASET.BUDGET_STYLE = 'object' # {'image', 'object'} 49 | # Depericated. Now the budget style is the same as AL.MODE 50 | _C.AL.DATASET.IMAGE_BUDGET = 20 51 | _C.AL.DATASET.OBJECT_BUDGET = 2000 52 | # Specifies the way to calculate the budget 53 | # If specify the BUDGET_STYLE as image, while using the object-level 54 | # Active Learning, we will convert the image_budget to object budget 55 | # by OBJECT_BUDGET = IMAGE_BUDGET * AVG_OBJ_IN_TRAINING. 56 | # Similarly, we have 57 | # IMAGE_BUDGET = OBJECT_BUDGET // AVG_OBJ_IN_TRAINING. 58 | _C.AL.DATASET.BUDGET_ALLOCATION = 'linear' 59 | _C.AL.DATASET.SAMPLE_METHOD = 'top' # {'top', 'kmeans'} 60 | # The method to sample images when labeling 61 | 62 | _C.AL.OBJECT_FUSION = CN() 63 | # Specifies the configs to fuse model prediction and ground-truth (gt) 64 | _C.AL.OBJECT_FUSION.OVERLAPPING_METRIC = 'iou' # {'iou', 'dice_coefficient', 'overlap_coefficient'} 65 | # The function to calculate the overlapping between model pred and gt 66 | _C.AL.OBJECT_FUSION.OVERLAPPING_TH = 0.25 # Optional 67 | # The threshold for selecting the boxes 68 | _C.AL.OBJECT_FUSION.SELECTION_METHOD = 'top' # {'top', 'above', 'nonzero'} 69 | # For gt boxes with non-zero overlapping with the pred box, specify the 70 | # way to choose the gt boxes. 71 | # top: choose the one with the highest overlapping 72 | # above: choose the ones has the overlapping above the threshold specified above 73 | # nonzero: choose the gt boxes with non-zero overlapping 74 | _C.AL.OBJECT_FUSION.REMOVE_DUPLICATES = True 75 | _C.AL.OBJECT_FUSION.REMOVE_DUPLICATES_TH = 0.15 76 | # For the fused dataset, remove duplicated boxes with overlapping more than 77 | # the given threshold 78 | _C.AL.OBJECT_FUSION.RECOVER_MISSING_OBJECTS = True 79 | # If true, we recover the mis-identified objects during the process 80 | _C.AL.OBJECT_FUSION.INITIAL_RATIO = 0.85 81 | _C.AL.OBJECT_FUSION.LAST_RATIO = 0.25 82 | _C.AL.OBJECT_FUSION.DECAY = 'linear' 83 | # During the object fusion process, we decay the number of objects selected for inspection 84 | # as training goes on. 85 | _C.AL.OBJECT_FUSION.PRESELECTION_RAIO = 1.5 86 | _C.AL.OBJECT_FUSION.ENDSELECTION_RAIO = 1.25 87 | _C.AL.OBJECT_FUSION.SELECTION_RAIO_DECAY = 'linear' 88 | # During the object fusion process, we take the top x number of objects out of predictions. 89 | # The x is calculate as x = avg_object_per_image * SELECTION_RAIO 90 | # The purpose is not to bring too much useless model predictions in the fusion procedure. 91 | _C.AL.OBJECT_FUSION.RECOVER_ALMOST_CORRECT_PRED = True 92 | _C.AL.OBJECT_FUSION.BUDGET_ETA = 0.2 93 | 94 | 95 | _C.AL.TRAINING = CN() 96 | _C.AL.TRAINING.ROUNDS = 5 97 | # The number of rounds for performing AL dataset update 98 | _C.AL.TRAINING.EPOCHS_PER_ROUND_INITIAL = 500 99 | # The numbers of epochs for training during each round. 100 | # As Detectron2 does not support epochs natively, we will use the 101 | # following formula to convert the epochs to iterations after creating 102 | # the new dataset: 103 | # iterations = total_imgs / batch_size * epochs_per_round 104 | _C.AL.TRAINING.EPOCHS_PER_ROUND_DECAY = 'linear' 105 | _C.AL.TRAINING.EPOCHS_PER_ROUND_LAST = 50 106 | 107 | 108 | ############################################################## 109 | ### Provide config support for newer version of Detectron2 ### 110 | ### ------------------------------------------------------ ### 111 | ### Note: this is only for compatibility when loading ### 112 | ### model configs and weights, and the actual config ### 113 | ### won't be used ### 114 | ############################################################## 115 | 116 | _C.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE = "smooth_l1" 117 | _C.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_WEIGHT = 1.0 118 | _C.MODEL.RPN.BBOX_REG_LOSS_TYPE = "smooth_l1" 119 | _C.MODEL.RPN.BBOX_REG_LOSS_WEIGHT = 1.0 120 | _C.SOLVER.CLIP_GRADIENTS = CN({"ENABLED": False}) 121 | _C.SOLVER.CLIP_GRADIENTS.CLIP_TYPE = "value" 122 | _C.SOLVER.CLIP_GRADIENTS.CLIP_VALUE = 1.0 123 | _C.SOLVER.CLIP_GRADIENTS.NORM_TYPE = 2.0 124 | _C.SOLVER.NESTEROV = False 125 | _C.SOLVER.REFERENCE_WORLD_SIZE = 0 -------------------------------------------------------------------------------- /configs/base/faster_rcnn_R_101_FPN.yaml: -------------------------------------------------------------------------------- 1 | AL: 2 | DATASET: 3 | ANNO_PATH: '' 4 | BUDGET_ALLOCATION: linear 5 | BUDGET_STYLE: object 6 | CACHE_DIR: al_datasets 7 | IMAGE_BUDGET: 20 8 | IMG_ROOT: '' 9 | NAME: '' 10 | NAME_PREFIX: r 11 | OBJECT_BUDGET: 2000 12 | SAMPLE_METHOD: top 13 | IMAGE_SCORE_AGGREGATION: avg 14 | MODE: object 15 | OBJECT_FUSION: 16 | BUDGET_ETA: 0.2 17 | DECAY: linear 18 | ENDSELECTION_RAIO: 1.25 19 | INITIAL_RATIO: 0.85 20 | LAST_RATIO: 0.25 21 | OVERLAPPING_METRIC: iou 22 | OVERLAPPING_TH: 0.25 23 | PRESELECTION_RAIO: 1.5 24 | RECOVER_ALMOST_CORRECT_PRED: true 25 | RECOVER_MISSING_OBJECTS: true 26 | REMOVE_DUPLICATES: true 27 | REMOVE_DUPLICATES_TH: 0.15 28 | SELECTION_METHOD: top 29 | SELECTION_RAIO_DECAY: linear 30 | OBJECT_SCORING: 1vs2 31 | TRAINING: 32 | EPOCHS_PER_ROUND_DECAY: linear 33 | EPOCHS_PER_ROUND_INITIAL: 500 34 | EPOCHS_PER_ROUND_LAST: 50 35 | ROUNDS: 5 36 | CUDNN_BENCHMARK: false 37 | DATALOADER: 38 | ASPECT_RATIO_GROUPING: true 39 | FILTER_EMPTY_ANNOTATIONS: true 40 | NUM_WORKERS: 4 41 | REPEAT_THRESHOLD: 0.0 42 | SAMPLER_TRAIN: TrainingSampler 43 | DATASETS: 44 | PRECOMPUTED_PROPOSAL_TOPK_TEST: 1000 45 | PRECOMPUTED_PROPOSAL_TOPK_TRAIN: 2000 46 | PROPOSAL_FILES_TEST: [] 47 | PROPOSAL_FILES_TRAIN: [] 48 | TEST: [] 49 | TRAIN: [] 50 | GLOBAL: 51 | HACK: 1.0 52 | INPUT: 53 | CROP: 54 | ENABLED: true 55 | SIZE: 56 | - 0.9 57 | - 0.9 58 | TYPE: relative_range 59 | FORMAT: BGR 60 | MASK_FORMAT: polygon 61 | MAX_SIZE_TEST: 1333 62 | MAX_SIZE_TRAIN: 1333 63 | MIN_SIZE_TEST: 800 64 | MIN_SIZE_TRAIN: 65 | - 640 66 | - 672 67 | - 704 68 | - 736 69 | - 768 70 | - 800 71 | MIN_SIZE_TRAIN_SAMPLING: choice 72 | MODEL: 73 | ANCHOR_GENERATOR: 74 | ANGLES: 75 | - - -90 76 | - 0 77 | - 90 78 | ASPECT_RATIOS: 79 | - - 0.5 80 | - 1.0 81 | - 2.0 82 | NAME: DefaultAnchorGenerator 83 | OFFSET: 0.0 84 | SIZES: 85 | - - 32 86 | - - 64 87 | - - 128 88 | - - 256 89 | - - 512 90 | BACKBONE: 91 | FREEZE_AT: 2 92 | NAME: build_resnet_fpn_backbone 93 | DEVICE: cuda 94 | FPN: 95 | FUSE_TYPE: sum 96 | IN_FEATURES: 97 | - res2 98 | - res3 99 | - res4 100 | - res5 101 | NORM: '' 102 | OUT_CHANNELS: 256 103 | KEYPOINT_ON: false 104 | LOAD_PROPOSALS: false 105 | MASK_ON: false 106 | META_ARCHITECTURE: ActiveLearningRCNN 107 | PANOPTIC_FPN: 108 | COMBINE: 109 | ENABLED: true 110 | INSTANCES_CONFIDENCE_THRESH: 0.5 111 | OVERLAP_THRESH: 0.5 112 | STUFF_AREA_LIMIT: 4096 113 | INSTANCE_LOSS_WEIGHT: 1.0 114 | PIXEL_MEAN: 115 | - 103.53 116 | - 116.28 117 | - 123.675 118 | PIXEL_STD: 119 | - 1.0 120 | - 1.0 121 | - 1.0 122 | PROPOSAL_GENERATOR: 123 | MIN_SIZE: 0 124 | NAME: RPN 125 | RESNETS: 126 | DEFORM_MODULATED: false 127 | DEFORM_NUM_GROUPS: 1 128 | DEFORM_ON_PER_STAGE: 129 | - false 130 | - false 131 | - false 132 | - false 133 | DEPTH: 101 134 | NORM: FrozenBN 135 | NUM_GROUPS: 1 136 | OUT_FEATURES: 137 | - res2 138 | - res3 139 | - res4 140 | - res5 141 | RES2_OUT_CHANNELS: 256 142 | RES5_DILATION: 1 143 | STEM_OUT_CHANNELS: 64 144 | STRIDE_IN_1X1: true 145 | WIDTH_PER_GROUP: 64 146 | RETINANET: 147 | BBOX_REG_WEIGHTS: 148 | - 1.0 149 | - 1.0 150 | - 1.0 151 | - 1.0 152 | FOCAL_LOSS_ALPHA: 0.25 153 | FOCAL_LOSS_GAMMA: 2.0 154 | IN_FEATURES: 155 | - p3 156 | - p4 157 | - p5 158 | - p6 159 | - p7 160 | IOU_LABELS: 161 | - 0 162 | - -1 163 | - 1 164 | IOU_THRESHOLDS: 165 | - 0.4 166 | - 0.5 167 | NMS_THRESH_TEST: 0.5 168 | NUM_CLASSES: 80 169 | NUM_CONVS: 4 170 | PRIOR_PROB: 0.01 171 | SCORE_THRESH_TEST: 0.05 172 | SMOOTH_L1_LOSS_BETA: 0.1 173 | TOPK_CANDIDATES_TEST: 1000 174 | ROI_BOX_CASCADE_HEAD: 175 | BBOX_REG_WEIGHTS: 176 | - - 10.0 177 | - 10.0 178 | - 5.0 179 | - 5.0 180 | - - 20.0 181 | - 20.0 182 | - 10.0 183 | - 10.0 184 | - - 30.0 185 | - 30.0 186 | - 15.0 187 | - 15.0 188 | IOUS: 189 | - 0.5 190 | - 0.6 191 | - 0.7 192 | ROI_BOX_HEAD: 193 | BBOX_REG_WEIGHTS: 194 | - 10.0 195 | - 10.0 196 | - 5.0 197 | - 5.0 198 | CLS_AGNOSTIC_BBOX_REG: false 199 | CONV_DIM: 256 200 | FC_DIM: 1024 201 | NAME: FastRCNNConvFCHead 202 | NORM: '' 203 | NUM_CONV: 0 204 | NUM_FC: 2 205 | POOLER_RESOLUTION: 7 206 | POOLER_SAMPLING_RATIO: 0 207 | POOLER_TYPE: ROIAlignV2 208 | SMOOTH_L1_BETA: 0.0 209 | TRAIN_ON_PRED_BOXES: false 210 | ROI_HEADS: 211 | BATCH_SIZE_PER_IMAGE: 512 212 | IN_FEATURES: 213 | - p2 214 | - p3 215 | - p4 216 | - p5 217 | IOU_LABELS: 218 | - 0 219 | - 1 220 | IOU_THRESHOLDS: 221 | - 0.5 222 | NAME: ROIHeadsAL 223 | NMS_THRESH_TEST: 0.5 224 | NUM_CLASSES: 80 225 | POSITIVE_FRACTION: 0.25 226 | PROPOSAL_APPEND_GT: true 227 | SCORE_THRESH_TEST: 0.05 228 | ROI_KEYPOINT_HEAD: 229 | CONV_DIMS: 230 | - 512 231 | - 512 232 | - 512 233 | - 512 234 | - 512 235 | - 512 236 | - 512 237 | - 512 238 | LOSS_WEIGHT: 1.0 239 | MIN_KEYPOINTS_PER_IMAGE: 1 240 | NAME: KRCNNConvDeconvUpsampleHead 241 | NORMALIZE_LOSS_BY_VISIBLE_KEYPOINTS: true 242 | NUM_KEYPOINTS: 17 243 | POOLER_RESOLUTION: 14 244 | POOLER_SAMPLING_RATIO: 0 245 | POOLER_TYPE: ROIAlignV2 246 | ROI_MASK_HEAD: 247 | CLS_AGNOSTIC_MASK: false 248 | CONV_DIM: 256 249 | NAME: MaskRCNNConvUpsampleHead 250 | NORM: '' 251 | NUM_CONV: 4 252 | POOLER_RESOLUTION: 14 253 | POOLER_SAMPLING_RATIO: 0 254 | POOLER_TYPE: ROIAlignV2 255 | RPN: 256 | BATCH_SIZE_PER_IMAGE: 256 257 | BBOX_REG_WEIGHTS: 258 | - 1.0 259 | - 1.0 260 | - 1.0 261 | - 1.0 262 | BOUNDARY_THRESH: -1 263 | HEAD_NAME: StandardRPNHead 264 | IN_FEATURES: 265 | - p2 266 | - p3 267 | - p4 268 | - p5 269 | - p6 270 | IOU_LABELS: 271 | - 0 272 | - -1 273 | - 1 274 | IOU_THRESHOLDS: 275 | - 0.3 276 | - 0.7 277 | LOSS_WEIGHT: 1.0 278 | NMS_THRESH: 0.7 279 | POSITIVE_FRACTION: 0.5 280 | POST_NMS_TOPK_TEST: 1000 281 | POST_NMS_TOPK_TRAIN: 1000 282 | PRE_NMS_TOPK_TEST: 1000 283 | PRE_NMS_TOPK_TRAIN: 2000 284 | SMOOTH_L1_BETA: 0.0 285 | SEM_SEG_HEAD: 286 | COMMON_STRIDE: 4 287 | CONVS_DIM: 128 288 | IGNORE_VALUE: 255 289 | IN_FEATURES: 290 | - p2 291 | - p3 292 | - p4 293 | - p5 294 | LOSS_WEIGHT: 1.0 295 | NAME: SemSegFPNHead 296 | NORM: GN 297 | NUM_CLASSES: 54 298 | WEIGHTS: detectron2://ImageNetPretrained/MSRA/R-101.pkl 299 | OUTPUT_DIR: ./output 300 | SEED: -1 301 | SOLVER: 302 | BASE_LR: 0.00025 303 | BIAS_LR_FACTOR: 1.0 304 | CHECKPOINT_PERIOD: 20000 305 | GAMMA: 0.1 306 | IMS_PER_BATCH: 16 307 | LR_SCHEDULER_NAME: WarmupMultiStepLR 308 | MAX_ITER: 270000 309 | MOMENTUM: 0.9 310 | STEPS: 311 | - 210000 312 | - 250000 313 | WARMUP_FACTOR: 0.001 314 | WARMUP_ITERS: 1000 315 | WARMUP_METHOD: linear 316 | WEIGHT_DECAY: 0.0001 317 | WEIGHT_DECAY_BIAS: 0.0001 318 | WEIGHT_DECAY_NORM: 0.0 319 | TEST: 320 | AUG: 321 | ENABLED: false 322 | FLIP: true 323 | MAX_SIZE: 4000 324 | MIN_SIZES: 325 | - 400 326 | - 500 327 | - 600 328 | - 700 329 | - 800 330 | - 900 331 | - 1000 332 | - 1100 333 | - 1200 334 | DETECTIONS_PER_IMAGE: 100 335 | EVAL_PERIOD: 0 336 | EXPECTED_RESULTS: [] 337 | KEYPOINT_OKS_SIGMAS: [] 338 | PRECISE_BN: 339 | ENABLED: false 340 | NUM_ITER: 200 341 | VERSION: 2 342 | VIS_PERIOD: 0 343 | -------------------------------------------------------------------------------- /configs/base/faster_rcnn_R_50_FPN.yaml: -------------------------------------------------------------------------------- 1 | AL: 2 | DATASET: 3 | ANNO_PATH: '' 4 | BUDGET_ALLOCATION: linear 5 | BUDGET_STYLE: object 6 | CACHE_DIR: al_datasets 7 | IMAGE_BUDGET: 20 8 | IMG_ROOT: '' 9 | NAME: '' 10 | NAME_PREFIX: r 11 | OBJECT_BUDGET: 2000 12 | SAMPLE_METHOD: top 13 | IMAGE_SCORE_AGGREGATION: avg 14 | MODE: object 15 | OBJECT_FUSION: 16 | BUDGET_ETA: 0.2 17 | DECAY: linear 18 | ENDSELECTION_RAIO: 1.25 19 | INITIAL_RATIO: 0.85 20 | LAST_RATIO: 0.25 21 | OVERLAPPING_METRIC: iou 22 | OVERLAPPING_TH: 0.25 23 | PRESELECTION_RAIO: 1.5 24 | RECOVER_ALMOST_CORRECT_PRED: true 25 | RECOVER_MISSING_OBJECTS: true 26 | REMOVE_DUPLICATES: true 27 | REMOVE_DUPLICATES_TH: 0.15 28 | SELECTION_METHOD: top 29 | SELECTION_RAIO_DECAY: linear 30 | OBJECT_SCORING: 1vs2 31 | TRAINING: 32 | EPOCHS_PER_ROUND_DECAY: linear 33 | EPOCHS_PER_ROUND_INITIAL: 500 34 | EPOCHS_PER_ROUND_LAST: 50 35 | ROUNDS: 5 36 | CUDNN_BENCHMARK: false 37 | DATALOADER: 38 | ASPECT_RATIO_GROUPING: true 39 | FILTER_EMPTY_ANNOTATIONS: true 40 | NUM_WORKERS: 4 41 | REPEAT_THRESHOLD: 0.0 42 | SAMPLER_TRAIN: TrainingSampler 43 | DATASETS: 44 | PRECOMPUTED_PROPOSAL_TOPK_TEST: 1000 45 | PRECOMPUTED_PROPOSAL_TOPK_TRAIN: 2000 46 | PROPOSAL_FILES_TEST: [] 47 | PROPOSAL_FILES_TRAIN: [] 48 | TEST: [] 49 | TRAIN: [] 50 | GLOBAL: 51 | HACK: 1.0 52 | INPUT: 53 | CROP: 54 | ENABLED: true 55 | SIZE: 56 | - 0.9 57 | - 0.9 58 | TYPE: relative_range 59 | FORMAT: BGR 60 | MASK_FORMAT: polygon 61 | MAX_SIZE_TEST: 1333 62 | MAX_SIZE_TRAIN: 1333 63 | MIN_SIZE_TEST: 800 64 | MIN_SIZE_TRAIN: 65 | - 640 66 | - 672 67 | - 704 68 | - 736 69 | - 768 70 | - 800 71 | MIN_SIZE_TRAIN_SAMPLING: choice 72 | MODEL: 73 | ANCHOR_GENERATOR: 74 | ANGLES: 75 | - - -90 76 | - 0 77 | - 90 78 | ASPECT_RATIOS: 79 | - - 0.5 80 | - 1.0 81 | - 2.0 82 | NAME: DefaultAnchorGenerator 83 | OFFSET: 0.0 84 | SIZES: 85 | - - 32 86 | - - 64 87 | - - 128 88 | - - 256 89 | - - 512 90 | BACKBONE: 91 | FREEZE_AT: 2 92 | NAME: build_resnet_fpn_backbone 93 | DEVICE: cuda 94 | FPN: 95 | FUSE_TYPE: sum 96 | IN_FEATURES: 97 | - res2 98 | - res3 99 | - res4 100 | - res5 101 | NORM: '' 102 | OUT_CHANNELS: 256 103 | KEYPOINT_ON: false 104 | LOAD_PROPOSALS: false 105 | MASK_ON: false 106 | META_ARCHITECTURE: ActiveLearningRCNN 107 | PANOPTIC_FPN: 108 | COMBINE: 109 | ENABLED: true 110 | INSTANCES_CONFIDENCE_THRESH: 0.5 111 | OVERLAP_THRESH: 0.5 112 | STUFF_AREA_LIMIT: 4096 113 | INSTANCE_LOSS_WEIGHT: 1.0 114 | PIXEL_MEAN: 115 | - 103.53 116 | - 116.28 117 | - 123.675 118 | PIXEL_STD: 119 | - 1.0 120 | - 1.0 121 | - 1.0 122 | PROPOSAL_GENERATOR: 123 | MIN_SIZE: 0 124 | NAME: RPN 125 | RESNETS: 126 | DEFORM_MODULATED: false 127 | DEFORM_NUM_GROUPS: 1 128 | DEFORM_ON_PER_STAGE: 129 | - false 130 | - false 131 | - false 132 | - false 133 | DEPTH: 50 134 | NORM: FrozenBN 135 | NUM_GROUPS: 1 136 | OUT_FEATURES: 137 | - res2 138 | - res3 139 | - res4 140 | - res5 141 | RES2_OUT_CHANNELS: 256 142 | RES5_DILATION: 1 143 | STEM_OUT_CHANNELS: 64 144 | STRIDE_IN_1X1: true 145 | WIDTH_PER_GROUP: 64 146 | RETINANET: 147 | BBOX_REG_WEIGHTS: 148 | - 1.0 149 | - 1.0 150 | - 1.0 151 | - 1.0 152 | FOCAL_LOSS_ALPHA: 0.25 153 | FOCAL_LOSS_GAMMA: 2.0 154 | IN_FEATURES: 155 | - p3 156 | - p4 157 | - p5 158 | - p6 159 | - p7 160 | IOU_LABELS: 161 | - 0 162 | - -1 163 | - 1 164 | IOU_THRESHOLDS: 165 | - 0.4 166 | - 0.5 167 | NMS_THRESH_TEST: 0.5 168 | NUM_CLASSES: 80 169 | NUM_CONVS: 4 170 | PRIOR_PROB: 0.01 171 | SCORE_THRESH_TEST: 0.05 172 | SMOOTH_L1_LOSS_BETA: 0.1 173 | TOPK_CANDIDATES_TEST: 1000 174 | ROI_BOX_CASCADE_HEAD: 175 | BBOX_REG_WEIGHTS: 176 | - - 10.0 177 | - 10.0 178 | - 5.0 179 | - 5.0 180 | - - 20.0 181 | - 20.0 182 | - 10.0 183 | - 10.0 184 | - - 30.0 185 | - 30.0 186 | - 15.0 187 | - 15.0 188 | IOUS: 189 | - 0.5 190 | - 0.6 191 | - 0.7 192 | ROI_BOX_HEAD: 193 | BBOX_REG_WEIGHTS: 194 | - 10.0 195 | - 10.0 196 | - 5.0 197 | - 5.0 198 | CLS_AGNOSTIC_BBOX_REG: false 199 | CONV_DIM: 256 200 | FC_DIM: 1024 201 | NAME: FastRCNNConvFCHead 202 | NORM: '' 203 | NUM_CONV: 0 204 | NUM_FC: 2 205 | POOLER_RESOLUTION: 7 206 | POOLER_SAMPLING_RATIO: 0 207 | POOLER_TYPE: ROIAlignV2 208 | SMOOTH_L1_BETA: 0.0 209 | TRAIN_ON_PRED_BOXES: false 210 | ROI_HEADS: 211 | BATCH_SIZE_PER_IMAGE: 512 212 | IN_FEATURES: 213 | - p2 214 | - p3 215 | - p4 216 | - p5 217 | IOU_LABELS: 218 | - 0 219 | - 1 220 | IOU_THRESHOLDS: 221 | - 0.5 222 | NAME: ROIHeadsAL 223 | NMS_THRESH_TEST: 0.5 224 | NUM_CLASSES: 80 225 | POSITIVE_FRACTION: 0.25 226 | PROPOSAL_APPEND_GT: true 227 | SCORE_THRESH_TEST: 0.05 228 | ROI_KEYPOINT_HEAD: 229 | CONV_DIMS: 230 | - 512 231 | - 512 232 | - 512 233 | - 512 234 | - 512 235 | - 512 236 | - 512 237 | - 512 238 | LOSS_WEIGHT: 1.0 239 | MIN_KEYPOINTS_PER_IMAGE: 1 240 | NAME: KRCNNConvDeconvUpsampleHead 241 | NORMALIZE_LOSS_BY_VISIBLE_KEYPOINTS: true 242 | NUM_KEYPOINTS: 17 243 | POOLER_RESOLUTION: 14 244 | POOLER_SAMPLING_RATIO: 0 245 | POOLER_TYPE: ROIAlignV2 246 | ROI_MASK_HEAD: 247 | CLS_AGNOSTIC_MASK: false 248 | CONV_DIM: 256 249 | NAME: MaskRCNNConvUpsampleHead 250 | NORM: '' 251 | NUM_CONV: 4 252 | POOLER_RESOLUTION: 14 253 | POOLER_SAMPLING_RATIO: 0 254 | POOLER_TYPE: ROIAlignV2 255 | RPN: 256 | BATCH_SIZE_PER_IMAGE: 256 257 | BBOX_REG_WEIGHTS: 258 | - 1.0 259 | - 1.0 260 | - 1.0 261 | - 1.0 262 | BOUNDARY_THRESH: -1 263 | HEAD_NAME: StandardRPNHead 264 | IN_FEATURES: 265 | - p2 266 | - p3 267 | - p4 268 | - p5 269 | - p6 270 | IOU_LABELS: 271 | - 0 272 | - -1 273 | - 1 274 | IOU_THRESHOLDS: 275 | - 0.3 276 | - 0.7 277 | LOSS_WEIGHT: 1.0 278 | NMS_THRESH: 0.7 279 | POSITIVE_FRACTION: 0.5 280 | POST_NMS_TOPK_TEST: 1000 281 | POST_NMS_TOPK_TRAIN: 1000 282 | PRE_NMS_TOPK_TEST: 1000 283 | PRE_NMS_TOPK_TRAIN: 2000 284 | SMOOTH_L1_BETA: 0.0 285 | SEM_SEG_HEAD: 286 | COMMON_STRIDE: 4 287 | CONVS_DIM: 128 288 | IGNORE_VALUE: 255 289 | IN_FEATURES: 290 | - p2 291 | - p3 292 | - p4 293 | - p5 294 | LOSS_WEIGHT: 1.0 295 | NAME: SemSegFPNHead 296 | NORM: GN 297 | NUM_CLASSES: 54 298 | WEIGHTS: detectron2://ImageNetPretrained/MSRA/R-50.pkl 299 | OUTPUT_DIR: ./output 300 | SEED: -1 301 | SOLVER: 302 | BASE_LR: 0.00025 303 | BIAS_LR_FACTOR: 1.0 304 | CHECKPOINT_PERIOD: 20000 305 | GAMMA: 0.1 306 | IMS_PER_BATCH: 16 307 | LR_SCHEDULER_NAME: WarmupMultiStepLR 308 | MAX_ITER: 270000 309 | MOMENTUM: 0.9 310 | STEPS: 311 | - 210000 312 | - 250000 313 | WARMUP_FACTOR: 0.001 314 | WARMUP_ITERS: 1000 315 | WARMUP_METHOD: linear 316 | WEIGHT_DECAY: 0.0001 317 | WEIGHT_DECAY_BIAS: 0.0001 318 | WEIGHT_DECAY_NORM: 0.0 319 | TEST: 320 | AUG: 321 | ENABLED: false 322 | FLIP: true 323 | MAX_SIZE: 4000 324 | MIN_SIZES: 325 | - 400 326 | - 500 327 | - 600 328 | - 700 329 | - 800 330 | - 900 331 | - 1000 332 | - 1100 333 | - 1200 334 | DETECTIONS_PER_IMAGE: 100 335 | EVAL_PERIOD: 0 336 | EXPECTED_RESULTS: [] 337 | KEYPOINT_OKS_SIGMAS: [] 338 | PRECISE_BN: 339 | ENABLED: false 340 | NUM_ITER: 200 341 | VERSION: 2 342 | VIS_PERIOD: 0 343 | -------------------------------------------------------------------------------- /tools/convert_prima_to_coco.py: -------------------------------------------------------------------------------- 1 | import os, re, json 2 | import imagesize 3 | from glob import glob 4 | from bs4 import BeautifulSoup 5 | import numpy as np 6 | from PIL import Image 7 | import argparse 8 | from tqdm import tqdm 9 | import sys 10 | sys.path.append('..') 11 | 12 | class NpEncoder(json.JSONEncoder): 13 | def default(self, obj): 14 | if isinstance(obj, np.integer): 15 | return int(obj) 16 | elif isinstance(obj, np.floating): 17 | return float(obj) 18 | elif isinstance(obj, np.ndarray): 19 | return obj.tolist() 20 | else: 21 | return super(NpEncoder, self).default(obj) 22 | 23 | def cvt_coords_to_array(obj): 24 | 25 | return np.array( 26 | [(float(pt['x']), float(pt['y'])) 27 | for pt in obj.find_all("Point")] 28 | ) 29 | 30 | def cal_ployarea(points): 31 | x = points[:,0] 32 | y = points[:,1] 33 | return 0.5*np.abs(np.dot(x,np.roll(y,1))-np.dot(y,np.roll(x,1))) 34 | 35 | def _create_category(schema=0): 36 | 37 | if schema==0: 38 | 39 | categories = \ 40 | [{"supercategory": "layout", "id": 0, "name": "Background"}, 41 | {"supercategory": "layout", "id": 1, "name": "TextRegion"}, 42 | {"supercategory": "layout", "id": 2, "name": "ImageRegion"}, 43 | {"supercategory": "layout", "id": 3, "name": "TableRegion"}, 44 | {"supercategory": "layout", "id": 4, "name": "MathsRegion"}, 45 | {"supercategory": "layout", "id": 5, "name": "SeparatorRegion"}, 46 | {"supercategory": "layout", "id": 6, "name": "OtherRegion"}] 47 | 48 | find_categories = lambda name: \ 49 | [val["id"] for val in categories if val['name'] == name][0] 50 | 51 | conversion = \ 52 | { 53 | 'TextRegion': find_categories("TextRegion"), 54 | 'TableRegion': find_categories("TableRegion"), 55 | 'MathsRegion': find_categories("MathsRegion"), 56 | 'ChartRegion': find_categories("ImageRegion"), 57 | 'GraphicRegion': find_categories("ImageRegion"), 58 | 'ImageRegion': find_categories("ImageRegion"), 59 | 'LineDrawingRegion':find_categories("OtherRegion"), 60 | 'SeparatorRegion': find_categories("SeparatorRegion"), 61 | 'NoiseRegion': find_categories("OtherRegion"), 62 | 'FrameRegion': find_categories("OtherRegion"), 63 | } 64 | 65 | return categories, conversion 66 | 67 | _categories, _categories_conversion = _create_category(schema=0) 68 | 69 | _info = { 70 | "description": "PRIMA Layout Analysis Dataset", 71 | "url": "https://www.primaresearch.org/datasets/Layout_Analysis", 72 | "version": "1.0", 73 | "year": 2010, 74 | "contributor": "PRIMA Research", 75 | "date_created": "2020/09/01", 76 | } 77 | 78 | def _load_soup(filename): 79 | with open(filename, "r") as fp: 80 | soup = BeautifulSoup(fp.read(),'xml') 81 | 82 | return soup 83 | 84 | def _image_template(image_id, image_path): 85 | 86 | width, height = imagesize.get(image_path) 87 | 88 | return { 89 | "file_name": os.path.basename(image_path), 90 | "height": height, 91 | "width": width, 92 | "id": int(image_id) 93 | } 94 | 95 | def _anno_template(anno_id, image_id, pts, obj_tag): 96 | 97 | x_1, x_2 = pts[:,0].min(), pts[:,0].max() 98 | y_1, y_2 = pts[:,1].min(), pts[:,1].max() 99 | height = y_2 - y_1 100 | width = x_2 - x_1 101 | 102 | return { 103 | "segmentation": [pts.flatten().tolist()], 104 | "area": cal_ployarea(pts), 105 | "iscrowd": 0, 106 | "image_id": image_id, 107 | "bbox": [x_1, y_1, width, height], 108 | "category_id": _categories_conversion[obj_tag], 109 | "id": anno_id 110 | } 111 | 112 | class PRIMADataset(): 113 | 114 | def __init__(self, base_path, anno_path='XML', 115 | image_path='Images', 116 | drop_categories=[]): 117 | 118 | self.base_path = base_path 119 | self.anno_path = os.path.join(base_path, anno_path) 120 | self.image_path = os.path.join(base_path, image_path) 121 | 122 | self._ids = self.find_all_image_ids() 123 | self.drop_categories = drop_categories 124 | 125 | def __len__(self): 126 | return len(self.ids) 127 | 128 | def __getitem__(self, idx): 129 | return self.load_image_and_annotaiton(idx) 130 | 131 | def find_all_annotation_files(self): 132 | return glob(os.path.join(self.anno_path, '*.xml')) 133 | 134 | def find_all_image_ids(self): 135 | replacer = lambda s: os.path.basename(s).replace('pc-', '').replace('.xml', '') 136 | return [replacer(s) for s in self.find_all_annotation_files()] 137 | 138 | def load_image_and_annotaiton(self, idx): 139 | 140 | image_id = self._ids[idx] 141 | 142 | image_path = os.path.join(self.image_path, f'{image_id}.tif') 143 | image = Image.open(image_path) 144 | 145 | anno = self.load_annotation(idx) 146 | 147 | return image, anno 148 | 149 | def load_annotation(self, idx): 150 | image_id = self._ids[idx] 151 | 152 | anno_path = os.path.join(self.anno_path, f'pc-{image_id}.xml') 153 | # A dirtly hack to load the files w/wo pc- simualtaneously 154 | if not os.path.exists(anno_path): 155 | anno_path = os.path.join(self.anno_path, f'{image_id}.xml') 156 | assert os.path.exists(anno_path), "Invalid path" 157 | anno = _load_soup(anno_path) 158 | 159 | return anno 160 | 161 | def convert_to_COCO(self, save_path): 162 | 163 | all_image_infos = [] 164 | all_anno_infos = [] 165 | anno_id = 0 166 | 167 | for idx, image_id in enumerate(tqdm(self._ids)): 168 | 169 | # We use the idx as the image id 170 | 171 | image_path = os.path.join(self.image_path, f'{image_id}.tif') 172 | image_info = _image_template(idx, image_path) 173 | all_image_infos.append(image_info) 174 | 175 | anno = self.load_annotation(idx) 176 | 177 | for item in anno.find_all(re.compile(".*Region")): 178 | 179 | pts = cvt_coords_to_array(item.Coords) 180 | if 0 not in pts.shape: 181 | # Sometimes there will be polygons with less 182 | # than 4 edges, and they could not be appropriately 183 | # handled by the COCO format. So we just drop them. 184 | if pts.shape[0] >= 4: 185 | anno_info = _anno_template(anno_id, idx, pts, item.name) 186 | if anno_info['category_id'] in self.drop_categories: continue 187 | all_anno_infos.append(anno_info) 188 | anno_id += 1 189 | 190 | 191 | final_annotation = { 192 | "info": _info, 193 | "licenses": [], 194 | "images": all_image_infos, 195 | "annotations": all_anno_infos, 196 | "categories": _categories} 197 | 198 | with open(save_path, 'w') as fp: 199 | json.dump(final_annotation, fp, cls=NpEncoder) 200 | 201 | return final_annotation 202 | 203 | 204 | parser = argparse.ArgumentParser() 205 | 206 | parser.add_argument('--prima_datapath', type=str, default='./data/prima', help='the path to the prima data folders') 207 | parser.add_argument('--anno_savepath', type=str, default='./annotations.json', help='the path to save the new annotations') 208 | parser.add_argument('--drop_categories', type=int, nargs='+', default=[]) 209 | 210 | 211 | if __name__ == "__main__": 212 | args = parser.parse_args() 213 | 214 | print("Start running the conversion script") 215 | 216 | print(f"Loading the information from the path {args.prima_datapath}") 217 | dataset = PRIMADataset(args.prima_datapath, drop_categories=args.drop_categories) 218 | 219 | print(f"Saving the annotation to {args.anno_savepath}") 220 | res = dataset.convert_to_COCO(args.anno_savepath) -------------------------------------------------------------------------------- /annotating/backend_model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.optim as optim 4 | import time 5 | import os 6 | import numpy as np 7 | import requests 8 | import io 9 | import hashlib 10 | import urllib 11 | import cv2 12 | 13 | from PIL import Image 14 | from torch.utils.data import Dataset, DataLoader 15 | from torchvision import models, transforms 16 | 17 | from label_studio.ml import LabelStudioMLBase 18 | from label_studio.ml.utils import get_single_tag_keys, get_choice, is_skipped 19 | 20 | 21 | import sys 22 | sys.path.append('./src') 23 | from detectron2_al.configs import get_cfg 24 | from detectron2_al.engine.al_engine import ActiveLearningPredictor 25 | from detectron2_al.modeling import * 26 | import layoutparser as lp 27 | from fvcore.common.file_io import PathManager 28 | 29 | device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') 30 | 31 | 32 | image_cache_dir = os.path.join(os.path.dirname(__file__), 'image-cache') 33 | os.makedirs(image_cache_dir, exist_ok=True) 34 | 35 | 36 | def load_image_from_url(url): 37 | # is_local_file = url.startswith('http://localhost:') and '/data/' in url 38 | is_local_file = True 39 | if is_local_file: 40 | filename, dir_path = url.split('/data/')[1].split('?d=') 41 | dir_path = str(urllib.parse.unquote_plus(dir_path)) 42 | filepath = os.path.join(dir_path, filename) 43 | return cv2.imread(filepath) 44 | else: 45 | cached_file = os.path.join(image_cache_dir, hashlib.md5(url.encode()).hexdigest()) 46 | if os.path.exists(cached_file): 47 | with open(cached_file, mode='rb') as f: 48 | image = Image.open(f).convert('RGB') 49 | else: 50 | r = requests.get(url, stream=True) 51 | r.raise_for_status() 52 | with io.BytesIO(r.content) as f: 53 | image = Image.open(f).convert('RGB') 54 | with io.open(cached_file, mode='wb') as fout: 55 | fout.write(r.content) 56 | return image_transforms(image) 57 | 58 | def convert_block_to_value(block, image_height, image_width): 59 | 60 | 61 | block.block.x_1 = max(0, block.block.x_1) 62 | block.block.x_2 = min(block.block.x_2, image_width) 63 | block.block.y_1 = max(0, block.block.y_1) 64 | block.block.y_2 = min(block.block.y_2, image_height) 65 | 66 | return { 67 | "height": block.height / image_height*100, 68 | "rectanglelabels": [str(block.type)], 69 | "rotation": 0, 70 | "width": block.width / image_width*100, 71 | "x": block.coordinates[0] / image_width*100, 72 | "y": block.coordinates[1] / image_height*100, 73 | "score": block.score_al*100 74 | } 75 | 76 | 77 | class Detectron2LayoutModel(): 78 | 79 | def __init__(self, config_path, 80 | model_path = None, 81 | label_map = None, 82 | extra_config= []): 83 | 84 | cfg = get_cfg() 85 | config_path = PathManager.get_local_path(config_path) 86 | cfg.merge_from_file(config_path) 87 | cfg.merge_from_list(extra_config) 88 | 89 | cfg.MODEL.ROI_HEADS.NAME = 'ROIHeadsAL' 90 | cfg.MODEL.META_ARCHITECTURE = 'ActiveLearningRCNN' 91 | 92 | if model_path is not None: 93 | cfg.MODEL.WEIGHTS = model_path 94 | cfg.MODEL.DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu' 95 | 96 | self.cfg = cfg 97 | self.label_map = label_map 98 | self._create_model() 99 | 100 | def _create_model(self): 101 | self.model = ActiveLearningPredictor(self.cfg) 102 | 103 | def gather_output(self, outputs): 104 | 105 | instance_pred = outputs['instances'].to("cpu") 106 | 107 | layout = lp.Layout() 108 | scores = instance_pred.scores.tolist() 109 | boxes = instance_pred.pred_boxes.tensor.tolist() 110 | labels = instance_pred.pred_classes.tolist() 111 | 112 | for score, box, label in zip(scores, boxes, labels): 113 | x_1, y_1, x_2, y_2 = box 114 | 115 | if self.label_map is not None: 116 | label = self.label_map.get(label, label) 117 | 118 | cur_block = lp.TextBlock( 119 | lp.Rectangle(x_1, y_1, x_2, y_2), 120 | type=label, 121 | score=score) 122 | layout.append(cur_block) 123 | 124 | return layout 125 | 126 | def gather_output_with_stats(self, outputs): 127 | 128 | instance_pred = outputs['instances'].to("cpu") 129 | 130 | layout = lp.Layout() 131 | scores = instance_pred.scores.tolist() 132 | scores_al = instance_pred.scores_al.tolist() 133 | boxes = instance_pred.pred_boxes.tensor.tolist() 134 | labels = instance_pred.pred_classes.tolist() 135 | 136 | for score, box, label, score_al in zip(scores, boxes, labels, scores_al): 137 | x_1, y_1, x_2, y_2 = box 138 | 139 | if self.label_map is not None: 140 | label = self.label_map.get(label, label) 141 | 142 | cur_block = lp.TextBlock( 143 | lp.Rectangle(x_1, y_1, x_2, y_2), 144 | type=label, 145 | score=score) 146 | cur_block.score_al = score_al 147 | layout.append(cur_block) 148 | 149 | return layout 150 | 151 | def detect(self, image): 152 | outputs, _ = self.model(image) 153 | layout = self.gather_output(outputs) 154 | return layout 155 | 156 | def detect_al(self, image): 157 | pred = self.model(image) 158 | layout = self.gather_output_with_stats(pred) 159 | return layout 160 | 161 | class ObjectDetectionAPI(LabelStudioMLBase): 162 | 163 | def __init__(self, freeze_extractor=False, **kwargs): 164 | 165 | super(ObjectDetectionAPI, self).__init__(**kwargs) 166 | 167 | self.from_name, self.to_name, self.value, self.classes =\ 168 | get_single_tag_keys(self.parsed_label_config, 'RectangleLabels', 'Image') 169 | self.freeze_extractor = freeze_extractor 170 | 171 | self.model = Detectron2LayoutModel( 172 | config_path="https://www.dropbox.com/s/ta4777i1g1jjj18/config.yml?dl=1", 173 | model_path ="https://www.dropbox.com/s/f261qar6f75b9c0/model_final.pth?dl=1", 174 | label_map={1: "title", 2: "address", 3: "text", 4:"number"}, 175 | extra_config=["TEST.DETECTIONS_PER_IMAGE", 150, 176 | "MODEL.ROI_HEADS.SCORE_THRESH_TEST", 0.5, 177 | "MODEL.ROI_HEADS.NMS_THRESH_TEST", 0.75] 178 | ) 179 | 180 | def reset_model(self): 181 | ## self.model = ImageClassifier(len(self.classes), self.freeze_extractor) 182 | pass 183 | 184 | def predict(self, tasks, **kwargs): 185 | 186 | image_urls = [task['data'][self.value] for task in tasks] 187 | images = [load_image_from_url(url) for url in image_urls] 188 | layouts = [self.model.detect_al(image) for image in images] 189 | 190 | predictions = [] 191 | for image, layout in zip(images, layouts): 192 | height, width = image.shape[:2] 193 | 194 | result = [ 195 | { 196 | 'from_name': self.from_name, 197 | 'to_name': self.to_name, 198 | "original_height": height, 199 | "original_width": width, 200 | "source": "$image", 201 | 'type': 'rectanglelabels', 202 | "value": convert_block_to_value(block, height, width) 203 | } for block in layout 204 | ] 205 | 206 | predictions.append({'result': result}) 207 | 208 | return predictions 209 | 210 | def fit(self, completions, workdir=None, 211 | batch_size=32, num_epochs=10, **kwargs): 212 | image_urls, image_classes = [], [] 213 | print('Collecting completions...') 214 | # for completion in completions: 215 | # if is_skipped(completion): 216 | # continue 217 | # image_urls.append(completion['data'][self.value]) 218 | # image_classes.append(get_choice(completion)) 219 | 220 | print('Creating dataset...') 221 | # dataset = ImageClassifierDataset(image_urls, image_classes) 222 | # dataloader = DataLoader(dataset, shuffle=True, batch_size=batch_size) 223 | 224 | print('Train model...') 225 | # self.reset_model() 226 | # self.model.train(dataloader, num_epochs=num_epochs) 227 | 228 | print('Save model...') 229 | # model_path = os.path.join(workdir, 'model.pt') 230 | # self.model.save(model_path) 231 | 232 | return {'model_path': None, 'classes': None} -------------------------------------------------------------------------------- /src/detectron2_al/engine/al_engine.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import logging 3 | import time 4 | import weakref 5 | import os 6 | import pandas as pd 7 | from torch.nn.parallel import DistributedDataParallel 8 | from fvcore.nn.precise_bn import get_bn_modules 9 | 10 | from detectron2.checkpoint import DetectionCheckpointer 11 | from detectron2.utils.logger import setup_logger 12 | from detectron2.utils import comm 13 | from detectron2.evaluation import verify_results 14 | from detectron2.engine import DefaultPredictor, DefaultTrainer, HookBase, hooks 15 | from detectron2.evaluation import COCOEvaluator 16 | 17 | 18 | from ..dataset.al_dataset import build_al_dataset 19 | from ..dataset.object_fusion import ObjectFusion 20 | 21 | 22 | __all__ = ["build_al_trainer", 23 | "ActiveLearningTrainer", 24 | "ImageActiveLearningTrainer", 25 | "ObjectActiveLearningTrainer"] 26 | 27 | def build_al_trainer(cfg): 28 | 29 | logger = logging.getLogger("detectron2") 30 | if not logger.isEnabledFor(logging.INFO): # setup_logger is not called for d2 31 | setup_logger() 32 | 33 | logger.info("Creating Active Learning trainer for {} mode".format(cfg.AL.MODE)) 34 | if cfg.AL.MODE == 'image': 35 | return ImageActiveLearningTrainer(cfg) 36 | elif cfg.AL.MODE == 'object': 37 | return ObjectActiveLearningTrainer(cfg) 38 | else: 39 | raise ValueError(f'Unknown active learning mode {cfg.AL.MODE}') 40 | 41 | 42 | class ActiveLearningTrainer(DefaultTrainer): 43 | 44 | """ 45 | Modified based on DefaultTrainer to support active 46 | learning functions. 47 | """ 48 | 49 | def __init__(self, cfg): 50 | 51 | self.logger = logging.getLogger("detectron2") 52 | 53 | model = self.build_model(cfg) 54 | optimizer = self.build_optimizer(cfg, model) 55 | 56 | self.model = model 57 | self.optimizer = optimizer 58 | self.al_dataset = self.build_al_dataset(cfg) 59 | self.object_fusion = ObjectFusion(cfg) 60 | # It should be moved to ObjectActiveLearningTrainer later when 61 | 62 | # For training, wrap with DDP. But don't need this for inference. 63 | if comm.get_world_size() > 1: 64 | model = DistributedDataParallel( 65 | model, device_ids=[comm.get_local_rank()], broadcast_buffers=False 66 | ) 67 | 68 | self.scheduler = self.build_lr_scheduler(cfg, optimizer) 69 | # Assume no other objects need to be checkpointed. 70 | # We can later make it checkpoint the stateful hooks 71 | self.checkpointer = DetectionCheckpointer( 72 | # Assume you want to save checkpoints together with logs/statistics 73 | model, 74 | cfg.OUTPUT_DIR, 75 | optimizer=optimizer, 76 | scheduler=self.scheduler, 77 | ) 78 | self.cfg = cfg 79 | 80 | @classmethod 81 | def build_al_dataset(cls, cfg): 82 | return build_al_dataset(cfg) 83 | 84 | @classmethod 85 | def build_evaluator(cls, cfg, dataset_name, output_folder=None): 86 | return COCOEvaluator(dataset_name, cfg, True, output_folder) 87 | 88 | def build_hooks(self): 89 | cfg = self.cfg.clone() 90 | cfg.defrost() 91 | cfg.DATALOADER.NUM_WORKERS = 0 # save some memory and time for PreciseBN 92 | 93 | ret = [ 94 | hooks.IterationTimer(), 95 | hooks.LRScheduler(self.optimizer, self.scheduler), 96 | hooks.PreciseBN( 97 | # Run at the same freq as (but before) evaluation. 98 | cfg.TEST.EVAL_PERIOD, 99 | self.model, 100 | # Build a new data loader to not affect training 101 | self.build_train_loader(cfg), 102 | cfg.TEST.PRECISE_BN.NUM_ITER, 103 | ) 104 | if cfg.TEST.PRECISE_BN.ENABLED and get_bn_modules(self.model) 105 | else None, 106 | ] 107 | 108 | if comm.is_main_process(): 109 | ret.append(hooks.PeriodicCheckpointer(self.checkpointer, cfg.SOLVER.CHECKPOINT_PERIOD)) 110 | 111 | def test_and_save_results(): 112 | res = self._last_eval_results = self.test(self.cfg, self.model) 113 | eval_dir = os.path.join(self.cfg.OUTPUT_DIR, 'evals') 114 | os.makedirs(eval_dir, exist_ok=True) 115 | pd.DataFrame(res).to_csv(os.path.join(eval_dir, f'{self.round}.csv')) 116 | return self._last_eval_results 117 | 118 | ret.append(hooks.EvalHook(cfg.TEST.EVAL_PERIOD, test_and_save_results)) 119 | 120 | if comm.is_main_process(): 121 | # run writers in the end, so that evaluation metrics are written 122 | ret.append(hooks.PeriodicWriter(self.build_writers(), period=20)) 123 | return ret 124 | 125 | def register_hooks(self, hooks): 126 | hooks = [h for h in hooks if h is not None] 127 | for h in hooks: 128 | assert isinstance(h, HookBase) 129 | h.trainer = weakref.proxy(self) 130 | self._hooks = hooks 131 | 132 | def train_al(self): 133 | """ 134 | Run training for active learning 135 | 136 | Returns: 137 | OrderedDict of results, if evaluation is enabled. Otherwise None. 138 | """ 139 | 140 | self.al_dataset.create_initial_dataset() 141 | 142 | self.logger.info("The estimated total number of iterations is {}".format(self.al_dataset.calculate_estimated_total_iterations())) 143 | total_round = self.cfg.AL.TRAINING.ROUNDS - 1 144 | for self.round in range(self.cfg.AL.TRAINING.ROUNDS): 145 | 146 | self.logger.info("Started training for round:{}/{}".format(self.round, total_round)) 147 | # Initialize the dataloader and the training steps 148 | dataloader, max_iter = self.al_dataset.get_training_dataloader() 149 | self._data_loader_iter = iter(dataloader) 150 | self.start_iter, self.max_iter = 0, max_iter 151 | 152 | # Build the hooks for each round 153 | self.register_hooks(self.build_hooks()) 154 | 155 | # Run the main training loop 156 | self.train() 157 | 158 | if self.round != total_round: 159 | # Run the scoring pass and create the new dataset 160 | # except for the last round 161 | self.model.eval() 162 | self.logger.info("Started running scoring for round:{}/{}".format(self.round, total_round)) 163 | self.run_scoring_step() 164 | self.model.train() 165 | 166 | self.al_dataset.save_history() 167 | 168 | def run_scoring_step(self): 169 | """ 170 | Run image/object scoring step in active learning 171 | And create the new dataset. 172 | The implementation is different for image-level 173 | and object-level active learning. 174 | """ 175 | pass 176 | 177 | 178 | class ImageActiveLearningTrainer(ActiveLearningTrainer): 179 | 180 | def run_scoring_step(self): 181 | """ 182 | For image-level active learning dataset, it will perform 183 | image-level scoring and update the dataset 184 | """ 185 | 186 | oracle_dataloader, max_iter = self.al_dataset.get_oracle_dataloader() 187 | oracle_dataloader_iter = iter(oracle_dataloader) 188 | 189 | image_scores = [] 190 | for _iter in range(max_iter): 191 | data = next(oracle_dataloader_iter) 192 | image_scores.extend(self.model.generate_image_al_scores(data)) 193 | if _iter % 100 == 0: 194 | self.logger.info("Running scoring functions for round {}. Step:{}/{}".format(self.round, _iter, max_iter)) 195 | 196 | self.al_dataset.create_new_dataset(image_scores) 197 | 198 | 199 | class ObjectActiveLearningTrainer(ActiveLearningTrainer): 200 | 201 | def run_scoring_step(self): 202 | """ 203 | For object-level active learning dataset, it will perform 204 | object-level scoring and update the dataset 205 | """ 206 | 207 | oracle_dataloader, max_iter = self.al_dataset.get_oracle_dataloader() 208 | oracle_dataloader_iter = iter(oracle_dataloader) 209 | 210 | num_imgs = [info.num_images for info in self.al_dataset._history] 211 | num_objs = [info.num_objects for info in self.al_dataset._history] 212 | ave_num_objects_per_image = sum(num_objs) // sum(num_imgs) 213 | 214 | fused_results = [] 215 | for _iter in range(max_iter): 216 | data = next(oracle_dataloader_iter) 217 | preds = self.model.generate_object_al_scores(data) 218 | 219 | for gt, pred in zip(data, preds): 220 | fused_results.append( 221 | self.object_fusion.combine(pred, gt, 222 | self.round, ave_num_objects_per_image) 223 | ) 224 | if _iter % 100 == 0: 225 | self.logger.info("Running scoring functions for round {}. Step:{}/{}".format(self.round, _iter, max_iter)) 226 | 227 | self.al_dataset.create_new_dataset(fused_results) 228 | 229 | 230 | class ActiveLearningPredictor(DefaultPredictor): 231 | """ 232 | Create an active learning predictor with the given config that runs on 233 | single device for a single input image. 234 | """ 235 | 236 | def __call__(self, original_image): 237 | """ 238 | Args: 239 | original_image (np.ndarray): an image of shape (H, W, C) (in BGR order). 240 | 241 | Returns: 242 | predictions (dict): 243 | the output of the model for one image only. 244 | See :doc:`/tutorials/models` for details about the format. 245 | """ 246 | with torch.no_grad(): # https://github.com/sphinx-doc/sphinx/issues/4258 247 | # Apply pre-processing to image. 248 | if self.input_format == "RGB": 249 | # whether the model expects BGR inputs or RGB 250 | original_image = original_image[:, :, ::-1] 251 | height, width = original_image.shape[:2] 252 | image = self.transform_gen.get_transform(original_image).apply_image(original_image) 253 | image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) 254 | 255 | inputs = {"image": image, "height": height, "width": width} 256 | predictions = self.model.forward_al([inputs]) 257 | return predictions[0] 258 | -------------------------------------------------------------------------------- /tools/show_results.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import sys 4 | from glob import glob 5 | from itertools import cycle 6 | import numpy as np 7 | import pandas as pd 8 | 9 | import seaborn as sns 10 | import matplotlib.pyplot as plt 11 | import matplotlib.ticker as ticker 12 | from matplotlib import rcParams 13 | from matplotlib.font_manager import FontProperties 14 | 15 | sys.path.append('../src') 16 | from detectron2_al.configs import get_cfg 17 | 18 | import argparse 19 | 20 | AP_FEATURES = ['AP', 'AP50', 'AP75', 'APl', 'APm', 'APs'] 21 | LB_FEATURES = ['num_images', 'num_objects', 'training_iter'] 22 | 23 | def load_cfg(config_file_path): 24 | cfg = get_cfg() 25 | cfg.merge_from_file(config_file_path) 26 | return cfg 27 | 28 | def load_json(json_path): 29 | with open(json_path, 'r') as fp: 30 | return json.load(fp) 31 | 32 | def create_color_palette(show=False, style=0): 33 | if style == 0: 34 | object_plot_color = sns.color_palette("GnBu_d") 35 | #sns.light_palette((210, 90, 60), input="husl") 36 | 37 | image_plot_color = sns.color_palette("RdPu") 38 | #sns.light_palette((45, 90, 60), input="husl") 39 | 40 | else: 41 | image_plot_color = sns.color_palette("GnBu_d") 42 | object_plot_color = sns.color_palette("Paired")[1:] 43 | 44 | if show: 45 | sns.palplot(object_plot_color) 46 | sns.palplot(image_plot_color) 47 | 48 | return {"object": object_plot_color[1:], "image":image_plot_color[1:]} 49 | 50 | 51 | class Experiment: 52 | 53 | def __init__(self, name, 54 | base_path, 55 | config_name = 'config.yaml', 56 | history_name = 'labeling_history.json', 57 | evaluation_folder = 'evals', 58 | allow_unfinished = True): 59 | """For a single run of a dataset experiment. 60 | """ 61 | self.name = name 62 | 63 | self.base_path = base_path 64 | self.cfg = load_cfg(os.path.join(self.base_path, config_name)) 65 | self.history_dir = os.path.join(self.base_path, self.cfg.AL.DATASET.CACHE_DIR) 66 | self.eval_dir = os.path.join(self.base_path, evaluation_folder) 67 | 68 | self.evals = self.load_all_evals() 69 | self.labeling_history = load_json(os.path.join(self.base_path, self.cfg.AL.DATASET.CACHE_DIR, history_name)) 70 | 71 | def load_all_evals(self): 72 | num_all_evals = len(glob(os.path.join(self.eval_dir,'*.csv'))) 73 | 74 | # Ensure the evals are ordered correctly 75 | # Assuming the experiment start from 0 to num_all_evals-1 76 | return [ 77 | pd.read_csv(os.path.join(self.eval_dir, f'{idx}.csv'), index_col=0) for idx in range(num_all_evals) 78 | ] 79 | 80 | 81 | def load_training_stats(self): 82 | 83 | res = [] 84 | 85 | for idx, (eval_res, history) in enumerate(zip(self.evals, self.labeling_history)): 86 | 87 | ap_info = eval_res.loc[AP_FEATURES,'bbox'].to_list() 88 | labeling_info = [history[feat] for feat in LB_FEATURES] 89 | res.append([idx] + ap_info + labeling_info) 90 | 91 | df = pd.DataFrame(res, columns=['round']+ AP_FEATURES + LB_FEATURES) 92 | 93 | for feat in LB_FEATURES: 94 | df[f'cum_{feat}'] = df[feat].cumsum() 95 | 96 | return df 97 | 98 | 99 | def load_history_dataset_acc(self): 100 | res = [] 101 | for idx, filename in enumerate(sorted(glob(self.history_dir + '/*eval.csv'))): 102 | ap_score = pd.read_csv(filename, index_col=0, header=None).loc['AP'].iloc[0] 103 | res.append([idx+1, ap_score]) 104 | res = pd.DataFrame(res, columns=['round', 'AP']) 105 | return res 106 | 107 | 108 | class ExperimentCV: 109 | 110 | def __init__(self, name, 111 | base_path, 112 | fold_number = None, 113 | config_name = 'config.yaml', 114 | history_name = 'labeling_history.json', 115 | evaluation_folder = 'evals', 116 | allow_unfinished = True, 117 | agg_table = True): 118 | 119 | """For all runs within a cross validation experiment for a dataset. 120 | """ 121 | 122 | self.name = name 123 | self.base_path = base_path 124 | self.exps = {} 125 | self.agg_table = agg_table 126 | 127 | if fold_number is None: 128 | fold_number = len(os.listdir(self.base_path)) 129 | 130 | self.fold_number = fold_number 131 | 132 | for idx in range(fold_number): 133 | try: 134 | exp = Experiment(name = f'{self.name}-{idx}', 135 | base_path= f'{self.base_path}/{idx}', 136 | config_name = config_name, 137 | history_name = history_name, 138 | evaluation_folder = evaluation_folder, 139 | allow_unfinished = allow_unfinished) 140 | self.exps[idx] = exp 141 | except: 142 | print(f"Fold [{idx}/{fold_number}] hasn't been successfully loaded.") 143 | 144 | def load_training_stats(self): 145 | 146 | df = pd.concat([exp.load_training_stats().assign(fold=idx) for idx, exp in self.exps.items()]) 147 | 148 | if not self.agg_table: 149 | return df 150 | else: 151 | return df.groupby(['round']).mean().reset_index() 152 | 153 | def load_history_dataset_acc(self): 154 | 155 | df = pd.concat([exp.load_history_dataset_acc().assign(fold=idx) for idx, exp in self.exps.items()]) 156 | 157 | if not self.agg_table: 158 | return df 159 | else: 160 | return df.groupby(['round']).mean().reset_index() 161 | 162 | class ExperimentGroup: 163 | 164 | exp_constructor = Experiment 165 | def __init__(self, base_path, 166 | dataset_name, 167 | select_img_exps = [], 168 | select_obj_exps = [], 169 | architecture_name = 'faster_rcnn_R_50_FPN'): 170 | 171 | """For all runs within experiments for a dataset. 172 | """ 173 | 174 | self.base_path = base_path 175 | self.dataset_name = dataset_name 176 | self.architecture_name = architecture_name 177 | self.select_exps = {'image': select_img_exps,'object': select_obj_exps} 178 | self.exps = {} 179 | for exp_cat in ['image', 'object']: 180 | self.exps[exp_cat] = self.load_experiments(f'{self.base_path}/{exp_cat}/{self.architecture_name}', self.select_exps[exp_cat]) 181 | 182 | def load_experiments(self, base_path, select_exps=[]): 183 | all_exps = [] 184 | for name in os.listdir(base_path): 185 | if select_exps != [] and name not in select_exps: 186 | print(f"Skip loading experiment {name}.") 187 | continue 188 | 189 | try: 190 | exp = self.exp_constructor(name = name, base_path = os.path.join(base_path, name)) 191 | all_exps.append(exp) 192 | except: 193 | print(f"Experiment {name} was not successfully loaded.") 194 | return all_exps 195 | 196 | 197 | def plot_training_stats(self, xaxis='cum_num_objects', yaxis='AP'): 198 | 199 | names = [] 200 | 201 | color_maps = create_color_palette() 202 | 203 | for exp_cat in ['image', 'object']: 204 | for (exp, color) in zip(self.exps[exp_cat], cycle(color_maps[exp_cat])): 205 | df = exp.load_training_stats() 206 | ax = sns.lineplot(x=xaxis, y=yaxis, data=df, color=color) 207 | names.append(f'{exp_cat}/'+ exp.name) 208 | ax.yaxis.set_major_formatter(ticker.FormatStrFormatter('%d')) 209 | 210 | plt.legend(names) 211 | plt.title(f'{yaxis} on {self.dataset_name} during Labeling') 212 | 213 | def plot_dataset_acc(self): 214 | 215 | fig, axes = plt.subplots(2, 1, figsize=(7, 8), dpi=200) 216 | 217 | names = [] 218 | all_res = [] 219 | 220 | avg = 87.5 221 | for exp, color in zip(self.exps['object'], cycle(sns.color_palette("Paired")[3:])): 222 | res = exp.load_history_dataset_acc() 223 | res['name'] = exp.name 224 | acc = exp.load_training_stats() 225 | sns.lineplot(x='round', y='AP', data=acc, color=color, ax=axes[0]) 226 | names.append(exp.name) 227 | all_res.append(res) 228 | 229 | axes[0].legend(names, loc='lower right', bbox_to_anchor=(1, 0.355)) 230 | axes[0].set_xlabel('') 231 | axes[0].set_xticklabels([]) 232 | axes[0].set_ylabel('AP@IOU[0.50:0.95] of Model Predictions') 233 | axes[0].set_title(f'Influence of Error Fixing Methods on {self.dataset_name}') 234 | 235 | plt.subplots_adjust(hspace=0.1) 236 | all_res = pd.concat(all_res) 237 | 238 | sns.barplot(x='round', y='AP', hue='name', data=all_res, ax=axes[1], 239 | palette= sns.color_palette("Paired")[3:]) 240 | plt.ylabel("AP@IOU[0.50:0.95] of the Created Dataset") 241 | plt.yticks(list(range(10,110,10))) 242 | plt.axhline(y=avg, linestyle ="--", linewidth=0.75) 243 | plt.legend(loc='lower right', bbox_to_anchor=(1, 0.355)) 244 | 245 | 246 | class ExperimentCVGroup(ExperimentGroup): 247 | 248 | exp_constructor = ExperimentCV 249 | 250 | 251 | class Visualizer: 252 | 253 | def __init__(self, font_path, 254 | save_base_path, 255 | figure_size = (10, 5), 256 | figure_dpi = 200): 257 | 258 | """Handling the plotting configurations 259 | """ 260 | 261 | self.font_path = os.path.abspath(font_path) 262 | self.figure_size = figure_size 263 | self.figure_dpi = figure_dpi 264 | self.save_base_path = save_base_path 265 | 266 | def initialize_mpl_fonts(self): 267 | 268 | from matplotlib.font_manager import _rebuild; _rebuild() 269 | 270 | fp = FontProperties(fname=self.font_path) 271 | 272 | rcParams['font.serif'] = fp.get_name() 273 | rcParams['font.family'] = 'serif' 274 | 275 | def create_customized_plot(self, plot_function, save_name=None): 276 | 277 | with plt.style.context(['scatter', 'no-latex']): 278 | 279 | self.initialize_mpl_fonts() 280 | 281 | plot_function() 282 | 283 | if save_name is not None: 284 | os.makedirs(self.save_base_path, exist_ok=True) 285 | plt.savefig(f'{self.save_base_path}/{save_name}.png') 286 | else: 287 | plt.show() 288 | 289 | def create_simple_plot(self, plot_function, save_name=None): 290 | 291 | def wrapped_plot_function(): 292 | plt.figure(figsize=self.figure_size, dpi=self.figure_dpi) 293 | plot_function() 294 | 295 | self.create_customized_plot(wrapped_plot_function, save_name) 296 | 297 | 298 | if __name__ == "__main__": 299 | 300 | parser = argparse.ArgumentParser() 301 | parser.add_argument("--viz_save_base_path", type=str) 302 | parser.add_argument("--exp_base_path", type=str) 303 | parser.add_argument("--dataset_name", type=str) 304 | 305 | args = parser.parse_args() 306 | 307 | exp_set = ExperimentGroup(args.exp_base_path, args.dataset_name) 308 | visualizer = Visualizer(font_path='./RobotoCondensed-Regular.ttf', save_base_path=args.viz_save_base_path) 309 | 310 | visualizer.create_simple_plot(lambda: exp_set.plot_training_stats(xaxis='cum_num_images'), save_name='tc_images') 311 | visualizer.create_simple_plot(lambda: exp_set.plot_training_stats(xaxis='cum_num_objects'), save_name='tc_objects') 312 | visualizer.create_simple_plot(lambda: exp_set.plot_training_stats(xaxis='round'), save_name='tc_rounds') 313 | visualizer.create_customized_plot(lambda: exp_set.plot_dataset_acc(), save_name='da') 314 | -------------------------------------------------------------------------------- /src/detectron2_al/modeling/roi_heads.py: -------------------------------------------------------------------------------- 1 | import layoutparser as lp 2 | from typing import Dict, List, Optional, Tuple, Union 3 | import torch 4 | import numpy as np 5 | import random 6 | from copy import deepcopy 7 | from itertools import product 8 | 9 | from detectron2.structures import Boxes, ImageList, Instances, pairwise_iou 10 | from detectron2.utils.registry import Registry 11 | from detectron2.modeling.roi_heads.fast_rcnn import FastRCNNOutputLayers, FastRCNNOutputs 12 | from detectron2.modeling.roi_heads.roi_heads import ROIHeads, StandardROIHeads, ROI_HEADS_REGISTRY 13 | from detectron2.structures.boxes import Boxes 14 | from .utils import * 15 | from ..scoring_utils import elementwise_iou 16 | 17 | __all__ = ['ROIHeadsAL'] 18 | 19 | 20 | def calculate_ce_scores(p, q, num_shifts): 21 | # use crossentropy for calculation diff 22 | diff = - (p * torch.log(q)).mean(dim=-1) 23 | # aggregate the statistics for each prediction 24 | diff = torch.Tensor([scores.mean() for scores in diff.split(num_shifts)]) 25 | 26 | return diff 27 | 28 | def calculate_kl_scores(p, q, num_shifts): 29 | # use kl divergence for calculation diff 30 | diff = - (p * torch.log(q/p)).mean(dim=-1) 31 | # aggregate the statistics for each prediction 32 | diff = torch.Tensor([scores.mean() for scores in diff.split(num_shifts)]) 33 | 34 | return diff 35 | 36 | def calculate_iou_scores(perturbed_box, raw_det, num_shifts, num_bbox_reg_classes): 37 | reshaped_boxes = perturbed_box.reshape(-1, num_bbox_reg_classes, 4) 38 | cat_ids = raw_det.pred_classes.repeat_interleave(num_shifts, dim=0) 39 | perturbed_boxes = Boxes(torch.stack([reshaped_boxes[row_id, cat_id] for row_id, cat_id in enumerate(cat_ids)])) 40 | raw_boxes = Boxes(raw_det.pred_boxes.tensor.repeat_interleave(num_shifts, dim=0)) 41 | ious = elementwise_iou(raw_boxes, perturbed_boxes) 42 | iou_scores = torch.Tensor([scores.mean() for scores in ious.split(num_shifts)]) 43 | 44 | return iou_scores 45 | 46 | @ROI_HEADS_REGISTRY.register() 47 | class ROIHeadsAL(StandardROIHeads): 48 | 49 | def __init__(self, cfg, input_shape): 50 | 51 | super(ROIHeadsAL, self).__init__(cfg, input_shape) 52 | self.device = torch.device(cfg.MODEL.DEVICE) 53 | self.cfg = cfg 54 | self._init_al(cfg) 55 | 56 | def _init_al(self, cfg): 57 | 58 | # The scoring objective: max 59 | # The larger the more problematic the detection is 60 | if cfg.AL.OBJECT_SCORING == '1vs2': 61 | self.object_scoring_func = self._one_vs_two_scoring 62 | elif cfg.AL.OBJECT_SCORING == 'least_confidence': 63 | self.object_scoring_func = self._least_confidence_scoring 64 | elif cfg.AL.OBJECT_SCORING == 'random': 65 | self.object_scoring_func = self._random_scoring 66 | elif cfg.AL.OBJECT_SCORING == 'perturbation': 67 | self.object_scoring_func = self._perturbation_scoring 68 | else: 69 | raise NotImplementedError 70 | 71 | if cfg.AL.IMAGE_SCORE_AGGREGATION == 'avg': 72 | self.image_score_aggregation_func = torch.mean 73 | elif cfg.AL.IMAGE_SCORE_AGGREGATION == 'max': 74 | self.image_score_aggregation_func = torch.max 75 | elif cfg.AL.IMAGE_SCORE_AGGREGATION == 'sum': 76 | self.image_score_aggregation_func = torch.sum 77 | elif cfg.AL.IMAGE_SCORE_AGGREGATION == 'random': 78 | self.image_score_aggregation_func = lambda x: torch.rand(1) 79 | else: 80 | raise NotImplementedError 81 | 82 | def estimate_for_proposals(self, features, proposals): 83 | 84 | with torch.no_grad(): 85 | features = [features[f] for f in self.in_features] 86 | box_features = self.box_pooler(features, 87 | [x if isinstance(x, Boxes) \ 88 | else x.proposal_boxes for x in proposals]) 89 | box_features = self.box_head(box_features) 90 | pred_class_logits, pred_proposal_deltas = self.box_predictor(box_features) 91 | del box_features 92 | 93 | outputs = FastRCNNOutputs( 94 | self.box2box_transform, 95 | pred_class_logits, 96 | pred_proposal_deltas, 97 | proposals, 98 | self.smooth_l1_beta) 99 | 100 | return outputs 101 | 102 | def generate_image_scores(self, features, proposals): 103 | 104 | detected_objects_with_given_scores = \ 105 | self.generate_object_scores(features, proposals) 106 | 107 | image_scores = [] 108 | 109 | for ds in detected_objects_with_given_scores: 110 | if len(ds) == 0: 111 | image_scores.append(1.) 112 | else: 113 | image_scores.append(self.image_score_aggregation_func(ds.scores_al).item()) 114 | 115 | return image_scores 116 | 117 | def generate_object_scores(self, features, proposals, with_image_scores=False): 118 | 119 | outputs = self.estimate_for_proposals(features, proposals) 120 | 121 | detected_objects_with_given_scores = self.object_scoring_func(outputs, features=features) 122 | 123 | if not with_image_scores: 124 | return detected_objects_with_given_scores 125 | else: 126 | image_scores = [] 127 | 128 | for ds in detected_objects_with_given_scores: 129 | image_scores.append(self.image_score_aggregation_func(ds.scores_al).item()) 130 | 131 | return image_scores, detected_objects_with_given_scores 132 | 133 | ######################################## 134 | ### Class specific scoring functions ### 135 | ######################################## 136 | 137 | def _one_vs_two_scoring(self, outputs, **kwargs): 138 | """ 139 | Comput the one_vs_two scores for the objects in the fasterrcnn outputs 140 | """ 141 | 142 | cur_detections, filtered_indices = \ 143 | outputs.inference(self.test_score_thresh, self.test_nms_thresh, 144 | self.test_detections_per_img) 145 | 146 | pred_probs = outputs.predict_probs() 147 | # The predicted probabilities are a list of size batch_size 148 | 149 | object_scores = [one_vs_two_scoring(prob[idx]) for \ 150 | (idx, prob) in zip(filtered_indices, pred_probs)] 151 | 152 | for cur_detection, object_score in zip(cur_detections, object_scores): 153 | cur_detection.scores_al = object_score 154 | 155 | return cur_detections 156 | 157 | def _least_confidence_scoring(self, outputs, **kwargs): 158 | """ 159 | Comput the least_confidence_scoring scores for the objects in the fasterrcnn outputs 160 | """ 161 | 162 | cur_detections, filtered_indices = \ 163 | outputs.inference(self.test_score_thresh, self.test_nms_thresh, 164 | self.test_detections_per_img) 165 | 166 | for cur_detection in cur_detections: 167 | cur_detection.scores_al = (1-cur_detection.scores)**2 168 | 169 | return cur_detections 170 | 171 | def _random_scoring(self, outputs, **kwargs): 172 | """ 173 | Assign random active learning scores for each object 174 | """ 175 | 176 | cur_detections, filtered_indices = \ 177 | outputs.inference(self.test_score_thresh, self.test_nms_thresh, 178 | self.test_detections_per_img) 179 | 180 | device = cur_detections[0].scores.device 181 | for cur_detection in cur_detections: 182 | cur_detection.scores_al = torch.rand(cur_detection.scores.shape).to(device) 183 | 184 | return cur_detections 185 | 186 | def _create_translations(self, cfg): 187 | 188 | def _generate_individual_shift_matrix(alpha, beta): 189 | if cfg.AL.PERTURBATION.RANDOM: 190 | alpha = random.uniform(0, alpha) 191 | beta = random.uniform(0, beta) 192 | return torch.Tensor([ 193 | [(1-alpha), 0, -alpha, 0], 194 | [0, (1-beta), 0, -beta], 195 | [alpha, 0, (1+alpha), 0], 196 | [0, beta, 0, (1+beta)], 197 | ]) 198 | 199 | alphas = cfg.AL.PERTURBATION.ALPHAS 200 | betas = cfg.AL.PERTURBATION.BETAS 201 | 202 | derived_shift = [ 203 | [ 204 | [alpha, beta], 205 | [alpha, -beta], 206 | [-alpha, beta], 207 | [-alpha, -beta] 208 | ] 209 | for alpha, beta in product(alphas, betas) 210 | ] 211 | 212 | matrices = [_generate_individual_shift_matrix(*params) 213 | for params in sum(derived_shift, [])] 214 | return len(matrices), torch.stack(matrices, dim=-1).to(self.device) 215 | 216 | def _perturbation_scoring(self, raw_outputs, features): 217 | 218 | # Obtain the raw prediction boxes and probabilities 219 | raw_detections, raw_indices = \ 220 | raw_outputs.inference(self.test_score_thresh, self.test_nms_thresh, 221 | self.test_detections_per_img) 222 | 223 | raw_probs = [prob[idx] for (idx, prob) 224 | in zip(raw_indices, raw_outputs.predict_probs())] 225 | 226 | is_empty_raw_detections = [len(det)==0 for det in raw_detections] 227 | 228 | # Generate perturbed boxes 229 | num_shifts, shift_matrix = self._create_translations(self.cfg) 230 | 231 | all_new_proposals = [] 232 | for is_empty, det in zip(is_empty_raw_detections, raw_detections): 233 | 234 | if is_empty: 235 | # Create a dummy box for empty detections 236 | used_boxes = torch.zeros((1,4)).to(self.device) 237 | else: 238 | used_boxes = det.pred_boxes.tensor 239 | 240 | new_proposals = Instances( 241 | det.image_size, 242 | proposal_boxes=Boxes( 243 | torch.einsum('bi,ijc->bjc', 244 | used_boxes, 245 | shift_matrix) 246 | .permute(0,2,1) 247 | .reshape(-1,4) 248 | ) 249 | ) 250 | all_new_proposals.append(new_proposals) 251 | 252 | perturbed_outputs = self.estimate_for_proposals(features, all_new_proposals) 253 | perturbed_probs = perturbed_outputs.predict_probs() 254 | perturbed_boxes = perturbed_outputs.predict_boxes() 255 | num_bbox_reg_classes = perturbed_boxes[0].shape[1] // 4 256 | 257 | for is_empty, raw_det, raw_prob, perturbed_prob, perturbed_box in \ 258 | zip(is_empty_raw_detections, raw_detections, raw_probs, perturbed_probs, perturbed_boxes): 259 | 260 | if is_empty: 261 | raw_det.scores_al = raw_det.scores 262 | else: 263 | p = raw_prob.repeat_interleave(num_shifts, dim=0) 264 | q = perturbed_prob 265 | 266 | if self.cfg.AL.PERTURBATION.VERSION == 1: 267 | diff = calculate_ce_scores(p, q, num_shifts) 268 | 269 | elif self.cfg.AL.PERTURBATION.VERSION == 2: 270 | diff = calculate_kl_scores(p, q, num_shifts) 271 | 272 | elif self.cfg.AL.PERTURBATION.VERSION == 3: 273 | diff = calculate_iou_scores(perturbed_box, raw_det, num_shifts, num_bbox_reg_classes) 274 | 275 | elif self.cfg.AL.PERTURBATION.VERSION == 4: 276 | diff1 = calculate_iou_scores(perturbed_box, raw_det, num_shifts, num_bbox_reg_classes) 277 | diff2 = calculate_ce_scores(p, q, num_shifts) 278 | diff = diff1 + diff2 * self.cfg.AL.PERTURBATION.LAMBDA 279 | 280 | elif self.cfg.AL.PERTURBATION.VERSION == 5: 281 | diff1 = calculate_iou_scores(perturbed_box, raw_det, num_shifts, num_bbox_reg_classes) 282 | diff2 = calculate_kl_scores(p, q, num_shifts) 283 | diff = diff1 + diff2*3 * self.cfg.AL.PERTURBATION.LAMBDA 284 | 285 | raw_det.scores_al = diff 286 | 287 | return raw_detections -------------------------------------------------------------------------------- /src/detectron2_al/dataset/object_fusion.py: -------------------------------------------------------------------------------- 1 | from typing import List, Tuple, Dict, Any 2 | from functools import partial 3 | import torch 4 | import numpy as np 5 | 6 | from detectron2.structures import Boxes, Instances 7 | from detectron2.modeling.postprocessing import detector_postprocess 8 | 9 | from ..scoring_utils import * 10 | from ..scheduling_utils import DefaultSchedular 11 | 12 | __all__ = ['ObjectFusion', 'ObjectFusionRatioScheduler', 'ObjectSelectionNumberScheuler'] 13 | 14 | def _quantile(t, q): 15 | # As we are using pytorch 1.4, there is no native 16 | # Pytorch support for the quantile function. 17 | # This implementation is based on 18 | # https://gist.github.com/spezold/42a451682422beb42bc43ad0c0967a30 19 | return (t.view(-1) 20 | .kthvalue(1 + round(float(q) * (t.numel() - 1))) 21 | .values.item()) 22 | 23 | def _deselect(t, indices, return_mapping=False): 24 | """ 25 | Select the elements in t with index not in indices 26 | """ 27 | selected_indices = [i not in indices for i in range(len(t))] 28 | if not return_mapping: 29 | return t[selected_indices] 30 | else: 31 | return t[selected_indices], [i for i in range(len(t)) if i not in indices] 32 | 33 | 34 | class ObjectFusionRatioScheduler(DefaultSchedular): 35 | 36 | def __init__(self, cfg): 37 | steps = cfg.AL.TRAINING.ROUNDS 38 | start = cfg.AL.OBJECT_FUSION.INITIAL_RATIO 39 | mode = cfg.AL.OBJECT_FUSION.DECAY 40 | end = cfg.AL.OBJECT_FUSION.LAST_RATIO 41 | super().__init__(start, end, steps, mode) 42 | 43 | 44 | class ObjectSelectionNumberScheuler(DefaultSchedular): 45 | def __init__(self, cfg): 46 | steps = cfg.AL.TRAINING.ROUNDS 47 | start = cfg.AL.OBJECT_FUSION.PRESELECTION_RAIO 48 | mode = cfg.AL.OBJECT_FUSION.SELECTION_RAIO_DECAY 49 | end = cfg.AL.OBJECT_FUSION.ENDSELECTION_RAIO 50 | super().__init__(start, end, steps, mode) 51 | 52 | 53 | class ObjectFusion: 54 | 55 | OVERLAPPING_METRICS = { 56 | 'iou': pairwise_iou, 57 | 'dice_coefficient': pairwise_dice_coefficient, 58 | 'overlap_coefficient': pairwise_overlap_coefficient 59 | } 60 | 61 | OBJECT_SELECTOR = { 62 | 'top': select_top, 63 | 'above': select_above, 64 | 'nonzero': select_nonzero 65 | } 66 | 67 | def __init__(self, cfg): 68 | self._init_overlapping_funcs(cfg) 69 | self.remove_duplicates = cfg.AL.OBJECT_FUSION.REMOVE_DUPLICATES 70 | self.remove_duplicates_th = cfg.AL.OBJECT_FUSION.REMOVE_DUPLICATES_TH 71 | self.recover_missing_objects = cfg.AL.OBJECT_FUSION.RECOVER_MISSING_OBJECTS 72 | self.recover_almost_correct_predictions = cfg.AL.OBJECT_FUSION.RECOVER_ALMOST_CORRECT_PRED 73 | 74 | self.device = torch.device(cfg.MODEL.DEVICE) 75 | self.fusion_ratio = ObjectFusionRatioScheduler(cfg) 76 | self.selection_ratio = ObjectSelectionNumberScheuler(cfg) 77 | 78 | def _init_overlapping_funcs(self, cfg): 79 | self.overlapping_metric = self.OVERLAPPING_METRICS[cfg.AL.OBJECT_FUSION.OVERLAPPING_METRIC] 80 | self.overlapping_th = cfg.AL.OBJECT_FUSION.OVERLAPPING_TH 81 | self.gt_selector = self.OBJECT_SELECTOR[cfg.AL.OBJECT_FUSION.SELECTION_METHOD] 82 | if cfg.AL.OBJECT_FUSION.SELECTION_METHOD == 'above': 83 | self.gt_selector = partial(select_above, threshold=self.overlapping_th) 84 | 85 | def combine(self, pred: Instances, 86 | gt: Dict, 87 | round: int, 88 | ave_num_objects_per_image: int = None): 89 | """ 90 | Combine the model predictions with the ground-truth 91 | by replacing the objects in the pred with score_al of 92 | top replace_ratio. It will automatically move all the boxes 93 | to self.device, and the output will be saved back to cpu. 94 | """ 95 | if len(pred) <= 0: 96 | # There is no prediction from the model 97 | # Which means this image is very challenging 98 | # We will use all ground-truth data, and assgin high scores 99 | 100 | return self._duplicate_gt_as_output(gt) 101 | 102 | if ave_num_objects_per_image is not None: 103 | top_object_numbers = int(ave_num_objects_per_image*self.selection_ratio[round]) 104 | pred = pred[:top_object_numbers] 105 | 106 | fusion_ratio = self.fusion_ratio[round] 107 | gt_boxes = gt['instances'].gt_boxes.to(self.device) 108 | pred_boxes = pred.pred_boxes.to(self.device) 109 | 110 | score_al_th = _quantile(pred.scores_al, 1-fusion_ratio) 111 | # If fusion_ratio is 0.8, then we want to find all objects that 112 | # have scores more than 0.2(=1-0.8) quantile. 113 | selected_pred_indices = torch.where(pred.scores_al > score_al_th)[0].to(self.device) 114 | 115 | if len(selected_pred_indices)<= 0: 116 | # For some rounding issues, we might catch no indices. 117 | # We just add all the boxes then. 118 | selected_pred_indices = torch.arange(len(pred_boxes)).to(self.device) 119 | 120 | aggregated_score = pred.scores_al[selected_pred_indices].mean().item() 121 | 122 | overlapping_scores = self.overlapping_metric( 123 | pred_boxes[selected_pred_indices], 124 | gt_boxes) 125 | 126 | selected_gt_indices = self.gt_selector(overlapping_scores) 127 | selected_gt_indices = list(sum(selected_gt_indices, [])) 128 | selected_gt_indices = torch.LongTensor(selected_gt_indices).to(self.device) 129 | 130 | # Remove duplicated gt_boxes from the output 131 | selected_gt_indices, _cts = torch.unique(selected_gt_indices, return_counts=True) 132 | gt_indices_ct = {gt_idx.item():_ct.item() for (gt_idx, _ct) in zip(selected_gt_indices, _cts)} 133 | 134 | if self.recover_almost_correct_predictions: 135 | # Sometimes the model with generate pretty decent predictions. Therefore 136 | # annotators only need to visually check it without modifying the labels. 137 | # Therefore the budget cost is not the "full cost", but a discounted 138 | # partialy cost. In this step, we are trying to find these regions. 139 | max_scores = overlapping_scores.max(dim=-1) 140 | to_recover_pred_indices = torch.where(max_scores.values>0.925)[0] 141 | 142 | recovered_pred_indices = [] 143 | recovered_gt_indices = [] 144 | 145 | for _idx in to_recover_pred_indices: 146 | 147 | gt_idx = max_scores.indices[_idx] 148 | gt_class = gt['instances'].gt_classes[gt_idx].item() 149 | pred_idx = selected_pred_indices[_idx] 150 | pred_class = pred.pred_classes[pred_idx].item() 151 | if gt_class == pred_class: 152 | if gt_indices_ct[gt_idx.item()] != 1: 153 | # If some gt box appears more than once, then 154 | # annotators still need to fix that. Thus we 155 | # don't include this box as the paritaly 156 | # discounted (or recovered) boxes 157 | continue 158 | else: 159 | recovered_pred_indices.append(pred_idx) 160 | recovered_gt_indices.append(gt_idx) 161 | 162 | recovered_pred_indices = torch.LongTensor(recovered_pred_indices).to(self.device) 163 | recovered_gt_indices = torch.LongTensor(recovered_gt_indices).to(self.device) 164 | 165 | if self.remove_duplicates: 166 | # Sometimes the model will generate duplicated predictions, and some of the 167 | # duplicates won't be selected for matching with the gt. Thus, in this step, 168 | # we will find and eliminate these boxes using the identified ground-truth 169 | # boxes. 170 | selected_gt = gt_boxes[selected_gt_indices] 171 | selected_pred, index_mapping = _deselect(pred_boxes, selected_pred_indices, return_mapping=True) 172 | 173 | overlapping_scores = pairwise_overlap_coefficient(selected_pred, selected_gt) 174 | selected_pred_indices_extra, _ = torch.where(overlapping_scores>self.remove_duplicates_th) 175 | selected_pred_indices_extra = torch.LongTensor([index_mapping[i] for i in 176 | torch.unique(selected_pred_indices_extra)]).to(self.device) 177 | selected_pred_indices = torch.cat([selected_pred_indices, selected_pred_indices_extra]) 178 | 179 | if self.recover_missing_objects: 180 | # Sometimes the model won't generate predictions for some region. In this step, 181 | # we will reterive them by calculating the overlapping between the combined 182 | # boxes with all gt_boxes. If there's a gt_boxes without any combined boxes of 183 | # overlapping higher than the threshold (0.05), we add them to the gt boxes list. 184 | combined_boxes = self._join_elements_pred_with_gt(pred_boxes, 185 | selected_pred_indices, 186 | gt_boxes, 187 | selected_gt_indices) 188 | 189 | if len(combined_boxes)<=0: 190 | return self._duplicate_gt_as_output(gt) 191 | 192 | max_overlapping_for_gt_boxes = pairwise_iou(combined_boxes, gt_boxes).max(dim=0).values 193 | missing_gt_boxes_indices = torch.where(max_overlapping_for_gt_boxes<=0.05)[0] 194 | selected_gt_indices = torch.cat([selected_gt_indices, missing_gt_boxes_indices]) 195 | 196 | if self.recover_almost_correct_predictions: 197 | # During the remove_duplicates and recover_missing_objects process, we use the 198 | # gt boxes as an delegate for processing. And it's time to switch them back to 199 | # the pred boxes. This might cause very tiny inaccuray in these process, but as 200 | # we've set very high matching accuracy (0.925), the inaccuracy should be negligible. 201 | modified_pred_indices = torch.LongTensor([idx for idx in selected_pred_indices if idx not in recovered_pred_indices]) 202 | modified_gt_indices = torch.LongTensor([idx for idx in selected_gt_indices if idx not in recovered_gt_indices]) 203 | combined_instances = self._fuse_pred_with_gt(pred, modified_pred_indices, 204 | gt, modified_gt_indices) 205 | result = self._postprocess(combined_instances, gt) 206 | result['image_score'] = aggregated_score 207 | result['labeled_inst_from_gt'] = len(modified_gt_indices) 208 | result['dropped_inst_from_pred'] = len(modified_pred_indices) 209 | result['recovered_inst'] = len(recovered_pred_indices) 210 | 211 | else: 212 | combined_instances = self._fuse_pred_with_gt(pred, selected_pred_indices, 213 | gt, selected_gt_indices) 214 | if len(combined_instances)<=0: 215 | return self._duplicate_gt_as_output(gt) 216 | 217 | result = self._postprocess(combined_instances, gt) 218 | result['image_score'] = aggregated_score 219 | result['labeled_inst_from_gt'] = len(selected_gt_indices) 220 | result['dropped_inst_from_pred'] = len(selected_pred_indices) 221 | result['recovered_inst'] = 0 222 | 223 | del gt_boxes 224 | del pred_boxes 225 | 226 | return result 227 | 228 | def _fuse_pred_with_gt(self, pred, pred_indices, gt, gt_indices): 229 | boxes = self._join_elements_pred_with_gt(pred.pred_boxes.to('cpu'), pred_indices.to('cpu'), 230 | gt['instances'].gt_boxes.to('cpu'), gt_indices.to('cpu')) 231 | 232 | classes = self._join_elements_pred_with_gt(pred.pred_classes.to('cpu'), pred_indices.to('cpu'), 233 | gt['instances'].gt_classes.to('cpu'), gt_indices.to('cpu')) 234 | 235 | return Instances(pred.image_size, 236 | pred_boxes = boxes, 237 | pred_classes = classes, 238 | scores = torch.ones_like(classes).float()) 239 | 240 | @staticmethod 241 | def _join_elements_pred_with_gt(pred_ele, pred_indices, gt_ele ,gt_indices): 242 | if isinstance(pred_ele, Boxes): 243 | return Boxes.cat([ 244 | _deselect(pred_ele, pred_indices), 245 | gt_ele[gt_indices] 246 | ]) 247 | else: 248 | return torch.cat([ 249 | _deselect(pred_ele, pred_indices), 250 | gt_ele[gt_indices] 251 | ]) 252 | 253 | @staticmethod 254 | def _postprocess(instances, gt): 255 | 256 | height = gt.get("height") 257 | width = gt.get("width") 258 | r = detector_postprocess(instances, height, width) 259 | return {'instances': r, 'image_id': gt['image_id']} 260 | 261 | def _duplicate_gt_as_output(self, gt): 262 | 263 | copied = Instances(gt['instances'].image_size, 264 | pred_boxes = gt['instances'].gt_boxes.to('cpu'), 265 | pred_classes = gt['instances'].gt_classes.to('cpu'), 266 | scores = torch.ones_like(gt['instances'].gt_classes).float()) 267 | 268 | result = self._postprocess(copied, gt) 269 | result['image_score'] = 1 270 | result['labeled_inst_from_gt'] = len(gt['instances']) 271 | result['dropped_inst_from_pred'] = 0 272 | result['recovered_inst'] = 0 273 | return result -------------------------------------------------------------------------------- /src/detectron2_al/dataset/al_dataset.py: -------------------------------------------------------------------------------- 1 | import json 2 | import random 3 | import os 4 | import numpy as np 5 | import pandas as pd 6 | from typing import Dict, Tuple, List 7 | from dataclasses import dataclass, field 8 | 9 | from pycocotools.coco import COCO 10 | from pycocotools.cocoeval import COCOeval 11 | from detectron2.data.datasets.coco import load_coco_json 12 | from detectron2.data.build import build_detection_train_loader 13 | from detectron2.data.datasets import register_coco_instances 14 | from detectron2.data import DatasetCatalog 15 | from detectron2.evaluation.coco_evaluation import instances_to_coco_json 16 | 17 | from .object_fusion import ObjectFusion 18 | from .dataset_mapper import DatasetMapperAL 19 | from .utils import build_detection_train_loader_drop_ids 20 | from ..scheduling_utils import * 21 | 22 | __all__ = ['build_al_dataset', 23 | 'HandyCOCO', 24 | 'Budget', 25 | 'DatasetHistory', 26 | 'EpochsPerRound', 27 | 'ActiveLearningDataset', 28 | 'ImageActiveLearningDataset', 29 | 'ObjectActiveLearningDataset'] 30 | 31 | 32 | def _write_json(data, filename): 33 | 34 | with open(filename, 'w') as fp: 35 | json.dump(data, fp) 36 | 37 | def _calculate_iterations(num_imgs, batch_size, num_epochs): 38 | return int(num_imgs*num_epochs/batch_size) 39 | 40 | def build_al_dataset(cfg): 41 | 42 | if cfg.AL.MODE == 'image': 43 | return ImageActiveLearningDataset(cfg) 44 | elif cfg.AL.MODE == 'object': 45 | return ObjectActiveLearningDataset(cfg) 46 | else: 47 | raise ValueError(f'Unknown active learning mode {cfg.AL.MODE}') 48 | 49 | class HandyCOCO(COCO): 50 | 51 | def subsample_with_image_ids(self, imgIds): 52 | 53 | imgs = [self.imgs[idx] for idx in imgIds] 54 | anns = self.loadAnns(ids=self.getAnnIds(imgIds=imgIds)) 55 | 56 | dataset = { 57 | 'images': imgs, 58 | 'annotations': anns, 59 | 'categories': self.dataset['categories'], 60 | 'info': self.dataset.get('info', {}), 61 | 'license': self.dataset.get('licenses', {}) 62 | } 63 | 64 | return dataset 65 | 66 | def subsample_and_save_with_image_ids(self, imgIds, filename): 67 | 68 | dataset = self.subsample_with_image_ids(imgIds) 69 | _write_json(dataset, filename) 70 | 71 | return dataset 72 | 73 | def avg_object_per_image(self): 74 | return np.mean([len(self.getAnnIds([img])) for img in self.imgs]) 75 | 76 | 77 | class Budget(object): 78 | 79 | def __init__(self, cfg, avg_object_per_image): 80 | """ 81 | A handy class for controlling budgets 82 | """ 83 | 84 | self.style = cfg.AL.MODE 85 | assert self.style in ['object', 'image'] 86 | 87 | self.allocation_method = cfg.AL.DATASET.BUDGET_ALLOCATION 88 | self.total_rounds = cfg.AL.TRAINING.ROUNDS 89 | self.eta = cfg.AL.OBJECT_FUSION.BUDGET_ETA 90 | 91 | self.avg_object_per_image = avg_object_per_image 92 | if self.style == 'object': 93 | self._initial = self._remaining = cfg.AL.DATASET.IMAGE_BUDGET*avg_object_per_image 94 | elif self.style == 'image': 95 | self._initial = self._remaining = cfg.AL.DATASET.IMAGE_BUDGET 96 | 97 | if self.allocation_method == 'linear': 98 | self._allocations = np.ones(self.total_rounds)*round(self.initial / self.total_rounds) 99 | else: 100 | raise NotImplementedError 101 | self._allocations = self._allocations.astype('int') 102 | 103 | @property 104 | def remaining_object_budget(self): 105 | if self.style == 'image': 106 | return self.remaining * self.avg_object_per_image 107 | else: 108 | return self.remaining 109 | 110 | @property 111 | def remaining_image_budget(self): 112 | if self.style == 'image': 113 | return self.remaining 114 | else: 115 | return int(self.remaining // self.avg_object_per_image) 116 | 117 | @property 118 | def remaining(self): 119 | """ 120 | The remaining budget 121 | """ 122 | return self._remaining 123 | 124 | @property 125 | def initial(self): 126 | """ 127 | The initial budget 128 | """ 129 | return self._initial 130 | 131 | def all_allocations(self, _as=None): 132 | 133 | if _as is not None and _as != self.style: 134 | if _as == 'object': 135 | # originally image 136 | return self._allocations * self.avg_object_per_image 137 | elif _as == 'image': 138 | # originally object 139 | return self._allocations // self.avg_object_per_image 140 | else: 141 | return self._allocations 142 | 143 | def allocate(self, _round, _as=None): 144 | 145 | allocated = self._allocations[_round] 146 | 147 | self._remaining -= allocated 148 | if _as is not None and _as != self.style: 149 | if _as == 'object': 150 | # originally image 151 | allocated = int(allocated * self.avg_object_per_image) 152 | elif _as == 'image': 153 | # originally object 154 | allocated = int(round(allocated / self.avg_object_per_image)) 155 | return allocated 156 | 157 | 158 | @dataclass 159 | class DatasetInfo: 160 | name: str 161 | json_path: str 162 | num_images: int 163 | num_objects: int 164 | image_ids: List 165 | anno_details: List = field(default_factory=list) 166 | training_iter: int = 0 167 | 168 | class DatasetHistory(list): 169 | 170 | @property 171 | def all_dataset_names(self): 172 | return [ele.name for ele in self] 173 | 174 | def save(self, filename): 175 | _write_json([vars(ele) for ele in self], filename) 176 | 177 | 178 | class EpochsPerRound(IntegerSchedular): 179 | 180 | def __init__(self, cfg): 181 | 182 | steps = cfg.AL.TRAINING.ROUNDS 183 | start = cfg.AL.TRAINING.EPOCHS_PER_ROUND_INITIAL 184 | mode = cfg.AL.TRAINING.EPOCHS_PER_ROUND_DECAY 185 | end = cfg.AL.TRAINING.EPOCHS_PER_ROUND_LAST 186 | super().__init__(start, end, steps, mode) 187 | 188 | 189 | class ActiveLearningDataset: 190 | 191 | def __init__(self, cfg): 192 | 193 | # Dataset configurations 194 | self.name = cfg.AL.DATASET.NAME 195 | self.image_root = cfg.AL.DATASET.IMG_ROOT 196 | self.anno_path = cfg.AL.DATASET.ANNO_PATH 197 | self.cache_dir = os.path.join(cfg.OUTPUT_DIR, cfg.AL.DATASET.CACHE_DIR) 198 | if not os.path.exists(self.cache_dir): 199 | os.makedirs(self.cache_dir) 200 | self.coco = HandyCOCO(self.anno_path) 201 | self.name_prefix = cfg.AL.DATASET.NAME_PREFIX 202 | 203 | # Add the base version of the dataset to the catalog 204 | self.add_new_dataset_into_catalogs(self.name, self.anno_path) 205 | 206 | # Scheduling 207 | self.total_rounds = cfg.AL.TRAINING.ROUNDS 208 | self.budget = Budget(cfg, self.coco.avg_object_per_image()) 209 | self.epochs_per_round = EpochsPerRound(cfg) 210 | 211 | # Sampling method during AL 212 | self.sampling_method = cfg.AL.DATASET.SAMPLE_METHOD 213 | 214 | # Internal Storage 215 | self._history = DatasetHistory() 216 | self._round = -1 217 | self._cfg = cfg.clone() 218 | self._cfg.freeze() 219 | 220 | def calculate_iterations_for_cur_datasets(self): 221 | return _calculate_iterations( 222 | num_imgs = sum([info.num_images for info in self._history]), 223 | batch_size = self._cfg.SOLVER.IMS_PER_BATCH, 224 | num_epochs = self.epochs_per_round[self._round] 225 | ) 226 | 227 | def calculate_estimated_total_iterations(self): 228 | bs = self._cfg.SOLVER.IMS_PER_BATCH 229 | num_imgs = self.budget.all_allocations(_as='image') 230 | num_epochs = [self.epochs_per_round[i] for i in range(self.total_rounds)] 231 | return sum( 232 | [ 233 | _calculate_iterations(num_img, bs, num_epoch) \ 234 | for (num_img, num_epoch) in zip(num_imgs, num_epochs) 235 | ] 236 | ) 237 | 238 | def dataset_name_at(self, round): 239 | return f'{self.name}-{self.name_prefix}{round:02d}' 240 | 241 | def dataset_jsonpath_at(self, round): 242 | return os.path.join( 243 | self.cache_dir, self.dataset_name_at(round) + '.json' 244 | ) 245 | 246 | @property 247 | def cur_dataset_name(self): 248 | return self.dataset_name_at(self._round) 249 | 250 | @property 251 | def cur_dataset_jsonpath(self): 252 | return self.dataset_jsonpath_at(self._round) 253 | 254 | def add_new_dataset_into_catalogs(self, name, json_path): 255 | 256 | if name not in DatasetCatalog.list(): 257 | 258 | register_coco_instances( 259 | name = name, 260 | metadata = {}, 261 | json_file = os.path.abspath(json_path), 262 | image_root = self.image_root 263 | ) 264 | 265 | def get_training_dataloader(self, dataset_name=None): 266 | """ 267 | Get the dataloader with the given datasetname. 268 | By default it will create the dataset using the last 269 | created dataset. 270 | """ 271 | 272 | if dataset_name is None: 273 | dataset_name = self._history[-1].name 274 | max_iter = self.calculate_iterations_for_cur_datasets() 275 | else: 276 | assert dataset_name in self._history.all_dataset_names 277 | max_iter = self._cfg.SOLVER.MAX_ITER / self.total_rounds 278 | 279 | self._history[-1].training_iter = max_iter 280 | 281 | cfg = self._cfg.clone() 282 | cfg.defrost() 283 | cfg.DATASETS.TRAIN = tuple(self._history.all_dataset_names) 284 | dataloader = build_detection_train_loader(cfg) 285 | 286 | return dataloader, max_iter 287 | 288 | def get_oracle_dataloader(self, drop_existing=True): 289 | """ 290 | Get the dataloader with all the images. Only used 291 | for evaluating all the image/object scores and create 292 | new datasets. 293 | """ 294 | cfg = self._cfg.clone() 295 | cfg.defrost() 296 | cfg.SOLVER.IMS_PER_BATCH = 1 # Avoid Dropped Images 297 | cfg.DATASETS.TRAIN = (self.name,) 298 | 299 | if not drop_existing: 300 | dropped_image_ids = [] 301 | else: 302 | dropped_image_ids = sum([info.image_ids for info in self._history], []) 303 | 304 | dataloader = build_detection_train_loader_drop_ids(cfg, dropped_image_ids, 305 | DatasetMapperAL(cfg, True)) 306 | 307 | total_image_num = len(self.coco.imgs)-len(dropped_image_ids) 308 | max_iter = _calculate_iterations(total_image_num, cfg.SOLVER.IMS_PER_BATCH, 1) 309 | 310 | return dataloader, max_iter 311 | 312 | def create_dataset_with_image_ids(self, image_ids): 313 | 314 | cur_json_path = self.cur_dataset_jsonpath 315 | cur_data_name = self.cur_dataset_name 316 | 317 | dataset = self.coco.subsample_and_save_with_image_ids( 318 | image_ids, 319 | cur_json_path 320 | ) 321 | self.add_new_dataset_into_catalogs(cur_data_name, cur_json_path) 322 | 323 | self._history.append( 324 | DatasetInfo( 325 | name = cur_data_name, 326 | json_path = cur_json_path, 327 | num_images = len(dataset['images']), 328 | num_objects = len(dataset['annotations']), 329 | image_ids = image_ids 330 | ) 331 | ) 332 | 333 | def create_dataset_with_annotations(self, annotations, image_ids, 334 | labeling_history, num_objects=None): 335 | 336 | cur_json_path = self.cur_dataset_jsonpath 337 | cur_data_name = self.cur_dataset_name 338 | 339 | # Save an original dataset for further reference 340 | dataset = self.coco.subsample_and_save_with_image_ids( 341 | image_ids, 342 | cur_json_path.replace('.json', '-orig.json') 343 | ) 344 | 345 | # Modify the annotations, save, and register 346 | dataset['annotations'] = self.coco.loadRes(annotations).dataset['annotations'] 347 | # To ensure the annotations is compatible with the coco format 348 | _write_json(dataset, cur_json_path) 349 | self.add_new_dataset_into_catalogs(cur_data_name, cur_json_path) 350 | 351 | self._history.append( 352 | DatasetInfo( 353 | name = cur_data_name, 354 | json_path = cur_json_path, 355 | num_images = len(dataset['images']), 356 | num_objects = len(dataset['annotations']) if num_objects is None else num_objects, 357 | anno_details = labeling_history, 358 | image_ids = image_ids 359 | ) 360 | ) 361 | 362 | def create_initial_dataset(self): 363 | """ 364 | Create the initial AL dataset. 365 | The implementation is depend on the active learning schema. 366 | """ 367 | 368 | self._round += 1 369 | allocated_budget = self.budget.allocate(self._round, _as='image') 370 | selected_image_ids = random.sample(list(self.coco.imgs.keys()), allocated_budget) 371 | self.create_dataset_with_image_ids(selected_image_ids) 372 | 373 | def create_new_dataset(self): 374 | """ 375 | Create a new dataset based on the round and the given budget. 376 | The implementation is depend on the active learning schema. 377 | """ 378 | assert self._round <= self.total_rounds 379 | self._round += 1 380 | 381 | def save_history(self): 382 | self._history.save(os.path.join(self.cache_dir, 'labeling_history.json')) 383 | 384 | class ImageActiveLearningDataset(ActiveLearningDataset): 385 | 386 | def create_new_dataset(self, image_scores:List[Tuple]): 387 | """ 388 | args: 389 | images_scores: 390 | a list of tuples (image_score, image_id) 391 | """ 392 | 393 | super().create_new_dataset() 394 | 395 | allocated_budget = self.budget.allocate(self._round, _as='object') 396 | if self.sampling_method == 'top': 397 | top_image_scores = sorted(image_scores) 398 | selected_image_ids = [] 399 | 400 | while allocated_budget>0: 401 | score, idx = top_image_scores.pop() 402 | selected_image_ids.append(idx) 403 | 404 | num_objects = len(self.coco.getAnnIds([idx])) 405 | allocated_budget -= num_objects 406 | else: 407 | raise NotImplementedError 408 | 409 | self.create_dataset_with_image_ids(selected_image_ids) 410 | 411 | class ObjectActiveLearningDataset(ActiveLearningDataset): 412 | 413 | def create_new_dataset(self, fused_results: List[Dict]): 414 | """ 415 | Args: 416 | fused_results: 417 | the predictions results on the oracle dataset with 418 | """ 419 | super().create_new_dataset() 420 | 421 | image_scores = [fs['image_score'] for fs in fused_results] 422 | 423 | selected_image_ids = [] 424 | selected_annotations = [] 425 | allocated_budget = self.budget.allocate(self._round) 426 | 427 | used_budget = 0 428 | labeling_history = [] 429 | if self.sampling_method == 'top': 430 | sorted_image_scores = np.argsort(image_scores).tolist() 431 | 432 | while allocated_budget>used_budget and sorted_image_scores!=[]: 433 | 434 | idx = sorted_image_scores.pop() 435 | image_id = fused_results[idx]['image_id'] 436 | instances = fused_results[idx]['instances'] 437 | annotations = instances_to_coco_json(instances, image_id) 438 | selected_image_ids.append(image_id) 439 | selected_annotations.extend(annotations) 440 | # Currently, there will be an 'score' field in each of the 441 | # annotations, and it will be saved in the JSON. The existence 442 | # of this field won't affect the coco loading, and will make 443 | # it easier to compute the score. 444 | cur_cost = fused_results[idx]['labeled_inst_from_gt'] + \ 445 | self.budget.eta * fused_results[idx]['recovered_inst'] 446 | used_budget += round(cur_cost) 447 | 448 | labeling_history.append({ 449 | "image_id": fused_results[idx]['image_id'], 450 | "labeled_inst_from_gt":fused_results[idx]['labeled_inst_from_gt'], 451 | "used_inst_from_pred": fused_results[idx]['dropped_inst_from_pred'], 452 | "recovered_inst": fused_results[idx]['recovered_inst'] 453 | }) 454 | else: 455 | raise NotImplementedError 456 | 457 | self.create_dataset_with_annotations(selected_annotations, 458 | selected_image_ids, 459 | labeling_history, 460 | num_objects=round(used_budget)) 461 | dataset_eval = self.evaluate_merged_dataset(self._round) 462 | pd.Series(dataset_eval).to_csv(self.cur_dataset_jsonpath.replace('.json', 'eval.csv')) 463 | 464 | def evaluate_merged_dataset(self, round, iou_type='bbox'): 465 | 466 | dt_json_path = self.dataset_jsonpath_at(round) 467 | 468 | coco_dt = COCO(dt_json_path) 469 | coco_gt = COCO(dt_json_path.replace('.json', '-orig.json')) 470 | coco_eval = COCOeval(coco_gt, coco_dt, iouType=iou_type) 471 | class_names = [val['name'] for _, val in coco_gt.cats.items()] 472 | 473 | coco_eval.evaluate() 474 | coco_eval.accumulate() 475 | coco_eval.summarize() 476 | 477 | # A simplified version of COCOEvaluator._derive_coco_results 478 | metrics = { 479 | "bbox": ["AP", "AP50", "AP75", "APs", "APm", "APl"], 480 | "segm": ["AP", "AP50", "AP75", "APs", "APm", "APl"], 481 | "keypoints": ["AP", "AP50", "AP75", "APm", "APl"], 482 | }[iou_type] 483 | 484 | results = { 485 | metric: float(coco_eval.stats[idx] * 100 if coco_eval.stats[idx] >= 0 else "nan") 486 | for idx, metric in enumerate(metrics) 487 | } 488 | 489 | if class_names is None or len(class_names) <= 1: 490 | return results 491 | # Compute per-category AP 492 | # from https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L222-L252 # noqa 493 | precisions = coco_eval.eval["precision"] 494 | # precision has dims (iou, recall, cls, area range, max dets) 495 | assert len(class_names) == precisions.shape[2] 496 | 497 | results_per_category = [] 498 | for idx, name in enumerate(class_names): 499 | # area range index 0: all area ranges 500 | # max dets index -1: typically 100 per image 501 | precision = precisions[:, :, idx, 0, -1] 502 | precision = precision[precision > -1] 503 | ap = np.mean(precision) if precision.size else float("nan") 504 | results_per_category.append(("{}".format(name), float(ap * 100))) 505 | 506 | results.update({"AP-" + name: ap for name, ap in results_per_category}) 507 | return results --------------------------------------------------------------------------------