├── assets └── Fig1.png ├── install_for_aaa.sh ├── algorithms ├── random.py ├── without_offline.py ├── mcct.py ├── aaa.py ├── hdt.py └── aaa_util.py ├── track_expert.py ├── experts ├── staple.py ├── siamrpn.py ├── siamfc.py ├── kys.py ├── atom.py ├── dimp.py ├── prdimp.py ├── roam.py ├── memdtc.py ├── memtrack.py ├── dasiamrpn.py ├── siamfcpp.py ├── rpt.py ├── siamban.py ├── siamrpnpp.py ├── siamcar.py ├── spm.py ├── drol.py ├── siamdw.py ├── ocean.py ├── siamrpnpp_group.py ├── siammcf.py ├── siamdw_group.py ├── siamrcnn.py ├── thor.py └── gradnet.py ├── local.py ├── track_algorithm.py ├── LICENSE ├── run_tuning.sh ├── run_hdt.sh ├── run_mcct.sh ├── run_algorithm.sh ├── track_tuning.py ├── run_parameters.sh ├── datasets ├── otbnoisydataset.py ├── got10kdataset.py ├── votdataset.py ├── trackingnetdataset.py ├── oxuvadataset.py ├── data.py └── lasotdataset.py ├── run_experts.sh ├── .gitignore ├── run_baselines.sh ├── track_dataset.py ├── install_for_experts.sh ├── base_tracker.py ├── path_config.py ├── evaluations ├── eval_trackers.py └── ope_benchmark.py ├── select_options.py ├── visualizes └── draw_tables.py └── README.md /assets/Fig1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songheony/A3T/HEAD/assets/Fig1.png -------------------------------------------------------------------------------- /install_for_aaa.sh: -------------------------------------------------------------------------------- 1 | # install libraries 2 | conda install -y pytorch==1.6.0 torchvision==0.7.0 cudatoolkit=10.1 -c pytorch 3 | pip install opencv-python opencv-contrib-python Cython seaborn sympy 4 | 5 | # make directory for external libraries 6 | mkdir external 7 | cd external 8 | 9 | # clone frameworks 10 | git clone https://github.com/songheony/pytracking.git 11 | git clone https://github.com/StrangerZhang/pysot-toolkit 12 | 13 | # install region 14 | cd pysot-toolkit/pysot/utils/ 15 | python setup.py build_ext --inplace -------------------------------------------------------------------------------- /algorithms/random.py: -------------------------------------------------------------------------------- 1 | import random 2 | import numpy as np 3 | from base_tracker import BaseTracker 4 | 5 | 6 | class Random(BaseTracker): 7 | def __init__(self, n_experts, mode): 8 | super(Random, self).__init__(f"Random/{mode}") 9 | 10 | def initialize(self, image, box): 11 | pass 12 | 13 | def track(self, image, boxes): 14 | id = random.randint(0, len(boxes) - 1) 15 | weight = np.zeros((len(boxes))) 16 | weight[id] = 1 17 | return (boxes[id], [boxes[id]], weight) 18 | -------------------------------------------------------------------------------- /track_expert.py: -------------------------------------------------------------------------------- 1 | from track_dataset import run 2 | from select_options import select_expert 3 | 4 | 5 | def main(tracker_name, dataset_name): 6 | tracker = select_expert(tracker_name) 7 | 8 | run(tracker, dataset_name, experts=None) 9 | 10 | 11 | if __name__ == "__main__": 12 | import argparse 13 | 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument("-e", "--expert", default="RPT", type=str, help="expert") 16 | parser.add_argument("-d", "--dataset", default="OTB2015", type=str, help="dataset") 17 | args = parser.parse_args() 18 | 19 | main(args.expert, args.dataset) 20 | -------------------------------------------------------------------------------- /experts/staple.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import cv2 3 | from base_tracker import BaseTracker 4 | 5 | sys.path.append("external/pyCFTrackers") 6 | from cftracker.staple import Staple as Tracker 7 | from cftracker.config import staple_config 8 | 9 | 10 | class Staple(BaseTracker): 11 | def __init__(self): 12 | super(Staple, self).__init__("Staple") 13 | self.tracker = Tracker(config=staple_config.StapleConfig()) 14 | 15 | def initialize(self, image_file, box): 16 | image = cv2.imread(image_file) 17 | self.tracker.init(image, box) 18 | 19 | def track(self, image_file): 20 | image = cv2.imread(image_file) 21 | bbox = self.tracker.update(image) 22 | return bbox 23 | -------------------------------------------------------------------------------- /experts/siamrpn.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from PIL import Image 3 | from base_tracker import BaseTracker 4 | import path_config 5 | 6 | sys.path.append("external/siamrpn-pytorch") 7 | from siamrpn import TrackerSiamRPN 8 | 9 | 10 | class SiamRPN(BaseTracker): 11 | def __init__(self): 12 | super(SiamRPN, self).__init__("SiamRPN") 13 | self.net_file = path_config.SIAMRPN_MODEL 14 | self.tracker = TrackerSiamRPN(net_path=self.net_file) 15 | 16 | def initialize(self, image_file, box): 17 | image = Image.open(image_file).convert("RGB") 18 | self.tracker.init(image, box) 19 | 20 | def track(self, image_file): 21 | image = Image.open(image_file).convert("RGB") 22 | return self.tracker.update(image) 23 | -------------------------------------------------------------------------------- /experts/siamfc.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from PIL import Image 3 | from base_tracker import BaseTracker 4 | import path_config 5 | 6 | sys.path.append("external/siamfc") 7 | from siamfc import TrackerSiamFC 8 | 9 | 10 | class SiamFC(BaseTracker): 11 | def __init__(self): 12 | super(SiamFC, self).__init__("SiamFC") 13 | # TODO: edit this path 14 | self.net_file = path_config.SIAMFC_MODEL 15 | self.tracker = TrackerSiamFC(net_path=self.net_file) 16 | 17 | def initialize(self, image_file, box): 18 | image = Image.open(image_file).convert("RGB") 19 | self.tracker.init(image, box) 20 | 21 | def track(self, image_file): 22 | image = Image.open(image_file).convert("RGB") 23 | return self.tracker.update(image) 24 | -------------------------------------------------------------------------------- /experts/kys.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import cv2 3 | from base_tracker import BaseTracker 4 | 5 | sys.path.append("external/pytracking/") 6 | from pytracking.tracker.kys.kys import KYS as Tracker 7 | from pytracking.parameter.kys.default import parameters 8 | 9 | 10 | class KYS(BaseTracker): 11 | def __init__(self): 12 | super(KYS, self).__init__("KYS") 13 | self.tracker = Tracker(parameters()) 14 | 15 | def initialize(self, image_file, box): 16 | image = cv2.cvtColor(cv2.imread(image_file), cv2.COLOR_BGR2RGB) 17 | state = {"init_bbox": box} 18 | self.tracker.initialize(image, state) 19 | 20 | def track(self, image_file): 21 | image = cv2.cvtColor(cv2.imread(image_file), cv2.COLOR_BGR2RGB) 22 | return self.tracker.track(image)["target_bbox"] 23 | -------------------------------------------------------------------------------- /experts/atom.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import cv2 3 | from base_tracker import BaseTracker 4 | 5 | sys.path.append("external/pytracking/") 6 | from pytracking.tracker.atom.atom import ATOM as Tracker 7 | from pytracking.parameter.atom.default import parameters 8 | 9 | 10 | class ATOM(BaseTracker): 11 | def __init__(self): 12 | super(ATOM, self).__init__("ATOM") 13 | self.tracker = Tracker(parameters()) 14 | 15 | def initialize(self, image_file, box): 16 | image = cv2.cvtColor(cv2.imread(image_file), cv2.COLOR_BGR2RGB) 17 | state = {"init_bbox": box} 18 | self.tracker.initialize(image, state) 19 | 20 | def track(self, image_file): 21 | image = cv2.cvtColor(cv2.imread(image_file), cv2.COLOR_BGR2RGB) 22 | return self.tracker.track(image)["target_bbox"] 23 | -------------------------------------------------------------------------------- /experts/dimp.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import cv2 3 | from base_tracker import BaseTracker 4 | 5 | sys.path.append("external/pytracking/") 6 | from pytracking.tracker.dimp.dimp import DiMP as Tracker 7 | from pytracking.parameter.dimp.dimp50 import parameters as dimp50param 8 | 9 | 10 | class DiMP(BaseTracker): 11 | def __init__(self): 12 | super(DiMP, self).__init__("DiMP") 13 | self.tracker = Tracker(dimp50param()) 14 | 15 | def initialize(self, image_file, box): 16 | image = cv2.cvtColor(cv2.imread(image_file), cv2.COLOR_BGR2RGB) 17 | state = {"init_bbox": box} 18 | self.tracker.initialize(image, state) 19 | 20 | def track(self, image_file): 21 | image = cv2.cvtColor(cv2.imread(image_file), cv2.COLOR_BGR2RGB) 22 | return self.tracker.track(image)["target_bbox"] 23 | -------------------------------------------------------------------------------- /experts/prdimp.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import cv2 3 | from base_tracker import BaseTracker 4 | 5 | sys.path.append("external/pytracking/") 6 | from pytracking.tracker.dimp.dimp import DiMP as Tracker 7 | from pytracking.parameter.dimp.prdimp50 import parameters as prdimp50param 8 | 9 | 10 | class PrDiMP(BaseTracker): 11 | def __init__(self): 12 | super(PrDiMP, self).__init__("PrDiMP") 13 | self.tracker = Tracker(prdimp50param()) 14 | 15 | def initialize(self, image_file, box): 16 | image = cv2.cvtColor(cv2.imread(image_file), cv2.COLOR_BGR2RGB) 17 | state = {"init_bbox": box} 18 | self.tracker.initialize(image, state) 19 | 20 | def track(self, image_file): 21 | image = cv2.cvtColor(cv2.imread(image_file), cv2.COLOR_BGR2RGB) 22 | return self.tracker.track(image)["target_bbox"] 23 | -------------------------------------------------------------------------------- /local.py: -------------------------------------------------------------------------------- 1 | from pytracking.evaluation.environment import EnvSettings 2 | 3 | 4 | def local_env_settings(): 5 | settings = EnvSettings() 6 | 7 | # Set your local paths here. 8 | 9 | settings.davis_dir = "" 10 | settings.got10k_path = "" 11 | settings.got_packed_results_path = "" 12 | settings.got_reports_path = "" 13 | settings.lasot_path = "" 14 | settings.network_path = "weights/pytracking" # Where tracking networks are stored. 15 | settings.nfs_path = "" 16 | settings.otb_path = "" 17 | settings.result_plot_path = "" 18 | settings.results_path = "" # Where to store tracking results 19 | settings.segmentation_path = "" 20 | settings.tn_packed_results_path = "" 21 | settings.tpl_path = "" 22 | settings.trackingnet_path = "" 23 | settings.uav_path = "" 24 | settings.vot_path = "" 25 | settings.youtubevos_dir = "" 26 | 27 | return settings 28 | -------------------------------------------------------------------------------- /experts/roam.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from PIL import Image 4 | import numpy as np 5 | import path_config 6 | from base_tracker import BaseTracker 7 | 8 | sys.path.append("external/ROAM") 9 | from utils import list_models 10 | from networks import FeatureExtractor 11 | from tracker import Tracker 12 | 13 | 14 | class ROAM(BaseTracker): 15 | def __init__(self): 16 | super(ROAM, self).__init__("ROAM") 17 | feat_extractor = FeatureExtractor(path_config.ROAM_FEAT_DIR) 18 | self.tracker = Tracker(feat_extractor, is_debug=False) 19 | models = list_models(os.path.abspath(path_config.ROAM_MODEL_DIR)) 20 | self.tracker.load_models(models[-1]) 21 | 22 | def initialize(self, image_file, box): 23 | img = np.array(Image.open(image_file).convert("RGB")) 24 | self.tracker.initialize(img, box) 25 | 26 | def track(self, image_file): 27 | img = np.array(Image.open(image_file).convert("RGB")) 28 | return self.tracker.track(img) 29 | -------------------------------------------------------------------------------- /experts/memdtc.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" 4 | os.environ["C_CPP_MIN_LOG_LEVEL"] = "3" 5 | import sys 6 | import tensorflow as tf 7 | import path_config 8 | 9 | tf.get_logger().setLevel("INFO") 10 | from base_tracker import BaseTracker 11 | 12 | sys.path.append("external/MemDTC/") 13 | from tracking.tracker import Tracker, Model 14 | 15 | 16 | class MemDTC(BaseTracker): 17 | def __init__(self): 18 | super(MemDTC, self).__init__("MemDTC") 19 | config_proto = tf.ConfigProto() 20 | config_proto.gpu_options.allow_growth = True 21 | ckpt = tf.train.get_checkpoint_state(path_config.MEMDTC_MODEL) 22 | sess = tf.Session(config=config_proto) 23 | model = Model(sess, ckpt.model_checkpoint_path) 24 | self.tracker = Tracker(model) 25 | 26 | def initialize(self, image_file, box): 27 | self.tracker.initialize(image_file, box) 28 | 29 | def track(self, image_file): 30 | bbox, _ = self.tracker.track(image_file) 31 | return bbox 32 | -------------------------------------------------------------------------------- /experts/memtrack.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" 4 | os.environ["C_CPP_MIN_LOG_LEVEL"] = "3" 5 | import sys 6 | import tensorflow as tf 7 | import path_config 8 | 9 | tf.get_logger().setLevel("INFO") 10 | from base_tracker import BaseTracker 11 | 12 | sys.path.append("external/MemTrack/") 13 | from tracking.tracker import Tracker, Model 14 | 15 | 16 | class MemTrack(BaseTracker): 17 | def __init__(self): 18 | super(MemTrack, self).__init__("MemTrack") 19 | config_proto = tf.ConfigProto() 20 | config_proto.gpu_options.allow_growth = True 21 | sess = tf.Session(config=config_proto) 22 | ckpt = tf.train.get_checkpoint_state(path_config.MEMTRACK_MODEL) 23 | model = Model(sess, ckpt.model_checkpoint_path) 24 | self.tracker = Tracker(model) 25 | 26 | def initialize(self, image_file, box): 27 | self.tracker.initialize(image_file, box) 28 | 29 | def track(self, image_file): 30 | bbox, _ = self.tracker.track(image_file) 31 | return bbox 32 | -------------------------------------------------------------------------------- /track_algorithm.py: -------------------------------------------------------------------------------- 1 | from track_dataset import run 2 | from select_options import select_algorithms 3 | 4 | 5 | def main(algorithm_name, experts, dataset_name, **kwargs): 6 | algorithm = select_algorithms(algorithm_name, experts, **kwargs) 7 | 8 | run(algorithm, dataset_name, experts=experts, debug=False) 9 | 10 | 11 | if __name__ == "__main__": 12 | import argparse 13 | 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument("-a", "--algorithm", default="AAA", type=str) 16 | parser.add_argument( 17 | "-e", "--experts", default=["DaSiamRPN", "SiamDW", "SiamRPN", "SPM"], nargs="+" 18 | ) 19 | parser.add_argument("-d", "--dataset", default="OTB2015", type=str) 20 | parser.add_argument("-m", "--mode", default="SuperFast", type=str) 21 | parser.add_argument("-t", "--threshold", default=0.8, type=float) 22 | args = parser.parse_args() 23 | 24 | main( 25 | args.algorithm, 26 | args.experts, 27 | args.dataset, 28 | mode=args.mode, 29 | threshold=args.threshold, 30 | ) 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Heon Song 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /experts/dasiamrpn.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import cv2 3 | import torch 4 | import numpy as np 5 | from base_tracker import BaseTracker 6 | import path_config 7 | 8 | sys.path.append("external/DaSiamRPN/code") 9 | from net import SiamRPNotb 10 | from run_SiamRPN import SiamRPN_init, SiamRPN_track 11 | from utils import cxy_wh_2_rect 12 | 13 | 14 | class DaSiamRPN(BaseTracker): 15 | def __init__(self): 16 | super(DaSiamRPN, self).__init__(name="DaSiamRPN") 17 | self.net_file = path_config.DASIAMRPN_MODEL 18 | 19 | def initialize(self, image_file, box): 20 | self.net = SiamRPNotb() 21 | self.net.load_state_dict(torch.load(self.net_file)) 22 | self.net.eval().cuda() 23 | 24 | image = cv2.imread(image_file) 25 | box = box - np.array([1, 1, 0, 0]) 26 | self.state = SiamRPN_init( 27 | image, box[:2] + box[2:] / 2.0, box[2:], self.net 28 | ) # init tracker 29 | 30 | def track(self, image_file): 31 | image = cv2.imread(image_file) 32 | self.state = SiamRPN_track(self.state, image) # track 33 | center = self.state["target_pos"] + 1 34 | target_sz = self.state["target_sz"] 35 | box = cxy_wh_2_rect(center, target_sz) 36 | return box 37 | -------------------------------------------------------------------------------- /experts/siamfcpp.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import cv2 3 | import torch 4 | import path_config 5 | from base_tracker import BaseTracker 6 | 7 | sys.path.append("external/video_analyst") 8 | from videoanalyst.config.config import cfg as root_cfg 9 | from videoanalyst.model import builder as model_builder 10 | from videoanalyst.pipeline import builder as pipeline_builder 11 | 12 | 13 | class SiamFCPP(BaseTracker): 14 | def __init__(self): 15 | super(SiamFCPP, self).__init__("SiamFC++") 16 | 17 | root_cfg.merge_from_file(path_config.SIAMFCPP_CONFIG) 18 | 19 | task = "track" 20 | task_cfg = root_cfg["test"][task] 21 | task_cfg.freeze() 22 | 23 | # build model 24 | model = model_builder.build(task, task_cfg.model) 25 | # build pipeline 26 | self.pipeline = pipeline_builder.build(task, task_cfg.pipeline, model) 27 | dev = torch.device("cuda") 28 | self.pipeline.set_device(dev) 29 | 30 | def initialize(self, image_file, box): 31 | frame = cv2.imread(image_file, cv2.IMREAD_COLOR) 32 | self.pipeline.init(frame, box) 33 | 34 | def track(self, image_file): 35 | frame = cv2.imread(image_file, cv2.IMREAD_COLOR) 36 | rect_pred = self.pipeline.update(frame) 37 | return rect_pred 38 | -------------------------------------------------------------------------------- /run_tuning.sh: -------------------------------------------------------------------------------- 1 | super_fast_experts=("DaSiamRPN" "SiamDW" "SiamRPN" "SPM") 2 | python ./track_tuning.py -e ${super_fast_experts[@]} -m SuperFast 3 | python ./track_tuning.py -a HDT -e ${super_fast_experts[@]} -m SuperFast 4 | 5 | fast_experts=("Ocean" "SiamBAN" "SiamCAR" "SiamFC++" "SiamRPN++") 6 | python ./track_tuning.py -e ${fast_experts[@]} -m Fast 7 | python ./track_tuning.py -a HDT -e ${fast_experts[@]} -m Fast 8 | 9 | normal_experts=("DiMP" "DROL" "KYS" "PrDiMP" "RPT") 10 | python ./track_tuning.py -e ${normal_experts[@]} -m Normal 11 | python ./track_tuning.py -a HDT -e ${normal_experts[@]} -m Normal 12 | 13 | siamdw_experts=("SiamDWGroup/SiamFCRes22/OTB" "SiamDWGroup/SiamFCIncep22/OTB" "SiamDWGroup/SiamFCNext22/OTB" "SiamDWGroup/SiamRPNRes22/OTB" "SiamDWGroup/SiamFCRes22/VOT" "SiamDWGroup/SiamFCIncep22/VOT" "SiamDWGroup/SiamFCNext22/VOT" "SiamDWGroup/SiamRPNRes22/VOT") 14 | python ./track_tuning.py -e ${siamdw_experts[@]} -m SiamDW 15 | python ./track_tuning.py -a HDT -e ${siamdw_experts[@]} -m SiamDW 16 | 17 | siamrpn_experts=("SiamRPN++Group/AlexNet/VOT" "SiamRPN++Group/AlexNet/OTB" "SiamRPN++Group/ResNet-50/VOT" "SiamRPN++Group/ResNet-50/OTB" "SiamRPN++Group/ResNet-50/VOTLT" "SiamRPN++Group/MobileNetV2/VOT" "SiamRPN++Group/SiamMask/VOT") 18 | python ./track_tuning.py -e ${siamrpn_experts[@]} -m SiamRPN++ 19 | python ./track_tuning.py -a HDT -e ${siamrpn_experts[@]} -m SiamRPN++ -------------------------------------------------------------------------------- /experts/rpt.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import cv2 3 | import numpy as np 4 | from base_tracker import BaseTracker 5 | import path_config 6 | 7 | sys.path.append("external/RPT") 8 | from siamreppoints.core.config import cfg 9 | from siamreppoints.models.model_builder import ModelBuilder 10 | from siamreppoints.tracker.tracker_builder import build_tracker 11 | from siamreppoints.utils.model_load import load_pretrain 12 | from siamreppoints.utils.bbox import get_axis_aligned_bbox 13 | 14 | 15 | class RPT(BaseTracker): 16 | def __init__(self): 17 | super(RPT, self).__init__("RPT") 18 | 19 | # load config 20 | cfg.merge_from_file(path_config.RPT_CONFIG) 21 | 22 | # create model 23 | model = ModelBuilder() 24 | 25 | # load model 26 | model = load_pretrain(model, path_config.RPT_SNAPSHOT).cuda().eval() 27 | 28 | # build tracker 29 | self.tracker = build_tracker(model) 30 | 31 | def initialize(self, image_file, box): 32 | image = cv2.imread(image_file) 33 | cx, cy, w, h = get_axis_aligned_bbox(np.array(box)) 34 | gt_bbox_ = [cx - (w - 1) / 2, cy - (h - 1) / 2, w, h] 35 | self.tracker.init(image, gt_bbox_) 36 | 37 | def track(self, image_file): 38 | image = cv2.imread(image_file) 39 | outputs = self.tracker.track(image) 40 | pred_bbox = outputs["bbox"] 41 | return pred_bbox 42 | -------------------------------------------------------------------------------- /experts/siamban.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import cv2 3 | import numpy as np 4 | from base_tracker import BaseTracker 5 | import path_config 6 | 7 | sys.path.append("external/siamban") 8 | from siamban.core.config import cfg 9 | from siamban.models.model_builder import ModelBuilder 10 | from siamban.tracker.tracker_builder import build_tracker 11 | from siamban.utils.bbox import get_axis_aligned_bbox 12 | from siamban.utils.model_load import load_pretrain 13 | 14 | 15 | class SiamBAN(BaseTracker): 16 | def __init__(self): 17 | super(SiamBAN, self).__init__("SiamBAN") 18 | 19 | # load config 20 | cfg.merge_from_file(path_config.SIAMBAN_CONFIG) 21 | 22 | # create model 23 | model = ModelBuilder() 24 | 25 | # load model 26 | model = load_pretrain(model, path_config.SIAMBAN_SNAPSHOT).cuda().eval() 27 | 28 | # build tracker 29 | self.tracker = build_tracker(model) 30 | 31 | def initialize(self, image_file, box): 32 | image = cv2.imread(image_file) 33 | cx, cy, w, h = get_axis_aligned_bbox(np.array(box)) 34 | gt_bbox_ = [cx - (w - 1) / 2, cy - (h - 1) / 2, w, h] 35 | self.tracker.init(image, gt_bbox_) 36 | 37 | def track(self, image_file): 38 | image = cv2.imread(image_file) 39 | outputs = self.tracker.track(image) 40 | pred_bbox = outputs["bbox"] 41 | return pred_bbox 42 | -------------------------------------------------------------------------------- /experts/siamrpnpp.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import cv2 3 | import torch 4 | from base_tracker import BaseTracker 5 | import path_config 6 | 7 | sys.path.append("external/pysot") 8 | from pysot.core.config import cfg 9 | from pysot.models.model_builder import ModelBuilder 10 | from pysot.tracker.tracker_builder import build_tracker 11 | 12 | 13 | class SiamRPNPP(BaseTracker): 14 | def __init__(self): 15 | super(SiamRPNPP, self).__init__("SiamRPN++") 16 | config = path_config.SIAMRPNPP_CONFIG 17 | snapshot = path_config.SIAMRPNPP_SNAPSHOT 18 | 19 | # load config 20 | cfg.merge_from_file(config) 21 | cfg.CUDA = torch.cuda.is_available() 22 | device = torch.device("cuda" if cfg.CUDA else "cpu") 23 | 24 | # create model 25 | self.model = ModelBuilder() 26 | 27 | # load model 28 | self.model.load_state_dict( 29 | torch.load(snapshot, map_location=lambda storage, loc: storage.cpu()) 30 | ) 31 | self.model.eval().to(device) 32 | 33 | # build tracker 34 | self.tracker = build_tracker(self.model) 35 | 36 | def initialize(self, image_file, box): 37 | image = cv2.imread(image_file) 38 | self.tracker.init(image, box) 39 | 40 | def track(self, image_file): 41 | image = cv2.imread(image_file) 42 | bbox = self.tracker.track(image)["bbox"] 43 | return bbox 44 | -------------------------------------------------------------------------------- /experts/siamcar.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import cv2 3 | import numpy as np 4 | from base_tracker import BaseTracker 5 | import path_config 6 | 7 | sys.path.append("external/SiamCAR/") 8 | from pysot.core.config import cfg 9 | from pysot.tracker.siamcar_tracker import SiamCARTracker 10 | from pysot.utils.bbox import get_axis_aligned_bbox 11 | from pysot.utils.model_load import load_pretrain 12 | from pysot.models.model_builder import ModelBuilder 13 | 14 | 15 | class SiamCAR(BaseTracker): 16 | def __init__(self): 17 | super(SiamCAR, self).__init__("SiamCAR") 18 | 19 | # load config 20 | cfg.merge_from_file(path_config.SIAMCAR_CONFIG) 21 | 22 | # hp_search 23 | params = getattr(cfg.HP_SEARCH, "OTB100") 24 | self.hp = {"lr": params[0], "penalty_k": params[1], "window_lr": params[2]} 25 | 26 | model = ModelBuilder() 27 | 28 | # load model 29 | model = load_pretrain(model, path_config.SIAMCAR_SNAPSHOT).cuda().eval() 30 | 31 | # build tracker 32 | self.tracker = SiamCARTracker(model, cfg.TRACK) 33 | 34 | def initialize(self, image_file, box): 35 | image = cv2.imread(image_file) 36 | cx, cy, w, h = get_axis_aligned_bbox(np.array(box)) 37 | gt_bbox_ = [cx - (w - 1) / 2, cy - (h - 1) / 2, w, h] 38 | self.tracker.init(image, gt_bbox_) 39 | 40 | def track(self, image_file): 41 | image = cv2.imread(image_file) 42 | outputs = self.tracker.track(image, self.hp) 43 | pred_bbox = outputs["bbox"] 44 | return pred_bbox 45 | -------------------------------------------------------------------------------- /run_hdt.sh: -------------------------------------------------------------------------------- 1 | datasets=("OTB2015" "NFS" "UAV123" "TColor128" "VOT2018" "LaSOT") 2 | 3 | super_fast_experts=("DaSiamRPN" "SiamDW" "SiamRPN" "SPM") 4 | for (( i=0; i<${#datasets[@]}; i++ )); do 5 | python ./track_algorithm.py -a HDT -m SuperFast -d ${datasets[$i]} -e ${super_fast_experts[@]} -t 0.65 6 | done 7 | 8 | fast_experts=("Ocean" "SiamBAN" "SiamCAR" "SiamFC++" "SiamRPN++") 9 | for (( i=0; i<${#datasets[@]}; i++ )); do 10 | python ./track_algorithm.py -a HDT -m Fast -d ${datasets[$i]} -e ${fast_experts[@]} -t 0.65 11 | done 12 | 13 | normal_experts=("DiMP" "DROL" "KYS" "PrDiMP" "RPT") 14 | for (( i=0; i<${#datasets[@]}; i++ )); do 15 | python ./track_algorithm.py -a HDT -m Normal -d ${datasets[$i]} -e ${normal_experts[@]} -t 0.65 16 | done 17 | 18 | siamdw_experts=("SiamDWGroup/SiamFCRes22/OTB" "SiamDWGroup/SiamFCIncep22/OTB" "SiamDWGroup/SiamFCNext22/OTB" "SiamDWGroup/SiamRPNRes22/OTB" "SiamDWGroup/SiamFCRes22/VOT" "SiamDWGroup/SiamFCIncep22/VOT" "SiamDWGroup/SiamFCNext22/VOT" "SiamDWGroup/SiamRPNRes22/VOT") 19 | for (( i=0; i<${#datasets[@]}; i++ )); do 20 | python ./track_algorithm.py -a HDT -m SiamDW -d ${datasets[$i]} -e ${siamdw_experts[@]} -t 0.65 21 | done 22 | 23 | siamrpn_experts=("SiamRPN++Group/AlexNet/VOT" "SiamRPN++Group/AlexNet/OTB" "SiamRPN++Group/ResNet-50/VOT" "SiamRPN++Group/ResNet-50/OTB" "SiamRPN++Group/ResNet-50/VOTLT" "SiamRPN++Group/MobileNetV2/VOT" "SiamRPN++Group/SiamMask/VOT") 24 | for (( i=0; i<${#datasets[@]}; i++ )); do 25 | python ./track_algorithm.py -a HDT -m SiamRPN++ -d ${datasets[$i]} -e ${siamrpn_experts[@]} -t 0.65 26 | done -------------------------------------------------------------------------------- /experts/spm.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import cv2 3 | import torch 4 | import numpy as np 5 | from base_tracker import BaseTracker 6 | import path_config 7 | 8 | sys.path.append("external/SPM-Tracker/") 9 | from siam_tracker.core.inference import SiamTracker 10 | from siam_tracker.core.config import merge_cfg_from_file 11 | 12 | 13 | def img2tensor(img, device): 14 | """ Convert numpy.ndarry to torch.Tensor """ 15 | img_tensor = torch.from_numpy(img.transpose((2, 0, 1))).float().to(device) 16 | img_tensor = img_tensor.unsqueeze(0) 17 | return img_tensor 18 | 19 | 20 | class SPM(BaseTracker): 21 | def __init__(self): 22 | super(SPM, self).__init__("SPM") 23 | cfg_path = path_config.SPM_CONFIG 24 | gpu_id = 0 25 | merge_cfg_from_file(cfg_path) 26 | self.device = torch.device("cuda:{}".format(gpu_id)) 27 | self.model = SiamTracker(self.device) 28 | 29 | def initialize(self, image_file, box): 30 | frame = cv2.imread(image_file) 31 | img_tensor = img2tensor(frame, self.device) 32 | box = [(box[0]), (box[1]), (box[2] + box[0]), (box[3] + box[1])] 33 | self.model.tracker.init_tracker(img_tensor, box) 34 | self.current_box = box 35 | 36 | def track(self, image_file): 37 | frame = cv2.imread(image_file) 38 | img_tensor = img2tensor(frame, self.device) 39 | self.current_box = self.model.tracker.predict_next_frame( 40 | img_tensor, self.current_box 41 | ) 42 | bbox = np.array(self.current_box[:]) 43 | bbox[2:] -= bbox[:2] 44 | return bbox 45 | -------------------------------------------------------------------------------- /run_mcct.sh: -------------------------------------------------------------------------------- 1 | datasets=("OTB2015" "NFS" "UAV123" "TColor128" "VOT2018" "LaSOT") 2 | 3 | super_fast_experts=("DaSiamRPN" "SiamDW" "SiamRPN" "SPM") 4 | for (( i=0; i<${#datasets[@]}; i++ )); do 5 | python ./track_algorithm.py -a MCCT -m SuperFast -d ${datasets[$i]} -e ${super_fast_experts[@]} -t 0.1 6 | done 7 | 8 | fast_experts=("Ocean" "SiamBAN" "SiamCAR" "SiamFC++" "SiamRPN++") 9 | for (( i=0; i<${#datasets[@]}; i++ )); do 10 | python ./track_algorithm.py -a MCCT -m Fast -d ${datasets[$i]} -e ${fast_experts[@]} -t 0.1 11 | done 12 | 13 | normal_experts=("DiMP" "DROL" "KYS" "PrDiMP" "RPT") 14 | for (( i=0; i<${#datasets[@]}; i++ )); do 15 | python ./track_algorithm.py -a MCCT -m Normal -d ${datasets[$i]} -e ${normal_experts[@]} -t 0.1 16 | done 17 | 18 | siamdw_experts=("SiamDWGroup/SiamFCRes22/OTB" "SiamDWGroup/SiamFCIncep22/OTB" "SiamDWGroup/SiamFCNext22/OTB" "SiamDWGroup/SiamRPNRes22/OTB" "SiamDWGroup/SiamFCRes22/VOT" "SiamDWGroup/SiamFCIncep22/VOT" "SiamDWGroup/SiamFCNext22/VOT" "SiamDWGroup/SiamRPNRes22/VOT") 19 | for (( i=0; i<${#datasets[@]}; i++ )); do 20 | python ./track_algorithm.py -a MCCT -m SiamDW -d ${datasets[$i]} -e ${siamdw_experts[@]} -t 0.1 21 | done 22 | 23 | siamrpn_experts=("SiamRPN++Group/AlexNet/VOT" "SiamRPN++Group/AlexNet/OTB" "SiamRPN++Group/ResNet-50/VOT" "SiamRPN++Group/ResNet-50/OTB" "SiamRPN++Group/ResNet-50/VOTLT" "SiamRPN++Group/MobileNetV2/VOT" "SiamRPN++Group/SiamMask/VOT") 24 | for (( i=0; i<${#datasets[@]}; i++ )); do 25 | python ./track_algorithm.py -a MCCT -m SiamRPN++ -d ${datasets[$i]} -e ${siamrpn_experts[@]} -t 0.1 26 | done -------------------------------------------------------------------------------- /algorithms/without_offline.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from PIL import Image 4 | from base_tracker import BaseTracker 5 | from .aaa_util import FeatureExtractor, AnchorDetector 6 | 7 | 8 | class WithoutOffline(BaseTracker): 9 | def __init__(self, n_experts, mode): 10 | super(WithoutOffline, self).__init__(f"WithoutOffline/{mode}") 11 | 12 | self.n_experts = n_experts 13 | 14 | # Anchor extractor 15 | self.detector = AnchorDetector(threshold=0.0) 16 | 17 | # Feature extractor 18 | use_cuda = torch.cuda.is_available() 19 | device = torch.device("cuda" if use_cuda else "cpu") 20 | self.extractor = FeatureExtractor(device) 21 | 22 | def initialize(self, image_file, box): 23 | image = Image.open(image_file).convert("RGB") 24 | 25 | # Extract target image 26 | self.target_feature = self.extractor.extract(image, [box])[0] 27 | 28 | # Init detector with target feature 29 | self.detector.init(self.target_feature) 30 | 31 | def track(self, image_file, boxes): 32 | image = Image.open(image_file).convert("RGB") 33 | 34 | # Extract features from boxes 35 | features = self.extractor.extract(image, boxes) 36 | 37 | # Detect if it is anchor frame 38 | detected, feature_scores = self.detector.detect(features) 39 | 40 | # Get the index of maximum feature score 41 | max_id = np.argmax(feature_scores) 42 | weight = np.zeros((len(boxes))) 43 | weight[max_id] = 1 44 | 45 | return boxes[max_id], [boxes[max_id]], weight 46 | -------------------------------------------------------------------------------- /run_algorithm.sh: -------------------------------------------------------------------------------- 1 | datasets=("OTB2015" "NFS" "UAV123" "TColor128" "VOT2018" "LaSOT") 2 | 3 | super_fast_experts=("DaSiamRPN" "SiamDW" "SiamRPN" "SPM") 4 | for (( i=0; i<${#datasets[@]}; i++ )); do 5 | python ./track_algorithm.py -a AAA -d ${datasets[$i]} -m SuperFast -e ${super_fast_experts[@]} -t 0.70 6 | done 7 | 8 | fast_experts=("Ocean" "SiamBAN" "SiamCAR" "SiamFC++" "SiamRPN++") 9 | for (( i=0; i<${#datasets[@]}; i++ )); do 10 | python ./track_algorithm.py -a AAA -d ${datasets[$i]} -m Fast -e ${fast_experts[@]} -t 0.72 11 | done 12 | 13 | normal_experts=("DiMP" "DROL" "KYS" "PrDiMP" "RPT") 14 | for (( i=0; i<${#datasets[@]}; i++ )); do 15 | python ./track_algorithm.py -a AAA -d ${datasets[$i]} -m Normal -e ${normal_experts[@]} -t 0.74 16 | done 17 | 18 | siamdw_experts=("SiamDWGroup/SiamFCRes22/OTB" "SiamDWGroup/SiamFCIncep22/OTB" "SiamDWGroup/SiamFCNext22/OTB" "SiamDWGroup/SiamRPNRes22/OTB" "SiamDWGroup/SiamFCRes22/VOT" "SiamDWGroup/SiamFCIncep22/VOT" "SiamDWGroup/SiamFCNext22/VOT" "SiamDWGroup/SiamRPNRes22/VOT") 19 | for (( i=0; i<${#datasets[@]}; i++ )); do 20 | python ./track_algorithm.py -a AAA -d ${datasets[$i]} -m SiamDW -e ${siamdw_experts[@]} -t 0.73 21 | done 22 | 23 | siamrpn_experts=("SiamRPN++Group/AlexNet/VOT" "SiamRPN++Group/AlexNet/OTB" "SiamRPN++Group/ResNet-50/VOT" "SiamRPN++Group/ResNet-50/OTB" "SiamRPN++Group/ResNet-50/VOTLT" "SiamRPN++Group/MobileNetV2/VOT" "SiamRPN++Group/SiamMask/VOT") 24 | for (( i=0; i<${#datasets[@]}; i++ )); do 25 | python ./track_algorithm.py -a AAA -d ${datasets[$i]} -m SiamRPN++ -e ${siamrpn_experts[@]} -t 0.75 26 | done 27 | -------------------------------------------------------------------------------- /track_tuning.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | from track_dataset import run_dataset 4 | from select_options import select_algorithms, select_datasets 5 | import path_config 6 | 7 | 8 | def main(algorithm_name, experts, save_dir, mode): 9 | algorithms = [] 10 | dataset_name = "Got10K" 11 | dataset = select_datasets(dataset_name) 12 | 13 | if algorithm_name == "AAA": 14 | thresholds = [0.70, 0.71, 0.72, 0.73, 0.74, 0.75, 0.76, 0.77, 0.78, 0.79] 15 | elif algorithm_name == "HDT": 16 | thresholds = [0.00, 0.05, 0.10, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40, 0.45, 0.50, 0.55, 0.60, 0.65, 0.70, 0.75, 0.80, 0.85, 0.90, 0.95] 17 | 18 | for threshold in thresholds: 19 | 20 | algorithm = select_algorithms( 21 | algorithm_name, experts, mode=mode, threshold=threshold, 22 | ) 23 | run_dataset(dataset, dataset_name, [algorithm], experts=experts, threads=8) 24 | algorithms.append(algorithm.name) 25 | 26 | 27 | if __name__ == "__main__": 28 | import argparse 29 | 30 | parser = argparse.ArgumentParser() 31 | parser.add_argument("-a", "--algorithm", default="AAA", type=str) 32 | parser.add_argument( 33 | "-e", "--experts", default=["DaSiamRPN", "SiamDW", "SiamRPN", "SPM"], nargs="+" 34 | ) 35 | parser.add_argument("-m", "--mode", default="SuperFast", type=str) 36 | args = parser.parse_args() 37 | 38 | save_dir = Path(f"./{path_config.EVALUATION_PATH}") 39 | os.makedirs(save_dir, exist_ok=True) 40 | 41 | main( 42 | args.algorithm, args.experts, save_dir, args.mode, 43 | ) 44 | -------------------------------------------------------------------------------- /run_parameters.sh: -------------------------------------------------------------------------------- 1 | thresholds=(0.70 0.71 0.72 0.73 0.74 0.75 0.76 0.77 0.78 0.79) 2 | 3 | super_fast_experts=("DaSiamRPN" "SiamDW" "SiamRPN" "SPM") 4 | for (( i=0; i<${#thresholds[@]}; i++ )); do 5 | python ./track_algorithm.py -a AAA -d OTB2015 -m SuperFast -e ${super_fast_experts[@]} -t ${thresholds[$i]} 6 | done 7 | 8 | fast_experts=("Ocean" "SiamBAN" "SiamCAR" "SiamFC++" "SiamRPN++") 9 | for (( i=0; i<${#thresholds[@]}; i++ )); do 10 | python ./track_algorithm.py -a AAA -d OTB2015 -m Fast -e ${fast_experts[@]} -t ${thresholds[$i]} 11 | done 12 | 13 | normal_experts=("DiMP" "DROL" "KYS" "PrDiMP" "RPT") 14 | for (( i=0; i<${#thresholds[@]}; i++ )); do 15 | python ./track_algorithm.py -a AAA -d OTB2015 -m Normal -e ${normal_experts[@]} -t ${thresholds[$i]} 16 | done 17 | 18 | siamdw_experts=("SiamDWGroup/SiamFCRes22/OTB" "SiamDWGroup/SiamFCIncep22/OTB" "SiamDWGroup/SiamFCNext22/OTB" "SiamDWGroup/SiamRPNRes22/OTB" "SiamDWGroup/SiamFCRes22/VOT" "SiamDWGroup/SiamFCIncep22/VOT" "SiamDWGroup/SiamFCNext22/VOT" "SiamDWGroup/SiamRPNRes22/VOT") 19 | for (( i=0; i<${#thresholds[@]}; i++ )); do 20 | python ./track_algorithm.py -a AAA -d OTB2015 -m SiamDW -e ${siamdw_experts[@]} -t ${thresholds[$i]} 21 | done 22 | 23 | siamrpn_experts=("SiamRPN++Group/AlexNet/VOT" "SiamRPN++Group/AlexNet/OTB" "SiamRPN++Group/ResNet-50/VOT" "SiamRPN++Group/ResNet-50/OTB" "SiamRPN++Group/ResNet-50/VOTLT" "SiamRPN++Group/MobileNetV2/VOT" "SiamRPN++Group/SiamMask/VOT") 24 | for (( i=0; i<${#thresholds[@]}; i++ )); do 25 | python ./track_algorithm.py -a AAA -d OTB2015 -m SiamRPN++ -e ${siamrpn_experts[@]} -t ${thresholds[$i]} 26 | done 27 | -------------------------------------------------------------------------------- /datasets/otbnoisydataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | import numpy as np 4 | import path_config 5 | from datasets.data import SequenceList 6 | from datasets.otbdataset import OTBDatasetClass 7 | 8 | 9 | def OTBNoisyDataset(frame_ratio=1.0): 10 | return OTBNoisyDatasetClass(frame_ratio).get_sequence_list() 11 | 12 | 13 | class OTBNoisyDatasetClass(OTBDatasetClass): 14 | """ OTB-2015 dataset with noise 15 | 16 | """ 17 | 18 | def __init__(self, frame_ratio): 19 | super().__init__() 20 | 21 | self.frame_ratio = frame_ratio 22 | self.idx_dir = Path(path_config.OTB_NOISY_PATH) / str(self.frame_ratio) 23 | os.makedirs(self.idx_dir, exist_ok=True) 24 | 25 | def get_sequence_list(self): 26 | return SequenceList([self.noisy_sequence(s) for s in self.sequence_info_list]) 27 | 28 | def noisy_sequence(self, sequence_info): 29 | seq = self._construct_sequence(sequence_info) 30 | 31 | idx_path = self.idx_dir / f"{seq.name}.txt" 32 | if idx_path.exists(): 33 | idxs = np.loadtxt(idx_path, dtype=int) 34 | else: 35 | n_frames = len(seq.frames) - 1 36 | selected_frames = int(n_frames * self.frame_ratio) 37 | idxs = np.sort(np.random.choice(n_frames, selected_frames, replace=False)) 38 | 39 | # add 0 40 | idxs += 1 41 | idxs = np.insert(idxs, 0, 0) 42 | 43 | idxs = idxs.astype(int) 44 | np.savetxt(idx_path, idxs, fmt="%i") 45 | 46 | seq.frames = [seq.frames[idx] for idx in idxs] 47 | seq.ground_truth_rect = seq.ground_truth_rect[idxs] 48 | seq.name = f"{seq.name}_{self.frame_ratio}" 49 | 50 | return seq 51 | -------------------------------------------------------------------------------- /experts/drol.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import random 3 | import os 4 | import torch 5 | import cv2 6 | import numpy as np 7 | from base_tracker import BaseTracker 8 | import path_config 9 | 10 | sys.path.append("external/DROL") 11 | from pysot.core.config import cfg 12 | from pysot.models.model_builder import ModelBuilder 13 | from pysot.tracker.tracker_builder import build_tracker 14 | from pysot.utils.bbox import get_axis_aligned_bbox 15 | from pysot.utils.model_load import load_pretrain 16 | 17 | 18 | def seed_torch(seed=0): 19 | random.seed(seed) 20 | os.environ["PYTHONHASHSEED"] = str(seed) 21 | np.random.seed(seed) 22 | torch.manual_seed(seed) 23 | torch.cuda.manual_seed(seed) 24 | torch.backends.cudnn.deterministic = True 25 | torch.backends.cudnn.benchmark = True 26 | 27 | 28 | class DROL(BaseTracker): 29 | def __init__(self): 30 | super(DROL, self).__init__("DROL") 31 | 32 | # load config 33 | cfg.merge_from_file(path_config.DROL_CONFIG) 34 | seed_torch(cfg.TRACK.SEED) 35 | 36 | # create model 37 | model = ModelBuilder() 38 | 39 | # load model 40 | model = load_pretrain(model, path_config.DROL_SNAPSHOT).cuda().eval() 41 | 42 | # build tracker 43 | self.tracker = build_tracker(model) 44 | 45 | def initialize(self, image_file, box): 46 | image = cv2.imread(image_file) 47 | cx, cy, w, h = get_axis_aligned_bbox(np.array(box)) 48 | gt_bbox_ = [cx - (w - 1) / 2, cy - (h - 1) / 2, w, h] 49 | self.tracker.init(image, gt_bbox_) 50 | 51 | def track(self, image_file): 52 | image = cv2.imread(image_file) 53 | outputs = self.tracker.track(image) 54 | pred_bbox = outputs["bbox"] 55 | return pred_bbox 56 | -------------------------------------------------------------------------------- /experts/siamdw.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import numpy as np 3 | import cv2 4 | from easydict import EasyDict as edict 5 | from base_tracker import BaseTracker 6 | import path_config 7 | 8 | sys.path.append("external/SiamDW/lib") 9 | from tracker.siamrpn import SiamRPN 10 | import models.models as models 11 | from utils.utils import load_pretrain 12 | 13 | 14 | class SiamDW(BaseTracker): 15 | def __init__(self): 16 | super().__init__("SiamDW") 17 | net_file = path_config.SIAMDW_MODEL 18 | info = edict() 19 | info.arch = "SiamRPNRes22" 20 | info.dataset = "OTB2015" 21 | info.epoch_test = False 22 | info.cls_type = "thinner" 23 | self.tracker = SiamRPN(info) 24 | self.net = models.__dict__[info.arch](anchors_nums=5, cls_type=info.cls_type) 25 | self.net = load_pretrain(self.net, net_file) 26 | self.net.eval() 27 | self.net = self.net.cuda() 28 | 29 | def initialize(self, image_file, box): 30 | image = cv2.imread(image_file) 31 | if len(image.shape) == 2: 32 | image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR) 33 | center = np.array([box[0] + (box[2] - 1) / 2, box[1] + (box[3] - 1) / 2]) 34 | size = np.array([box[2], box[3]]) 35 | self.state = self.tracker.init(image, center, size, self.net) 36 | 37 | def track(self, image_file): 38 | image = cv2.imread(image_file) 39 | if len(image.shape) == 2: 40 | image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR) 41 | self.state = self.tracker.track(self.state, image) 42 | center = self.state["target_pos"] 43 | size = self.state["target_sz"] 44 | bbox = (center[0] - size[0] / 2, center[1] - size[1] / 2, size[0], size[1]) 45 | return bbox 46 | -------------------------------------------------------------------------------- /run_experts.sh: -------------------------------------------------------------------------------- 1 | datasets=("OTB2015" "NFS" "UAV123" "TColor128" "VOT2018" "LaSOT" "Got10K") 2 | 3 | super_fast_experts=("DaSiamRPN" "SiamDW" "SiamRPN" "SPM") 4 | for (( j=0; j<${#datasets[@]}; j++ )); do 5 | for (( i=0; i<${#super_fast_experts[@]}; i++ )); do 6 | python ./track_expert.py -e ${super_fast_experts[$i]} -d ${datasets[$j]} 7 | done 8 | done 9 | 10 | fast_experts=("Ocean" "SiamBAN" "SiamCAR" "SiamFC++" "SiamRPN++") 11 | for (( j=0; j<${#datasets[@]}; j++ )); do 12 | for (( i=0; i<${#fast_experts[@]}; i++ )); do 13 | python ./track_expert.py -e ${fast_experts[$i]} -d ${datasets[$j]} 14 | done 15 | done 16 | 17 | normal_experts=("DiMP" "DROL" "KYS" "PrDiMP" "RPT") 18 | for (( j=0; j<${#datasets[@]}; j++ )); do 19 | for (( i=0; i<${#normal_experts[@]}; i++ )); do 20 | python ./track_expert.py -e ${normal_experts[$i]} -d ${datasets[$j]} 21 | done 22 | done 23 | 24 | siamdw_experts=("SiamDWGroup/SiamFCRes22/OTB" "SiamDWGroup/SiamFCIncep22/OTB" "SiamDWGroup/SiamFCNext22/OTB" "SiamDWGroup/SiamRPNRes22/OTB" "SiamDWGroup/SiamFCRes22/VOT" "SiamDWGroup/SiamFCIncep22/VOT" "SiamDWGroup/SiamFCNext22/VOT" "SiamDWGroup/SiamRPNRes22/VOT") 25 | for (( j=0; j<${#datasets[@]}; j++ )); do 26 | for (( i=0; i<${#siamdw_experts[@]}; i++ )); do 27 | python ./track_expert.py -e ${siamdw_experts[$i]} -d ${datasets[$j]} 28 | done 29 | done 30 | 31 | siamrpn_experts=("SiamRPN++Group/AlexNet/VOT" "SiamRPN++Group/AlexNet/OTB" "SiamRPN++Group/ResNet-50/VOT" "SiamRPN++Group/ResNet-50/OTB" "SiamRPN++Group/ResNet-50/VOTLT" "SiamRPN++Group/MobileNetV2/VOT" "SiamRPN++Group/SiamMask/VOT") 32 | for (( j=0; j<${#datasets[@]}; j++ )); do 33 | for (( i=0; i<${#siamrpn_experts[@]}; i++ )); do 34 | python ./track_expert.py -e ${siamrpn_experts[$i]} -d ${datasets[$j]} 35 | done 36 | done 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # vscode 107 | .vscode/ 108 | 109 | # do not need 110 | external/ 111 | *_results*/ 112 | *.zip 113 | backup/ 114 | weights/ 115 | train_log/ 116 | experiments/ 117 | dump/ 118 | noisy_idx/ -------------------------------------------------------------------------------- /run_baselines.sh: -------------------------------------------------------------------------------- 1 | datasets=("OTB2015" "NFS" "UAV123" "TColor128" "VOT2018" "LaSOT") 2 | baselines=("Random" "WithoutDelay") 3 | 4 | super_fast_experts=("DaSiamRPN" "SiamDW" "SiamRPN" "SPM") 5 | for (( i=0; i<${#datasets[@]}; i++ )); do 6 | for (( j=0; j<${#baselines[@]}; j++ )); do 7 | python ./track_algorithm.py -a ${baselines[$j]} -m SuperFast -d ${datasets[$i]} -e ${super_fast_experts[@]} 8 | done 9 | done 10 | 11 | fast_experts=("Ocean" "SiamBAN" "SiamCAR" "SiamFC++" "SiamRPN++") 12 | for (( i=0; i<${#datasets[@]}; i++ )); do 13 | for (( j=0; j<${#baselines[@]}; j++ )); do 14 | python ./track_algorithm.py -a ${baselines[$j]} -m Fast -d ${datasets[$i]} -e ${fast_experts[@]} 15 | done 16 | done 17 | 18 | normal_experts=("DiMP" "DROL" "KYS" "PrDiMP" "RPT") 19 | for (( i=0; i<${#datasets[@]}; i++ )); do 20 | for (( j=0; j<${#baselines[@]}; j++ )); do 21 | python ./track_algorithm.py -a ${baselines[$j]} -m Normal -d ${datasets[$i]} -e ${normal_experts[@]} 22 | done 23 | done 24 | 25 | siamdw_experts=("SiamDWGroup/SiamFCRes22/OTB" "SiamDWGroup/SiamFCIncep22/OTB" "SiamDWGroup/SiamFCNext22/OTB" "SiamDWGroup/SiamRPNRes22/OTB" "SiamDWGroup/SiamFCRes22/VOT" "SiamDWGroup/SiamFCIncep22/VOT" "SiamDWGroup/SiamFCNext22/VOT" "SiamDWGroup/SiamRPNRes22/VOT") 26 | for (( i=0; i<${#datasets[@]}; i++ )); do 27 | for (( j=0; j<${#baselines[@]}; j++ )); do 28 | python ./track_algorithm.py -a ${baselines[$j]} -m SiamDW -d ${datasets[$i]} -e ${siamdw_experts[@]} 29 | done 30 | done 31 | 32 | siamrpn_experts=("SiamRPN++Group/AlexNet/VOT" "SiamRPN++Group/AlexNet/OTB" "SiamRPN++Group/ResNet-50/VOT" "SiamRPN++Group/ResNet-50/OTB" "SiamRPN++Group/ResNet-50/VOTLT" "SiamRPN++Group/MobileNetV2/VOT" "SiamRPN++Group/SiamMask/VOT") 33 | for (( i=0; i<${#datasets[@]}; i++ )); do 34 | for (( j=0; j<${#baselines[@]}; j++ )); do 35 | python ./track_algorithm.py -a ${baselines[$j]} -m SiamRPN++ -d ${datasets[$i]} -e ${siamrpn_experts[@]} 36 | done 37 | done -------------------------------------------------------------------------------- /experts/ocean.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from tqdm import tqdm 3 | import numpy as np 4 | import torch 5 | import cv2 6 | from easydict import EasyDict as edict 7 | from base_tracker import BaseTracker 8 | import path_config 9 | 10 | sys.path.append("external/TracKit/lib") 11 | from tracker.ocean import Ocean as Tracker 12 | from utils.utils import load_pretrain, cxy_wh_2_rect, get_axis_aligned_bbox 13 | import models.models as models 14 | 15 | 16 | class Ocean(BaseTracker): 17 | def __init__(self): 18 | super(Ocean, self).__init__("Ocean") 19 | 20 | siam_info = edict() 21 | siam_info.arch = "Ocean" 22 | siam_info.dataset = "OTB2015" 23 | siam_info.online = False 24 | siam_info.epoch_test = False 25 | siam_info.TRT = False 26 | siam_info.align = False 27 | 28 | self.siam_tracker = Tracker(siam_info) 29 | self.siam_net = models.__dict__["Ocean"](align=siam_info.align, online=False) 30 | self.siam_net = load_pretrain(self.siam_net, path_config.OCEAN_MODEL) 31 | self.siam_net.eval() 32 | self.siam_net = self.siam_net.cuda() 33 | 34 | # warmup 35 | for i in tqdm(range(100)): 36 | self.siam_net.template(torch.rand(1, 3, 127, 127).cuda()) 37 | self.siam_net.track(torch.rand(1, 3, 255, 255).cuda()) 38 | 39 | def initialize(self, image_file, box): 40 | im = cv2.imread(image_file) 41 | if len(im.shape) == 2: 42 | im = cv2.cvtColor(im, cv2.COLOR_GRAY2BGR) # align with training 43 | 44 | cx, cy, w, h = get_axis_aligned_bbox(box) 45 | target_pos = np.array([cx, cy]) 46 | target_sz = np.array([w, h]) 47 | 48 | self.state = self.siam_tracker.init( 49 | im, target_pos, target_sz, self.siam_net 50 | ) # init tracker 51 | 52 | def track(self, image_file): 53 | im = cv2.imread(image_file) 54 | if len(im.shape) == 2: 55 | im = cv2.cvtColor(im, cv2.COLOR_GRAY2BGR) # align with training 56 | 57 | self.state = self.siam_tracker.track(self.state, im) 58 | location = cxy_wh_2_rect(self.state["target_pos"], self.state["target_sz"]) 59 | return location 60 | -------------------------------------------------------------------------------- /experts/siamrpnpp_group.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import cv2 3 | import torch 4 | from base_tracker import BaseTracker 5 | import path_config 6 | 7 | sys.path.append("external/pysot") 8 | from pysot.core.config import cfg 9 | from pysot.models.model_builder import ModelBuilder 10 | from pysot.tracker.tracker_builder import build_tracker 11 | 12 | 13 | class SiamRPNPPGroup(BaseTracker): 14 | def __init__(self, backbone, target): 15 | super(SiamRPNPPGroup, self).__init__(f"SiamRPN++Group/{backbone}/{target}") 16 | 17 | if backbone == "AlexNet" and target == "OTB": 18 | config = path_config.SIAMRPNPP_ALEXNET_OTB_CONFIG 19 | snapshot = path_config.SIAMRPNPP_ALEXNET_OTB_SNAPSHOT 20 | elif backbone == "AlexNet" and target == "VOT": 21 | config = path_config.SIAMRPNPP_ALEXNET_CONFIG 22 | snapshot = path_config.SIAMRPNPP_ALEXNET_SNAPSHOT 23 | elif backbone == "ResNet-50" and target == "OTB": 24 | config = path_config.SIAMRPNPP_RESNET_OTB_CONFIG 25 | snapshot = path_config.SIAMRPNPP_RESNET_OTB_SNAPSHOT 26 | elif backbone == "ResNet-50" and target == "VOT": 27 | config = path_config.SIAMRPNPP_RESNET_CONFIG 28 | snapshot = path_config.SIAMRPNPP_RESNET_SNAPSHOT 29 | elif backbone == "ResNet-50" and target == "VOTLT": 30 | config = path_config.SIAMRPNPP_RESNET_LT_CONFIG 31 | snapshot = path_config.SIAMRPNPP_RESNET_LT_SNAPSHOT 32 | elif backbone == "MobileNetV2" and target == "VOT": 33 | config = path_config.SIAMRPNPP_MOBILENET_CONFIG 34 | snapshot = path_config.SIAMRPNPP_MOBILENET_SNAPSHOT 35 | elif backbone == "SiamMask" and target == "VOT": 36 | config = path_config.SIAMPRNPP_SIAMMASK_CONFIG 37 | snapshot = path_config.SIAMPRNPP_SIAMMASK_SNAPSHOT 38 | else: 39 | raise ValueError("Invalid backbone and target") 40 | 41 | # load config 42 | cfg.merge_from_file(config) 43 | cfg.CUDA = torch.cuda.is_available() 44 | device = torch.device("cuda" if cfg.CUDA else "cpu") 45 | 46 | # create model 47 | self.model = ModelBuilder() 48 | 49 | # load model 50 | self.model.load_state_dict( 51 | torch.load(snapshot, map_location=lambda storage, loc: storage.cpu()) 52 | ) 53 | self.model.eval().to(device) 54 | 55 | # build tracker 56 | self.tracker = build_tracker(self.model) 57 | 58 | def initialize(self, image_file, box): 59 | image = cv2.imread(image_file) 60 | self.tracker.init(image, box) 61 | 62 | def track(self, image_file): 63 | image = cv2.imread(image_file) 64 | bbox = self.tracker.track(image)["bbox"] 65 | return bbox 66 | -------------------------------------------------------------------------------- /experts/siammcf.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" 4 | os.environ["C_CPP_MIN_LOG_LEVEL"] = "3" 5 | import sys 6 | from base_tracker import BaseTracker 7 | import tensorflow as tf 8 | import path_config 9 | 10 | tf.get_logger().setLevel("INFO") 11 | 12 | sys.path.append("external/siam-mcf") 13 | from src.parse_arguments import parse_arguments 14 | from src.siam_mcf.siam_mcf_tracker import SiamMcfTracker 15 | import src.siamese as siam 16 | from src.region_to_bbox import region_to_bbox 17 | 18 | 19 | class SiamMCF(BaseTracker): 20 | def __init__(self): 21 | super(SiamMCF, self).__init__("SiamMCF") 22 | root_dir = path_config.SIAMMCF_ROOT_DIR 23 | self.hp, self.evaluation, self.env, self.design = parse_arguments(root_dir) 24 | self.final_score_sz = self.hp.response_up * (self.design.score_sz - 1) + 1 25 | # build TF graph once for all 26 | ( 27 | self.filename, 28 | self.image, 29 | self.templates_x, 30 | self.templates_z, 31 | self.scores_list, 32 | ) = siam.build_tracking_graph( 33 | root_dir, self.final_score_sz, self.design, self.env, self.hp 34 | ) 35 | config = tf.ConfigProto() 36 | config.gpu_options.allow_growth = True 37 | self.sess = tf.Session(config=config) 38 | tf.global_variables_initializer().run(session=self.sess) 39 | vars_to_load = [] 40 | for v in tf.global_variables(): 41 | if "postnorm" not in v.name: 42 | vars_to_load.append(v) 43 | 44 | siam_ckpt_name = path_config.SIAMMCF_MODEL 45 | siam_saver = tf.train.Saver(vars_to_load) 46 | siam_saver.restore(self.sess, siam_ckpt_name) 47 | 48 | def initialize(self, image_file, box): 49 | pos_x, pos_y, target_w, target_h = region_to_bbox(box) 50 | self.tracker = SiamMcfTracker( 51 | self.design.context, 52 | self.design.exemplar_sz, 53 | self.design.search_sz, 54 | self.hp.scale_step, 55 | self.hp.scale_num, 56 | self.hp.scale_penalty, 57 | self.hp.scale_lr, 58 | self.hp.window_influence, 59 | self.design.tot_stride, 60 | self.hp.response_up, 61 | self.final_score_sz, 62 | pos_x, 63 | pos_y, 64 | target_w, 65 | target_h, 66 | image_file, 67 | self.sess, 68 | self.templates_z, 69 | self.filename, 70 | ) 71 | 72 | def track(self, image_file): 73 | bbox = self.tracker.track( 74 | image_file, 75 | self.sess, 76 | self.templates_z, 77 | self.templates_x, 78 | self.scores_list, 79 | self.filename, 80 | ) 81 | return bbox 82 | -------------------------------------------------------------------------------- /experts/siamdw_group.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import numpy as np 3 | import cv2 4 | from easydict import EasyDict as edict 5 | from base_tracker import BaseTracker 6 | import path_config 7 | 8 | sys.path.append("external/SiamDW/lib") 9 | from tracker.siamfc import SiamFC 10 | from tracker.siamrpn import SiamRPN 11 | import models.models as models 12 | from utils.utils import load_pretrain 13 | 14 | 15 | class SiamDWGroup(BaseTracker): 16 | def __init__(self, backbone, target): 17 | super(SiamDWGroup, self).__init__(f"SiamDWGroup/{backbone}/{target}") 18 | info = edict() 19 | info.arch = backbone 20 | info.dataset = target 21 | info.epoch_test = False 22 | 23 | if target == "OTB": 24 | info.dataset = "OTB2015" 25 | elif target == "VOT": 26 | info.dataset = "VOT2017" 27 | else: 28 | raise ValueError("Invalid target") 29 | 30 | if backbone == "SiamFCIncep22": 31 | net_file = path_config.SIAMDW_CIRINCEP22_MODEL 32 | self.tracker = SiamFC(info) 33 | self.net = models.__dict__[info.arch]() 34 | elif backbone == "SiamFCNext22": 35 | net_file = path_config.SIAMDW_CIRNEXT22_MODEL 36 | self.tracker = SiamFC(info) 37 | self.net = models.__dict__[info.arch]() 38 | elif backbone == "SiamFCRes22": 39 | net_file = path_config.SIAMDW_CIRESNET22_MODEL 40 | self.tracker = SiamFC(info) 41 | self.net = models.__dict__[info.arch]() 42 | elif backbone == "SiamRPNRes22": 43 | net_file = path_config.SIAMDW_CIRESNET22_RPN_MODEL 44 | info.cls_type = "thinner" 45 | self.tracker = SiamRPN(info) 46 | self.net = models.__dict__[info.arch]( 47 | anchors_nums=5, cls_type=info.cls_type 48 | ) 49 | else: 50 | raise ValueError("Invalid backbone") 51 | 52 | self.net = load_pretrain(self.net, net_file) 53 | self.net.eval() 54 | self.net = self.net.cuda() 55 | 56 | def initialize(self, image_file, box): 57 | image = cv2.imread(image_file) 58 | if len(image.shape) == 2: 59 | image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR) 60 | center = np.array([box[0] + (box[2] - 1) / 2, box[1] + (box[3] - 1) / 2]) 61 | size = np.array([box[2], box[3]]) 62 | self.state = self.tracker.init(image, center, size, self.net) 63 | 64 | def track(self, image_file): 65 | image = cv2.imread(image_file) 66 | if len(image.shape) == 2: 67 | image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR) 68 | self.state = self.tracker.track(self.state, image) 69 | center = self.state["target_pos"] 70 | size = self.state["target_sz"] 71 | bbox = (center[0] - size[0] / 2, center[1] - size[1] / 2, size[0], size[1]) 72 | return bbox 73 | -------------------------------------------------------------------------------- /datasets/got10kdataset.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | import path_config 4 | from datasets.data import Sequence, BaseDataset, SequenceList 5 | 6 | 7 | def GOT10KDatasetTest(): 8 | """ GOT-10k official test set""" 9 | return GOT10KDatasetClass("test").get_sequence_list() 10 | 11 | 12 | def GOT10KDatasetVal(): 13 | """ GOT-10k official val set""" 14 | return GOT10KDatasetClass("val").get_sequence_list() 15 | 16 | 17 | class GOT10KDatasetClass(BaseDataset): 18 | """ GOT-10k dataset. 19 | 20 | Publication: 21 | GOT-10k: A Large High-Diversity Benchmark for Generic Object Tracking in the Wild 22 | Lianghua Huang, Xin Zhao, and Kaiqi Huang 23 | arXiv:1810.11981, 2018 24 | https://arxiv.org/pdf/1810.11981.pdf 25 | 26 | Download dataset from http://got-10k.aitestunion.com/downloads 27 | """ 28 | 29 | def __init__(self, split): 30 | """ 31 | args: 32 | split - Split to use. Can be i) 'test': official test set, ii) 'val': official val set, and iii) 'ltrval': 33 | a custom validation set, a subset of the official train set. 34 | """ 35 | super().__init__() 36 | # Split can be test, val, or ltrval 37 | if split == "test" or split == "val": 38 | self.base_path = os.path.join(path_config.GOT10K_PATH, split) 39 | else: 40 | self.base_path = os.path.join(path_config.GOT10K_PATH, "train") 41 | 42 | self.sequence_list = self._get_sequence_list(split) 43 | self.split = split 44 | 45 | def get_sequence_list(self): 46 | return SequenceList([self._construct_sequence(s) for s in self.sequence_list]) 47 | 48 | def _construct_sequence(self, sequence_name): 49 | anno_path = "{}/{}/groundtruth.txt".format(self.base_path, sequence_name) 50 | try: 51 | ground_truth_rect = np.loadtxt(str(anno_path), dtype=np.float64) 52 | except Exception: 53 | ground_truth_rect = np.loadtxt( 54 | str(anno_path), delimiter=",", dtype=np.float64 55 | ) 56 | 57 | frames_path = "{}/{}".format(self.base_path, sequence_name) 58 | frame_list = [ 59 | frame for frame in os.listdir(frames_path) if frame.endswith(".jpg") 60 | ] 61 | frame_list.sort(key=lambda f: int(f[:-4])) 62 | frames_list = [os.path.join(frames_path, frame) for frame in frame_list] 63 | 64 | return Sequence( 65 | sequence_name, frames_list, "got10k", ground_truth_rect.reshape(-1, 4) 66 | ) 67 | 68 | def __len__(self): 69 | """Overload this function in your evaluation. This should return number of sequences in the evaluation """ 70 | return len(self.sequence_list) 71 | 72 | def _get_sequence_list(self, split): 73 | with open("{}/list.txt".format(self.base_path)) as f: 74 | sequence_list = f.read().splitlines() 75 | 76 | return sequence_list 77 | -------------------------------------------------------------------------------- /track_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import multiprocessing 4 | from itertools import product 5 | import random 6 | import numpy as np 7 | import torch 8 | from select_options import select_datasets 9 | 10 | random.seed(42) 11 | np.random.seed(42) 12 | torch.random.manual_seed(42) 13 | 14 | 15 | def run_sequence(dataset_name, seq, tracker, experts=None, debug=False): 16 | """Runs a tracker on a sequence.""" 17 | 18 | dataset_path = "{}/{}".format(tracker.results_dir, dataset_name) 19 | os.makedirs(dataset_path, exist_ok=True) 20 | 21 | base_results_path = "{}/{}".format(dataset_path, seq.name) 22 | results_path = "{}.txt".format(base_results_path) 23 | times_path = "{}_time.txt".format(base_results_path) 24 | weights_path = "{}_weight.txt".format(base_results_path) 25 | offline_path = "{}_offline.pkl".format(base_results_path) 26 | 27 | if os.path.isfile(results_path): 28 | return 29 | 30 | print("Tracker: {}, Sequence: {}".format(tracker.name, seq.name)) 31 | 32 | if debug: 33 | tracked_bb, offline_bb, weights, exec_times = tracker.run( 34 | dataset_name, seq, experts 35 | ) 36 | else: 37 | try: 38 | tracked_bb, offline_bb, weights, exec_times = tracker.run( 39 | dataset_name, seq, experts 40 | ) 41 | except Exception as e: 42 | print(e) 43 | return 44 | 45 | tracked_bb = np.array(tracked_bb).astype(float) 46 | exec_times = np.array(exec_times).astype(float) 47 | 48 | if experts is not None: 49 | print( 50 | "FPS: {} Anchor: {}".format( 51 | len(exec_times) / exec_times.sum(), 52 | (sum(x is not None for x in offline_bb) + 1) / len(tracked_bb), 53 | ) 54 | ) 55 | else: 56 | print("FPS: {}".format(len(exec_times) / exec_times.sum())) 57 | if not debug: 58 | np.savetxt(results_path, tracked_bb, delimiter="\t", fmt="%f") 59 | np.savetxt(times_path, exec_times, delimiter="\t", fmt="%f") 60 | if experts is not None: 61 | np.savetxt(weights_path, weights, delimiter="\t", fmt="%f") 62 | with open(offline_path, "wb") as fp: 63 | pickle.dump(offline_bb, fp) 64 | 65 | 66 | def run_dataset(dataset, dataset_name, trackers, experts=None, threads=0, debug=False): 67 | """Runs a list of experts on a dataset. 68 | args: 69 | dataset: List of Sequence instances, forming a dataset. 70 | experts: List of Tracker instances. 71 | debug: Debug level. 72 | """ 73 | multiprocessing.set_start_method("spawn", force=True) 74 | 75 | if threads == 0: 76 | for seq in dataset: 77 | for tracker_info in trackers: 78 | run_sequence( 79 | dataset_name, seq, tracker_info, experts=experts, debug=debug 80 | ) 81 | else: 82 | param_list = [ 83 | (dataset_name, seq, tracker_info, experts, debug) 84 | for seq, tracker_info in product(dataset, trackers) 85 | ] 86 | with multiprocessing.Pool(processes=threads) as pool: 87 | pool.starmap(run_sequence, param_list) 88 | print("Done") 89 | 90 | 91 | def run(tracker, dataset_name, experts=None, debug=False): 92 | dataset = select_datasets(dataset_name) 93 | 94 | run_dataset( 95 | dataset, dataset_name, [tracker], experts=experts, threads=0, debug=debug 96 | ) 97 | -------------------------------------------------------------------------------- /experts/siamrcnn.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from types import SimpleNamespace 3 | from PIL import Image 4 | from base_tracker import BaseTracker 5 | 6 | sys.path.append("external/SiamR-CNN/") 7 | sys.path.append("external/SiamR-CNN/tensorpack") 8 | sys.path.append("external/SiamR-CNN/got10k-toolkit") 9 | from tracking.argmax_tracker import ArgmaxTracker 10 | from tracking.three_stage_tracker import ThreeStageTracker 11 | 12 | 13 | def build_tracker(args): 14 | if args.tracker == "ArgmaxTracker": 15 | return ArgmaxTracker() 16 | elif args.tracker == "ThreeStageTracker": 17 | pass 18 | else: 19 | assert False, ("Unknown tracker", args.tracker) 20 | 21 | tracklet_param_str = ( 22 | str(args.tracklet_distance_threshold) 23 | + "_" 24 | + str(args.tracklet_merging_threshold) 25 | + "_" 26 | + str(args.tracklet_merging_second_best_relative_threshold) 27 | ) 28 | if args.n_proposals is not None: 29 | tracklet_param_str += "_proposals" + str(args.n_proposals) 30 | if args.resolution is not None: 31 | tracklet_param_str += "_resolution-" + str(args.resolution) 32 | if args.model != "best": 33 | tracklet_param_str = args.model + "_" + tracklet_param_str 34 | if args.visualize_tracker: 35 | tracklet_param_str2 = "viz_" + tracklet_param_str 36 | else: 37 | tracklet_param_str2 = tracklet_param_str 38 | param_str = ( 39 | tracklet_param_str2 40 | + "_" 41 | + str(args.ff_gt_score_weight) 42 | + "_" 43 | + str(args.ff_gt_tracklet_score_weight) 44 | + "_" 45 | + str(args.location_score_weight) 46 | ) 47 | 48 | name = "ThreeStageTracker_" + param_str 49 | tracker = ThreeStageTracker( 50 | tracklet_distance_threshold=args.tracklet_distance_threshold, 51 | tracklet_merging_threshold=args.tracklet_merging_threshold, 52 | tracklet_merging_second_best_relative_threshold=args.tracklet_merging_second_best_relative_threshold, 53 | ff_gt_score_weight=args.ff_gt_score_weight, 54 | ff_gt_tracklet_score_weight=args.ff_gt_tracklet_score_weight, 55 | location_score_weight=args.location_score_weight, 56 | name=name, 57 | do_viz=args.visualize_tracker, 58 | model=args.model, 59 | n_proposals=args.n_proposals, 60 | resolution=args.resolution, 61 | ) 62 | return tracker 63 | 64 | 65 | class SiamRCNN(BaseTracker): 66 | def __init__(self): 67 | super(SiamRCNN, self).__init__("SiamR-CNN") 68 | conf = { 69 | "tracklet_distance_threshold": 0.06, 70 | "tracklet_merging_threshold": 0.3, 71 | "tracklet_merging_second_best_relative_threshold": 0.3, 72 | "ff_gt_score_weight": 0.1, 73 | "ff_gt_tracklet_score_weight": 0.9, 74 | "location_score_weight": 7.0, 75 | "model": "best", 76 | "tracker": "ThreeStageTracker", 77 | "n_proposals": None, 78 | "resolution": None, 79 | "visualize_tracker": False, 80 | } 81 | self.args = SimpleNamespace(**conf) 82 | 83 | def initialize(self, image_file, box): 84 | image = Image.open(image_file).convert("RGB") 85 | self.tracker = build_tracker(self.args) 86 | self.tracker.init(image, box) 87 | 88 | def track(self, image_file): 89 | image = Image.open(image_file).convert("RGB") 90 | return self.tracker.update(image) 91 | -------------------------------------------------------------------------------- /algorithms/mcct.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import numpy as np 3 | from base_tracker import BaseTracker 4 | 5 | sys.path.append("external/pyCFTrackers") 6 | sys.path.append("external/pysot-toolkit") 7 | from pysot.utils import overlap_ratio 8 | from cftracker.config.mccth_staple_config import MCCTHOTBConfig 9 | 10 | 11 | class Expert: 12 | def __init__(self): 13 | self.rect_positions = [] 14 | self.centers = [] 15 | self.smooth_scores = [] 16 | 17 | 18 | class MCCT(BaseTracker): 19 | def __init__(self, n_experts, mode, mu): 20 | super(MCCT, self).__init__(f"MCCT/{mode}/{mu:.2f}") 21 | 22 | self.period = MCCTHOTBConfig().period 23 | self.expert_num = n_experts 24 | self.mu = mu 25 | 26 | def initialize(self, image_file, box): 27 | weight_num = np.arange(self.period) 28 | self.weight = 1.1 ** weight_num 29 | self.psr_score = [0] 30 | self.id_ensemble = np.ones(self.expert_num) 31 | self.frame_idx = 0 32 | self.experts = [Expert() for _ in range(self.expert_num)] 33 | 34 | center = box[:2] + box[2:] / 2 35 | for i in range(self.expert_num): 36 | self.experts[i].rect_positions.append(box) 37 | self.experts[i].smooth_scores.append(1) 38 | self.experts[i].centers.append(center) 39 | 40 | def track(self, image_file, boxes): 41 | self.frame_idx += 1 42 | 43 | for i in range(self.expert_num): 44 | center = boxes[i][:2] + boxes[i][2:] / 2 45 | pre_center = self.experts[i].centers[-1] 46 | self.experts[i].rect_positions.append(boxes[i]) 47 | self.experts[i].centers.append(center) 48 | 49 | smooth = np.linalg.norm(center - pre_center) 50 | avg_dim = np.sum(boxes[i][2:]) / 2 51 | self.experts[i].smooth_scores.append(np.exp(-((smooth / avg_dim) ** 2) / 2)) 52 | 53 | if self.frame_idx >= self.period - 1: 54 | for i in range(self.expert_num): 55 | self.id_ensemble[i] = self.robustness_eva(i) 56 | 57 | idx = np.argmax(self.id_ensemble) 58 | self.box = boxes[idx] 59 | else: 60 | self.box = boxes[0] 61 | 62 | return (self.box, [self.box], self.id_ensemble) 63 | 64 | def robustness_eva(self, num): 65 | overlap_score = np.zeros((self.period, self.expert_num)) 66 | src_bboxes = np.array(self.experts[num].rect_positions[-self.period :]) 67 | for i in range(self.expert_num): 68 | target_bboxes = np.array(self.experts[i].rect_positions[-self.period :]) 69 | overlaps = overlap_ratio(src_bboxes, target_bboxes) 70 | overlap_score[:, i] = np.exp(-((1 - overlaps) ** 2)) 71 | avg_overlap = np.mean(overlap_score, axis=1) 72 | expert_avg_overlap = np.mean(overlap_score, axis=0) 73 | var_overlap = np.sqrt( 74 | np.mean((overlap_score - expert_avg_overlap[np.newaxis, :]) ** 2, axis=1) 75 | ) 76 | norm_factor = 1 / np.sum(self.weight) 77 | weight_avg_overlap = norm_factor * (self.weight.dot(avg_overlap)) 78 | weight_var_overlap = norm_factor * (self.weight.dot(var_overlap)) 79 | pair_score = weight_avg_overlap / (weight_var_overlap + 0.008) 80 | 81 | smooth_score = self.experts[num].smooth_scores[-self.period :] 82 | self_score = norm_factor * self.weight.dot(smooth_score) 83 | 84 | reliability = self.mu * pair_score + (1 - self.mu) * self_score 85 | return reliability 86 | -------------------------------------------------------------------------------- /install_for_experts.sh: -------------------------------------------------------------------------------- 1 | # install libraries 2 | conda install -y tensorflow-gpu==1.14 3 | pip install matplotlib pandas tqdm visdom scikit-image tikzplotlib pycocotools lvis jpeg4py pyyaml yacs colorama tb-nightly future optuna shapely scipy easydict tensorboardX mpi4py==2.0.0 gaft hyperopt ray==0.6.3 requests pillow msgpack msgpack_numpy tabulate xmltodict zmq annoy wget protobuf cupy-cuda101 mxnet-cu101 h5py pyzmq numba ipdb loguru scikit-learn got10k 4 | 5 | # make directory for external libraries 6 | cd external 7 | 8 | # clone experts 9 | # ATOM 10 | git clone https://github.com/ClementPinard/Pytorch-Correlation-extension 11 | # DaSiamRPN 12 | git clone https://github.com/songheony/DaSiamRPN 13 | # DROL 14 | git clone https://github.com/shallowtoil/DROL 15 | # GradNet 16 | git clone https://github.com/LPXTT/GradNet-Tensorflow 17 | # MemDTC 18 | git clone https://github.com/skyoung/MemDTC 19 | # MemTrack 20 | git clone https://github.com/skyoung/MemTrack 21 | # Ocean 22 | git clone https://github.com/researchmm/TracKit 23 | # ROAM 24 | git clone https://github.com/skyoung/ROAM 25 | # RPT 26 | git clone https://github.com/songheony/RPT 27 | # SiamBAN 28 | git clone https://github.com/hqucv/siamban 29 | # SiamCAR 30 | git clone https://github.com/ohhhyeahhh/SiamCAR 31 | # SiamDW 32 | git clone https://github.com/researchmm/SiamDW 33 | # SiamFC 34 | git clone https://github.com/got-10k/siamfc 35 | # SiamFC++ 36 | git clone https://github.com/MegviiDetection/video_analyst 37 | # SiamMCF 38 | git clone https://github.com/hmorimitsu/siam-mcf 39 | # SiamR-CNN 40 | git clone https://github.com/VisualComputingInstitute/SiamR-CNN 41 | # SiamRPN 42 | git clone https://github.com/huanglianghua/siamrpn-pytorch 43 | # SiamRPN++ 44 | git clone https://github.com/STVIR/pysot 45 | # SPM 46 | git clone https://github.com/songheony/SPM-Tracker 47 | # Staple 48 | git clone https://github.com/wwdguu/pyCFTrackers 49 | # THOR 50 | git clone https://github.com/xl-sr/THOR 51 | 52 | # edit network path of ATOM, DiMP, PrDiMP, KYS 53 | cd pytracking 54 | cp ../../local.py pytracking/evaluation/local.py 55 | python -c "from ltr.admin.environment import create_default_local_file; create_default_local_file()" 56 | cd ../ 57 | 58 | # For ATOM 59 | cd pytracking 60 | git submodule update --init 61 | cd ../ 62 | cd Pytorch-Correlation-extension 63 | pip install -e . 64 | cd ../ 65 | 66 | # For DROL 67 | cd DROL 68 | python setup.py build_ext --inplace 69 | cd ../ 70 | 71 | # For RLS-RTMDNet 72 | cd RLS-RTMDNet/modules/roi_align 73 | python setup.py build_ext --inplace 74 | cd ../../../ 75 | 76 | # For RPT 77 | cd RPT 78 | python setup.py build_ext --inplace 79 | cd siamreppoints 80 | python ./setup.py build_ext --inplace 81 | cd ../../ 82 | 83 | # For SiamBAN 84 | cd siamban 85 | python setup.py build_ext --inplace 86 | cd ../ 87 | 88 | # For SiamR-CNN 89 | cd SiamR-CNN 90 | git clone https://github.com/pvoigtlaender/got10k-toolkit.git 91 | git clone https://github.com/tensorpack/tensorpack.git 92 | cd tensorpack 93 | git checkout d24a9230d50b1dea1712a4c2765a11876f1e193c 94 | cd ../../ 95 | 96 | # For SiamRPN++ 97 | cd pysot 98 | python setup.py build_ext --inplace 99 | cd ../ 100 | 101 | # For SPM 102 | cd SPM-Tracker 103 | bash compile.sh 104 | cd ../ 105 | 106 | # For Staple 107 | cd pyCFTrackers/lib/pysot/utils 108 | python setup.py build_ext --inplace 109 | cd ../../../../ 110 | cd pyCFTrackers/lib/eco/features/ 111 | python setup.py build_ext --inplace 112 | cd ../../../../ 113 | 114 | # For THOR 115 | cd THOR 116 | bash benchmark/make_toolkits.sh 117 | cd ../ -------------------------------------------------------------------------------- /base_tracker.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import time 3 | import os 4 | import numpy as np 5 | import cv2 6 | import path_config 7 | 8 | 9 | class BaseTracker(object): 10 | """Base class for all trackers.""" 11 | 12 | def __init__(self, name): 13 | self.name = name 14 | self.results_dir = "{}/{}".format(path_config.RESULTS_PATH, self.name) 15 | if not os.path.exists(self.results_dir): 16 | os.makedirs(self.results_dir) 17 | 18 | def initialize(self, image, state): 19 | """Overload this function in your tracker. This should initialize the model.""" 20 | raise NotImplementedError 21 | 22 | def track(self, image, boxes): 23 | """Overload this function in your tracker. This should track in the frame and update the model.""" 24 | raise NotImplementedError 25 | 26 | def track_sequence(self, dataset_name, sequence, experts=None): 27 | """Run tracker on a sequence.""" 28 | 29 | if experts is not None: 30 | boxes = np.zeros((len(experts), len(sequence.ground_truth_rect), 4)) 31 | tracker_times = np.zeros((len(experts), len(sequence.ground_truth_rect))) 32 | for n, tracker_name in enumerate(experts): 33 | results_dir = "{}/{}/{}".format( 34 | path_config.RESULTS_PATH, tracker_name, dataset_name 35 | ) 36 | base_results_path = "{}/{}".format(results_dir, sequence.name) 37 | results_path = "{}.txt".format(base_results_path) 38 | tracker_traj = np.loadtxt(results_path, delimiter="\t", dtype=float) 39 | times_path = "{}_time.txt".format(base_results_path) 40 | tracker_time = np.loadtxt(times_path, delimiter="\t", dtype=float) 41 | boxes[n] = tracker_traj 42 | tracker_times[n] = tracker_time 43 | 44 | times = [] 45 | start_time = time.time() 46 | self.initialize(sequence.frames[0], np.array(sequence.init_bbox())) 47 | init_time = time.time() - start_time 48 | times.append(init_time) 49 | 50 | # Track 51 | tracked_bb = [sequence.init_bbox()] 52 | offline_bb = [] 53 | weights = [] 54 | for n, frame in enumerate(sequence.frames[1:]): 55 | if experts is not None: 56 | start_time = time.time() 57 | state, offline, weight = self.track(frame, boxes[:, n + 1, :]) 58 | calc_time = time.time() - start_time 59 | last_time = np.max(tracker_times[:, n + 1]) 60 | duration = calc_time + last_time 61 | else: 62 | start_time = time.time() 63 | state = self.track(frame) 64 | duration = time.time() - start_time 65 | offline = None 66 | weight = None 67 | times.append(duration) 68 | tracked_bb.append(copy.deepcopy(state)) 69 | offline_bb.append(copy.deepcopy(offline)) 70 | weights.append(copy.deepcopy(weight)) 71 | 72 | return tracked_bb, offline_bb, weights, times 73 | 74 | def _read_image(self, image_file: str): 75 | return cv2.cvtColor(cv2.imread(image_file), cv2.COLOR_BGR2RGB) 76 | 77 | def run(self, dataset_name, seq, trackers): 78 | """Run tracker on sequence. 79 | args: 80 | dataset_name: Name of the dataset 81 | seq: Sequence to run the tracker on. 82 | """ 83 | 84 | output_bb, offline_bb, weights, execution_times = self.track_sequence( 85 | dataset_name, seq, trackers 86 | ) 87 | 88 | return output_bb, offline_bb, weights, execution_times 89 | -------------------------------------------------------------------------------- /datasets/votdataset.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import path_config 3 | from datasets.data import Sequence, BaseDataset, SequenceList 4 | 5 | 6 | def VOTDataset(): 7 | return VOTDatasetClass().get_sequence_list() 8 | 9 | 10 | class VOTDatasetClass(BaseDataset): 11 | """VOT2018 dataset 12 | 13 | Publication: 14 | The sixth Visual Object Tracking VOT2018 challenge results. 15 | Matej Kristan, Ales Leonardis, Jiri Matas, Michael Felsberg, Roman Pfugfelder, Luka Cehovin Zajc, Tomas Vojir, 16 | Goutam Bhat, Alan Lukezic et al. 17 | ECCV, 2018 18 | https://prints.vicos.si/publications/365 19 | 20 | Download the dataset from http://www.votchallenge.net/vot2018/dataset.html""" 21 | 22 | def __init__(self): 23 | super().__init__() 24 | self.base_path = path_config.VOT_PATH 25 | self.sequence_list = self._get_sequence_list() 26 | 27 | def get_sequence_list(self): 28 | return SequenceList([self._construct_sequence(s) for s in self.sequence_list]) 29 | 30 | def _construct_sequence(self, sequence_name): 31 | sequence_path = sequence_name 32 | nz = 8 33 | ext = "jpg" 34 | start_frame = 1 35 | 36 | anno_path = "{}/{}/groundtruth.txt".format(self.base_path, sequence_name) 37 | try: 38 | ground_truth_rect = np.loadtxt(str(anno_path), dtype=np.float64) 39 | except Exception: 40 | ground_truth_rect = np.loadtxt( 41 | str(anno_path), delimiter=",", dtype=np.float64 42 | ) 43 | 44 | end_frame = ground_truth_rect.shape[0] 45 | 46 | frames = [ 47 | "{base_path}/{sequence_path}/{frame:0{nz}}.{ext}".format( 48 | base_path=self.base_path, 49 | sequence_path=sequence_path, 50 | frame=frame_num, 51 | nz=nz, 52 | ext=ext, 53 | ) 54 | for frame_num in range(start_frame, end_frame + 1) 55 | ] 56 | 57 | # Convert gt 58 | if ground_truth_rect.shape[1] > 4: 59 | gt_x_all = ground_truth_rect[:, [0, 2, 4, 6]] 60 | gt_y_all = ground_truth_rect[:, [1, 3, 5, 7]] 61 | 62 | x1 = np.amin(gt_x_all, 1).reshape(-1, 1) 63 | y1 = np.amin(gt_y_all, 1).reshape(-1, 1) 64 | x2 = np.amax(gt_x_all, 1).reshape(-1, 1) 65 | y2 = np.amax(gt_y_all, 1).reshape(-1, 1) 66 | 67 | ground_truth_rect = np.concatenate((x1, y1, x2 - x1, y2 - y1), 1) 68 | return Sequence(sequence_name, frames, "vot", ground_truth_rect) 69 | 70 | def __len__(self): 71 | return len(self.sequence_list) 72 | 73 | def _get_sequence_list(self): 74 | sequence_list = [ 75 | "ants1", 76 | "ants3", 77 | "bag", 78 | "ball1", 79 | "ball2", 80 | "basketball", 81 | "birds1", 82 | "blanket", 83 | "bmx", 84 | "bolt1", 85 | "bolt2", 86 | "book", 87 | "butterfly", 88 | "car1", 89 | "conduction1", 90 | "crabs1", 91 | "crossing", 92 | "dinosaur", 93 | "drone_across", 94 | "drone_flip", 95 | "drone1", 96 | "fernando", 97 | "fish1", 98 | "fish2", 99 | "fish3", 100 | "flamingo1", 101 | "frisbee", 102 | "girl", 103 | "glove", 104 | "godfather", 105 | "graduate", 106 | "gymnastics1", 107 | "gymnastics2", 108 | "gymnastics3", 109 | "hand", 110 | "handball1", 111 | "handball2", 112 | "helicopter", 113 | "iceskater1", 114 | "iceskater2", 115 | "leaves", 116 | "matrix", 117 | "motocross1", 118 | "motocross2", 119 | "nature", 120 | "pedestrian1", 121 | "rabbit", 122 | "racing", 123 | "road", 124 | "shaking", 125 | "sheep", 126 | "singer2", 127 | "singer3", 128 | "soccer1", 129 | "soccer2", 130 | "soldier", 131 | "tiger", 132 | "traffic", 133 | "wiper", 134 | "zebrafish1", 135 | ] 136 | 137 | return sequence_list 138 | -------------------------------------------------------------------------------- /path_config.py: -------------------------------------------------------------------------------- 1 | RESULTS_PATH = "tracking_results" 2 | EVALUATION_PATH = "evaluation_results" 3 | VISUALIZATION_PATH = "visualization_results" 4 | 5 | # Dataset 6 | GOT10K_PATH = "/home/heonsong/Disk2/Dataset/Got10K" 7 | LASOT_PATH = "/home/heonsong/Disk2/Dataset/LaSOT" 8 | NFS_PATH = "/home/heonsong/Disk2/Dataset/NFS" 9 | OTB_PATH = "/home/heonsong/Disk2/Dataset/OTB" 10 | OXUVA_PATH = "/home/heonsong/Disk2/Dataset/OxUvA" 11 | TPL_PATH = "/home/heonsong/Disk2/Dataset/TColor128" 12 | UAV_PATH = "/home/heonsong/Disk2/Dataset/UAV123" 13 | TRACKINGNET_PATH = "/home/heonsong/Disk2/Dataset/TrackingNet" 14 | VOT_PATH = "/home/heonsong/Disk2/Dataset/VOT2018" 15 | OTB_NOISY_PATH = "noisy_idx" 16 | 17 | # DaSiamRPN 18 | DASIAMRPN_MODEL = "weights/DaSiamRPN/SiamRPNOTB.model" 19 | 20 | # DROL 21 | DROL_CONFIG = "weights/DROL/siamrpn_r50_l234_dwxcorr_otb/config.yaml" 22 | DROL_SNAPSHOT = "weights/DROL/siamrpn_r50_l234_dwxcorr_otb/model.pth" 23 | 24 | # GradNet 25 | GRADNET_MODEL = "weights/GradNet/ckpt/base_l5_1t_49/model_epoch49.ckpt" 26 | 27 | # MemDTC 28 | MEMDTC_MODEL = "weights/MemDTC/models" 29 | 30 | # MemTrack 31 | MEMTRACK_MODEL = "weights/MemTrack/models" 32 | 33 | # Ocean 34 | OCEAN_MODEL = "weights/Ocean/OceanO.pth" 35 | 36 | # RLS-RTMDNet 37 | RLS_RTMDNET_MODEL = "weights/RLS-RTMDNet/rt-mdnet.pth" 38 | 39 | # ROAM 40 | ROAM_FEAT_DIR = "" 41 | ROAM_MODEL_DIR = "" 42 | 43 | # RPT 44 | RPT_CONFIG = "weights/RPT/config_vot2018_offline.yaml" 45 | RPT_SNAPSHOT = "weights/RPT/siamreppoints.model" 46 | 47 | # SiamBAN 48 | SIAMBAN_CONFIG = "weights/SiamBAN/siamban_r50_l234_otb/config.yaml" 49 | SIAMBAN_SNAPSHOT = "weights/SiamBAN/siamban_r50_l234_otb/model.pth" 50 | 51 | # SiamCAR 52 | SIAMCAR_CONFIG = "weights/SiamCAR/siamcar_r50/config.yaml" 53 | SIAMCAR_SNAPSHOT = "weights/SiamCAR/model_general.pth" 54 | 55 | # SiamDW 56 | SIAMDW_MODEL = "weights/SiamDW/CIResNet22_RPN.pth" 57 | SIAMDW_CIRINCEP22_MODEL = "weights/SiamDW/CIRIncep22.pth" 58 | SIAMDW_CIRNEXT22_MODEL = "weights/SiamDW/CIRNext22.pth" 59 | SIAMDW_CIRESNET22FC_G_MODEL = "weights/SiamDW/CIResNet22FC_G.pth" 60 | SIAMDW_CIRESNET22_MODEL = "weights/SiamDW/CIResNet22.pth" 61 | SIAMDW_SIAMFCRES22W_MODEL = "weights/SiamDW/SiamFCRes22W.pth" 62 | SIAMDW_CIRESNET22_RPN_MODEL = "weights/SiamDW/CIResNet22_RPN.pth" 63 | 64 | # SiamFC 65 | SIAMFC_MODEL = "weights/SiamFC/model.pth" 66 | 67 | # SiamFC++ 68 | SIAMFCPP_CONFIG = "weights/SiamFC++/otb/siamfcpp_googlenet-otb.yaml" 69 | 70 | # SiamMFC 71 | SIAMMCF_ROOT_DIR = "external/siam-mcf/" 72 | SIAMMCF_MODEL = "weights/SiamMCF/pretrained/siam_mcf.ckpt-50000" 73 | 74 | # SiamRPN 75 | SIAMRPN_MODEL = "weights/SiamRPN/model.pth" 76 | 77 | # SiamRPN++ 78 | SIAMRPNPP_CONFIG = "weights/SiamRPN++/siamrpn_r50_l234_dwxcorr_otb/config.yaml" 79 | SIAMRPNPP_SNAPSHOT = "weights/SiamRPN++/siamrpn_r50_l234_dwxcorr_otb/model.pth" 80 | SIAMRPNPP_ALEXNET_OTB_CONFIG = "weights/SiamRPN++/siamrpn_alex_dwxcorr_otb/config.yaml" 81 | SIAMRPNPP_ALEXNET_OTB_SNAPSHOT = "weights/SiamRPN++/siamrpn_alex_dwxcorr_otb/model.pth" 82 | SIAMRPNPP_ALEXNET_CONFIG = "weights/SiamRPN++/siamrpn_alex_dwxcorr/config.yaml" 83 | SIAMRPNPP_ALEXNET_SNAPSHOT = "weights/SiamRPN++/siamrpn_alex_dwxcorr/model.pth" 84 | SIAMRPNPP_MOBILENET_CONFIG = ( 85 | "weights/SiamRPN++/siamrpn_mobilev2_l234_dwxcorr/config.yaml" 86 | ) 87 | SIAMRPNPP_MOBILENET_SNAPSHOT = ( 88 | "weights/SiamRPN++/siamrpn_mobilev2_l234_dwxcorr/model.pth" 89 | ) 90 | SIAMRPNPP_RESNET_LT_CONFIG = "weights/SiamRPN++/siamrpn_r50_l234_dwxcorr_lt/config.yaml" 91 | SIAMRPNPP_RESNET_LT_SNAPSHOT = "weights/SiamRPN++/siamrpn_r50_l234_dwxcorr_lt/model.pth" 92 | SIAMRPNPP_RESNET_OTB_CONFIG = ( 93 | "weights/SiamRPN++/siamrpn_r50_l234_dwxcorr_otb/config.yaml" 94 | ) 95 | SIAMRPNPP_RESNET_OTB_SNAPSHOT = ( 96 | "weights/SiamRPN++/siamrpn_r50_l234_dwxcorr_otb/model.pth" 97 | ) 98 | SIAMRPNPP_RESNET_CONFIG = "weights/SiamRPN++/siamrpn_r50_l234_dwxcorr/config.yaml" 99 | SIAMRPNPP_RESNET_SNAPSHOT = "weights/SiamRPN++/siamrpn_r50_l234_dwxcorr/model.pth" 100 | SIAMPRNPP_SIAMMASK_CONFIG = "weights/SiamRPN++/siammask_r50_l3/config.yaml" 101 | SIAMPRNPP_SIAMMASK_SNAPSHOT = "weights/SiamRPN++/siammask_r50_l3/model.pth" 102 | 103 | # SPM 104 | SPM_CONFIG = "weights/SPM/alexnet_c42_otb.yaml" 105 | 106 | # THOR 107 | THOR_CONFIG = "weights/THOR" 108 | THOR_SIAMFC_MODEL = "weights/THOR/SiamFC/model.pth" 109 | THOR_SIAMRPN_MODEL = "weights/THOR/SiamRPN/SiamRPNBIG.model" 110 | THOR_SIAMMASK_MODEL = "weights/THOR/SiamMask/model.pth" 111 | 112 | # TRAS 113 | TRAS_MODEL = "weights/TRAS/Student.weights" 114 | -------------------------------------------------------------------------------- /datasets/trackingnetdataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import pandas as pd 4 | import path_config 5 | from datasets.data import Sequence, BaseDataset, SequenceList 6 | 7 | 8 | def load_text_numpy(path, delimiter, dtype): 9 | if isinstance(delimiter, (tuple, list)): 10 | for d in delimiter: 11 | try: 12 | ground_truth_rect = np.loadtxt(path, delimiter=d, dtype=dtype) 13 | return ground_truth_rect 14 | except Exception: 15 | pass 16 | 17 | raise Exception("Could not read file {}".format(path)) 18 | else: 19 | ground_truth_rect = np.loadtxt(path, delimiter=delimiter, dtype=dtype) 20 | return ground_truth_rect 21 | 22 | 23 | def load_text_pandas(path, delimiter, dtype): 24 | if isinstance(delimiter, (tuple, list)): 25 | for d in delimiter: 26 | try: 27 | ground_truth_rect = pd.read_csv( 28 | path, 29 | delimiter=d, 30 | header=None, 31 | dtype=dtype, 32 | na_filter=False, 33 | low_memory=False, 34 | ).values 35 | return ground_truth_rect 36 | except Exception: 37 | pass 38 | 39 | raise Exception("Could not read file {}".format(path)) 40 | else: 41 | ground_truth_rect = pd.read_csv( 42 | path, 43 | delimiter=delimiter, 44 | header=None, 45 | dtype=dtype, 46 | na_filter=False, 47 | low_memory=False, 48 | ).values 49 | return ground_truth_rect 50 | 51 | 52 | def load_text(path, delimiter=" ", dtype=np.float32, backend="numpy"): 53 | if backend == "numpy": 54 | return load_text_numpy(path, delimiter, dtype) 55 | elif backend == "pandas": 56 | return load_text_pandas(path, delimiter, dtype) 57 | 58 | 59 | def TrackingNetDataset(): 60 | return TrackingNetClass().get_sequence_list() 61 | 62 | 63 | class TrackingNetClass(BaseDataset): 64 | """ TrackingNet test set. 65 | 66 | Publication: 67 | TrackingNet: A Large-Scale Dataset and Benchmark for Object Tracking in the Wild. 68 | Matthias Mueller,Adel Bibi, Silvio Giancola, Salman Al-Subaihi and Bernard Ghanem 69 | ECCV, 2018 70 | https://ivul.kaust.edu.sa/Documents/Publications/2018/TrackingNet%20A%20Large%20Scale%20Dataset%20and%20Benchmark%20for%20Object%20Tracking%20in%20the%20Wild.pdf 71 | 72 | Download the dataset using the toolkit https://github.com/SilvioGiancola/TrackingNet-devkit. 73 | """ 74 | 75 | def __init__(self): 76 | super().__init__() 77 | self.base_path = path_config.TRACKINGNET_PATH 78 | 79 | sets = "TEST" 80 | if not isinstance(sets, (list, tuple)): 81 | if sets == "TEST": 82 | sets = ["TEST"] 83 | elif sets == "TRAIN": 84 | sets = ["TRAIN_{}".format(i) for i in range(5)] 85 | 86 | self.sequence_list = self._list_sequences(self.base_path, sets) 87 | 88 | def get_sequence_list(self): 89 | return SequenceList( 90 | [ 91 | self._construct_sequence(set, seq_name) 92 | for set, seq_name in self.sequence_list 93 | ] 94 | ) 95 | 96 | def _construct_sequence(self, set, sequence_name): 97 | anno_path = "{}/{}/anno/{}.txt".format(self.base_path, set, sequence_name) 98 | 99 | ground_truth_rect = load_text( 100 | str(anno_path), delimiter=",", dtype=np.float64, backend="numpy" 101 | ) 102 | 103 | frames_path = "{}/{}/frames/{}".format(self.base_path, set, sequence_name) 104 | frame_list = [ 105 | frame for frame in os.listdir(frames_path) if frame.endswith(".jpg") 106 | ] 107 | frame_list.sort(key=lambda f: int(f[:-4])) 108 | frames_list = [os.path.join(frames_path, frame) for frame in frame_list] 109 | 110 | return Sequence( 111 | sequence_name, frames_list, "trackingnet", ground_truth_rect.reshape(-1, 4) 112 | ) 113 | 114 | def __len__(self): 115 | return len(self.sequence_list) 116 | 117 | def _list_sequences(self, root, set_ids): 118 | sequence_list = [] 119 | 120 | for s in set_ids: 121 | anno_dir = os.path.join(root, s, "anno") 122 | sequences_cur_set = [ 123 | (s, os.path.splitext(f)[0]) 124 | for f in os.listdir(anno_dir) 125 | if f.endswith(".txt") 126 | ] 127 | 128 | sequence_list += sequences_cur_set 129 | 130 | return sequence_list 131 | -------------------------------------------------------------------------------- /algorithms/aaa.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import random 3 | import numpy as np 4 | from PIL import Image 5 | from base_tracker import BaseTracker 6 | from .aaa_util import ( 7 | FeatureExtractor, 8 | ShortestPathTracker, 9 | WAADelayed, 10 | AnchorDetector, 11 | calc_overlap, 12 | ) 13 | 14 | 15 | class AAA(BaseTracker): 16 | def __init__( 17 | self, n_experts, mode="SuperFast", threshold=0.0, 18 | ): 19 | super(AAA, self).__init__( 20 | f"AAA/{mode}/{threshold:.2f}" if threshold > 0 else f"WithoutDelay/{mode}" 21 | ) 22 | 23 | # The number of experts 24 | self.n_experts = n_experts 25 | 26 | # Anchor extractor 27 | self.detector = AnchorDetector(threshold=threshold) 28 | 29 | # Feature extractor 30 | use_cuda = torch.cuda.is_available() 31 | device = torch.device("cuda" if use_cuda else "cpu") 32 | self.extractor = FeatureExtractor(device) 33 | 34 | # Offline tracker 35 | self.offline = ShortestPathTracker() 36 | 37 | # If offline tracker is reset 38 | self.reset_offline = True 39 | 40 | # Online learner 41 | self.learner = WAADelayed() 42 | 43 | def initialize(self, image_file, box): 44 | image = Image.open(image_file).convert("RGB") 45 | 46 | # Previous boxes of experts 47 | self.prev_boxes = [] 48 | 49 | # Extract target image 50 | self.target_feature = self.extractor.extract(image, [box]) 51 | 52 | # Init detector with target feature 53 | self.detector.init(self.target_feature) 54 | 55 | # Init offline tracker with target feature 56 | self.offline.initialize(box, self.target_feature) 57 | 58 | # Init online learner 59 | self.learner.init(self.n_experts) 60 | 61 | def track(self, image_file, boxes): 62 | image = Image.open(image_file).convert("RGB") 63 | 64 | # Save box of experts 65 | self.prev_boxes.append(boxes) 66 | 67 | # Extract features from boxes 68 | features = self.extractor.extract(image, boxes) 69 | 70 | # Detect if it is anchor frame 71 | detected, feature_scores = self.detector.detect(features) 72 | anchor = len(detected) > 0 73 | 74 | # If it is anchor frame, 75 | if anchor: 76 | # Add only boxes whose score is over than threshold to offline tracker 77 | self.offline.track( 78 | boxes, features, feature_scores 79 | ) 80 | 81 | # Caluclate optimal path 82 | path = self.offline.run(detected) 83 | 84 | # Get the last box's id 85 | final_box_id = path[-1][1] 86 | 87 | # Change to ndarray 88 | self.prev_boxes = np.stack(self.prev_boxes) 89 | 90 | if self.reset_offline: 91 | # Reset offline tracker 92 | self.offline.initialize(boxes[final_box_id], features[final_box_id]) 93 | 94 | # Get offline tracking results 95 | offline_results = np.array( 96 | [self.prev_boxes[frame, ind[1]] for frame, ind in enumerate(path)] 97 | ) 98 | 99 | else: 100 | offline_results = np.array( 101 | [self.prev_boxes[frame, ind[1]] for frame, ind in enumerate(path[-len(self.prev_boxes):])] 102 | ) 103 | 104 | # Calc losses of experts 105 | gradient_losses = self._calc_expert_losses(offline_results) 106 | 107 | # Clean previous boxes 108 | self.prev_boxes = [] 109 | 110 | # Update weight of experts 111 | self.learner.update(gradient_losses) 112 | 113 | # Return last box of offline results 114 | predict = boxes[final_box_id] 115 | 116 | # Otherwise 117 | else: 118 | # Add all boxes to offline tracker 119 | self.offline.track(boxes, features, feature_scores) 120 | 121 | # No offline result here 122 | offline_results = None 123 | 124 | # Return box with aggrogating experts' box 125 | predict = random.choices(boxes, weights=self.learner.w)[0] 126 | 127 | return predict, offline_results, self.learner.w 128 | 129 | def _calc_expert_losses(self, offline_results): 130 | """ 131 | offline_results = #frames X 4 132 | """ 133 | 134 | expert_gradient_losses = np.zeros((self.n_experts, len(offline_results))) 135 | 136 | for i in range(self.n_experts): 137 | expert_results = self.prev_boxes[:, i, :] 138 | expert_gradient_losses[i] = 1 - calc_overlap( 139 | expert_results, offline_results 140 | ) 141 | 142 | return expert_gradient_losses 143 | -------------------------------------------------------------------------------- /algorithms/hdt.py: -------------------------------------------------------------------------------- 1 | from PIL import Image 2 | import numpy as np 3 | import torch 4 | from base_tracker import BaseTracker 5 | from .aaa_util import FeatureExtractor, calc_similarity 6 | 7 | 8 | def avgnh(r, c, A): 9 | n = r.size 10 | T = r + A 11 | T[T < 0] = 0 12 | w = np.exp(T / c) 13 | total = (1 / n) * np.sum(w) - 2.72 14 | return total 15 | 16 | 17 | def find_nh_scale(regrets, A): 18 | clower = 1.0 19 | counter = 0 20 | while avgnh(regrets, clower, A) < 0 and counter < 30: 21 | clower *= 0.5 22 | counter += 1 23 | 24 | cupper = 1.0 25 | counter = 0 26 | while avgnh(regrets, cupper, A) > 0 and counter < 30: 27 | cupper *= 2 28 | counter += 1 29 | 30 | cmid = (cupper + clower) / 2 31 | counter = 0 32 | while np.abs(avgnh(regrets, cmid, A)) > 1e-2 and counter < 30: 33 | if avgnh(regrets, cmid, A) > 1e-2: 34 | clower = cmid 35 | cmid = (cmid + cupper) / 2 36 | else: 37 | cupper = cmid 38 | cmid = (cmid + clower) / 2 39 | counter += 1 40 | 41 | return cmid 42 | 43 | 44 | def nnhedge_weights(r, scale, is_tpami, A): 45 | n = r.size 46 | w = np.zeros((n)) 47 | 48 | T = r + A 49 | for i in range(n): 50 | if T[i] <= 0: 51 | w[i] = 2.2204e-16 52 | else: 53 | if is_tpami: 54 | w[i] = np.exp(T[i] / scale) / scale 55 | else: 56 | w[i] = T[i] / scale * np.exp(T[i] * T[i] / scale / 2) 57 | return w 58 | 59 | 60 | class HDT(BaseTracker): 61 | def __init__(self, n_experts, mode, beta): 62 | super(HDT, self).__init__(f"HDT/{mode}/{beta:.2f}") 63 | self.n_experts = n_experts 64 | self.beta = beta 65 | 66 | # Feature extractor 67 | use_cuda = torch.cuda.is_available() 68 | device = torch.device("cuda" if use_cuda else "cpu") 69 | self.extractor = FeatureExtractor(device) 70 | 71 | self.delta_t = 5 72 | self.scale_gamma = 0.25 73 | self.is_tpami = True 74 | self.A = 0.011 75 | 76 | def initialize(self, image_file, box): 77 | self.frame_idx = -1 78 | image = Image.open(image_file).convert("RGB") 79 | self.target_feature = self.extractor.extract(image, [box])[0] 80 | 81 | self.experts_loss = np.zeros((self.n_experts, self.delta_t + 1)) 82 | self.experts_regret = np.zeros((self.n_experts)) 83 | self.weights = np.ones((1, self.n_experts)) / self.n_experts 84 | self.center = box[:2] + box[2:] / 2 85 | self.size = box[2:] 86 | 87 | def track(self, image_file, boxes): 88 | self.frame_idx += 1 89 | 90 | experts_center = boxes[:, :2] + boxes[:, 2:] / 2 91 | 92 | if self.frame_idx > 0: 93 | self.center = self.weights.dot(experts_center) 94 | 95 | image = Image.open(image_file).convert("RGB") 96 | features = self.extractor.extract(image, boxes) 97 | 98 | distance = np.linalg.norm(self.center - experts_center, axis=1) 99 | distance_loss = distance / np.sum(distance) 100 | 101 | similarity = calc_similarity(self.target_feature, features)[0] 102 | similarity_loss = 1 - similarity 103 | similarity_loss = similarity_loss / np.sum(similarity_loss) 104 | 105 | self.experts_loss[:, -1] = ( 106 | 1 - self.beta 107 | ) * similarity_loss + self.beta * distance_loss 108 | expected_loss = self.weights.dot(self.experts_loss[:, -1]) 109 | 110 | mu = np.mean(self.experts_loss[:, :-1], axis=1) 111 | sigma = np.std(self.experts_loss[:, :-1], axis=1) 112 | mu[mu < 1e-4] = 0 113 | sigma[sigma < 1e-4] = 0 114 | 115 | if self.is_tpami: 116 | s = (self.experts_loss[:, -1] - mu) / (sigma + 2.2204e-16) 117 | self.experts_regret += ( 118 | expected_loss - np.tanh(self.scale_gamma * s) * self.experts_loss[:, -1] 119 | ) 120 | else: 121 | curDiff = self.experts_loss[:, -1] - mu 122 | alpha = 0.97 * np.exp((-10 * np.abs(curDiff) / (sigma + 2.2204e-16))) 123 | alpha[alpha > 0.9] = 0.97 124 | alpha[alpha < 0.12] = 0.119 125 | 126 | self.experts_regret = alpha * self.experts_regret + (1 - alpha) * ( 127 | expected_loss - self.experts_loss[:, -1] 128 | ) 129 | 130 | loss_idx = self.frame_idx % self.delta_t 131 | self.experts_loss[:, loss_idx] = self.experts_loss[:, -1] 132 | 133 | c = find_nh_scale(self.experts_regret, self.A) 134 | self.weights = nnhedge_weights(self.experts_regret, c, self.is_tpami, self.A) 135 | self.weights /= np.sum(self.weights) 136 | 137 | box = np.zeros((4)) 138 | box[:2] = self.center - self.size / 2 139 | box[2:] = self.size 140 | 141 | return (box, [box], self.weights) 142 | -------------------------------------------------------------------------------- /evaluations/eval_trackers.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | from pathlib import Path 4 | from path_config import EVALUATION_PATH 5 | from evaluations.ope_benchmark import OPEBenchmark 6 | 7 | 8 | def save_pickle(dir_path, filename, func, *args): 9 | file_path = dir_path / f"{filename}.pkl" 10 | 11 | if file_path.exists(): 12 | data = pickle.loads(file_path.read_bytes()) 13 | else: 14 | data = func(*args) 15 | file_path.write_bytes(pickle.dumps(data)) 16 | 17 | return data 18 | 19 | 20 | def evaluate(datasets, datasets_name, experts, baselines, algorithm, save_dir=None): 21 | if save_dir is None: 22 | save_dir = Path(EVALUATION_PATH) 23 | os.makedirs(save_dir, exist_ok=True) 24 | 25 | if algorithm is not None: 26 | eval_trackers = experts + baselines + [algorithm] 27 | else: 28 | eval_trackers = experts 29 | 30 | tracking_time_rets = {} 31 | success_rets = {} 32 | precision_rets = {} 33 | norm_precision_rets = {} 34 | anchor_success_rets = {} 35 | anchor_precision_rets = {} 36 | anchor_norm_precision_rets = {} 37 | error_rets = {} 38 | loss_rets = {} 39 | offline_success_rets = {} 40 | offline_precision_rets = {} 41 | anchor_frame_rets = {} 42 | 43 | for dataset, dataset_name in zip(datasets, datasets_name): 44 | ope = OPEBenchmark(dataset, dataset_name) 45 | 46 | tracking_time_rets[dataset_name] = {} 47 | success_rets[dataset_name] = {} 48 | precision_rets[dataset_name] = {} 49 | norm_precision_rets[dataset_name] = {} 50 | anchor_success_rets[dataset_name] = {} 51 | anchor_precision_rets[dataset_name] = {} 52 | anchor_norm_precision_rets[dataset_name] = {} 53 | error_rets[dataset_name] = {} 54 | loss_rets[dataset_name] = {} 55 | offline_success_rets[dataset_name] = {} 56 | offline_precision_rets[dataset_name] = {} 57 | 58 | if algorithm is not None: 59 | anchor_frame_rets[dataset_name] = ope.get_anchor_frames(algorithm) 60 | 61 | for tracker_name in eval_trackers: 62 | tracker_dir = save_dir / tracker_name / dataset_name 63 | os.makedirs(tracker_dir, exist_ok=True) 64 | 65 | tracking_time = save_pickle( 66 | tracker_dir, "tracking_time", ope.eval_times, tracker_name 67 | ) 68 | tracking_time_rets[dataset_name][tracker_name] = tracking_time 69 | 70 | success = save_pickle( 71 | tracker_dir, "success", ope.eval_success, tracker_name 72 | ) 73 | success_rets[dataset_name][tracker_name] = success 74 | 75 | precision = save_pickle( 76 | tracker_dir, "precision", ope.eval_precision, tracker_name 77 | ) 78 | precision_rets[dataset_name][tracker_name] = precision 79 | 80 | norm_precision = save_pickle( 81 | tracker_dir, "norm_precision", ope.eval_norm_precision, tracker_name, 82 | ) 83 | norm_precision_rets[dataset_name][tracker_name] = norm_precision 84 | 85 | if algorithm is not None: 86 | anchor_success = save_pickle( 87 | tracker_dir, 88 | "anchor_success", 89 | ope.eval_success, 90 | tracker_name, 91 | anchor_frame_rets[dataset_name], 92 | ) 93 | anchor_success_rets[dataset_name][tracker_name] = anchor_success 94 | 95 | anchor_precision = save_pickle( 96 | tracker_dir, 97 | "anchor_precision", 98 | ope.eval_precision, 99 | tracker_name, 100 | anchor_frame_rets[dataset_name], 101 | ) 102 | anchor_precision_rets[dataset_name][tracker_name] = anchor_precision 103 | 104 | anchor_norm_precision = save_pickle( 105 | tracker_dir, 106 | "anchor_norm_precision", 107 | ope.eval_norm_precision, 108 | tracker_name, 109 | anchor_frame_rets[dataset_name], 110 | ) 111 | anchor_norm_precision_rets[dataset_name][ 112 | tracker_name 113 | ] = anchor_norm_precision 114 | 115 | error, loss = save_pickle( 116 | tracker_dir, "loss", ope.eval_loss, algorithm, tracker_name, 117 | ) 118 | error_rets[dataset_name][tracker_name] = error 119 | loss_rets[dataset_name][tracker_name] = loss 120 | 121 | return ( 122 | tracking_time_rets, 123 | success_rets, 124 | precision_rets, 125 | norm_precision_rets, 126 | anchor_success_rets, 127 | anchor_precision_rets, 128 | anchor_norm_precision_rets, 129 | error_rets, 130 | loss_rets, 131 | offline_success_rets, 132 | offline_precision_rets, 133 | anchor_frame_rets, 134 | ) 135 | 136 | 137 | if __name__ == "__main__": 138 | import argparse 139 | 140 | parser = argparse.ArgumentParser() 141 | parser.add_argument("-a", "--algorithm", default=None, type=str) 142 | parser.add_argument("-e", "--experts", default=list(), nargs="+") 143 | parser.add_argument("-b", "--baselines", default=list(), nargs="+") 144 | args = parser.parse_args() 145 | 146 | eval_dir = Path("./evaluation_results") 147 | evaluate(args.experts, args.baselines, args.algorithm, eval_dir) 148 | -------------------------------------------------------------------------------- /experts/thor.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | import cv2 4 | import torch 5 | from base_tracker import BaseTracker 6 | import path_config 7 | 8 | sys.path.append("external/THOR/") 9 | from trackers.THOR_modules.wrapper import THOR_SiamFC, THOR_SiamRPN, THOR_SiamMask 10 | 11 | # SiamFC import 12 | from trackers.SiamFC.net import SiamFC 13 | from trackers.SiamFC.siamfc import SiamFC_init, SiamFC_track 14 | 15 | # SiamRPN Imports 16 | from trackers.SiamRPN.net import SiamRPN 17 | from trackers.SiamRPN.siamrpn import SiamRPN_init, SiamRPN_track 18 | 19 | # SiamMask Imports 20 | from trackers.SiamMask.net import SiamMaskCustom 21 | from trackers.SiamMask.siammask import SiamMask_init, SiamMask_track 22 | from trackers.SiamMask.utils.load_helper import load_pretrain 23 | 24 | sys.path.append("external/THOR/benchmark") 25 | from bench_utils.bbox_helper import cxy_wh_2_rect, rect_2_cxy_wh 26 | 27 | 28 | class THOR(BaseTracker): 29 | def __init__(self): 30 | super(THOR, self).__init__("THOR") 31 | tracker = "SiamRPN" 32 | dataset = "OTB2015" 33 | vanilla = False 34 | lb_type = "dynamic" # [dynamic, ensemble] 35 | json_path = f"{path_config.THOR_CONFIG}/{tracker}/" 36 | json_path += f"{dataset}_" 37 | if vanilla: 38 | json_path += "vanilla.json" 39 | else: 40 | json_path += f"THOR_{lb_type}.json" 41 | cfg = json.load(open(json_path)) 42 | cfg["THOR"]["viz"] = False 43 | cfg["THOR"]["verbose"] = False 44 | 45 | if tracker == "SiamFC": 46 | self.tracker = SiamFC_Tracker(cfg, path_config.THOR_SIAMFC_MODEL) 47 | elif tracker == "SiamRPN": 48 | self.tracker = SiamRPN_Tracker(cfg, path_config.THOR_SIAMRPN_MODEL) 49 | elif tracker == "SiamMask": 50 | self.tracker = SiamMask_Tracker(cfg, path_config.THOR_SIAMMASK_MODEL) 51 | else: 52 | raise ValueError(f"Tracker {tracker} does not exist.") 53 | 54 | def initialize(self, image_file, box): 55 | image = cv2.imread(image_file) 56 | init_pos, init_sz = rect_2_cxy_wh(box) 57 | self.state = self.tracker.setup(image, init_pos, init_sz) 58 | 59 | def track(self, image_file): 60 | image = cv2.imread(image_file) 61 | self.state = self.tracker.track(image, self.state) 62 | bbox = cxy_wh_2_rect(self.state["target_pos"], self.state["target_sz"]) 63 | return bbox 64 | 65 | 66 | class Tracker: 67 | def __init__(self): 68 | use_cuda = torch.cuda.is_available() 69 | self.device = torch.device("cuda" if use_cuda else "cpu") 70 | self.mask = False 71 | self.temp_mem = None 72 | 73 | def init_func(self, im, pos, sz): 74 | raise NotImplementedError 75 | 76 | def track_func(self, state, im): 77 | raise NotImplementedError 78 | 79 | def setup(self, im, target_pos, target_sz): 80 | state = self.init_func(im, target_pos, target_sz) 81 | self.temp_mem.setup(im, target_pos, target_sz) 82 | return state 83 | 84 | def track(self, im, state): 85 | state = self.track_func(state, im) 86 | self.temp_mem.update(im, state["crop"], state["target_pos"], state["target_sz"]) 87 | return state 88 | 89 | 90 | class SiamFC_Tracker(Tracker): 91 | def __init__(self, cfg, model_path): 92 | super(SiamFC_Tracker, self).__init__() 93 | self.cfg = cfg 94 | 95 | # setting up the tracker 96 | # model_path = dirname(abspath(__file__)) + '/SiamFC/model.pth' 97 | model = SiamFC() 98 | model.load_state_dict(torch.load(model_path)) 99 | self.model = model.eval().to(self.device) 100 | 101 | # set up template memory 102 | self.temp_mem = THOR_SiamFC(cfg=cfg["THOR"], net=self.model) 103 | 104 | def init_func(self, im, pos, sz): 105 | return SiamFC_init(im, pos, sz, self.cfg["tracker"]) 106 | 107 | def track_func(self, state, im): 108 | return SiamFC_track(state, im, self.temp_mem) 109 | 110 | 111 | class SiamRPN_Tracker(Tracker): 112 | def __init__(self, cfg, model_path): 113 | super(SiamRPN_Tracker, self).__init__() 114 | self.cfg = cfg 115 | 116 | # setting up the model 117 | # model_path = dirname(abspath(__file__)) + '/SiamRPN/model.pth' 118 | model = SiamRPN() 119 | model.load_state_dict( 120 | torch.load( 121 | model_path, map_location=("cpu" if str(self.device) == "cpu" else None) 122 | ) 123 | ) 124 | self.model = model.eval().to(self.device) 125 | 126 | # set up template memory 127 | self.temp_mem = THOR_SiamRPN(cfg=cfg["THOR"], net=self.model) 128 | 129 | def init_func(self, im, pos, sz): 130 | return SiamRPN_init(im, pos, sz, self.cfg["tracker"]) 131 | 132 | def track_func(self, state, im): 133 | return SiamRPN_track(state, im, self.temp_mem) 134 | 135 | 136 | class SiamMask_Tracker(Tracker): 137 | def __init__(self, cfg, model_path): 138 | super(SiamMask_Tracker, self).__init__() 139 | self.cfg = cfg 140 | self.mask = True 141 | 142 | # setting up the model 143 | # model_path = dirname(abspath(__file__)) + '/SiamMask/model.pth' 144 | model = SiamMaskCustom(anchors=cfg["anchors"]) 145 | model = load_pretrain(model, model_path) 146 | self.model = model.eval().to(self.device) 147 | 148 | # set up template memory 149 | self.temp_mem = THOR_SiamMask(cfg=cfg["THOR"], net=self.model) 150 | 151 | def init_func(self, im, pos, sz): 152 | return SiamMask_init(im, pos, sz, self.model, self.cfg["tracker"]) 153 | 154 | def track_func(self, state, im): 155 | return SiamMask_track(state, im, self.temp_mem) 156 | -------------------------------------------------------------------------------- /select_options.py: -------------------------------------------------------------------------------- 1 | from datasets.otbdataset import OTBDataset 2 | from datasets.otbnoisydataset import OTBNoisyDataset 3 | from datasets.votdataset import VOTDataset 4 | from datasets.tpldataset import TPLDataset 5 | from datasets.trackingnetdataset import TrackingNetDataset 6 | from datasets.uavdataset import UAVDataset 7 | from datasets.nfsdataset import NFSDataset 8 | from datasets.lasotdataset import LaSOTDataset 9 | from datasets.got10kdataset import GOT10KDatasetVal 10 | 11 | 12 | def select_expert(tracker_name): 13 | if tracker_name == "ATOM": 14 | from experts.atom import ATOM 15 | 16 | tracker = ATOM() 17 | elif tracker_name == "DaSiamRPN": 18 | from experts.dasiamrpn import DaSiamRPN 19 | 20 | tracker = DaSiamRPN() 21 | elif tracker_name == "DiMP": 22 | from experts.dimp import DiMP 23 | 24 | tracker = DiMP() 25 | elif tracker_name == "DROL": 26 | from experts.drol import DROL 27 | 28 | tracker = DROL() 29 | elif tracker_name == "GradNet": 30 | from experts.gradnet import GradNet 31 | 32 | tracker = GradNet() 33 | elif tracker_name == "KYS": 34 | from experts.kys import KYS 35 | 36 | tracker = KYS() 37 | elif tracker_name == "MemDTC": 38 | from experts.memdtc import MemDTC 39 | 40 | tracker = MemDTC() 41 | elif tracker_name == "MemTrack": 42 | from experts.memtrack import MemTrack 43 | 44 | tracker = MemTrack() 45 | elif tracker_name == "Ocean": 46 | from experts.ocean import Ocean 47 | 48 | tracker = Ocean() 49 | elif tracker_name == "PrDiMP": 50 | from experts.prdimp import PrDiMP 51 | 52 | tracker = PrDiMP() 53 | elif tracker_name == "RLS-RTMDNet": 54 | from experts.rls_rtmdnet import RLS_RTMDNet 55 | 56 | tracker = RLS_RTMDNet() 57 | elif tracker_name == "RPT": 58 | from experts.rpt import RPT 59 | 60 | tracker = RPT() 61 | elif tracker_name == "SiamBAN": 62 | from experts.siamban import SiamBAN 63 | 64 | tracker = SiamBAN() 65 | elif tracker_name == "SiamCAR": 66 | from experts.siamcar import SiamCAR 67 | 68 | tracker = SiamCAR() 69 | elif tracker_name == "SiamDW": 70 | from experts.siamdw import SiamDW 71 | 72 | tracker = SiamDW() 73 | elif tracker_name.startswith("SiamDWGroup"): 74 | from experts.siamdw_group import SiamDWGroup 75 | 76 | parameter = tracker_name.split("/") 77 | 78 | tracker = SiamDWGroup(parameter[1], parameter[2]) 79 | elif tracker_name == "SiamFC": 80 | from experts.siamfc import SiamFC 81 | 82 | tracker = SiamFC() 83 | elif tracker_name == "SiamFC++": 84 | from experts.siamfcpp import SiamFCPP 85 | 86 | tracker = SiamFCPP() 87 | elif tracker_name == "SiamMCF": 88 | from experts.siammcf import SiamMCF 89 | 90 | tracker = SiamMCF() 91 | elif tracker_name == "SiamR-CNN": 92 | from experts.siamrcnn import SiamRCNN 93 | 94 | tracker = SiamRCNN() 95 | elif tracker_name == "SiamRPN": 96 | from experts.siamrpn import SiamRPN 97 | 98 | tracker = SiamRPN() 99 | elif tracker_name == "SiamRPN++": 100 | from experts.siamrpnpp import SiamRPNPP 101 | 102 | tracker = SiamRPNPP() 103 | elif tracker_name.startswith("SiamRPN++Group"): 104 | from experts.siamrpnpp_group import SiamRPNPPGroup 105 | 106 | parameter = tracker_name.split("/") 107 | 108 | tracker = SiamRPNPPGroup(parameter[1], parameter[2]) 109 | elif tracker_name == "SPM": 110 | from experts.spm import SPM 111 | 112 | tracker = SPM() 113 | elif tracker_name == "Staple": 114 | from experts.staple import Staple 115 | 116 | tracker = Staple() 117 | elif tracker_name == "THOR": 118 | from experts.thor import THOR 119 | 120 | tracker = THOR() 121 | elif tracker_name == "TRAS": 122 | from experts.tras import ETRAS 123 | 124 | tracker = ETRAS() 125 | elif tracker_name == "TRAST": 126 | from experts.tras import ETRAST 127 | 128 | tracker = ETRAST() 129 | elif tracker_name == "TRASFUST": 130 | from experts.tras import ETRASFUST 131 | 132 | tracker = ETRASFUST() 133 | else: 134 | raise ValueError("Unknown expert name") 135 | 136 | return tracker 137 | 138 | 139 | def select_algorithms(algorithm_name, experts, **kwargs): 140 | n_experts = len(experts) 141 | if algorithm_name == "AAA": 142 | from algorithms.aaa import AAA 143 | 144 | algorithm = AAA(n_experts, **kwargs) 145 | elif algorithm_name == "WithoutDelay": 146 | from algorithms.aaa import AAA 147 | 148 | kwargs["threshold"] = 0 149 | algorithm = AAA(n_experts, **kwargs) 150 | elif algorithm_name == "WithoutOffline": 151 | from algorithms.without_offline import WithoutOffline 152 | 153 | algorithm = WithoutOffline(n_experts, mode=kwargs["mode"]) 154 | elif algorithm_name == "Random": 155 | from algorithms.random import Random 156 | 157 | algorithm = Random(n_experts, mode=kwargs["mode"]) 158 | elif algorithm_name == "MCCT": 159 | from algorithms.mcct import MCCT 160 | 161 | algorithm = MCCT(n_experts, mode=kwargs["mode"], mu=kwargs["threshold"]) 162 | elif algorithm_name == "HDT": 163 | from algorithms.hdt import HDT 164 | 165 | algorithm = HDT(n_experts, mode=kwargs["mode"], beta=kwargs["threshold"]) 166 | else: 167 | raise ValueError(f"Unknown algorithm name: {algorithm_name}") 168 | 169 | return algorithm 170 | 171 | 172 | def select_datasets(dataset_name): 173 | if dataset_name == "OTB2015": 174 | dataset = OTBDataset() 175 | elif dataset_name == "OTB2015-80%": 176 | dataset = OTBNoisyDataset(0.8) 177 | elif dataset_name == "OTB2015-60%": 178 | dataset = OTBNoisyDataset(0.6) 179 | elif dataset_name == "OTB2015-40%": 180 | dataset = OTBNoisyDataset(0.4) 181 | elif dataset_name == "OTB2015-20%": 182 | dataset = OTBNoisyDataset(0.2) 183 | elif dataset_name == "NFS": 184 | dataset = NFSDataset() 185 | elif dataset_name == "UAV123": 186 | dataset = UAVDataset() 187 | elif dataset_name == "TColor128": 188 | dataset = TPLDataset() 189 | elif dataset_name == "TrackingNet": 190 | dataset = TrackingNetDataset() 191 | elif dataset_name == "VOT2018": 192 | dataset = VOTDataset() 193 | elif dataset_name == "LaSOT": 194 | dataset = LaSOTDataset() 195 | elif dataset_name == "Got10K": 196 | dataset = GOT10KDatasetVal() 197 | else: 198 | raise ValueError("Unknown dataset name") 199 | 200 | return dataset 201 | -------------------------------------------------------------------------------- /algorithms/aaa_util.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import torch 4 | from torchvision import models 5 | from torchvision import transforms 6 | import torch.nn as nn 7 | import scipy.special as sc 8 | 9 | 10 | def calc_overlap(rect1, rect2): 11 | r"""Generalized Intersection over Union 12 | https://giou.stanford.edu/ 13 | """ 14 | if rect1.ndim == 1: 15 | rect1 = rect1[None, :] 16 | if rect2.ndim == 1: 17 | rect2 = rect2[None, :] 18 | 19 | left = np.maximum(rect1[:, 0], rect2[:, 0]) 20 | right = np.minimum(rect1[:, 0] + rect1[:, 2], rect2[:, 0] + rect2[:, 2]) 21 | top = np.maximum(rect1[:, 1], rect2[:, 1]) 22 | bottom = np.minimum(rect1[:, 1] + rect1[:, 3], rect2[:, 1] + rect2[:, 3]) 23 | 24 | intersect = np.maximum(0, right - left) * np.maximum(0, bottom - top) 25 | union = rect1[:, 2] * rect1[:, 3] + rect2[:, 2] * rect2[:, 3] - intersect 26 | iou = np.clip(intersect / union, 0, 1) 27 | 28 | left_min = np.minimum(rect1[:, 0], rect2[:, 0]) 29 | right_max = np.maximum(rect1[:, 0] + rect1[:, 2], rect2[:, 0] + rect2[:, 2]) 30 | top_min = np.minimum(rect1[:, 1], rect2[:, 1]) 31 | bottom_max = np.maximum(rect1[:, 1] + rect1[:, 3], rect2[:, 1] + rect2[:, 3]) 32 | closure_width = np.maximum(0, right_max - left_min) 33 | closure_height = np.maximum(0, bottom_max - top_min) 34 | 35 | closure = closure_width * closure_height 36 | g_iou = iou - (closure - union) / closure 37 | 38 | g_iou = (1 + g_iou) / 2 39 | return g_iou 40 | 41 | 42 | def calc_similarity(feature1, feature2): 43 | with torch.no_grad(): 44 | if feature1.ndim == 1: 45 | feature1 = feature1.unsqueeze(0) 46 | if feature2.ndim == 1: 47 | feature2 = feature2.unsqueeze(0) 48 | dot_product = torch.matmul(feature1, feature2.T) 49 | norm_feature1 = torch.norm(feature1, dim=1, keepdim=True) + 1e-7 50 | norm_feature2 = torch.norm(feature2, dim=1, keepdim=True).T + 1e-7 51 | sim = dot_product / norm_feature1 / norm_feature2 52 | score = (1 + sim) / 2 53 | return score.cpu().numpy() 54 | 55 | 56 | class WAADelayed: 57 | def __init__(self): 58 | pass 59 | 60 | def init(self, n): 61 | self.w = np.ones(n) / n 62 | self.Z = 1 63 | self.TD = 0 64 | self.lnN = np.log(len(self.w)) 65 | 66 | def update(self, gradient_losses): 67 | """ 68 | gradient_losses should be n * len(dt) 69 | """ 70 | # add t 71 | self.TD += gradient_losses.shape[1] 72 | 73 | # add D 74 | self.TD += ((gradient_losses.shape[1] + 1) * gradient_losses.shape[1]) // 2 75 | 76 | # update estimated Z 77 | while self.Z < self.TD: 78 | self.Z *= 2 79 | 80 | lr = np.sqrt(self.lnN / self.Z) 81 | changes = lr * gradient_losses.sum(axis=1) 82 | log_multiple = np.log(self.w + 1e-14) - changes 83 | self.w = np.exp(log_multiple - sc.logsumexp(log_multiple)) 84 | 85 | 86 | class AnchorDetector: 87 | def __init__(self, threshold=0.0): 88 | # Threshold for detecting anchor frame 89 | self.threshold = threshold 90 | 91 | def init(self, target_feature): 92 | self.target_feature = target_feature 93 | 94 | def detect(self, features): 95 | feature_scores = calc_similarity(self.target_feature, features)[0] 96 | detected = np.where(feature_scores >= self.threshold)[0] 97 | return detected, feature_scores 98 | 99 | 100 | class ShortestPathTracker: 101 | def __init__(self): 102 | pass 103 | 104 | def initialize(self, box, target_feature): 105 | self.frame_id = -1 106 | 107 | self.prev_boxes = box[None, :] 108 | self.prev_features = target_feature 109 | self.shortest_cost = None 110 | self.shortest_path = None 111 | 112 | def track( 113 | self, curr_boxes, curr_features, curr_feature_scores, 114 | ): 115 | self.frame_id += 1 116 | 117 | prob_prev_similarity = calc_similarity(self.prev_features, curr_features) 118 | cost_feature = -np.log(prob_prev_similarity + 1e-7) 119 | cost_template = -np.log(curr_feature_scores + 1e-7) 120 | costs = cost_feature + cost_template 121 | 122 | if self.shortest_path is None: 123 | self.shortest_path = [[[self.frame_id, curr_idx]] for curr_idx in range(len(curr_boxes))] 124 | self.shortest_cost = costs[0, :] 125 | else: 126 | new_shortest_path = [[[self.frame_id, curr_idx]] for curr_idx in range(len(curr_boxes))] 127 | new_shortest_cost = np.zeros_like(self.shortest_cost) 128 | for curr_idx in range(len(curr_boxes)): 129 | shortest_prev_idx = np.argmin(self.shortest_cost + costs[:, curr_idx]) 130 | new_shortest_path[curr_idx] = self.shortest_path[shortest_prev_idx] + new_shortest_path[curr_idx] 131 | new_shortest_cost[curr_idx] = self.shortest_cost[shortest_prev_idx] + costs[shortest_prev_idx, curr_idx] 132 | self.shortest_path = new_shortest_path 133 | self.shortest_cost = new_shortest_cost 134 | 135 | self.prev_boxes = curr_boxes 136 | self.prev_features = curr_features 137 | 138 | def run(self, valid_idx): 139 | shortest_idx = np.argmin(self.shortest_cost[valid_idx]) 140 | shortest_idx = valid_idx[shortest_idx] 141 | 142 | ids = self.shortest_path[shortest_idx] 143 | return ids 144 | 145 | 146 | class FeatureExtractor: 147 | def __init__(self, device): 148 | self.device = device 149 | model = models.resnet18(pretrained=True) 150 | feature_map = list(model.children()) 151 | self.extractor = nn.Sequential(*feature_map[:-2]).to(self.device).eval() 152 | self.transform = transforms.Compose( 153 | [ 154 | transforms.Resize((224, 224)), 155 | transforms.ToTensor(), 156 | transforms.Normalize( 157 | mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225) 158 | ), 159 | ] 160 | ) 161 | 162 | def extract(self, image, bboxes): 163 | features = [] 164 | croped_images = [] 165 | 166 | norm_bboxes = np.array(bboxes) 167 | norm_bboxes[:, 2:] += norm_bboxes[:, :2] 168 | norm_bboxes[:, 0] = np.maximum(norm_bboxes[:, 0], 0) 169 | norm_bboxes[:, 1] = np.maximum(norm_bboxes[:, 1], 0) 170 | norm_bboxes[:, 2] = np.minimum(norm_bboxes[:, 2], image.size[0]) 171 | norm_bboxes[:, 3] = np.minimum(norm_bboxes[:, 3], image.size[1]) 172 | for bbox in norm_bboxes: 173 | croped_image = image.crop(bbox) 174 | croped_image = self.transform(croped_image) 175 | croped_images.append(croped_image) 176 | croped_images = torch.stack(croped_images) 177 | 178 | with torch.no_grad(): 179 | croped_images = croped_images.to(self.device) 180 | features = ( 181 | self.extractor(croped_images).view(croped_images.shape[0], -1).detach() 182 | ) 183 | return features 184 | -------------------------------------------------------------------------------- /datasets/oxuvadataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import cv2 4 | import csv 5 | import path_config 6 | from datasets.data import Sequence, BaseDataset, SequenceList 7 | 8 | 9 | TASK_FIELDS = [ 10 | "video_id", 11 | "object_id", 12 | "init_frame", 13 | "last_frame", 14 | "xmin", 15 | "xmax", 16 | "ymin", 17 | "ymax", 18 | ] 19 | 20 | 21 | class Task(object): 22 | """Describes a tracking task with optional ground-truth annotations.""" 23 | 24 | def __init__( 25 | self, init_time, init_rect, labels=None, last_time=None, attributes=None 26 | ): 27 | """Create a trasking task. 28 | Args: 29 | init_time -- Time of supervision (in frames). 30 | init_rect -- Rectangle dict. 31 | labels -- SparseTimeSeries of frame annotation dicts. 32 | Does not include first frame. 33 | last_time -- Time of last frame of interest, inclusive (optional). 34 | Consider frames init_time <= t <= last_time. 35 | attributes -- Dictionary with extra attributes. 36 | If last_time is None, then the last frame of labels will be used. 37 | """ 38 | self.init_time = init_time 39 | self.init_rect = init_rect 40 | if labels: 41 | if init_time in labels: 42 | raise ValueError("labels should not contain init time") 43 | self.labels = labels 44 | if last_time is None and labels is not None: 45 | self.last_time = labels.sorted_keys()[-1] 46 | else: 47 | self.last_time = last_time 48 | self.attributes = attributes or {} 49 | 50 | def len(self): 51 | return self.last_time - self.init_time + 1 52 | 53 | 54 | class VideoObjectDict(object): 55 | """Represents map video -> object -> element. 56 | Behaves as a dictionary with keys of (video, object) tuples. 57 | Example: 58 | for key in tracks.keys(): 59 | print(tracks[key]) 60 | tracks = VideoObjectDict() 61 | ... 62 | for vid in tracks.videos(): 63 | for obj in tracks.objects(vid): 64 | print(tracks[(vid, obj)]) 65 | """ 66 | 67 | def __init__(self, elems=None): 68 | if elems is None: 69 | self._elems = dict() 70 | elif isinstance(elems, VideoObjectDict): 71 | self._elems = dict(elems._elems) 72 | else: 73 | self._elems = dict(elems) 74 | 75 | def videos(self): 76 | return set([vid for vid, obj in self._elems.keys()]) 77 | 78 | def objects(self, vid): 79 | # TODO: This is somewhat inefficient if called for all videos. 80 | return [obj_i for vid_i, obj_i in self._elems.keys() if vid_i == vid] 81 | 82 | def __len__(self): 83 | return len(self._elems) 84 | 85 | def __getitem__(self, key): 86 | return self._elems[key] 87 | 88 | def __setitem__(self, key, value): 89 | self._elems[key] = value 90 | 91 | def __delitem__(self, key): 92 | del self._elems[key] 93 | 94 | def keys(self): 95 | return self._elems.keys() 96 | 97 | def values(self): 98 | return self._elems.values() 99 | 100 | def items(self): 101 | return self._elems.items() 102 | 103 | def __iter__(self): 104 | for k in self._elems.keys(): 105 | yield k 106 | 107 | def to_nested_dict(self): 108 | elems = {} 109 | for (vid, obj), elem in self._elems.items(): 110 | elems.setdefault(vid, {})[obj] = elem 111 | return elems 112 | 113 | def update_from_nested_dict(self, elems): 114 | for vid, vid_elems in elems.items(): 115 | for obj, elem in vid_elems.items(): 116 | self._elems[(vid, obj)] = elem 117 | 118 | 119 | def load_dataset_tasks_csv(fp): 120 | """Loads the problem definitions for an entire dataset from one CSV file.""" 121 | reader = csv.DictReader(fp, fieldnames=TASK_FIELDS) 122 | rows = [row for row in reader] 123 | 124 | tasks = VideoObjectDict() 125 | for row in rows: 126 | key = (row["video_id"], row["object_id"]) 127 | tasks[key] = Task( 128 | init_time=int(row["init_frame"]), 129 | last_time=int(row["last_frame"]), 130 | init_rect={ 131 | "xmin": float(row["xmin"]), 132 | "xmax": float(row["xmax"]), 133 | "ymin": float(row["ymin"]), 134 | "ymax": float(row["ymax"]), 135 | }, 136 | ) 137 | return tasks 138 | 139 | 140 | def rect_to_opencv(rect, imsize_hw): 141 | imheight, imwidth = imsize_hw 142 | xmin_abs = rect["xmin"] * imwidth 143 | ymin_abs = rect["ymin"] * imheight 144 | xmax_abs = rect["xmax"] * imwidth 145 | ymax_abs = rect["ymax"] * imheight 146 | return (xmin_abs, ymin_abs, xmax_abs - xmin_abs, ymax_abs - ymin_abs) 147 | 148 | 149 | def rect_from_opencv(rect, imsize_hw): 150 | imheight, imwidth = imsize_hw 151 | xmin_abs, ymin_abs, width_abs, height_abs = rect 152 | xmax_abs = xmin_abs + width_abs 153 | ymax_abs = ymin_abs + height_abs 154 | return { 155 | "xmin": xmin_abs / imwidth, 156 | "ymin": ymin_abs / imheight, 157 | "xmax": xmax_abs / imwidth, 158 | "ymax": ymax_abs / imheight, 159 | } 160 | 161 | 162 | def OxUvADataset(): 163 | return OxUvADatasetClass().get_sequence_list() 164 | 165 | 166 | class OxUvADatasetClass(BaseDataset): 167 | """ OxUvA test set. 168 | 169 | Publication: 170 | Long-term Tracking in the Wild: A Benchmark 171 | Jack Valmadre, Luca Bertinetto, João F. Henriques, Ran Tao, Andrea Vedaldi, Arnold Smeulders, Philip Torr and Efstratios Gavves 172 | ECCV, 2018 173 | https://arxiv.org/abs/1803.09502 174 | 175 | Download the dataset using the toolkit https://oxuva.github.io/long-term-tracking-benchmark/. 176 | """ 177 | 178 | def __init__(self): 179 | super().__init__() 180 | self.base_path = path_config.OXUVA_PATH 181 | self.tasks = self._get_tasks(self.base_path) 182 | self.imfile = lambda vid, t: os.path.join( 183 | self.base_path, "images", "test", vid, "{:06d}.jpeg".format(t) 184 | ) 185 | 186 | def get_sequence_list(self): 187 | return SequenceList( 188 | [self._construct_sequence(key, task) for key, task in self.tasks.items()] 189 | ) 190 | 191 | def _construct_sequence(self, key, task): 192 | vid, obj = key 193 | 194 | init_image = self.imfile(vid, task.init_time) 195 | im = cv2.imread(init_image, cv2.IMREAD_COLOR) 196 | imheight, imwidth, _ = im.shape 197 | 198 | frames_list = [ 199 | self.imfile(vid, t) for t in range(task.init_time, task.last_time + 1) 200 | ] 201 | 202 | ground_truth_rect = np.zeros((len(frames_list), 4)) 203 | init_box = rect_to_opencv(task.init_rect, imsize_hw=(imheight, imwidth)) 204 | ground_truth_rect[0, :] = init_box 205 | 206 | return Sequence(vid, frames_list, "oxuva", ground_truth_rect) 207 | 208 | def __len__(self): 209 | return len(self.sequence_list) 210 | 211 | def _get_tasks(self, root): 212 | tasks_file = os.path.join(root, "test.csv") 213 | with open(tasks_file, "r") as fp: 214 | tasks = load_dataset_tasks_csv(fp) 215 | 216 | return tasks 217 | -------------------------------------------------------------------------------- /datasets/data.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | 4 | class BaseDataset: 5 | """Base class for all datasets.""" 6 | 7 | def __init__(self): 8 | pass 9 | 10 | def __len__(self): 11 | """Overload this function in your dataset. This should return number of sequences in the dataset.""" 12 | raise NotImplementedError 13 | 14 | def get_sequence_list(self): 15 | """Overload this in your dataset. Should return the list of sequences in the dataset.""" 16 | raise NotImplementedError 17 | 18 | 19 | class Sequence: 20 | """Class for the sequence in an evaluation.""" 21 | 22 | def __init__( 23 | self, 24 | name, 25 | frames, 26 | dataset, 27 | ground_truth_rect, 28 | ground_truth_seg=None, 29 | init_data=None, 30 | object_class=None, 31 | target_visible=None, 32 | object_ids=None, 33 | multiobj_mode=False, 34 | ): 35 | self.name = name 36 | self.frames = frames 37 | self.dataset = dataset 38 | self.ground_truth_rect = ground_truth_rect 39 | self.ground_truth_seg = ground_truth_seg 40 | self.object_class = object_class 41 | self.target_visible = target_visible 42 | self.object_ids = object_ids 43 | self.multiobj_mode = multiobj_mode 44 | self.init_data = self._construct_init_data(init_data) 45 | self._ensure_start_frame() 46 | 47 | def _ensure_start_frame(self): 48 | # Ensure start frame is 0 49 | start_frame = min(list(self.init_data.keys())) 50 | if start_frame > 0: 51 | self.frames = self.frames[start_frame:] 52 | if self.ground_truth_rect is not None: 53 | if isinstance(self.ground_truth_rect, (dict, OrderedDict)): 54 | for obj_id, gt in self.ground_truth_rect.items(): 55 | self.ground_truth_rect[obj_id] = gt[start_frame:, :] 56 | else: 57 | self.ground_truth_rect = self.ground_truth_rect[start_frame:, :] 58 | if self.ground_truth_seg is not None: 59 | self.ground_truth_seg = self.ground_truth_seg[start_frame:] 60 | assert len(self.frames) == len(self.ground_truth_seg) 61 | 62 | if self.target_visible is not None: 63 | self.target_visible = self.target_visible[start_frame:] 64 | self.init_data = { 65 | frame - start_frame: val for frame, val in self.init_data.items() 66 | } 67 | 68 | def _construct_init_data(self, init_data): 69 | if init_data is not None: 70 | if not self.multiobj_mode: 71 | assert self.object_ids is None or len(self.object_ids) == 1 72 | for frame, init_val in init_data.items(): 73 | if "bbox" in init_val and isinstance( 74 | init_val["bbox"], (dict, OrderedDict) 75 | ): 76 | init_val["bbox"] = init_val["bbox"][self.object_ids[0]] 77 | # convert to list 78 | for frame, init_val in init_data.items(): 79 | if "bbox" in init_val: 80 | if isinstance(init_val["bbox"], (dict, OrderedDict)): 81 | init_val["bbox"] = OrderedDict( 82 | { 83 | obj_id: list(init) 84 | for obj_id, init in init_val["bbox"].items() 85 | } 86 | ) 87 | else: 88 | init_val["bbox"] = list(init_val["bbox"]) 89 | else: 90 | init_data = {0: dict()} # Assume start from frame 0 91 | 92 | if self.object_ids is not None: 93 | init_data[0]["object_ids"] = self.object_ids 94 | 95 | if self.ground_truth_rect is not None: 96 | if self.multiobj_mode: 97 | assert isinstance(self.ground_truth_rect, (dict, OrderedDict)) 98 | init_data[0]["bbox"] = OrderedDict( 99 | { 100 | obj_id: list(gt[0, :]) 101 | for obj_id, gt in self.ground_truth_rect.items() 102 | } 103 | ) 104 | else: 105 | assert self.object_ids is None or len(self.object_ids) == 1 106 | if isinstance(self.ground_truth_rect, (dict, OrderedDict)): 107 | init_data[0]["bbox"] = list( 108 | self.ground_truth_rect[self.object_ids[0]][0, :] 109 | ) 110 | else: 111 | init_data[0]["bbox"] = list(self.ground_truth_rect[0, :]) 112 | 113 | if self.ground_truth_seg is not None: 114 | init_data[0]["mask"] = self.ground_truth_seg[0] 115 | 116 | return init_data 117 | 118 | def init_info(self): 119 | info = self.frame_info(frame_num=0) 120 | return info 121 | 122 | def frame_info(self, frame_num): 123 | info = self.object_init_data(frame_num=frame_num) 124 | return info 125 | 126 | def init_bbox(self, frame_num=0): 127 | return self.object_init_data(frame_num=frame_num).get("init_bbox") 128 | 129 | def init_mask(self, frame_num=0): 130 | return self.object_init_data(frame_num=frame_num).get("init_mask") 131 | 132 | def get_info(self, keys, frame_num=None): 133 | info = dict() 134 | for k in keys: 135 | val = self.get(k, frame_num=frame_num) 136 | if val is not None: 137 | info[k] = val 138 | return info 139 | 140 | def object_init_data(self, frame_num=None) -> dict: 141 | if frame_num is None: 142 | frame_num = 0 143 | if frame_num not in self.init_data: 144 | return dict() 145 | 146 | init_data = dict() 147 | for key, val in self.init_data[frame_num].items(): 148 | if val is None: 149 | continue 150 | init_data["init_" + key] = val 151 | 152 | if self.object_ids is not None: 153 | init_data["object_ids"] = self.object_ids 154 | init_data["sequence_object_ids"] = self.object_ids 155 | 156 | return init_data 157 | 158 | def target_class(self, frame_num=None): 159 | return self.object_class 160 | 161 | def get(self, name, frame_num=None): 162 | return getattr(self, name)(frame_num) 163 | 164 | def __repr__(self): 165 | return "{self.__class__.__name__} {self.name}, length={len} frames".format( 166 | self=self, len=len(self.frames) 167 | ) 168 | 169 | 170 | class SequenceList(list): 171 | """List of sequences. Supports the addition operator to concatenate sequence lists.""" 172 | 173 | def __getitem__(self, item): 174 | if isinstance(item, str): 175 | for seq in self: 176 | if seq.name == item: 177 | return seq 178 | raise IndexError("Sequence name not in the dataset.") 179 | elif isinstance(item, int): 180 | return super(SequenceList, self).__getitem__(item) 181 | elif isinstance(item, (tuple, list)): 182 | return SequenceList( 183 | [super(SequenceList, self).__getitem__(i) for i in item] 184 | ) 185 | else: 186 | return SequenceList(super(SequenceList, self).__getitem__(item)) 187 | 188 | def __add__(self, other): 189 | return SequenceList(super(SequenceList, self).__add__(other)) 190 | 191 | def copy(self): 192 | return SequenceList(super(SequenceList, self).copy()) 193 | -------------------------------------------------------------------------------- /visualizes/draw_tables.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sympy import preview 3 | 4 | 5 | def minmax(x): 6 | return (x - np.min(x)) / (np.max(x) - np.min(x)) 7 | 8 | 9 | def is_algorithm(tracker_name): 10 | for algorithm in ["HDT", "MCCT", "Random", "WithoutDelay", "A3T"]: 11 | if tracker_name.startswith(algorithm): 12 | return True 13 | return False 14 | 15 | 16 | def calc_rank(dataset_name, seq_names, trackers, rets, mean=True): 17 | ranks = [] 18 | for seq_name in seq_names: 19 | if mean: 20 | value = np.mean( 21 | [ 22 | rets[dataset_name][tracker_name][seq_name] 23 | for tracker_name in trackers 24 | ], 25 | axis=1, 26 | ) 27 | else: 28 | value = np.array( 29 | [ 30 | rets[dataset_name][tracker_name][seq_name] 31 | for tracker_name in trackers 32 | ] 33 | ) 34 | temp = value.argsort() 35 | rank = np.empty_like(temp) 36 | rank[temp] = np.arange(len(value)) 37 | rank = len(trackers) - rank 38 | ranks.append(rank) 39 | ranks = np.array(ranks) 40 | return ranks 41 | 42 | 43 | def get_mean_succ(trackers_name, datasets_name, success_rets): 44 | mean_succ = np.zeros((len(trackers_name), len(datasets_name))) 45 | for i, tracker_name in enumerate(trackers_name): 46 | for j, dataset_name in enumerate(datasets_name): 47 | succ = [ 48 | v 49 | for v in success_rets[dataset_name][tracker_name].values() 50 | if not np.any(np.isnan(v)) 51 | ] 52 | mean_succ[i, j] = np.mean(succ) 53 | 54 | return mean_succ 55 | 56 | 57 | def get_mean_prec(trackers_name, datasets_name, precision_rets): 58 | mean_prec = np.zeros((len(trackers_name), len(datasets_name))) 59 | for i, tracker_name in enumerate(trackers_name): 60 | for j, dataset_name in enumerate(datasets_name): 61 | prec = [ 62 | v 63 | for v in precision_rets[dataset_name][tracker_name].values() 64 | if not np.any(np.isnan(v)) 65 | ] 66 | mean_prec[i, j] = np.mean(prec, axis=0)[20] 67 | 68 | return mean_prec 69 | 70 | 71 | def get_mean_ratio(trackers_name, datasets_name, anchor_frames, gt_trajs): 72 | mean_ratio = np.zeros((len(trackers_name), len(datasets_name))) 73 | for i, tracker_name in enumerate(trackers_name): 74 | for j, dataset_name in enumerate(datasets_name): 75 | ratio = [ 76 | len(v) / len(gt_trajs[k]) 77 | for k, v in anchor_frames[dataset_name][tracker_name].items() 78 | ] 79 | mean_ratio[i, j] = np.mean(ratio) 80 | 81 | return mean_ratio 82 | 83 | 84 | def get_mean_fps(trackers_name, datasets_name, tracking_time_rets): 85 | mean_fps = np.zeros((len(trackers_name), len(datasets_name))) 86 | for i, tracker_name in enumerate(trackers_name): 87 | for j, dataset_name in enumerate(datasets_name): 88 | fps = [ 89 | 1 / v for v in tracking_time_rets[dataset_name][tracker_name].values() 90 | ] 91 | mean_fps[i, j] = np.mean(fps) 92 | 93 | return mean_fps 94 | 95 | 96 | def make_score_table( 97 | datasets_name, algorithms_name, experts_name, mean_values, save_dir, filename=None, 98 | ): 99 | trackers_name = experts_name + algorithms_name 100 | metrics_name = list(mean_values.keys()) 101 | 102 | latex = "\\begin{table*}\n" 103 | latex += "\\centering\n" 104 | latex += "\\begin{threeparttable}\n" 105 | 106 | header = "c" 107 | num_header = len(datasets_name) * len(metrics_name) 108 | for i in range(num_header): 109 | header += "|c" 110 | 111 | latex += f"\\begin{{tabular}}{{{header}}}\n" 112 | latex += "\\hline\n" 113 | 114 | if len(metrics_name) > 1: 115 | columns = "\\multirow{2}{*}{Tracker}" 116 | else: 117 | columns = "Tracker" 118 | 119 | for i in range(len(datasets_name)): 120 | dataset_name = datasets_name[i].replace("%", "\\%") 121 | if len(metrics_name) > 1: 122 | small_colunm = "c|" if i < len(datasets_name) - 1 else "c" 123 | columns += f" & \\multicolumn{{{len(metrics_name)}}}{{{small_colunm}}}{{{dataset_name}}}" 124 | else: 125 | columns += f" & {dataset_name}" 126 | latex += f"{columns} \\\\\n" 127 | 128 | if len(metrics_name) > 1: 129 | small_columns = " " 130 | for i in range(len(datasets_name)): 131 | for j in range(len(metrics_name)): 132 | small_columns += f" & {metrics_name[j]}" 133 | latex += f"{small_columns} \\\\\n" 134 | latex += "\\hline\\hline\n" 135 | 136 | for i in range(len(trackers_name)): 137 | if i == len(experts_name) or trackers_name[i] == "Best expert": 138 | latex += "\\hdashline\n" 139 | 140 | if is_algorithm(trackers_name[i]): 141 | line = trackers_name[i].split("/")[0] 142 | if line == "WithoutDelay": 143 | line = "AAA w/o delay" 144 | else: 145 | line = trackers_name[i] 146 | line = line.replace("Group", "") 147 | for j in range(len(datasets_name)): 148 | for metric_name in metrics_name: 149 | value = mean_values[metric_name] 150 | if metric_name == "FPS": 151 | line += f" & {value[i, j]}" 152 | else: 153 | if "Best expert" in trackers_name: 154 | sorted_idx = np.argsort(value[:-1, j]) 155 | else: 156 | sorted_idx = np.argsort(value[:, j]) 157 | 158 | if i == sorted_idx[-1]: 159 | line += f" & {{\\color{{red}} \\textbf{{{value[i, j]}}}}}" 160 | elif i == sorted_idx[-2]: 161 | line += f" & {{\\color{{blue}} \\textit{{{value[i, j]}}}}}" 162 | else: 163 | line += f" & {value[i, j]}" 164 | line += " \\\\\n" 165 | 166 | latex += f"{line}" 167 | 168 | latex += "\\hline\n" 169 | latex += "\\end{tabular}\n" 170 | latex += "\\end{threeparttable}\n" 171 | latex += "\\end{table*}\n" 172 | 173 | if filename is None: 174 | filename = "table" 175 | txt_file = save_dir / f"{filename}.txt" 176 | txt_file.write_text(latex) 177 | 178 | preview( 179 | latex, 180 | viewer="file", 181 | filename=save_dir / f"{filename}.png", 182 | packages=("multirow", "xcolor", "arydshln", "threeparttable"), 183 | ) 184 | 185 | 186 | def find_rank( 187 | dataset_names, algorithms, experts, success_rets, save_dir, filename="Ranking" 188 | ): 189 | text = "" 190 | for i, dataset_name in enumerate(dataset_names): 191 | text += f"{dataset_name}\n" 192 | text += "-" * 10 + "\n" 193 | seq_names = sorted(success_rets[dataset_name][experts[0]].keys()) 194 | for algorithm in algorithms: 195 | rank = calc_rank( 196 | dataset_name, seq_names, experts + [algorithm], success_rets 197 | )[:, -1] 198 | first = [seq for seq, cond in zip(seq_names, rank == 1) if cond] 199 | last = [ 200 | seq for seq, cond in zip(seq_names, rank == len(experts) + 1) if cond 201 | ] 202 | text += f"{algorithm.split('_')[0]}: best-{first} / worst-{last}\n" 203 | text += "\n" 204 | 205 | txt_file = save_dir / f"{filename}.txt" 206 | txt_file.write_text(text) 207 | 208 | 209 | def make_rank_table( 210 | datasets_name, experts_name, error_rets, loss_rets, save_dir, filename=None 211 | ): 212 | latex = "\\begin{table}\n" 213 | latex += "\\centering\n" 214 | latex += "\\begin{threeparttable}\n" 215 | 216 | header = "c|c|c" 217 | latex += f"\\begin{{tabular}}{{{header}}}\n" 218 | latex += "\\hline\n" 219 | 220 | columns = "Dataset & Rank & Diff" 221 | latex += f"{columns} \\\\\n" 222 | latex += "\\hline\\hline\n" 223 | 224 | for dataset_name in datasets_name: 225 | line = dataset_name 226 | 227 | seq_names = sorted(error_rets[dataset_name][experts_name[0]].keys()) 228 | error_rank = ( 229 | len(experts_name) 230 | + 1 231 | - calc_rank(dataset_name, seq_names, experts_name, error_rets, mean=False) 232 | ) 233 | loss_rank = ( 234 | len(experts_name) 235 | + 1 236 | - calc_rank(dataset_name, seq_names, experts_name, loss_rets, mean=False) 237 | ) 238 | 239 | best_expert = np.argmin(loss_rank, axis=1) 240 | best_expert_rank = np.take_along_axis(error_rank, best_expert[:, None], axis=1) 241 | line += f" & {np.mean(best_expert_rank):.2f}" 242 | 243 | errors = [] 244 | for seq_name, best in zip(seq_names, best_expert): 245 | error = np.array( 246 | [ 247 | error_rets[dataset_name][expert_name][seq_name] 248 | for expert_name in experts_name 249 | ] 250 | ) 251 | norm_error = minmax(error) 252 | errors.append(norm_error[best]) 253 | 254 | line += f" & {np.mean(errors):.2f}" 255 | line += " \\\\\n" 256 | 257 | latex += f"{line}" 258 | 259 | latex += "\\hline\n" 260 | latex += "\\end{tabular}\n" 261 | latex += "\\end{threeparttable}\n" 262 | latex += "\\end{table}\n" 263 | 264 | if filename is None: 265 | filename = "table" 266 | txt_file = save_dir / f"{filename}.txt" 267 | txt_file.write_text(latex) 268 | 269 | preview( 270 | latex, 271 | viewer="file", 272 | filename=save_dir / f"{filename}.png", 273 | packages=("multirow", "xcolor", "arydshln", "threeparttable"), 274 | ) 275 | -------------------------------------------------------------------------------- /evaluations/ope_benchmark.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import pickle 3 | import numpy as np 4 | from algorithms.aaa_util import calc_overlap 5 | import path_config 6 | 7 | sys.path.append("external/pysot-toolkit") 8 | from pysot.utils import success_overlap, success_error 9 | 10 | 11 | class OPEBenchmark: 12 | """ 13 | Args: 14 | result_path: result path of your tracker 15 | should the same format like VOT 16 | """ 17 | 18 | def __init__(self, dataset, dataset_name): 19 | self.dataset = dataset 20 | self.dataset_name = dataset_name 21 | 22 | def convert_bb_to_center(self, bboxes): 23 | return np.array( 24 | [ 25 | (bboxes[:, 0] + (bboxes[:, 2] - 1) / 2), 26 | (bboxes[:, 1] + (bboxes[:, 3] - 1) / 2), 27 | ] 28 | ).T 29 | 30 | def convert_bb_to_norm_center(self, bboxes, gt_wh): 31 | return self.convert_bb_to_center(bboxes) / (gt_wh + 1e-16) 32 | 33 | def get_tracker_traj(self, seq_name, tracker_name): 34 | results_dir = "{}/{}/{}".format( 35 | path_config.RESULTS_PATH, tracker_name, self.dataset_name 36 | ) 37 | base_results_path = "{}/{}".format(results_dir, seq_name) 38 | results_path = "{}.txt".format(base_results_path) 39 | tracker_traj = np.loadtxt(results_path, delimiter="\t") 40 | return tracker_traj 41 | 42 | def get_algorithm_data(self, seq_name, algorithm_name): 43 | results_dir = "{}/{}/{}".format( 44 | path_config.RESULTS_PATH, algorithm_name, self.dataset_name 45 | ) 46 | base_results_path = "{}/{}".format(results_dir, seq_name) 47 | offline_path = "{}_offline.pkl".format(base_results_path) 48 | with open(offline_path, "rb") as fp: 49 | offline_bb = pickle.load(fp) 50 | 51 | weights_path = "{}_weight.txt".format(base_results_path) 52 | tracker_weight = np.loadtxt(weights_path, delimiter="\t") 53 | 54 | return offline_bb, tracker_weight 55 | 56 | def get_anchor_frames(self, algorithm_name): 57 | anchor_frames = {} 58 | for seq in self.dataset: 59 | gt_traj = np.array(seq.ground_truth_rect) 60 | offline_bb, tracker_weight = self.get_algorithm_data( 61 | seq.name, algorithm_name 62 | ) 63 | offline_bb.insert(0, [gt_traj[0]]) 64 | anchor_frame = [ 65 | i for i in range(len(offline_bb)) if offline_bb[i] is not None 66 | ] 67 | anchor_frames[seq.name] = anchor_frame 68 | return anchor_frames 69 | 70 | def get_gt_trajs(self): 71 | gt_trajs = {} 72 | for seq in self.dataset: 73 | gt_traj = np.array(seq.ground_truth_rect) 74 | gt_trajs[seq.name] = gt_traj 75 | 76 | return gt_trajs 77 | 78 | def eval_times(self, tracker_name): 79 | time_ret = {} 80 | for seq in self.dataset: 81 | results_dir = "{}/{}/{}".format( 82 | path_config.RESULTS_PATH, tracker_name, self.dataset_name 83 | ) 84 | base_results_path = "{}/{}".format(results_dir, seq.name) 85 | times_path = "{}_time.txt".format(base_results_path) 86 | tracker_time = np.loadtxt(times_path, delimiter="\t", dtype=float) 87 | time_ret[seq.name] = np.mean(tracker_time[1:]) 88 | return time_ret 89 | 90 | def eval_success(self, tracker_name, anchor_frames=None): 91 | success_ret = {} 92 | for seq in self.dataset: 93 | gt_traj = np.array(seq.ground_truth_rect) 94 | valid_idx = ~np.isnan(gt_traj)[:, 0] 95 | tracker_traj = self.get_tracker_traj(seq.name, tracker_name) 96 | 97 | if anchor_frames is not None: 98 | anchor_frame_idx = np.zeros((len(gt_traj)), dtype=bool) 99 | anchor_frame_idx[anchor_frames[seq.name]] = 1 100 | valid_idx = valid_idx * anchor_frame_idx 101 | 102 | n_frame = sum(valid_idx) 103 | 104 | if n_frame > 0: 105 | success_ret[seq.name] = success_overlap( 106 | gt_traj[valid_idx], tracker_traj[valid_idx], n_frame 107 | ) 108 | else: 109 | success_ret[seq.name] = np.nan 110 | return success_ret 111 | 112 | def eval_precision(self, tracker_name, anchor_frames=None): 113 | precision_ret = {} 114 | for seq in self.dataset: 115 | gt_traj = np.array(seq.ground_truth_rect) 116 | valid_idx = ~np.isnan(gt_traj)[:, 0] 117 | tracker_traj = self.get_tracker_traj(seq.name, tracker_name) 118 | 119 | if anchor_frames is not None: 120 | anchor_frame_idx = np.zeros((len(gt_traj)), dtype=bool) 121 | anchor_frame_idx[anchor_frames[seq.name]] = 1 122 | valid_idx = valid_idx * anchor_frame_idx 123 | 124 | n_frame = sum(valid_idx) 125 | 126 | if n_frame > 0: 127 | gt_center = self.convert_bb_to_center(gt_traj[valid_idx]) 128 | tracker_center = self.convert_bb_to_center(tracker_traj[valid_idx]) 129 | thresholds = np.arange(0, 51, 1) 130 | precision_ret[seq.name] = success_error( 131 | gt_center, tracker_center, thresholds, n_frame 132 | ) 133 | else: 134 | precision_ret[seq.name] = np.nan 135 | return precision_ret 136 | 137 | def eval_norm_precision(self, tracker_name, anchor_frames=None): 138 | norm_precision_ret = {} 139 | for seq in self.dataset: 140 | gt_traj = np.array(seq.ground_truth_rect) 141 | valid_idx = ~np.isnan(gt_traj)[:, 0] 142 | tracker_traj = self.get_tracker_traj(seq.name, tracker_name) 143 | 144 | if anchor_frames is not None: 145 | anchor_frame_idx = np.zeros((len(gt_traj)), dtype=bool) 146 | anchor_frame_idx[anchor_frames[seq.name]] = 1 147 | valid_idx = valid_idx * anchor_frame_idx 148 | 149 | n_frame = sum(valid_idx) 150 | 151 | if n_frame > 0: 152 | gt_center_norm = self.convert_bb_to_norm_center( 153 | gt_traj[valid_idx], gt_traj[valid_idx, 2:4] 154 | ) 155 | tracker_center_norm = self.convert_bb_to_norm_center( 156 | tracker_traj[valid_idx], gt_traj[valid_idx, 2:4] 157 | ) 158 | thresholds = np.arange(0, 51, 1) / 100 159 | norm_precision_ret[seq.name] = success_error( 160 | gt_center_norm, tracker_center_norm, thresholds, n_frame 161 | ) 162 | else: 163 | norm_precision_ret[seq.name] = np.nan 164 | return norm_precision_ret 165 | 166 | def eval_loss(self, algorithm_name, tracker_name): 167 | error_ret = {} 168 | loss_ret = {} 169 | for seq in self.dataset: 170 | gt_traj = np.array(seq.ground_truth_rect) 171 | valid_idx = ~np.isnan(gt_traj)[:, 0] 172 | tracker_traj = self.get_tracker_traj(seq.name, tracker_name) 173 | 174 | offline_bb, tracker_weight = self.get_algorithm_data( 175 | seq.name, algorithm_name 176 | ) 177 | 178 | # flat offilne results 179 | offline_results = [gt_traj[0]] 180 | for box in offline_bb: 181 | if box is not None: 182 | if isinstance(box, np.ndarray): 183 | offline_results += box.tolist() 184 | else: 185 | offline_results += box 186 | offline_results = np.array(offline_results) 187 | 188 | # calc 189 | error_ret[seq.name] = 1 - calc_overlap(gt_traj[valid_idx], tracker_traj[valid_idx]) 190 | valid_results = tracker_traj[: len(offline_results)] 191 | if len(offline_results) > 0: 192 | loss_ret[seq.name] = 1 - calc_overlap(offline_results, valid_results) 193 | else: 194 | loss_ret[seq.name] = np.nan 195 | 196 | return error_ret, loss_ret 197 | 198 | def eval_offline(self, algorithm_name, tracker_name): 199 | success_ret = {} 200 | precision_ret = {} 201 | for seq in self.dataset: 202 | gt_traj = np.array(seq.ground_truth_rect) 203 | offline_bb, tracker_weight = self.get_algorithm_data( 204 | seq.name, algorithm_name 205 | ) 206 | tracker_traj = self.get_tracker_traj(seq.name, tracker_name) 207 | 208 | # flat offilne results 209 | offline_results = [gt_traj[0]] 210 | for box in offline_bb: 211 | if box is not None: 212 | if isinstance(box, np.ndarray): 213 | offline_results += box.tolist() 214 | else: 215 | offline_results += box 216 | offline_results = np.array(offline_results) 217 | 218 | valid_gt_traj = gt_traj[:len(offline_results)] 219 | valid_tracker_traj = tracker_traj[:len(offline_results)] 220 | valid_idx = ~np.isnan(valid_gt_traj)[:, 0] 221 | n_frame = sum(valid_idx) 222 | 223 | if algorithm_name == tracker_name: 224 | success_ret[seq.name] = success_overlap( 225 | valid_gt_traj[valid_idx], offline_results[valid_idx], n_frame 226 | ) 227 | gt_center = self.convert_bb_to_center(valid_gt_traj[valid_idx]) 228 | tracker_center = self.convert_bb_to_center(offline_results[valid_idx]) 229 | thresholds = np.arange(0, 51, 1) 230 | precision_ret[seq.name] = success_error( 231 | gt_center, tracker_center, thresholds, n_frame 232 | ) 233 | else: 234 | success_ret[seq.name] = success_overlap( 235 | valid_gt_traj[valid_idx], valid_tracker_traj[valid_idx], n_frame 236 | ) 237 | gt_center = self.convert_bb_to_center(valid_gt_traj[valid_idx]) 238 | tracker_center = self.convert_bb_to_center(valid_tracker_traj[valid_idx]) 239 | thresholds = np.arange(0, 51, 1) 240 | precision_ret[seq.name] = success_error( 241 | gt_center, tracker_center, thresholds, n_frame 242 | ) 243 | return success_ret, precision_ret 244 | -------------------------------------------------------------------------------- /datasets/lasotdataset.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | import path_config 4 | from datasets.data import Sequence, BaseDataset, SequenceList 5 | 6 | 7 | def LaSOTDataset(): 8 | return LaSOTDatasetClass().get_sequence_list() 9 | 10 | 11 | class LaSOTDatasetClass(BaseDataset): 12 | """ LaSOT test set consisting of 280 videos (see Protocol-II in the LaSOT paper) 13 | 14 | Publication: 15 | LaSOT: A High-quality Benchmark for Large-scale Single Object Tracking 16 | Heng Fan, Liting Lin, Fan Yang, Peng Chu, Ge Deng, Sijia Yu, Hexin Bai, Yong Xu, Chunyuan Liao and Haibin Ling 17 | CVPR, 2019 18 | https://arxiv.org/pdf/1809.07845.pdf 19 | 20 | Download the dataset from https://cis.temple.edu/lasot/download.html 21 | """ 22 | 23 | def __init__(self): 24 | super().__init__() 25 | self.base_path = path_config.LASOT_PATH 26 | self.sequence_list = self._get_sequence_list() 27 | self.clean_list = self.clean_seq_list() 28 | 29 | def clean_seq_list(self): 30 | clean_lst = [] 31 | for i in range(len(self.sequence_list)): 32 | cls, _ = self.sequence_list[i].split("-") 33 | clean_lst.append(cls) 34 | return clean_lst 35 | 36 | def get_sequence_list(self): 37 | return SequenceList([self._construct_sequence(s) for s in self.sequence_list]) 38 | 39 | def _construct_sequence(self, sequence_name): 40 | class_name = sequence_name.split("-")[0] 41 | anno_path = "{}/{}/{}/groundtruth.txt".format( 42 | self.base_path, class_name, sequence_name 43 | ) 44 | try: 45 | ground_truth_rect = np.loadtxt(str(anno_path), dtype=np.float64) 46 | except Exception: 47 | ground_truth_rect = np.loadtxt( 48 | str(anno_path), delimiter=",", dtype=np.float64 49 | ) 50 | 51 | frames_path = "{}/{}/{}/img".format(self.base_path, class_name, sequence_name) 52 | frame_list = [ 53 | frame for frame in os.listdir(frames_path) if frame.endswith(".jpg") 54 | ] 55 | frame_list.sort(key=lambda f: int(f[:-4])) 56 | frames_list = [os.path.join(frames_path, frame) for frame in frame_list] 57 | 58 | return Sequence( 59 | sequence_name, frames_list, "lasot", ground_truth_rect.reshape(-1, 4) 60 | ) 61 | 62 | def __len__(self): 63 | return len(self.sequence_list) 64 | 65 | def _get_sequence_list(self): 66 | sequence_list = [ 67 | "airplane-1", 68 | "airplane-9", 69 | "airplane-13", 70 | "airplane-15", 71 | "basketball-1", 72 | "basketball-6", 73 | "basketball-7", 74 | "basketball-11", 75 | "bear-2", 76 | "bear-4", 77 | "bear-6", 78 | "bear-17", 79 | "bicycle-2", 80 | "bicycle-7", 81 | "bicycle-9", 82 | "bicycle-18", 83 | "bird-2", 84 | "bird-3", 85 | "bird-15", 86 | "bird-17", 87 | "boat-3", 88 | "boat-4", 89 | "boat-12", 90 | "boat-17", 91 | "book-3", 92 | "book-10", 93 | "book-11", 94 | "book-19", 95 | "bottle-1", 96 | "bottle-12", 97 | "bottle-14", 98 | "bottle-18", 99 | "bus-2", 100 | "bus-5", 101 | "bus-17", 102 | "bus-19", 103 | "car-2", 104 | "car-6", 105 | "car-9", 106 | "car-17", 107 | "cat-1", 108 | "cat-3", 109 | "cat-18", 110 | "cat-20", 111 | "cattle-2", 112 | "cattle-7", 113 | "cattle-12", 114 | "cattle-13", 115 | "spider-14", 116 | "spider-16", 117 | "spider-18", 118 | "spider-20", 119 | "coin-3", 120 | "coin-6", 121 | "coin-7", 122 | "coin-18", 123 | "crab-3", 124 | "crab-6", 125 | "crab-12", 126 | "crab-18", 127 | "surfboard-12", 128 | "surfboard-4", 129 | "surfboard-5", 130 | "surfboard-8", 131 | "cup-1", 132 | "cup-4", 133 | "cup-7", 134 | "cup-17", 135 | "deer-4", 136 | "deer-8", 137 | "deer-10", 138 | "deer-14", 139 | "dog-1", 140 | "dog-7", 141 | "dog-15", 142 | "dog-19", 143 | "guitar-3", 144 | "guitar-8", 145 | "guitar-10", 146 | "guitar-16", 147 | "person-1", 148 | "person-5", 149 | "person-10", 150 | "person-12", 151 | "pig-2", 152 | "pig-10", 153 | "pig-13", 154 | "pig-18", 155 | "rubicCube-1", 156 | "rubicCube-6", 157 | "rubicCube-14", 158 | "rubicCube-19", 159 | "swing-10", 160 | "swing-14", 161 | "swing-17", 162 | "swing-20", 163 | "drone-13", 164 | "drone-15", 165 | "drone-2", 166 | "drone-7", 167 | "pool-12", 168 | "pool-15", 169 | "pool-3", 170 | "pool-7", 171 | "rabbit-10", 172 | "rabbit-13", 173 | "rabbit-17", 174 | "rabbit-19", 175 | "racing-10", 176 | "racing-15", 177 | "racing-16", 178 | "racing-20", 179 | "robot-1", 180 | "robot-19", 181 | "robot-5", 182 | "robot-8", 183 | "sepia-13", 184 | "sepia-16", 185 | "sepia-6", 186 | "sepia-8", 187 | "sheep-3", 188 | "sheep-5", 189 | "sheep-7", 190 | "sheep-9", 191 | "skateboard-16", 192 | "skateboard-19", 193 | "skateboard-3", 194 | "skateboard-8", 195 | "tank-14", 196 | "tank-16", 197 | "tank-6", 198 | "tank-9", 199 | "tiger-12", 200 | "tiger-18", 201 | "tiger-4", 202 | "tiger-6", 203 | "train-1", 204 | "train-11", 205 | "train-20", 206 | "train-7", 207 | "truck-16", 208 | "truck-3", 209 | "truck-6", 210 | "truck-7", 211 | "turtle-16", 212 | "turtle-5", 213 | "turtle-8", 214 | "turtle-9", 215 | "umbrella-17", 216 | "umbrella-19", 217 | "umbrella-2", 218 | "umbrella-9", 219 | "yoyo-15", 220 | "yoyo-17", 221 | "yoyo-19", 222 | "yoyo-7", 223 | "zebra-10", 224 | "zebra-14", 225 | "zebra-16", 226 | "zebra-17", 227 | "elephant-1", 228 | "elephant-12", 229 | "elephant-16", 230 | "elephant-18", 231 | "goldfish-3", 232 | "goldfish-7", 233 | "goldfish-8", 234 | "goldfish-10", 235 | "hat-1", 236 | "hat-2", 237 | "hat-5", 238 | "hat-18", 239 | "kite-4", 240 | "kite-6", 241 | "kite-10", 242 | "kite-15", 243 | "motorcycle-1", 244 | "motorcycle-3", 245 | "motorcycle-9", 246 | "motorcycle-18", 247 | "mouse-1", 248 | "mouse-8", 249 | "mouse-9", 250 | "mouse-17", 251 | "flag-3", 252 | "flag-9", 253 | "flag-5", 254 | "flag-2", 255 | "frog-3", 256 | "frog-4", 257 | "frog-20", 258 | "frog-9", 259 | "gametarget-1", 260 | "gametarget-2", 261 | "gametarget-7", 262 | "gametarget-13", 263 | "hand-2", 264 | "hand-3", 265 | "hand-9", 266 | "hand-16", 267 | "helmet-5", 268 | "helmet-11", 269 | "helmet-19", 270 | "helmet-13", 271 | "licenseplate-6", 272 | "licenseplate-12", 273 | "licenseplate-13", 274 | "licenseplate-15", 275 | "electricfan-1", 276 | "electricfan-10", 277 | "electricfan-18", 278 | "electricfan-20", 279 | "chameleon-3", 280 | "chameleon-6", 281 | "chameleon-11", 282 | "chameleon-20", 283 | "crocodile-3", 284 | "crocodile-4", 285 | "crocodile-10", 286 | "crocodile-14", 287 | "gecko-1", 288 | "gecko-5", 289 | "gecko-16", 290 | "gecko-19", 291 | "fox-2", 292 | "fox-3", 293 | "fox-5", 294 | "fox-20", 295 | "giraffe-2", 296 | "giraffe-10", 297 | "giraffe-13", 298 | "giraffe-15", 299 | "gorilla-4", 300 | "gorilla-6", 301 | "gorilla-9", 302 | "gorilla-13", 303 | "hippo-1", 304 | "hippo-7", 305 | "hippo-9", 306 | "hippo-20", 307 | "horse-1", 308 | "horse-4", 309 | "horse-12", 310 | "horse-15", 311 | "kangaroo-2", 312 | "kangaroo-5", 313 | "kangaroo-11", 314 | "kangaroo-14", 315 | "leopard-1", 316 | "leopard-7", 317 | "leopard-16", 318 | "leopard-20", 319 | "lion-1", 320 | "lion-5", 321 | "lion-12", 322 | "lion-20", 323 | "lizard-1", 324 | "lizard-3", 325 | "lizard-6", 326 | "lizard-13", 327 | "microphone-2", 328 | "microphone-6", 329 | "microphone-14", 330 | "microphone-16", 331 | "monkey-3", 332 | "monkey-4", 333 | "monkey-9", 334 | "monkey-17", 335 | "shark-2", 336 | "shark-3", 337 | "shark-5", 338 | "shark-6", 339 | "squirrel-8", 340 | "squirrel-11", 341 | "squirrel-13", 342 | "squirrel-19", 343 | "volleyball-1", 344 | "volleyball-13", 345 | "volleyball-18", 346 | "volleyball-19", 347 | ] 348 | return sequence_list 349 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # [AAA: Adaptive Aggregation of Arbitrary Online Trackers with Theoretical Performance Guarantee](https://arxiv.org/abs/2009.09237) 2 | 3 | ![Figure 1](assets/Fig1.png?raw=true "Score") 4 | 5 | Heon Song, Daiki Suehiro, Seiichi Uchida 6 | 7 | > For visual object tracking, it is difficult to realize an almighty online tracker due to the huge variations of target appearance depending on an image sequence. This paper proposes an online tracking method that adaptively aggregates arbitrary multiple online trackers. The performance of the proposed method is theoretically guaranteed to be comparable to that of the best tracker for any image sequence, although the best expert is unknown during tracking. The experimental study on the large variations of benchmark datasets and aggregated trackers demonstrates that the proposed method can achieve state-of-the-art performance. 8 | 9 | ## Experts 10 | 11 | In this repository, we implemented or edited the following trackers to use as experts. 12 | **You can use the trackers with just a few lines of code.** 13 | 14 | | Tracker | Link | 15 | |-----------|---------------------| 16 | | ATOM (CVPR 2019) | [Paper](https://arxiv.org/abs/1811.07628) / [Original Repo](https://github.com/visionml/pytracking) | 17 | | DaSiamRPN (ECCV 2018) | [Paper](https://arxiv.org/abs/1808.06048) / [Original Repo](https://github.com/foolwood/DaSiamRPN) | 18 | | DiMP (ICCV 2019) | [Paper](https://arxiv.org/abs/1904.07220) / [Original Repo](https://github.com/visionml/pytracking) | 19 | | DROL (AAAI 2020) | [Paper](https://arxiv.org/abs/1909.02959) / [Original Repo](https://github.com/shallowtoil/DROL) | 20 | | GradNet (ICCV 2019) | [Paper](https://arxiv.org/abs/1909.06800) / [Original Repo](https://github.com/LPXTT/GradNet-Tensorflow) | 21 | | KYS (ECCV 2020) | [Paper](https://arxiv.org/abs/2003.11014) / [Original Repo](https://github.com/visionml/pytracking) | 22 | | MemDTC (TPAMI 2019) | [Paper](https://arxiv.org/abs/1907.07613) / [Original Repo](https://github.com/skyoung/MemDTC) | 23 | | MemTrack (ECCV 2018) | [Paper](https://arxiv.org/abs/1803.07268) / [Original Repo](https://github.com/skyoung/MemTrack) | 24 | | Ocean (ECCV 2020) | [Paper](https://arxiv.org/abs/2006.10721) / [Original Repo](https://github.com/researchmm/TracKit) | 25 | | PrDiMP (CVPR 2020) | [Paper](https://arxiv.org/abs/2003.12565) / [Original Repo](https://github.com/visionml/pytracking) | 26 | | RLS-RTMDNet (CVPR 2020) | [Paper](https://openaccess.thecvf.com/content_CVPR_2020/html/Gao_Recursive_Least-Squares_Estimator-Aided_Online_Learning_for_Visual_Tracking_CVPR_2020_paper.html) / [Original Repo](https://github.com/Amgao/RLS-RTMDNet) | 27 | | ROAM (CVPR 2020) | [Paper](https://arxiv.org/abs/1907.12006) / [Original Repo](https://github.com/skyoung/ROAM) | 28 | | RPT (CVPR 2020) | [Paper](https://arxiv.org/abs/2008.03467) / [Original Repo](https://github.com/songheony/RPT) | 29 | | SiamBAN (CVPR 2020) | [Paper](https://arxiv.org/abs/2003.06761) / [Original Repo](https://github.com/hqucv/siamban) | 30 | | SiamCAR (CVPR 2020) | [Paper](https://arxiv.org/abs/1911.07241) / [Original Repo](https://github.com/ohhhyeahhh/SiamCAR) | 31 | | SiamDW (CVPR 2019) | [Paper](https://arxiv.org/abs/1901.01660) / [Original Repo](https://github.com/researchmm/SiamDW) | 32 | | SiamFC (ECCVW 2016) | [Paper](https://arxiv.org/abs/1606.09549) / [Original Repo](https://github.com/got-10k/siamfc) | 33 | | SiamFC++ (AAAI 2020) | [Paper](https://arxiv.org/abs/1911.06188) / [Original Repo](https://github.com/MegviiDetection/video_analyst) | 34 | | SiamMCF (ECCVW 2018) | [Paper](https://link.springer.com/chapter/10.1007/978-3-030-11009-3_6) / [Original Repo](https://github.com/hmorimitsu/siam-mcf) | 35 | | SiamR-CNN (CVPR 2020) | [Paper](https://arxiv.org/abs/1911.12836) / [Original Repo](https://github.com/VisualComputingInstitute/SiamR-CNN) | 36 | | SiamRPN (CVPR 2018) | [Paper](http://openaccess.thecvf.com/content_cvpr_2018/papers/Li_High_Performance_Visual_CVPR_2018_paper.pdf) / [Original Repo](https://github.com/huanglianghua/siamrpn-pytorch) | 37 | | SiamRPN++ (CVPR 2019) | [Paper](https://arxiv.org/abs/1812.11703) / [Original Repo](https://github.com/STVIR/pysot) | 38 | | SPM (CVPR 2019) | [Paper](https://arxiv.org/abs/1904.04452) / [Original Repo](https://github.com/microsoft/SPM-Tracker) | 39 | | Staple (CVPR 2016) | [Paper](https://arxiv.org/abs/1512.01355) / [Original Repo](https://github.com/wwdguu/pyCFTrackers) | 40 | | THOR (BMVC 2019) | [Paper](https://arxiv.org/abs/1907.12920) / [Original Repo](https://github.com/xl-sr/THOR) | 41 | 42 | For DaSiamRPN, RLS-RTMDNet, RPT and SPM, we've slightly modified the code to be compatible with Python3 and Pytorch >= 1.3. 43 | 44 | ## Datasets 45 | 46 | We evaluated the performance of the experts and AAA on the following datasets. 47 | 48 | * [OTB2015](https://ieeexplore.ieee.org/document/7001050)[] 49 | * [NFS](https://arxiv.org/abs/1703.05884)[] 50 | * [UAV123](https://ivul.kaust.edu.sa/Pages/pub-benchmark-simulator-uav.aspx)[] 51 | * [TColor128](https://ieeexplore.ieee.org/document/7277070)[] 52 | * [TrackingNet](https://arxiv.org/abs/1803.10794)[] 53 | * [VOT2018](https://link.springer.com/chapter/10.1007/978-3-030-11009-3_1)[] 54 | * [LaSOT](https://arxiv.org/abs/1809.07845)[] 55 | * [Got10K](https://arxiv.org/abs/1810.11981)[] 56 | 57 | VOT2018 is evaluated in unsupervised experiment as same as other datasets. 58 | 59 | ## Frameworks 60 | 61 | The following frameworks were used to conveniently track videos and evaluate trackers. 62 | 63 | * pytracking[] for tracking datasets. 64 | * pysot-toolkit[] for evaluating trackers. 65 | 66 | ## Requirements 67 | 68 | We strongly recommend using a virtual environment like Anaconda or Docker. 69 | The following is how to build the virtual environment for AAA using anaconda. 70 | 71 | ```sh 72 | # clone this repository 73 | git clone https://github.com/songheony/AAA-journal 74 | cd AAA-journal 75 | 76 | # create and activate anaconda environment 77 | conda create -y -n [ENV_NAME] python=[PYTHON_VERSION>=3] 78 | conda activate [ENV_NAME] 79 | 80 | # install requirements 81 | bash install_for_aaa.sh 82 | ``` 83 | 84 | ## Tracking 85 | 86 | If you want to apply AAA to your own project, 87 | simply make the following python script: 88 | 89 | ```python 90 | from algorithms.aaa import AAA 91 | 92 | img_paths = [] # list of image file paths 93 | initial_bbox = [x, y, w, h] # left x, top y, width, height of the initial target bbox 94 | n_experts = 6 # the number of experts you are using 95 | 96 | # define AAA 97 | theta, gamma = 0.92, 11 # you can tune hyperparameters by running run_tuning.sh 98 | algorithm = AAA(n_experts, mode="LOG_DIR", threshold=theta, feature_factor=gamma) 99 | 100 | # initialize AAA 101 | algorith.initialize(img_paths[0], initial_bbox) 102 | 103 | # track the target 104 | for img_path in img_paths[1:]: 105 | experts_result = np.zeros((n_experts, 4)) # the matrix of experts' estimation 106 | 107 | # state is the prediction of target bbox. 108 | # if the frame is not anchor frame, offline is None. else offline will be offline tracking results. 109 | # weight is the weight of the experts. 110 | state, offline, weight = self.track(img_path, experts_result) 111 | ``` 112 | 113 | In addition, trackers that we have implemented can be easily executed the following python script. 114 | 115 | ```python 116 | from select_options import select_expert 117 | 118 | img_paths = [] # list of image file paths 119 | initial_bbox = [x, y, w, h] # left x, top y, width, height of the initial target bbox 120 | 121 | # define Expert 122 | tracker_name = "DiMP" 123 | tracker = select_expert(tracker_name) 124 | 125 | # initialize Expert 126 | tracker.initialize(img_paths[0], initial_bbox) 127 | 128 | # track the target 129 | for img_path in img_paths[1:]: 130 | # state is the prediction of target bbox. 131 | state = self.track(img_path) 132 | ``` 133 | 134 | ## Requirements for experts 135 | 136 | * PyTorch 1.6.0 137 | * Tensorflow 1.14.0 138 | * CUDA 10.1 139 | * GCC 8 140 | 141 | First, metafiles including pretrained weights are need to be donloaded. 142 | And, the path of metafiles in ``path_config.py`` and ``local.py`` file must be edited. 143 | 144 | In order to run experts, you need to install additional libraries. 145 | We offer install script to make it easy to run experts: 146 | 147 | ```sh 148 | # Only for Ubuntu 149 | sudo apt install -y libopenmpi-dev libgl1-mesa-glx ninja-build 150 | 151 | # activate anaconda environment 152 | conda activate [ENV_NAME] 153 | 154 | # install requirements 155 | bash install_for_experts.sh 156 | ``` 157 | 158 | ## Reproduce our results 159 | 160 | We provide scripts to reproduce all results, figures, and tables in our paper. 161 | In addition, we provide the following files in case you don't have time to run all the scripts yourself. 162 | [Experts tracking results](##) 163 | [AAA tuning results](##) 164 | [AAA tracking results](##) 165 | [HDT tracking results](##) 166 | [MCCT tracking results](##) 167 | [Baselines tracking results](##) 168 | 169 | ```sh 170 | # Run experts 171 | # If you've downloaded Experts tracking results, you can skip this command 172 | bash run_experts.sh 173 | 174 | # Tune the hyperparameter 175 | # If you've downloaded AAA tuning results, you can skip this command 176 | bash run_tuning.sh 177 | 178 | # Run AAA 179 | # If you've download AAA tracking results, you can skip this command 180 | bash run_algorithm.sh 181 | 182 | # Run HDT 183 | # If you've download HDT tracking results, you can skip this command 184 | bash run_hdt.sh 185 | 186 | # Run MCCT 187 | # If you've download MCCT tracking results, you can skip this command 188 | bash run_mcct.sh 189 | 190 | # Run simple baselines 191 | # If you've download Baselines tracking results, you can skip this command 192 | bash run_baselines.sh 193 | 194 | # Visualize figures and tables in our paper 195 | python visualize_figure.py 196 | ``` 197 | 198 | The code is supposed to run algorithms after running experts for test. 199 | However, it is easy to modify the code to do both simultaneously. 200 | 201 | ## Citation 202 | 203 | If you find AAA useful in your work, please cite our paper: 204 | 205 | ```none 206 | @article{song2020aaa, 207 | title={AAA: Adaptive Aggregation of Arbitrary Online Trackers with Theoretical Performance Guarantee}, 208 | author={Song, Heon and Suehiro, Daiki and Uchida, Seiichi}, 209 | journal={arXiv preprint arXiv:2009.09237}, 210 | year={2020} 211 | } 212 | ``` 213 | 214 | ## Author 215 | 216 | 👤 **Heon Song** 217 | 218 | * Github: [@songheony](https://github.com/songheony) 219 | * Contact: songheony@gmail.com 220 | -------------------------------------------------------------------------------- /experts/gradnet.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" 4 | os.environ["C_CPP_MIN_LOG_LEVEL"] = "3" 5 | import sys 6 | import cv2 7 | import numpy as np 8 | import path_config 9 | import tensorflow as tf 10 | 11 | tf.get_logger().setLevel("INFO") 12 | from base_tracker import BaseTracker 13 | 14 | sys.path.append("external/GradNet-Tensorflow") 15 | from parameters import configParams 16 | from track import ( 17 | getOpts, 18 | createLabels, 19 | makeScalePyramid, 20 | getSubWinTracking, 21 | trackerEval, 22 | ) 23 | from siamese import SiameseNet 24 | from region_to_bbox import region_to_bbox 25 | 26 | 27 | class GradNet(BaseTracker): 28 | def __init__(self): 29 | super(GradNet, self).__init__("GradNet") 30 | self.opts = configParams() 31 | self.opts = getOpts(self.opts) 32 | 33 | """define input tensors and network""" 34 | self.exemplarOp_init = tf.placeholder( 35 | tf.float32, [1, self.opts["exemplarSize"], self.opts["exemplarSize"], 3] 36 | ) 37 | self.instanceOp_init = tf.placeholder( 38 | tf.float32, [1, self.opts["instanceSize"], self.opts["instanceSize"], 3] 39 | ) 40 | self.instanceOp = tf.placeholder( 41 | tf.float32, [3, self.opts["instanceSize"], self.opts["instanceSize"], 3] 42 | ) 43 | self.template_Op = tf.placeholder(tf.float32, [1, 6, 6, 256]) 44 | self.search_tr_Op = tf.placeholder(tf.float32, [3, 22, 22, 32]) 45 | self.isTrainingOp = tf.convert_to_tensor( 46 | False, dtype="bool", name="is_training" 47 | ) 48 | self.lr = tf.constant(0.0001, dtype="float32") 49 | self.sn = SiameseNet() 50 | 51 | """build the model""" 52 | # initial embedding 53 | with tf.variable_scope("siamese") as scope: 54 | self.zFeat2Op_init, self.zFeat5Op_init = self.sn.extract_gra_fea_template( 55 | self.exemplarOp_init, self.opts, self.isTrainingOp 56 | ) 57 | self.scoreOp_init = self.sn.response_map_cal( 58 | self.instanceOp_init, self.zFeat5Op_init, self.opts, self.isTrainingOp 59 | ) 60 | # gradient calculation 61 | self.labels = np.ones([8], dtype=np.float32) 62 | self.respSz = int(self.scoreOp_init.get_shape()[1]) 63 | self.respSz = [self.respSz, self.respSz] 64 | self.respStride = 8 65 | self.fixedLabel, self.instanceWeight = createLabels( 66 | self.respSz, 67 | self.opts["lossRPos"] / self.respStride, 68 | self.opts["lossRNeg"] / self.respStride, 69 | 1, 70 | ) 71 | self.instanceWeightOp = tf.constant(self.instanceWeight, dtype=tf.float32) 72 | self.yOp = tf.constant(self.fixedLabel, dtype=tf.float32) 73 | with tf.name_scope("logistic_loss"): 74 | self.lossOp_init = self.sn.loss( 75 | self.scoreOp_init, self.yOp, self.instanceWeightOp 76 | ) 77 | self.grad_init = tf.gradients(self.lossOp_init, self.zFeat2Op_init) 78 | # template update and get score map 79 | with tf.variable_scope("siamese") as scope: 80 | self.zFeat5Op_gra, self.zFeat2Op_gra = self.sn.template_update_based_grad( 81 | self.zFeat2Op_init, self.grad_init[0], self.opts, self.isTrainingOp 82 | ) 83 | scope.reuse_variables() 84 | self.zFeat5Op_sia = self.sn.extract_sia_fea_template( 85 | self.exemplarOp_init, self.opts, self.isTrainingOp 86 | ) 87 | self.scoreOp_sia = self.sn.response_map_cal( 88 | self.instanceOp, self.zFeat5Op_sia, self.opts, self.isTrainingOp 89 | ) 90 | self.scoreOp_gra = self.sn.response_map_cal( 91 | tf.expand_dims(self.instanceOp[1], 0), 92 | self.zFeat5Op_gra, 93 | self.opts, 94 | self.isTrainingOp, 95 | ) 96 | 97 | """restore pretrained network""" 98 | self.saver = tf.train.Saver() 99 | self.config = tf.ConfigProto() 100 | self.config.gpu_options.allow_growth = True 101 | self.sess = tf.Session(config=self.config) 102 | self.saver.restore(self.sess, path_config.GRADNET_MODEL) 103 | 104 | def initialize(self, image_file, box): 105 | im = cv2.imread(image_file, cv2.IMREAD_COLOR) 106 | cx, cy, w, h = region_to_bbox(box) 107 | self.targetSize = np.array([h, w]) 108 | self.targetPosition = np.array([cy, cx]) 109 | self.avgChans = np.mean( 110 | im, axis=(0, 1) 111 | ) # [np.mean(np.mean(img[:, :, 0])), np.mean(np.mean(img[:, :, 1])), np.mean(np.mean(img[:, :, 2]))] 112 | self.wcz = self.targetSize[1] + self.opts["contextAmount"] * np.sum( 113 | self.targetSize 114 | ) 115 | self.hcz = self.targetSize[0] + self.opts["contextAmount"] * np.sum( 116 | self.targetSize 117 | ) 118 | self.sz = np.sqrt(self.wcz * self.hcz) 119 | self.scalez = self.opts["exemplarSize"] / self.sz 120 | 121 | self.zCrop, _ = getSubWinTracking( 122 | im, 123 | self.targetPosition, 124 | (self.opts["exemplarSize"], self.opts["exemplarSize"]), 125 | (np.around(self.sz), np.around(self.sz)), 126 | self.avgChans, 127 | ) 128 | 129 | if self.opts["subMean"]: 130 | pass 131 | 132 | self.dSearch = (self.opts["instanceSize"] - self.opts["exemplarSize"]) / 2 133 | self.pad = self.dSearch / self.scalez 134 | self.sx = self.sz + 2 * self.pad 135 | 136 | self.minSx = 0.2 * self.sx 137 | self.maxSx = 5.0 * self.sx 138 | 139 | self.winSz = self.opts["scoreSize"] * self.opts["responseUp"] 140 | if self.opts["windowing"] == "cosine": 141 | self.hann = np.hanning(self.winSz).reshape(self.winSz, 1) 142 | self.window = self.hann.dot(self.hann.T) 143 | elif self.opts["windowing"] == "uniform": 144 | self.window = np.ones((self.winSz, self.winSz), dtype=np.float32) 145 | 146 | self.window = self.window / np.sum(self.window) 147 | self.scales = np.array( 148 | [ 149 | self.opts["scaleStep"] ** i 150 | for i in range( 151 | int(np.ceil(self.opts["numScale"] / 2.0) - self.opts["numScale"]), 152 | int(np.floor(self.opts["numScale"] / 2.0) + 1), 153 | ) 154 | ] 155 | ) 156 | 157 | """initialization at the first frame""" 158 | self.xCrops = makeScalePyramid( 159 | im, 160 | self.targetPosition, 161 | self.sx * self.scales, 162 | self.opts["instanceSize"], 163 | self.avgChans, 164 | None, 165 | self.opts, 166 | ) 167 | self.xCrops0 = np.expand_dims(self.xCrops[1], 0) 168 | self.zCrop = np.expand_dims(self.zCrop, axis=0) 169 | self.zCrop0 = np.copy(self.zCrop) 170 | 171 | ( 172 | self.zFeat5_gra_init, 173 | self.zFeat2_gra_init, 174 | self.zFeat5_sia_init, 175 | ) = self.sess.run( 176 | [self.zFeat5Op_gra, self.zFeat2Op_gra, self.zFeat5Op_sia], 177 | feed_dict={ 178 | self.exemplarOp_init: self.zCrop0, 179 | self.instanceOp_init: self.xCrops0, 180 | self.instanceOp: self.xCrops, 181 | }, 182 | ) 183 | self.template_gra = np.copy(self.zFeat5_gra_init) 184 | self.template_sia = np.copy(self.zFeat5_sia_init) 185 | self.hid_gra = np.copy(self.zFeat2_gra_init) 186 | 187 | self.train_all = [] 188 | self.frame_all = [] 189 | self.F_max_all = 0 190 | self.A_all = [] 191 | self.F_max_thred = 0 192 | self.F_max = 0 193 | self.train_all.append(self.xCrops0) 194 | self.A_all.append(0) 195 | self.frame_all.append(0) 196 | self.updata_features = [] 197 | self.updata_features_score = [] 198 | self.updata_features_frame = [] 199 | self.no_cos = 1 200 | self.refind = 0 201 | 202 | self.frame = 0 203 | 204 | """tracking results""" 205 | self.rectPosition = self.targetPosition - self.targetSize / 2.0 206 | self.Position_now = np.concatenate( 207 | [ 208 | np.round(self.rectPosition).astype(int)[::-1], 209 | np.round(self.targetSize).astype(int)[::-1], 210 | ], 211 | 0, 212 | ) 213 | 214 | if ( 215 | self.Position_now[0] + self.Position_now[2] > im.shape[1] 216 | and self.F_max < self.F_max_thred * 0.5 217 | ): 218 | self.refind = 1 219 | 220 | """if you want use groundtruth""" 221 | 222 | # region = np.copy(gt[i]) 223 | 224 | # cx, cy, w, h = getAxisAlignedBB(region) 225 | # pos = np.array([cy, cx]) 226 | # targetSz = np.array([h, w]) 227 | # iou_ = _compute_distance(region, Position_now) 228 | # 229 | 230 | """save the reliable training sample""" 231 | if self.F_max >= min( 232 | self.F_max_thred * 0.5, np.mean(self.updata_features_score) 233 | ): 234 | self.scaledInstance = self.sx * self.scales 235 | self.xCrops = makeScalePyramid( 236 | im, 237 | self.targetPosition, 238 | self.scaledInstance, 239 | self.opts["instanceSize"], 240 | self.avgChans, 241 | None, 242 | self.opts, 243 | ) 244 | self.updata_features.append(self.xCrops) 245 | self.updata_features_score.append(self.F_max) 246 | self.updata_features_frame.append(self.frame) 247 | if self.updata_features_score.__len__() > 5: 248 | del self.updata_features_score[0] 249 | del self.updata_features[0] 250 | del self.updata_features_frame[0] 251 | else: 252 | if self.frame < 10 and self.F_max < self.F_max_thred * 0.4: 253 | self.scaledInstance = self.sx * self.scales 254 | self.xCrops = makeScalePyramid( 255 | im, 256 | self.targetPosition, 257 | self.scaledInstance, 258 | self.opts["instanceSize"], 259 | self.avgChans, 260 | None, 261 | self.opts, 262 | ) 263 | self.template_gra, self.zFeat2_gra = self.sess.run( 264 | [self.zFeat5Op_gra, self.zFeat2Op_gra], 265 | feed_dict={ 266 | self.zFeat2Op_init: self.hid_gra, 267 | self.instanceOp_init: np.expand_dims(self.xCrops[1], 0), 268 | }, 269 | ) 270 | self.hid_gra = np.copy(0.3 * self.hid_gra + 0.7 * self.zFeat2_gra) 271 | 272 | """update the template every 5 frames""" 273 | 274 | if self.frame % 5 == 0: 275 | self.template_gra, self.zFeat2_gra = self.sess.run( 276 | [self.zFeat5Op_gra, self.zFeat2Op_gra], 277 | feed_dict={ 278 | self.zFeat2Op_init: self.hid_gra, 279 | self.instanceOp_init: np.expand_dims( 280 | self.updata_features[np.argmax(self.updata_features_score)][1], 281 | 0, 282 | ), 283 | }, 284 | ) 285 | self.hid_gra = np.copy(0.4 * self.hid_gra + 0.6 * self.zFeat2_gra) 286 | 287 | def track(self, image_file): 288 | im = cv2.imread(image_file, cv2.IMREAD_COLOR) 289 | self.frame += 1 290 | 291 | if self.frame - self.updata_features_frame[-1] == 9 and self.no_cos: 292 | self.opts["wInfluence"] = 0 293 | self.no_cos = 0 294 | else: 295 | self.opts["wInfluence"] = 0.25 296 | 297 | if im.shape[-1] == 1: 298 | tmp = np.zeros([im.shape[0], im.shape[1], 3], dtype=np.float32) 299 | tmp[:, :, 0] = tmp[:, :, 1] = tmp[:, :, 2] = np.squeeze(im) 300 | im = tmp 301 | 302 | self.scaledInstance = self.sx * self.scales 303 | self.scaledTarget = np.array([self.targetSize * scale for scale in self.scales]) 304 | 305 | self.xCrops = makeScalePyramid( 306 | im, 307 | self.targetPosition, 308 | self.scaledInstance, 309 | self.opts["instanceSize"], 310 | self.avgChans, 311 | None, 312 | self.opts, 313 | ) 314 | 315 | self.score_gra, self.score_sia = self.sess.run( 316 | [self.scoreOp_gra, self.scoreOp_sia], 317 | feed_dict={ 318 | self.zFeat5Op_gra: self.template_gra, 319 | self.zFeat5Op_sia: self.template_sia, 320 | self.instanceOp: self.xCrops, 321 | }, 322 | ) 323 | # sio.savemat('score.mat', {'score': score}) 324 | # score_gra = np.copy(np.expand_dims(score_sia[1],0)) 325 | 326 | self.newTargetPosition, self.newScale = trackerEval( 327 | self.score_sia, 328 | self.score_gra, 329 | round(self.sx), 330 | self.targetPosition, 331 | self.window, 332 | self.opts, 333 | ) 334 | 335 | self.targetPosition = self.newTargetPosition 336 | self.sx = max( 337 | self.minSx, 338 | min( 339 | self.maxSx, 340 | (1 - self.opts["scaleLr"]) * self.sx 341 | + self.opts["scaleLr"] * self.scaledInstance[self.newScale], 342 | ), 343 | ) 344 | self.F_max = np.max(self.score_sia) 345 | self.targetSize = (1 - self.opts["scaleLr"]) * self.targetSize + self.opts[ 346 | "scaleLr" 347 | ] * self.scaledTarget[self.newScale] 348 | # print('frame:%d--loss:%f--frame_now:%d' %(i, np.max(score),frame_now)) 349 | 350 | if self.refind: 351 | 352 | self.xCrops = makeScalePyramid( 353 | im, 354 | np.array([im.shape[0] / 2, im.shape[1] / 2]), 355 | self.scaledInstance, 356 | self.opts["instanceSize"], 357 | self.avgChans, 358 | None, 359 | self.opts, 360 | ) 361 | 362 | self.score_gra, self.score_sia = self.sess.run( 363 | [self.scoreOp_gra, self.scoreOp_sia], 364 | feed_dict={ 365 | self.zFeat5Op_gra: self.template_gra, 366 | self.zFeat5Op_sia: self.template_sia, 367 | self.instanceOp: self.xCrops, 368 | }, 369 | ) 370 | self.F_max2 = np.max(self.score_sia) 371 | self.F_max3 = np.max(self.score_gra) 372 | if self.F_max2 > self.F_max and self.F_max3 > self.F_max: 373 | self.newTargetPosition, self.newScale = trackerEval( 374 | self.score_sia, 375 | self.score_gra, 376 | round(self.sx), 377 | np.array([im.shape[0] / 2, im.shape[1] / 2]), 378 | self.window, 379 | self.opts, 380 | ) 381 | 382 | self.targetPosition = self.newTargetPosition 383 | self.sx = max( 384 | self.minSx, 385 | min( 386 | self.maxSx, 387 | (1 - self.opts["scaleLr"]) * self.sx 388 | + self.opts["scaleLr"] * self.scaledInstance[self.newScale], 389 | ), 390 | ) 391 | self.F_max = np.max(self.score_sia) 392 | self.targetSize = ( 393 | (1 - self.opts["scaleLr"]) * self.targetSize 394 | + self.opts["scaleLr"] * self.scaledTarget[self.newScale] 395 | ) 396 | 397 | self.refind = 0 398 | 399 | """use the average of the first five frames to set the threshold""" 400 | if self.frame < 6: 401 | self.F_max_all = self.F_max_all + self.F_max 402 | if self.frame == 5: 403 | self.F_max_thred = self.F_max_all / 5.0 404 | 405 | """tracking results""" 406 | self.rectPosition = self.targetPosition - self.targetSize / 2.0 407 | self.Position_now = np.concatenate( 408 | [ 409 | np.round(self.rectPosition).astype(int)[::-1], 410 | np.round(self.targetSize).astype(int)[::-1], 411 | ], 412 | 0, 413 | ) 414 | bbox = self.Position_now[:] 415 | 416 | if ( 417 | self.Position_now[0] + self.Position_now[2] > im.shape[1] 418 | and self.F_max < self.F_max_thred * 0.5 419 | ): 420 | self.refind = 1 421 | 422 | """if you want use groundtruth""" 423 | 424 | # region = np.copy(gt[i]) 425 | 426 | # cx, cy, w, h = getAxisAlignedBB(region) 427 | # pos = np.array([cy, cx]) 428 | # targetSz = np.array([h, w]) 429 | # iou_ = _compute_distance(region, Position_now) 430 | # 431 | 432 | """save the reliable training sample""" 433 | if self.F_max >= min( 434 | self.F_max_thred * 0.5, np.mean(self.updata_features_score) 435 | ): 436 | self.scaledInstance = self.sx * self.scales 437 | self.xCrops = makeScalePyramid( 438 | im, 439 | self.targetPosition, 440 | self.scaledInstance, 441 | self.opts["instanceSize"], 442 | self.avgChans, 443 | None, 444 | self.opts, 445 | ) 446 | self.updata_features.append(self.xCrops) 447 | self.updata_features_score.append(self.F_max) 448 | self.updata_features_frame.append(self.frame) 449 | if self.updata_features_score.__len__() > 5: 450 | del self.updata_features_score[0] 451 | del self.updata_features[0] 452 | del self.updata_features_frame[0] 453 | else: 454 | if self.frame < 10 and self.F_max < self.F_max_thred * 0.4: 455 | self.scaledInstance = self.sx * self.scales 456 | self.xCrops = makeScalePyramid( 457 | im, 458 | self.targetPosition, 459 | self.scaledInstance, 460 | self.opts["instanceSize"], 461 | self.avgChans, 462 | None, 463 | self.opts, 464 | ) 465 | self.template_gra, self.zFeat2_gra = self.sess.run( 466 | [self.zFeat5Op_gra, self.zFeat2Op_gra], 467 | feed_dict={ 468 | self.zFeat2Op_init: self.hid_gra, 469 | self.instanceOp_init: np.expand_dims(self.xCrops[1], 0), 470 | }, 471 | ) 472 | self.hid_gra = np.copy(0.3 * self.hid_gra + 0.7 * self.zFeat2_gra) 473 | 474 | """update the template every 5 frames""" 475 | 476 | if self.frame % 5 == 0: 477 | self.template_gra, self.zFeat2_gra = self.sess.run( 478 | [self.zFeat5Op_gra, self.zFeat2Op_gra], 479 | feed_dict={ 480 | self.zFeat2Op_init: self.hid_gra, 481 | self.instanceOp_init: np.expand_dims( 482 | self.updata_features[np.argmax(self.updata_features_score)][1], 483 | 0, 484 | ), 485 | }, 486 | ) 487 | self.hid_gra = np.copy(0.4 * self.hid_gra + 0.6 * self.zFeat2_gra) 488 | 489 | return bbox 490 | --------------------------------------------------------------------------------