├── .gitignore ├── LICENSE ├── README.md ├── code ├── MergeTrack │ ├── ReID_net_functions.py │ ├── merge.py │ ├── merge_functions.py │ ├── oldmerge.py │ ├── print_max_reid_distance.py │ └── refinement_net_functions.py ├── ReID_net │ ├── Config.py │ ├── Constants.py │ ├── Engine.py │ ├── Forwarding │ │ ├── CMC_Validator.py │ │ ├── COCOInstanceForwarder.py │ │ ├── ClusteringForwarder.py │ │ ├── ClusteringForwarder_old.py │ │ ├── DatasetSpeedtestForwarder.py │ │ ├── Forwarder.py │ │ ├── InteractiveImageForwarder.py │ │ ├── IterativeImageForwarder.py │ │ ├── MARKET1501Forwarder.py │ │ ├── MOT15Forwarder.py │ │ ├── MaskTransferForwarder.py │ │ ├── OfflineAdaptingForwarder.py │ │ ├── OneshotForwarder.py │ │ ├── OnlineAdaptingForwarder.py │ │ ├── PascalVOCInstanceForwarder.py │ │ ├── ReIDForwarding.py │ │ └── __init__.py │ ├── Interface.py │ ├── Log.py │ ├── Measures.py │ ├── README │ ├── Trainer.py │ ├── Util.py │ ├── __init__.py │ ├── configs │ │ ├── live │ │ ├── old_reference_configs │ │ │ ├── FORWARD_davis │ │ │ ├── FORWARD_live │ │ │ ├── FORWARD_test │ │ │ ├── FORWARD_valid │ │ │ ├── TRAIN │ │ │ ├── TRAIN-ytvos │ │ │ ├── TRAIN-ytvos2 │ │ │ ├── forward │ │ │ └── forward_temp │ │ └── run │ ├── datasets │ │ ├── Augmentors.py │ │ ├── COCO │ │ │ ├── COCO.py │ │ │ ├── COCO_detection.py │ │ │ ├── COCO_instance.py │ │ │ ├── COCO_interactive.py │ │ │ ├── COCO_objectness.py │ │ │ └── __init__.py │ │ ├── Custom │ │ │ ├── Custom.py │ │ │ └── __init__.py │ │ ├── DAVIS │ │ │ ├── DAVIS.py │ │ │ ├── DAVIS2017_oneshot.py │ │ │ ├── DAVIS_instance.py │ │ │ ├── DAVIS_iterative.py │ │ │ ├── DAVIS_masktransfer.py │ │ │ ├── DAVIS_oneshot.py │ │ │ └── __init__.py │ │ ├── Dataset.py │ │ ├── DetectionFeedDataset.py │ │ ├── FeedDataset.py │ │ ├── Forward.py │ │ ├── InteractiveEval.py │ │ ├── InteractiveFeedDataset.py │ │ ├── Loader.py │ │ ├── Similarity │ │ │ ├── COCO_Similarity.py │ │ │ ├── DAVIS_Forward_Feed.py │ │ │ ├── DAVIS_Forward_Similarity.py │ │ │ ├── DAVIS_Similarity.py │ │ │ ├── Similarity.py │ │ │ └── __init__.py │ │ ├── Util │ │ │ ├── Batch.py │ │ │ ├── Input.py │ │ │ ├── MaskDamager.py │ │ │ ├── Normalization.py │ │ │ ├── Reader.py │ │ │ ├── Resize.py │ │ │ ├── Timer.py │ │ │ ├── Util.py │ │ │ ├── Video.py │ │ │ ├── Webp.py │ │ │ ├── __init__.py │ │ │ ├── flo_Reader.py │ │ │ ├── pascal_colormap.py │ │ │ └── python_pfm.py │ │ └── __init__.py │ ├── main.py │ ├── network │ │ ├── Network.py │ │ ├── NetworkLayers.py │ │ ├── NetworkOutputLayers.py │ │ ├── NetworkSiameseLayers.py │ │ ├── Util_Network.py │ │ └── __init__.py │ ├── prepare_input.py │ └── scripts │ │ ├── cluster_optimizer.py │ │ ├── config │ │ ├── create_oneshot_trainsplit_configs.sh │ │ ├── duplicate_trainsplit_config.sh │ │ └── duplicate_trainsplit_config3.sh │ │ ├── dfext.py │ │ ├── eval │ │ ├── eval.py │ │ ├── eval_KITTI_det.sh │ │ ├── eval_coco.py │ │ ├── eval_youtube.py │ │ └── eval_youtube_nonfull.py │ │ ├── eval_pascal_instance │ │ ├── convert_voc_to_sharpmask.py │ │ └── eval_sharpmask_output.py │ │ ├── make_cluster_graphic.py │ │ ├── plot_learn_curve2.py │ │ ├── postproc │ │ ├── combine_single_object_predictions.py │ │ ├── convert_coco_detections_to_kitti_format.py │ │ ├── crf │ │ │ ├── combine_single_object_predictions_crf.py │ │ │ ├── crf_davis.py │ │ │ └── crf_youtube.py │ │ ├── ensemble.py │ │ ├── ensemble2017.py │ │ ├── postproc.py │ │ └── test_snapping.py │ │ ├── read_match_merge_files.py │ │ ├── rename_weights.py │ │ ├── rename_weights.py~ │ │ ├── stats │ │ ├── DAVIS_iou.py │ │ ├── compute_iou.sh │ │ ├── compute_iou_cv.sh │ │ ├── find_best_err.sh │ │ ├── find_best_score.sh │ │ ├── per_sequence.sh │ │ ├── print_mean_std_ddof.py │ │ ├── print_mean_std_ddof_pretty.py │ │ ├── print_stats.py │ │ ├── show_ious.sh │ │ └── show_ious_cv.sh │ │ ├── temp.py │ │ └── test_load_pairs.py ├── lucid_data_dreaming │ ├── readme.txt │ └── run.m ├── optical_flow_net-PWC-Net │ ├── LICENSE.md │ ├── README.md │ ├── external_packages │ │ └── correlation-pytorch-master │ │ │ ├── correlation-pytorch │ │ │ ├── __init__.py │ │ │ ├── build.py │ │ │ ├── build │ │ │ │ ├── lib.linux-x86_64-3.6 │ │ │ │ │ └── correlation_package │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── _ext │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── corr │ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ │ └── _corr.abi3.so │ │ │ │ │ │ ├── functions │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── corr.py │ │ │ │ │ │ └── modules │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── corr.py │ │ │ │ └── temp.linux-x86_64-3.6 │ │ │ │ │ ├── build │ │ │ │ │ └── temp.linux-x86_64-3.6 │ │ │ │ │ │ └── correlation_package._ext.corr._corr.o │ │ │ │ │ ├── correlation_package._ext.corr._corr.c │ │ │ │ │ └── home │ │ │ │ │ └── luiten │ │ │ │ │ └── vision │ │ │ │ │ └── youtubevos │ │ │ │ │ └── PWC-net │ │ │ │ │ └── PWC-Net-master │ │ │ │ │ └── PyTorch │ │ │ │ │ └── external_packages │ │ │ │ │ └── correlation-pytorch-master │ │ │ │ │ └── correlation-pytorch │ │ │ │ │ └── correlation_package │ │ │ │ │ └── src │ │ │ │ │ ├── corr.o │ │ │ │ │ ├── corr1d.o │ │ │ │ │ ├── corr1d_cuda.o │ │ │ │ │ └── corr_cuda.o │ │ │ ├── correlation_package.egg-info │ │ │ │ ├── PKG-INFO │ │ │ │ ├── SOURCES.txt │ │ │ │ ├── dependency_links.txt │ │ │ │ ├── requires.txt │ │ │ │ └── top_level.txt │ │ │ ├── correlation_package │ │ │ │ ├── __init__.py │ │ │ │ ├── _ext │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── corr │ │ │ │ │ │ └── __init__.py │ │ │ │ ├── functions │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── corr.py │ │ │ │ ├── modules │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── corr.py │ │ │ │ └── src │ │ │ │ │ ├── corr.c │ │ │ │ │ ├── corr.h │ │ │ │ │ ├── corr1d.c │ │ │ │ │ ├── corr1d.h │ │ │ │ │ ├── corr1d_cuda.c │ │ │ │ │ ├── corr1d_cuda.h │ │ │ │ │ ├── corr1d_cuda_kernel.cu │ │ │ │ │ ├── corr1d_cuda_kernel.cu.o │ │ │ │ │ ├── corr1d_cuda_kernel.h │ │ │ │ │ ├── corr_cuda.c │ │ │ │ │ ├── corr_cuda.h │ │ │ │ │ ├── corr_cuda_kernel.cu │ │ │ │ │ ├── corr_cuda_kernel.cu.o │ │ │ │ │ └── corr_cuda_kernel.h │ │ │ ├── dist │ │ │ │ └── correlation_package-0.1-py3.6-linux-x86_64.egg │ │ │ ├── setup.py │ │ │ └── test │ │ │ │ ├── test.ipynb │ │ │ │ └── test.py │ │ │ ├── make_cuda.sh │ │ │ └── readme.MD │ ├── models │ │ ├── PWCNet.py │ │ └── __init__.py │ ├── script_pwc.py │ └── script_pwc_multi.py ├── proposal_net │ ├── README │ ├── basemodel.py │ ├── coco.py │ ├── combine_general_and_specific.py │ ├── common.py │ ├── config.py │ ├── data.py │ ├── eval.py │ ├── forward_proto.py │ ├── hypotheses_pb2.py │ ├── model.py │ ├── train.py │ ├── utils │ │ ├── README.md │ │ ├── __init__.py │ │ ├── box_ops.py │ │ ├── generate_anchors.py │ │ └── np_box_ops.py │ └── viz.py └── refinement_net │ ├── README │ ├── __init__.py │ ├── configs │ ├── live │ ├── old_configs │ │ ├── mapillary_quarter_jitter005_deeplab │ │ ├── mapillary_quarter_jitter005_deeplab_FINETUNE │ │ ├── mapillary_quarter_jitter005_deeplab_TRAIN │ │ ├── old_reference_configs │ │ │ ├── forward │ │ │ ├── forward_temp │ │ │ ├── train │ │ │ ├── train_coco │ │ │ ├── train_davis_with_old_ldd │ │ │ ├── train_together │ │ │ ├── train_together_davis │ │ │ ├── train_ytvos │ │ │ ├── train_ytvos_orignal │ │ │ ├── ytvos1_davis │ │ │ ├── ytvos1_davis-with-ldd-weights │ │ │ ├── ytvos1_davis-with-ytvos-weights │ │ │ ├── ytvos1_davis_test │ │ │ ├── ytvos1_final │ │ │ ├── ytvos1_live │ │ │ ├── ytvos1_test │ │ │ └── ytvos1_val │ │ ├── refinement_network_FORWARD_ITERATIVE_template │ │ ├── refinement_network_FORWARD_template │ │ └── refinement_network_TRAIN_template │ └── run │ ├── core │ ├── Config.py │ ├── Engine.py │ ├── Extractions.py │ ├── Log.py │ ├── Measures.py │ ├── Saver.py │ ├── Timer.py │ ├── Trainer.py │ ├── Util.py │ └── __init__.py │ ├── datasets │ ├── Augmentors.py │ ├── COCO │ │ ├── COCO.py │ │ ├── COCO_detection.py │ │ ├── COCO_instance.py │ │ └── __init__.py │ ├── DAVIS │ │ ├── COCO_for_DAVIS.py │ │ ├── DAVIS.py │ │ ├── DAVIS_lucid.py │ │ ├── DAVISjono.py │ │ └── __init__.py │ ├── DataKeys.py │ ├── Dataset.py │ ├── FeedDataset.py │ ├── Loader.py │ ├── Mapillary │ │ ├── MapillaryLike_instance.py │ │ ├── Mapillary_instance.py │ │ └── __init__.py │ ├── Resize.py │ ├── __init__.py │ ├── few_shot_segmentation │ │ ├── DAVISFewShotSegmentationDataset.py │ │ ├── FewShotFeedSegmentationDataset.py │ │ ├── FewShotSegmentation.py │ │ └── __init__.py │ └── util │ │ ├── BoundingBox.py │ │ ├── DistanceTransform.py │ │ ├── Normalization.py │ │ ├── PascalColormap.py │ │ ├── Util.py │ │ └── __init__.py │ ├── forwarding │ ├── FewShotSegmentationForwarder.py │ ├── Forwarder.py │ └── __init__.py │ ├── main.py │ ├── network │ ├── ConvolutionalLayers.py │ ├── FullyConnected.py │ ├── Layer.py │ ├── Network.py │ ├── NetworkTower.py │ ├── Resnet.py │ ├── SegmentationOutputLayers.py │ ├── Util.py │ ├── __init__.py │ └── deeplab │ │ ├── DeepLabV3Plus.py │ │ ├── __init__.py │ │ ├── common.py │ │ ├── core │ │ ├── __init__.py │ │ ├── feature_extractor.py │ │ ├── preprocess_utils.py │ │ └── xception.py │ │ └── model.py │ └── scripts │ ├── DAVIS │ ├── __init__.py │ └── create_DAVIS_instance_list.py │ ├── __init__.py │ ├── eval │ ├── Datasets │ │ ├── EvalCOCO.py │ │ ├── EvalDAVIS.py │ │ ├── EvalGrabcut.py │ │ ├── EvalOSVOSWorst.py │ │ ├── EvalPascalMasked.py │ │ └── __init__.py │ ├── __init__.py │ ├── analyze_few_shot_results.py │ ├── create_KITTI_eval_config.sh │ ├── create_PASCAL_eval_config.sh │ ├── eval_KITTI_all.sh │ ├── eval_KITTI_testset.sh │ ├── eval_KITTI_tuning.sh │ ├── eval_Pascal_recursive.py │ └── eval_coco_detection.py │ ├── flownet_config_maker.py │ ├── prepare_tensorpack_checkpoints.py │ ├── preproc │ ├── KITTI │ │ ├── __init__.py │ │ └── create_KITTI_instance_list.py │ ├── __init__.py │ ├── cityscapes │ │ ├── __init__.py │ │ └── create_cityscapes_instance_list.py │ └── mapillary │ │ ├── __init__.py │ │ ├── create_mapillary_instance_list.py │ │ └── remove_dashes_at_beginning.py │ ├── rename_weights.py │ └── visualization │ ├── merge_KITTI_masks.py │ ├── plot_learn_curve.py │ └── visualize_coco_detections.py ├── data ├── README.txt └── teaser.png ├── output └── logs │ ├── ReID_net │ └── README.txt │ └── refinement_net │ └── README.txt ├── seq_to_run.txt ├── simple_run.sh └── weights └── README.txt /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.pyc 3 | .idea 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Jonathon Luiten 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /code/MergeTrack/ReID_net_functions.py: -------------------------------------------------------------------------------- 1 | from scipy.misc import imread 2 | import matplotlib.pyplot as plt 3 | 4 | from ReID_net.Engine import Engine 5 | from ReID_net.Config import Config 6 | from ReID_net.Log import log 7 | 8 | def init_log(config): 9 | log_dir = "../output/logs/ReID_net" 10 | model = config.str("model") 11 | 12 | print(log_dir,model,".log") 13 | 14 | filename = log_dir + model + ".log" 15 | verbosity = config.int("log_verbosity", 3) 16 | log.initialize([filename], [verbosity], []) 17 | 18 | def ReID_net_init(): 19 | config_path = "ReID_net/configs/live" 20 | config = Config(config_path) 21 | init_log(config) 22 | config.initialize() 23 | engine = Engine(config) 24 | return engine 25 | 26 | def add_ReID(proposals,image_fn,ReID_net): 27 | image = imread(image_fn) 28 | boxes = [prop['bbox'] for prop in proposals] 29 | network = ReID_net.test_network 30 | data = ReID_net.valid_data 31 | output = network.get_output_layer().outputs 32 | image_input = data.image 33 | boxes_input = data.boxes 34 | crop_list_output = data.crop_list 35 | # ReID_embeddings = ReID_net.session.run([output], feed_dict={image_input: image, boxes_input: boxes})[0][0] 36 | ReID_embeddings,crop_list = ReID_net.session.run([output,crop_list_output],feed_dict={image_input: image,boxes_input:boxes}) 37 | ReID_embeddings = ReID_embeddings[0] 38 | for idx,ReID in enumerate(ReID_embeddings): 39 | proposals[idx]["ReID"] = ReID.tolist() 40 | # print (ReID.shape) 41 | 42 | # for crop in crop_list: 43 | # plt.imshow(crop) 44 | # plt.show() 45 | 46 | return(proposals) -------------------------------------------------------------------------------- /code/MergeTrack/print_max_reid_distance.py: -------------------------------------------------------------------------------- 1 | import glob 2 | from numpy.linalg import norm 3 | import numpy as np 4 | from copy import deepcopy as copy 5 | 6 | from MergeTrack.merge_functions import read_ann,read_props 7 | from MergeTrack.ReID_net_functions import ReID_net_init, add_ReID 8 | 9 | input_images = "DAVIS/val17/" 10 | input_proposals = "DAVIS/ReID_props/" 11 | first_frame_anns = "DAVIS/val17-ff/" 12 | output_images = "DAVIS/final_results/" 13 | output_proposals = "DAVIS/final_props/" 14 | 15 | ReID_net = ReID_net_init() 16 | 17 | dataset_max_distances = [] 18 | for video_fn in sorted(glob.glob(input_images+"*/")): 19 | video_proposals = [] 20 | templates = [] 21 | for image_fn in sorted(glob.glob(video_fn+"*")): 22 | ann_fn = image_fn.replace(input_images,first_frame_anns).replace('.jpg','.png') 23 | if glob.glob(ann_fn): 24 | new_templates = read_ann(ann_fn) 25 | new_templates = add_ReID(new_templates, image_fn, ReID_net) 26 | 27 | # import json 28 | # ff_fn = image_fn.replace(input_images, "DAVIS/ff_test/").replace('.jpg', '.json') 29 | # with open(ff_fn, "r") as f: 30 | # new_templates = json.load(f) 31 | # for id, templ in enumerate(new_templates): 32 | # templ['ReID'] = np.array(templ['ReID']) 33 | # templ['id'] = id 34 | 35 | templates = templates + new_templates 36 | prop_fn = image_fn.replace(input_images,input_proposals).replace('.jpg','.json') 37 | proposals = read_props(prop_fn) 38 | video_proposals.append(proposals) 39 | 40 | ReIDs = [[prop['ReID'] for prop in props] for props in video_proposals] 41 | template_ReIDs = [templ['ReID'] for templ in templates] 42 | all_reid_distances = [np.array([[norm(c_reid - gt_reid) for c_reid in curr] for gt_reid in template_ReIDs]) for curr in ReIDs] 43 | all_reid_distances_no_inf = copy(all_reid_distances) 44 | 45 | for mat in all_reid_distances_no_inf: 46 | mat[np.isinf(mat)] = 0 47 | 48 | max_distances = np.array([mat.max(axis=1) if mat.shape[1]>0 else np.zeros((mat.shape[0])) for mat in all_reid_distances_no_inf]).max(axis=0) 49 | print(max_distances) 50 | dataset_max_distances.append(max_distances.max()) 51 | 52 | print(np.array(dataset_max_distances).max()) -------------------------------------------------------------------------------- /code/MergeTrack/refinement_net_functions.py: -------------------------------------------------------------------------------- 1 | from scipy.misc import imread 2 | from pycocotools.mask import encode, iou, area, decode, toBbox 3 | import numpy as np 4 | 5 | from refinement_net.core.Engine import Engine 6 | from refinement_net.core.Config import Config 7 | from refinement_net.core.Log import log 8 | from refinement_net.core import Extractions 9 | import refinement_net.datasets.DataKeys as DataKeys 10 | 11 | 12 | def init_log(config): 13 | log_dir = "../output/logs/refinement_net" 14 | model = config.string("model") 15 | filename = log_dir + model + ".log" 16 | verbosity = config.int("log_verbosity", 3) 17 | log.initialize([filename], [verbosity], []) 18 | 19 | def refinement_net_init(): 20 | config_path = "refinement_net/configs/live" 21 | config = Config(config_path) 22 | init_log(config) 23 | engine = Engine(config) 24 | return engine 25 | 26 | def extract(key,extractions): 27 | if key not in extractions: 28 | return None 29 | val = extractions[key] 30 | # for now assume we only use 1 gpu for forwarding 31 | assert len(val) == 1, len(val) 32 | val = val[0] 33 | # # for now assume, we use a batch size of 1 for forwarding 34 | assert val.shape[0] == 1, val.shape[0] 35 | val = val[0] 36 | return val 37 | 38 | def do_refinement(proposals,image_fn,refinement_net): 39 | image = imread(image_fn) 40 | boxes = [prop['bbox'] for prop in proposals] 41 | data = refinement_net.valid_data 42 | image_data = data.set_up_data_for_image(image, boxes) 43 | 44 | for idx in range(len(boxes)): 45 | feed_dict = data.get_feed_dict_for_next_step(image_data, idx) 46 | step_res = refinement_net.trainer.validation_step(feed_dict=feed_dict, extraction_keys=[Extractions.SEGMENTATION_POSTERIORS_ORIGINAL_SIZE, 47 | Extractions.SEGMENTATION_MASK_ORIGINAL_SIZE, DataKeys.OBJ_TAGS]) 48 | extractions = step_res[Extractions.EXTRACTIONS] 49 | predicted_segmentation = extract(Extractions.SEGMENTATION_MASK_ORIGINAL_SIZE,extractions) 50 | obj_tag = extract(DataKeys.OBJ_TAGS,extractions) 51 | obj_tag = int(obj_tag.decode('utf-8')) 52 | mask = predicted_segmentation.astype("uint8") * 255 53 | encoded_mask = encode(np.asfortranarray(mask)) 54 | encoded_mask['counts'] = encoded_mask['counts'].decode("utf-8") 55 | proposals[obj_tag]["segmentation"] = encoded_mask 56 | 57 | posteriors = extract(Extractions.SEGMENTATION_POSTERIORS_ORIGINAL_SIZE,extractions) 58 | conf_scores = posteriors.copy() 59 | conf_scores[predicted_segmentation == 0] = 1 - posteriors[predicted_segmentation == 0] 60 | conf_scores = 2 * conf_scores - 1 61 | conf_score = conf_scores[:].mean() 62 | proposals[obj_tag]["conf_score"] = str(conf_score) 63 | 64 | 65 | return(proposals) -------------------------------------------------------------------------------- /code/ReID_net/Constants.py: -------------------------------------------------------------------------------- 1 | CONFUSION_MATRIX = "confusion_matrix" 2 | AP = "ap" 3 | DETECTION_AP = "detection_mAP" 4 | DETECTION_APS = "detection_APs" 5 | MOTA = "MOTA" 6 | MOTP = "MOTP" 7 | FALSE_POSITIVES = "false_positives" 8 | FALSE_NEGATIVES = "false_negatives" 9 | ID_SWITCHES = "id_switches" 10 | AP_INTERPOLATED = "ap_interpolated" 11 | ERRORS = "errors" 12 | IOU = "iou" 13 | BINARY_IOU = "binary_IOU" 14 | RANKS = "ranks" 15 | DT_NEG = "u0" 16 | DT_POS = "u1" 17 | CLICKS = "clicks" 18 | FORWARD_INTERACTIVE = "forward_interactive" 19 | FORWARD_RECURSIVE = "forward_recursive" 20 | ONESHOT_INTERACTIVE = "oneshot_interactive" 21 | ITERATIVE_FORWARD = "iterative_forward" 22 | INTERACTIVE_DT_METHOD = "dt_method" 23 | DT_NEG_PARAMS = "dt_neg_params" 24 | DT_POS_PARAMS = "dt_pos_params" 25 | USE_CLICKS = "use_clicks" 26 | OLD_LABEL_AS_DT = "old_label_as_distance_transform" 27 | STRATEGY = "STRATEGY" 28 | IGNORE_CLASSES = "ignore_classes" 29 | DT_FN_ARGS = "distance_transform_fn_args" 30 | # Train recursively - add a click based on the previous output mask for the interactive correction network. 31 | RECURSIVE_TRAINING = "recursive_training" 32 | YS_ARGMAX = "ys_argmax" 33 | TAGS = "tags" 34 | BBOXES = "bboxes" 35 | IGNORE_REGIONS = "ignore_regions" 36 | CLASSES = "classes" 37 | IDS = "ids" 38 | UNNORMALIZED_IMG = "unnormalized_img" 39 | IMG_IDS = "img_ids" 40 | ORIGINAL_SIZES = "original_size" 41 | RESIZED_SIZES = "resized_size" 42 | CLUSTER_IDS = "cluster_ids" 43 | ORIGINAL_LABELS = "original_labels" 44 | ADJUSTED_MUTUAL_INFORMATION = "AMI" 45 | HOMOGENEITY = "homogeneity" 46 | COMPLETENESS = "completeness" 47 | EMBEDDING = "embedding" 48 | PREV_NEG_CLICKS = 'prev_neg_clicks' 49 | PREV_POS_CLICKS = 'prev_pos_clicks' 50 | SCENE_INFOS = "scene_infos" 51 | LABELS = "labels" 52 | INPUTS = "inputs" 53 | BYPASS_MASKS_FLAG = "bypass_masks_flag" 54 | -------------------------------------------------------------------------------- /code/ReID_net/Forwarding/COCOInstanceForwarder.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | from scipy.misc import imsave 3 | 4 | import numpy 5 | import tensorflow as tf 6 | 7 | from ReID_net.Forwarding.Forwarder import ImageForwarder 8 | from ReID_net.Measures import compute_measures_for_binary_segmentation 9 | 10 | 11 | class COCOInstanceForwarder(ImageForwarder): 12 | 13 | def _process_forward_result(self, y_argmax, logit, target, tag, extraction_vals, main_folder, save_results): 14 | #hack for avoiding storing logits for frames, which are not evaluated 15 | if "DO_NOT_STORE_LOGITS" in tag: 16 | logit = None 17 | tag = tag.replace("_DO_NOT_STORE_LOGITS", "") 18 | 19 | folder = main_folder + tag.split("/")[-2] + "/" 20 | tf.gfile.MakeDirs(folder) 21 | inst = tag.split(":")[-1] 22 | out_fn = folder + tag.split(":")[0].split("/")[-1].replace(".jpg", "_" + inst + ".png").replace(".bin", ".png") 23 | out_fn_logits = out_fn.replace(".png", ".pickle") 24 | 25 | target_fn = out_fn.replace(".png", "_target.png") 26 | measures = compute_measures_for_binary_segmentation(y_argmax, target) 27 | if save_results: 28 | y_scaled = (y_argmax ).astype("uint8") 29 | print(out_fn) 30 | imsave(out_fn, numpy.squeeze(y_scaled, axis=2)) 31 | # imsave(target_fn, numpy.squeeze(target_scaled, axis=2 )) 32 | if logit is not None: 33 | pickle.dump(logit, open(out_fn_logits, "w"), pickle.HIGHEST_PROTOCOL) 34 | for e in extraction_vals: 35 | assert e.shape[0] == 1 # batchs size should be 1 here for now 36 | for name, val in zip(self.extractions, extraction_vals): 37 | val = val[0] # remove batch dimension 38 | sp = out_fn.replace(".png", ".bin").split("/") 39 | sp[-1] = name + "_" + sp[-1] 40 | out_fn_extract = "/".join(sp) 41 | print(out_fn_extract) 42 | val.tofile(out_fn_extract) 43 | return measures 44 | -------------------------------------------------------------------------------- /code/ReID_net/Forwarding/DatasetSpeedtestForwarder.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from ReID_net.Forwarding.Forwarder import Forwarder 4 | from ReID_net.datasets.Util.Timer import Timer 5 | from ReID_net.Log import log 6 | 7 | 8 | class DatasetSpeedtestForwarder(Forwarder): 9 | def __init__(self, engine): 10 | super(DatasetSpeedtestForwarder, self).__init__(engine) 11 | 12 | def forward(self, network, data, save_results=True, save_logits=False): 13 | batch_size = self.config.int("batch_size", -1) 14 | assert batch_size != -1 15 | values = list(data.create_input_tensors_dict(batch_size).values()) 16 | tf.train.start_queue_runners(self.session) 17 | n_runs = 2000 18 | print("batch_size", batch_size, file=log.v1) 19 | with Timer("%s runs tooks" % n_runs): 20 | for idx in range(n_runs): 21 | print(idx, "/", n_runs, file=log.v5) 22 | self.session.run(values) 23 | -------------------------------------------------------------------------------- /code/ReID_net/Forwarding/InteractiveImageForwarder.py: -------------------------------------------------------------------------------- 1 | from ReID_net.Forwarding.Forwarder import ImageForwarder 2 | from ReID_net.Log import log 3 | from ReID_net.Measures import average_measures 4 | 5 | 6 | class InteractiveImageForwarder(ImageForwarder): 7 | def __init__(self, engine): 8 | super(InteractiveImageForwarder, self).__init__(engine) 9 | #hope moving this import here, does not break anything, otherwise move it back up again 10 | import matplotlib 11 | matplotlib.use('Agg') 12 | 13 | def forward(self, network, data, save_results=True, save_logits=False): 14 | n_total = data.num_examples_per_epoch() 15 | n_processed = 0 16 | targets = network.raw_labels 17 | ys = network.ys_resized 18 | 19 | # e.g. used for resizing 20 | # ys = self._adjust_results_to_targets(ys, targets) 21 | 22 | measures = [] 23 | ys_argmax_values = [] 24 | while n_processed < n_total: 25 | n, new_measures, ys_argmax_val, _, _ = self._process_forward_minibatch(data, network, save_logits, save_results, 26 | targets, ys, n_processed) 27 | measures += new_measures 28 | ys_argmax_values += list(ys_argmax_val) 29 | n_processed += n 30 | print(n_processed, "/", n_total, file=log.v5) 31 | if self.ignore_first_and_last_results: 32 | measures = measures[1:-1] 33 | elif self.ignore_first_result: 34 | measures = measures[1:] 35 | 36 | measures = average_measures(measures) 37 | if hasattr(data, "video_tag"): 38 | video_idx = data.get_video_idx() 39 | print("sequence", video_idx + 1, data.video_tag(video_idx), measures, file=log.v1) 40 | else: 41 | print(measures, file=log.v1) 42 | 43 | return ys_argmax_values, measures 44 | -------------------------------------------------------------------------------- /code/ReID_net/Forwarding/MARKET1501Forwarder.py: -------------------------------------------------------------------------------- 1 | from .Forwarder import Forwarder 2 | import numpy as np 3 | from ReID_net.Log import log 4 | import time 5 | # from ReID_net.Forwarding.CMC_Validator import do_cmc_validation_triplet 6 | # import ReID_net.Measures as Measures 7 | 8 | class MARKET1501Forwarder(Forwarder): 9 | def __init__(self, engine): 10 | super(MARKET1501Forwarder, self).__init__(engine) 11 | self.engine = engine 12 | 13 | def forward(self, network, data, save_results=True, save_logits=False): 14 | 15 | file = self.engine.valid_data.file 16 | 17 | output_dir = data.data_dir + "/" + file + "_output.txt" 18 | 19 | outfile = open(output_dir, 'w') 20 | 21 | idx_placeholder = data.idx_placeholder 22 | batch_size = network.batch_size 23 | 24 | # out_layer = network.tower_layers[0]["outputTriplet"] 25 | out_layer = network.tower_layers[0]["fc1"] 26 | assert len(out_layer.outputs) == 1 27 | out_feature = out_layer.outputs[0] 28 | 29 | features = np.empty([0, 128]) 30 | 31 | m = self.engine.valid_data.m 32 | idx = 0 33 | while idx < m: 34 | start = time.time() 35 | idx_value = [idx, min(idx + batch_size, m), 0, 0] 36 | 37 | feature_val,debug = self.engine.session.run([out_feature, network.tags], feed_dict={idx_placeholder: idx_value}) 38 | # print feature_val.shape 39 | # features = np.concatenate((features, feature_val), axis=0) 40 | np.savetxt(outfile, feature_val) 41 | # print debug 42 | 43 | end = time.time() 44 | elapsed = end - start 45 | print(min(idx + batch_size, m), '/', m, "elapsed", elapsed, file=log.v5) 46 | 47 | idx += batch_size 48 | 49 | outfile.close() 50 | 51 | # start = time.time() 52 | # valid_loss, valid_measures = do_cmc_validation_triplet(self.engine, self.engine.test_network, self.engine.valid_data) 53 | # end = time.time() 54 | # elapsed = end - start 55 | # # train_error_string = Measures.get_error_string(train_measures, "train") 56 | # valid_error_string = Measures.get_error_string(valid_measures, "valid") 57 | # print >> log.v1, "epoch", 1, "finished. elapsed:", "%.5f" % elapsed, "valid_score:", valid_loss, valid_error_string 58 | 59 | -------------------------------------------------------------------------------- /code/ReID_net/Forwarding/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/ReID_net/Forwarding/__init__.py -------------------------------------------------------------------------------- /code/ReID_net/README: -------------------------------------------------------------------------------- 1 | SAVITAR 1 codebase: 2 | 3 | Code base contributed to by: 4 | Paul Voigtlaender, Jonathon Luiten, Sabarinath Mahadevan. 5 | 6 | MIT license. 7 | 8 | 9 | -------------------------------------------------------------------------------- /code/ReID_net/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/ReID_net/__init__.py -------------------------------------------------------------------------------- /code/ReID_net/datasets/COCO/COCO_detection.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy 3 | 4 | from ReID_net.datasets.COCO.COCO import COCODataset 5 | import ReID_net.Constants as Constants 6 | 7 | NUM_CLASSES = 90 8 | N_MAX_DETECTIONS = 100 9 | 10 | 11 | class COCODetectionDataset(COCODataset): 12 | def __init__(self, config, subset, coord, fraction=1.0): 13 | super(COCODetectionDataset, self).__init__(config, subset, coord, fraction=fraction, num_classes=NUM_CLASSES) 14 | n_max_detections = max([len(x) for x in list(self.filename_to_anns.values())]) 15 | assert n_max_detections <= N_MAX_DETECTIONS, n_max_detections 16 | 17 | def label_load_fn(self, img_path, label_path): 18 | def my_create_labels(im_path): 19 | return self.create_labels(im_path) 20 | bboxes, ids, classes = tf.py_func(my_create_labels, [img_path], [tf.float32, tf.int32, tf.int32]) 21 | bboxes.set_shape((N_MAX_DETECTIONS, 4)) 22 | ids.set_shape((N_MAX_DETECTIONS,)) 23 | classes.set_shape((N_MAX_DETECTIONS,)) 24 | labels = {Constants.BBOXES: bboxes, Constants.IDS: ids, Constants.CLASSES: classes} 25 | return labels 26 | 27 | def create_labels(self, img_path): 28 | anns = self.filename_to_anns[img_path.split("/")[-1]] 29 | #they need to be padded to N_MAX_DETECTIONS 30 | bboxes = numpy.zeros((N_MAX_DETECTIONS, 4), dtype="float32") 31 | # to avoid divison by 0: 32 | bboxes[:, [1, 3]] = 1 33 | ids = numpy.zeros(N_MAX_DETECTIONS, dtype="int32") 34 | classes = numpy.zeros(N_MAX_DETECTIONS, dtype="int32") 35 | 36 | if len(anns) > N_MAX_DETECTIONS: 37 | print("N_MAX_DETECTIONS not set high enough!") 38 | assert False, "N_MAX_DETECTIONS not set high enough!" 39 | 40 | for idx, ann in enumerate(anns): 41 | #TODO: ann["iscrowd"], ann["area"] 42 | x1 = ann["bbox"][0] 43 | y1 = ann["bbox"][1] 44 | box_width = ann["bbox"][2] 45 | box_height = ann["bbox"][3] 46 | x2 = x1 + box_width 47 | y2 = y1 + box_height 48 | bboxes[idx] = [y1, y2, x1, x2] 49 | ids[idx] = idx + 1 50 | # categories are from 1 to 90 in the annotations -> map to 0 to 89 51 | classes[idx] = ann["category_id"] - 1 52 | return bboxes, ids, classes 53 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/COCO/COCO_instance.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from ReID_net.datasets.COCO.COCO import COCODataset 4 | from ReID_net.datasets.Util import Reader 5 | 6 | 7 | class COCOInstanceDataset(COCODataset): 8 | def __init__(self, config, subset, coord, fraction=1.0): 9 | super(COCOInstanceDataset, self).__init__(config, subset, coord, fraction=fraction) 10 | 11 | def build_filename_to_anns_dict(self): 12 | for ann in self.anns: 13 | ann_id = ann['id'] 14 | img_id = ann['image_id'] 15 | img = self.coco.loadImgs(img_id) 16 | file_name = img[0]['file_name'] 17 | 18 | if not ann['iscrowd'] and \ 19 | 'segmentation' in ann and \ 20 | 'bbox' in ann and ann['area'] > 200: 21 | file_name = file_name + ":" + repr(ann_id) 22 | if file_name in self.filename_to_anns: 23 | print("Ignoring instance as an instance with the same id exists in filename_to_anns.") 24 | # self.filename_to_anns[file_name].append(ann) 25 | else: 26 | self.filename_to_anns[file_name] = [ann] 27 | # self.filename_to_anns[file_name] = ann 28 | 29 | def img_load_fn(self, img_path): 30 | path = tf.string_split([img_path], ':').values[0] 31 | path = tf.string_split([path], '/').values[-1] 32 | img_dir = tf.cond(tf.equal(tf.string_split([path], '_').values[1], tf.constant("train2014")), 33 | lambda: '%s/%s/' % (self.data_dir, "train2014"), 34 | lambda: '%s/%s/' % (self.data_dir, "val2014")) 35 | path = img_dir + path 36 | return Reader.load_img_default(path) 37 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/COCO/COCO_objectness.py: -------------------------------------------------------------------------------- 1 | from ReID_net.datasets.COCO.COCO import COCODataset 2 | import os 3 | import tensorflow as tf 4 | import numpy as np 5 | 6 | NUM_CLASSES = 2 7 | COCO_DEFAULT_PATH = "/fastwork/" + os.environ['USER'] + "/mywork/data/coco/" 8 | INPUT_SIZE = (None, None) 9 | COCO_VOID_LABEL = 255 10 | 11 | 12 | class CocoObjectnessDataset(COCODataset): 13 | def label_load_fn(self, img_path, label_path): 14 | def my_create_labels(im_path): 15 | return self.create_labels(im_path) 16 | label, old_label = tf.py_func(my_create_labels, [img_path], [tf.uint8, tf.uint8]) 17 | labels = {"label": label} 18 | return labels 19 | 20 | def create_labels(self, img_path): 21 | anns = self.filename_to_anns[img_path.split("/")[-1]] 22 | img = self.coco.loadImgs(anns[0]['image_id'])[0] 23 | height = img['height'] 24 | width = img['width'] 25 | 26 | label = np.zeros((height, width, 1)) 27 | old_label = np.zeros((height, width, 1)) 28 | 29 | for ann in anns: 30 | mask = self.coco.annToMask(ann)[:, :] 31 | label[mask == 1] = 1 32 | 33 | return label.astype(np.uint8), old_label.astype(np.uint8) 34 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/COCO/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/ReID_net/datasets/COCO/__init__.py -------------------------------------------------------------------------------- /code/ReID_net/datasets/Custom/Custom.py: -------------------------------------------------------------------------------- 1 | from ReID_net.datasets.Dataset import ImageDataset 2 | import tensorflow as tf 3 | 4 | 5 | def zero_label(img_path, label_path): 6 | #TODO: we load the image again just to get it's size which is kind of a waste (but should not matter for most cases) 7 | img_contents = tf.read_file(img_path) 8 | img = tf.image.decode_image(img_contents, channels=3) 9 | img = tf.image.convert_image_dtype(img, tf.float32) 10 | img.set_shape((None, None, 3)) 11 | label = tf.zeros_like(img, dtype=tf.uint8)[..., 0:1] 12 | res = {"label": label} 13 | return res 14 | 15 | 16 | class CustomDataset(ImageDataset): 17 | def __init__(self, config, subset, coord, fraction=1.0): 18 | super(CustomDataset, self).__init__("custom", "", 2, config, subset, coord, None, 255, fraction, 19 | label_load_fn=zero_label) 20 | self.file_list = config.str("file_list") 21 | 22 | def read_inputfile_lists(self): 23 | imgs = [x.strip() for x in open(self.file_list).readlines()] 24 | labels = ["" for _ in imgs] 25 | return [imgs, labels] 26 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/Custom/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/ReID_net/datasets/Custom/__init__.py -------------------------------------------------------------------------------- /code/ReID_net/datasets/DAVIS/DAVIS_instance.py: -------------------------------------------------------------------------------- 1 | from ReID_net.datasets.DAVIS.DAVIS_oneshot import DavisOneshotDataset 2 | from ReID_net.datasets.DAVIS.DAVIS import VOID_LABEL 3 | import numpy as np 4 | 5 | 6 | def get_bounding_box(mask, inst): 7 | mask = np.copy(mask) 8 | rows = np.where(mask == inst)[0] 9 | cols = np.where(mask == inst)[1] 10 | rmin = rows.min() 11 | rmax = rows.max() 12 | cmin = cols.min() 13 | cmax = cols.max() 14 | 15 | mask[rmin:rmax, cmin:cmax] = 1.0 16 | return mask 17 | 18 | 19 | class DAVISInstanceDataset(DavisOneshotDataset): 20 | def __init__(self, config, subset, use_old_label): 21 | self.use_old_label = use_old_label 22 | super(DAVISInstanceDataset, self).__init__(config, subset, use_old_label) 23 | 24 | def feed_dict_for_video_frame(self, frame_idx, with_annotations, old_mask=None, train_on_background=False): 25 | tensors = self._get_video_data()[frame_idx].copy() 26 | feed_dict = {self.img_placeholder: tensors["unnormalized_img"], self.tag_placeholder: tensors["tag"]} 27 | if with_annotations: 28 | if train_on_background: 29 | bbox = get_bounding_box(tensors["label"], 1) 30 | feed_dict[self.label_placeholder] = np.where(bbox == 1, VOID_LABEL, bbox) 31 | else: 32 | feed_dict[self.label_placeholder] = tensors["label"] 33 | 34 | assert "old_mask" not in tensors 35 | if old_mask is not None: 36 | feed_dict[self.old_label_placeholder] = old_mask 37 | 38 | if "flow_past" in tensors: 39 | feed_dict[self.flow_into_past_placeholder] = tensors["flow_past"] 40 | if "flow_future" in tensors: 41 | feed_dict[self.flow_into_future_placeholder] = tensors["flow_future"] 42 | 43 | if self.use_old_label: 44 | feed_dict[self.old_label_placeholder] = get_bounding_box(tensors["label"], 1) 45 | 46 | return feed_dict 47 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/DAVIS/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/ReID_net/datasets/DAVIS/__init__.py -------------------------------------------------------------------------------- /code/ReID_net/datasets/DetectionFeedDataset.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from ReID_net.datasets.Dataset import Dataset 4 | from ReID_net.datasets.Util.Reader import read_images_from_disk 5 | from ReID_net.datasets.Util.Batch import create_batch_dict 6 | from ReID_net.datasets.Util.Resize import ResizeMode 7 | import ReID_net.Constants as Constants 8 | 9 | 10 | def _create_labels(img_path, bboxes): 11 | # this dataset will not be used for training, so we can set ids and classes automatically 12 | ids = tf.range(tf.shape(bboxes)[0]) + 1 13 | classes = tf.zeros(tf.shape(ids), dtype=tf.int32) 14 | labels = {Constants.BBOXES: bboxes, Constants.IDS: ids, Constants.CLASSES: classes} 15 | return labels 16 | 17 | 18 | class DetectionFeedDataset(Dataset): 19 | def __init__(self, config, subset, coord): 20 | super(DetectionFeedDataset, self).__init__(subset) 21 | self.config = config 22 | self.img_filename_placeholder = tf.placeholder(tf.string, shape=(), name="img_filename_placeholder") 23 | self.bboxes_placeholder = tf.placeholder(tf.float32, shape=(None, 4), name="label_placeholder") 24 | 25 | def create_input_tensors_dict(self, batch_size): 26 | assert batch_size == 1 27 | #TODO: get this from ReID_net.Config 28 | resize_mode, input_size = self._get_resize_params(self.subset, [None, None], ResizeMode.DetectionFixedSizeForEval) 29 | queue = [self.img_filename_placeholder, self.bboxes_placeholder] 30 | tensors_dict, summaries = read_images_from_disk(queue, input_size, resize_mode, label_load_fn=_create_labels) 31 | tensors_dict, summ = create_batch_dict(batch_size, tensors_dict) 32 | tensors_dict["labels"] = (tensors_dict[Constants.BBOXES], tensors_dict[Constants.IDS], 33 | tensors_dict[Constants.CLASSES]) 34 | return tensors_dict 35 | 36 | def num_classes(self): 37 | # keep it consitent with coco for the moment 38 | return 90 39 | 40 | def num_examples_per_epoch(self): 41 | return 1 42 | 43 | def void_label(self): 44 | return 255 45 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/Similarity/COCO_Similarity.py: -------------------------------------------------------------------------------- 1 | from .Similarity import SimilarityDataset 2 | from ReID_net.datasets.COCO.COCO import COCODataset 3 | 4 | 5 | class COCOSimilarityDataset(SimilarityDataset): 6 | def __init__(self, config, subset, coord): 7 | self.coco_dataset = COCODataset(config, subset, coord) 8 | ann_id_to_filename = {} 9 | for filename, anns in list(self.coco_dataset.filename_to_anns.items()): 10 | for ann in anns: 11 | ann_id_to_filename[ann["id"]] = filename 12 | annotations = self.coco_dataset.anns 13 | for ann in annotations: 14 | file_name = ann_id_to_filename[ann["id"]] 15 | data_type = "train2014" if "train2014" in file_name else "val2014" 16 | img_dir = '%s/%s/' % (self.coco_dataset.data_dir, data_type) 17 | ann["img_file"] = img_dir + file_name 18 | cat_ids = self.coco_dataset.coco.getCatIds() 19 | for ann in annotations: 20 | ann["category_id"] = cat_ids.index(ann["category_id"]) + 1 21 | super(COCOSimilarityDataset, self).__init__(config, subset, coord, annotations, n_train_ids=80) 22 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/Similarity/DAVIS_Forward_Similarity.py: -------------------------------------------------------------------------------- 1 | from .Similarity import SimilarityDataset 2 | import glob 3 | import json 4 | from pycocotools.mask import toBbox 5 | 6 | class DAVISForwardSimilarityDataset(SimilarityDataset): 7 | def __init__(self, config, subset, coord): 8 | 9 | # old_proposal_directory = "/home/luiten/vision/PReMVOS/first_frame_no_ReID/%s/" 10 | old_proposal_directory = config.str("bb_input_dir", None) 11 | data_directory = config.str("image_input_dir", None) 12 | 13 | # old_proposal_directory = "/home/luiten/vision/PReMVOS/proposals_with_flow/%s/" 14 | # old_proposal_directory = "/home/luiten/vision/PReMVOS/post_proposal_expansion_json_with_flow/%s/" 15 | # sets = ['test-challenge/', 'val/', 'test-dev/'] 16 | # sets = ['val/',] 17 | annotations = [] 18 | 19 | # Read in all proposals 20 | # for set_id, set in enumerate(sets): 21 | # folders = sorted(glob.glob(old_proposal_directory.split('%s')[0] + set + '*/')) 22 | folders = sorted(glob.glob(old_proposal_directory + '*/')) 23 | for folder in folders: 24 | seq = folder.split('/')[-2] 25 | # name = set + seq 26 | name = seq 27 | # files = sorted(glob.glob(old_proposal_directory % name + "*.json")) 28 | files = sorted(glob.glob(old_proposal_directory + name + "/*.json")) 29 | for file in files: 30 | timestep = file.split('/')[-1].split('.json')[0] 31 | with open(file, "r") as f: 32 | proposals = json.load(f) 33 | for prop_id, proposal in enumerate(proposals): 34 | # img_file = "/home/luiten/vision/PReMVOS/home_data/"+name+"/images/"+timestep+".jpg" 35 | img_file = data_directory + name + "/" + timestep + ".jpg" 36 | catagory_id = 1 37 | tag = name+'/'+timestep+'___'+str(prop_id) 38 | segmentation = proposal["segmentation"] 39 | bbox = toBbox(segmentation) 40 | ann = {"img_file":img_file,"category_id":catagory_id,"bbox":bbox,"tag":tag} 41 | 42 | if bbox[2] <= 0 or bbox[3] <= 0: 43 | continue 44 | 45 | annotations.append(ann) 46 | 47 | super(DAVISForwardSimilarityDataset, self).__init__(config, subset, coord, annotations, n_train_ids=1) 48 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/Similarity/DAVIS_Similarity.py: -------------------------------------------------------------------------------- 1 | from .Similarity import SimilarityDataset 2 | import glob 3 | from ReID_net.datasets.Util.Reader import load_image_tensorflow 4 | from ReID_net.datasets.Augmentors import apply_augmentors 5 | from ReID_net.datasets.Util.Normalization import normalize 6 | 7 | DEFAULT_REID_DATA = '/home/luiten/vision/PReMVOS/data/Re-ID-data' 8 | DEFAULT_NUM_IDS = 242 9 | 10 | class DAVISSimilarityDataset(SimilarityDataset): 11 | def __init__(self, config, subset, coord): 12 | 13 | ReId_data = config.str('reid_input_dir',DEFAULT_REID_DATA) 14 | num_ids = config.int('num_reid_ids', DEFAULT_NUM_IDS) 15 | import time 16 | t = time.time() 17 | print("starting something else weirdly slow") 18 | filenames = glob.glob(ReId_data + '/*') 19 | print("finishing something else weirdly slow",time.time()-t) 20 | cat_ids = [int(f.split('/')[-1].split('-')[0])+1 for f in filenames] 21 | annotations = [{"img_file":f,"category_id":c,"bbox":0} for f,c in zip(filenames,cat_ids)] 22 | super(DAVISSimilarityDataset, self).__init__(config, subset, coord, annotations, n_train_ids=num_ids) 23 | 24 | def _load_crop_helper(self, img_file_name, img_bbox): 25 | img = load_image_tensorflow(img_file_name, jpg=self.jpg, channels=3) 26 | 27 | img.set_shape(self.input_size + (3,)) 28 | # augment and normalize 29 | tensors = {"unnormalized_img": img} 30 | tensors = apply_augmentors(tensors, self.augmentors) 31 | img = tensors["unnormalized_img"] 32 | img_norm = normalize(img) 33 | return img_norm, img, None 34 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/Similarity/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/ReID_net/datasets/Similarity/__init__.py -------------------------------------------------------------------------------- /code/ReID_net/datasets/Util/Batch.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | 4 | def create_batch_dict(batch_size, tensors_dict): 5 | if batch_size == 1: 6 | batch = {k: tf.expand_dims(t, axis=0) for k, t in list(tensors_dict.items())} 7 | summary = None 8 | else: 9 | keys = list(tensors_dict.keys()) 10 | values = list(tensors_dict.values()) 11 | values = tf.train.batch(values, batch_size, num_threads=8, capacity=5 * batch_size) 12 | batch = dict(list(zip(keys, values))) 13 | summary = tf.get_collection(tf.GraphKeys.SUMMARIES)[-1] 14 | assert "fraction_of_" in summary.name 15 | for t in list(batch.values()): 16 | t.set_shape([batch_size] + [None] * (t.get_shape().ndims - 1)) 17 | return batch, summary 18 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/Util/MaskDamager.py: -------------------------------------------------------------------------------- 1 | from skimage.transform import SimilarityTransform, warp 2 | from .Video import shift 3 | import numpy 4 | from math import sqrt 5 | from scipy.ndimage import grey_dilation 6 | 7 | 8 | def shift_mask_speed_absolute(mask, max_speed=30): 9 | speed = (numpy.random.rand() * 2 - 1.0) * max_speed 10 | offset = numpy.random.rand(2) 11 | offset /= numpy.linalg.norm(offset, 2) 12 | offset *= speed 13 | return shift(mask, offset, "constant", 0) 14 | 15 | 16 | #factor w.r.t. object size 17 | def shift_mask_speed_factor(mask, factor=0.1): 18 | #this method of determining the object size seems to be bad! 19 | #size = sqrt(mask.sum()) 20 | 21 | nzy, nzx, _ = mask.nonzero() 22 | if nzy.size == 0: 23 | return mask 24 | size = sqrt((nzy.max() - nzy.min()) * (nzx.max() - nzx.min())) 25 | max_speed = int(round(factor * size)) 26 | #print max_speed 27 | return shift_mask_speed_absolute(mask, max_speed) 28 | 29 | 30 | def scale_mask(mask, factor=1.05): 31 | nzy, nzx, _ = mask.nonzero() 32 | if nzy.size == 0: 33 | return mask 34 | #center_y, center_x = nzy.mean(), nzx.mean() 35 | #print center_y, center_x 36 | center_y, center_x = (nzy.max() + nzy.min()) / 2, (nzx.max() + nzx.min()) / 2 37 | #print center_y, center_x 38 | 39 | shift_ = SimilarityTransform(translation=[-center_x, -center_y]) 40 | shift_inv = SimilarityTransform(translation=[center_x, center_y]) 41 | 42 | A = SimilarityTransform(scale=(factor, factor)) 43 | mask_out = warp(mask, (shift_ + (A + shift_inv)).inverse) 44 | mask_out = (mask_out > 0.5).astype("float32") 45 | #import matplotlib.pyplot as plt 46 | #im = numpy.concatenate([mask, mask, mask_out],axis=2) 47 | #plt.imshow(im) 48 | #plt.show() 49 | return mask_out 50 | 51 | 52 | def damage_mask(mask, scale_factor, shift_absolute, shift_factor): 53 | assert not (shift_absolute != 0.0 and shift_factor != 0.0) 54 | mask = mask.astype("float32") 55 | 56 | if shift_absolute != 0.0: 57 | mask = shift_mask_speed_absolute(mask, max_speed=shift_absolute) 58 | elif shift_factor != 0.0: 59 | mask = shift_mask_speed_factor(mask, factor=shift_factor) 60 | 61 | if scale_factor != 0.0: 62 | scale = numpy.random.uniform(1 - scale_factor, 1 + scale_factor) 63 | mask = scale_mask(mask, scale) 64 | 65 | # dilation_size = 5 66 | # mask = grey_dilation(mask, size=(dilation_size, dilation_size, 1)) 67 | return mask 68 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/Util/Normalization.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | # IMAGENET_RGB_MEAN = numpy.array((124.0, 117.0, 104.0), dtype=numpy.float32) / 255.0 4 | # values from https://github.com/itijyou/ademxapp/blob/0239e6cf53c081b3803ccad109a7beb56e3a386f/iclass/ilsvrc.py 5 | IMAGENET_RGB_MEAN = numpy.array([0.485, 0.456, 0.406], dtype="float32") 6 | IMAGENET_RGB_STD = numpy.array([0.229, 0.224, 0.225], dtype="float32") 7 | 8 | 9 | def normalize(img, img_mean=IMAGENET_RGB_MEAN, img_std=IMAGENET_RGB_STD): 10 | if hasattr(img, "get_shape"): 11 | l = img.get_shape()[-1] 12 | if img_mean is not None and l != img_mean.size: 13 | img_mean = numpy.concatenate([img_mean, numpy.zeros(l - img_mean.size, dtype="float32")], axis=0) 14 | if img_std is not None and l != img_std.size: 15 | img_std = numpy.concatenate([img_std, numpy.ones(l - img_std.size, dtype="float32")], axis=0) 16 | 17 | if img_mean is not None: 18 | img -= img_mean 19 | if img_std is not None: 20 | img /= img_std 21 | return img 22 | 23 | 24 | def unnormalize(img, img_mean=IMAGENET_RGB_MEAN, img_std=IMAGENET_RGB_STD): 25 | if hasattr(img, "get_shape"): 26 | l = img.get_shape()[-1] 27 | if img_mean is not None and l != img_mean.size: 28 | img_mean = numpy.concatenate([img_mean, numpy.zeros(l - img_mean.size, dtype="float32")], axis=0) 29 | if img_std is not None and l != img_std.size: 30 | img_std = numpy.concatenate([img_std, numpy.ones(l - img_std.size, dtype="float32")], axis=0) 31 | 32 | if img_std is not None: 33 | img *= img_std 34 | if img_mean is not None: 35 | img += img_mean 36 | return img 37 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/Util/Timer.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | 4 | class Timer(object): 5 | def __init__(self, message="", stream=None): 6 | if stream is None: 7 | from ReID_net.Log import log 8 | stream = log.v4 9 | self.stream = stream 10 | self.start = None 11 | self.message = message 12 | 13 | def __enter__(self): 14 | self.start = time.time() 15 | 16 | def __exit__(self, exc_type, exc_val, exc_tb): 17 | end = time.time() 18 | start = self.start 19 | self.start = None 20 | elapsed = end - start 21 | print(self.message, "elapsed", elapsed, file=self.stream) 22 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/Util/Video.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | from ReID_net.Log import log 3 | 4 | 5 | def shift_im(im, offset): 6 | return shift(im, offset, "reflect", None) 7 | 8 | 9 | def shift_lab(lab, offset, void_label): 10 | return shift(lab, offset, "constant", void_label) 11 | 12 | 13 | def generate_video(im, lab, n_frames, max_speed, void_label): 14 | speed = (numpy.random.rand() * 2 - 1.0) * max_speed 15 | step_offset = numpy.random.rand(2) 16 | step_offset /= numpy.linalg.norm(step_offset, 2) 17 | step_offset *= speed 18 | 19 | video_ims = [im] 20 | video_labs = [lab] 21 | total_offset = numpy.array([0.0, 0.0]) 22 | for frame in range(n_frames - 1): 23 | total_offset += step_offset 24 | im_frame = shift_im(im, total_offset) 25 | lab_frame = shift_lab(lab, total_offset, void_label) 26 | video_ims.append(im_frame) 27 | video_labs.append(lab_frame) 28 | return video_ims, video_labs 29 | 30 | 31 | def shift(im, offset, mode, value=None): 32 | assert mode in ("reflect", "constant") 33 | if mode == "reflect": 34 | assert value is None 35 | else: 36 | assert value is not None 37 | 38 | offset = numpy.round(offset).astype("int32") 39 | 40 | start = numpy.maximum(-offset, 0) 41 | size = im.shape[:2] - numpy.abs(offset) 42 | 43 | # Extract the image region that is defined by the offset 44 | im = im[start[0]:start[0] + size[0], start[1]:start[1] + size[1]] 45 | 46 | # Pad the image on the opposite side 47 | padding = numpy.array([ 48 | [max(0, offset[0]), max(0, -offset[0])], 49 | [max(0, offset[1]), max(0, -offset[1])], 50 | [0, 0] 51 | ]) 52 | 53 | if mode == "reflect": 54 | im = numpy.pad(im, padding, mode) 55 | else: 56 | im = numpy.pad(im, padding, mode, constant_values=value) 57 | return im 58 | 59 | 60 | def make_chunks(fns, size): 61 | res = [] 62 | for seq_fns in fns: 63 | l = len(seq_fns) 64 | if l < size: 65 | print("warning, sequence", seq_fns[0], "too short for chunk size", size, file=log.v1) 66 | for i in range(l / size): 67 | chunk = seq_fns[size * i: size * (i + 1)] 68 | res.append(chunk) 69 | return res 70 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/Util/Webp.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import tensorflow as tf 3 | 4 | 5 | def load_webp_from_bytes(data): 6 | # needs sudo apt-get install python-webm 7 | from webm.decode import DecodeRGB 8 | x = DecodeRGB(data) 9 | res = numpy.array(x.bitmap).reshape((x.height, x.width, 3)) 10 | return res 11 | 12 | 13 | def load_webp_from_file(fn): 14 | with open(fn) as f: 15 | data = f.read() 16 | return load_webp_from_bytes(data) 17 | 18 | 19 | def decode_webp_tensorflow(fn): 20 | return tf.py_func(load_webp_from_bytes, [fn], tf.uint8) 21 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/Util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/ReID_net/datasets/Util/__init__.py -------------------------------------------------------------------------------- /code/ReID_net/datasets/Util/flo_Reader.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | 4 | #adapted from http://stackoverflow.com/questions/28013200/reading-middlebury-flow-files-with-python-bytes-array-numpy 5 | # WARNING: this will work on little-endian architectures (eg Intel x86) only! 6 | def read_flo_file(fp): 7 | with open(fp, 'rb') as f: 8 | magic = numpy.fromfile(f, numpy.float32, count=1) 9 | if 202021.25 != magic: 10 | print('Magic number incorrect. Invalid .flo file', fp) 11 | else: 12 | w = numpy.fromfile(f, numpy.int32, count=1)[0] 13 | h = numpy.fromfile(f, numpy.int32, count=1)[0] 14 | data = numpy.fromfile(f, numpy.float32, count=2 * w * h) 15 | data2D = data.reshape((h, w, 2)) 16 | return data2D 17 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/Util/python_pfm.py: -------------------------------------------------------------------------------- 1 | #from http://lmb.informatik.uni-freiburg.de/resources/datasets/SceneFlow/assets/code/python_pfm.py 2 | import re 3 | import numpy as np 4 | import sys 5 | 6 | 7 | def readPFM(file): 8 | file = open(file, 'rb') 9 | 10 | color = None 11 | width = None 12 | height = None 13 | scale = None 14 | endian = None 15 | 16 | header = file.readline().rstrip() 17 | if header == 'PF': 18 | color = True 19 | elif header == 'Pf': 20 | color = False 21 | else: 22 | raise Exception('Not a PFM file.') 23 | 24 | dim_match = re.match(r'^(\d+)\s(\d+)\s$', file.readline()) 25 | if dim_match: 26 | width, height = list(map(int, dim_match.groups())) 27 | else: 28 | raise Exception('Malformed PFM header.') 29 | 30 | scale = float(file.readline().rstrip()) 31 | if scale < 0: # little-endian 32 | endian = '<' 33 | scale = -scale 34 | else: 35 | endian = '>' # big-endian 36 | 37 | data = np.fromfile(file, endian + 'f') 38 | shape = (height, width, 3) if color else (height, width) 39 | 40 | data = np.reshape(data, shape) 41 | data = np.flipud(data) 42 | return data, scale 43 | 44 | 45 | def writePFM(file, image, scale=1): 46 | file = open(file, 'wb') 47 | 48 | color = None 49 | 50 | if image.dtype.name != 'float32': 51 | raise Exception('Image dtype must be float32.') 52 | 53 | image = np.flipud(image) 54 | 55 | if len(image.shape) == 3 and image.shape[2] == 3: # color image 56 | color = True 57 | elif len(image.shape) == 2 or len(image.shape) == 3 and image.shape[2] == 1: # greyscale 58 | color = False 59 | else: 60 | raise Exception('Image must have H x W x 3, H x W x 1 or H x W dimensions.') 61 | 62 | file.write('PF\n' if color else 'Pf\n') 63 | file.write('%d %d\n' % (image.shape[1], image.shape[0])) 64 | 65 | endian = image.dtype.byteorder 66 | 67 | if endian == '<' or endian == '=' and sys.byteorder == 'little': 68 | scale = -scale 69 | 70 | file.write('%f\n' % scale) 71 | 72 | image.tofile(file) 73 | -------------------------------------------------------------------------------- /code/ReID_net/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/ReID_net/datasets/__init__.py -------------------------------------------------------------------------------- /code/ReID_net/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import os 4 | 5 | from ReID_net.Engine import Engine 6 | from ReID_net.Config import Config 7 | from ReID_net.Log import log 8 | import tensorflow as tf 9 | 10 | 11 | def init_log(config): 12 | log_dir = config.dir("log_dir", "logs") 13 | model = config.str("model") 14 | filename = log_dir + model + ".log" 15 | verbosity = config.int("log_verbosity", 3) 16 | log.initialize([filename], [verbosity], []) 17 | 18 | 19 | def main(_): 20 | assert len(sys.argv) == 2, "usage: main.py " 21 | config_path = sys.argv[1] 22 | assert os.path.exists(config_path), config_path 23 | try: 24 | config = Config(config_path) 25 | except ValueError as e: 26 | print("Malformed config file:", e) 27 | return -1 28 | init_log(config) 29 | config.initialize() 30 | #dump the config into the log 31 | print(open(config_path).read()) 32 | engine = Engine(config) 33 | engine.run() 34 | 35 | if __name__ == '__main__': 36 | #for profiling. Note however that this will not be useful for the execution of the tensorflow graph, 37 | #only for stuff like initialization including creation of the graph, loading of weights, etc. 38 | #import cProfile 39 | #cProfile.run("tf.app.run(main)", sort="tottime") 40 | 41 | tf.app.run(main) 42 | -------------------------------------------------------------------------------- /code/ReID_net/network/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/ReID_net/network/__init__.py -------------------------------------------------------------------------------- /code/ReID_net/scripts/config/create_oneshot_trainsplit_configs.sh: -------------------------------------------------------------------------------- 1 | if [ "$#" -ne 1 ]; then 2 | echo "Usage ${0} logfile" 3 | exit 4 | fi 5 | 6 | config=$1 7 | config2=$(echo ${config} | sed s/trainsplit/trainsplit2/g) 8 | config3=$(echo ${config} | sed s/trainsplit/trainsplit3/g) 9 | 10 | 11 | if [ ! -e $config ]; then 12 | echo "${config} does not exist!" 13 | exit 1 14 | fi 15 | 16 | cp ${config} ${config2} 17 | sed -i s/_trainsplit/_trainsplit2/g ${config2} 18 | sed -i s/"trainsplit\": 1,"/"trainsplit\": 2,"/g ${config2} 19 | 20 | cp ${config} ${config3} 21 | sed -i s/_trainsplit/_trainsplit3/g ${config3} 22 | sed -i s/"trainsplit\": 1,"/"trainsplit\": 3,"/g ${config3} 23 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/config/duplicate_trainsplit_config.sh: -------------------------------------------------------------------------------- 1 | if [ "$#" -ne 1 ]; then 2 | echo "Usage ${0} logfile (without trainsplit)" 3 | exit 4 | fi 5 | 6 | NEW_IDX=2 7 | 8 | for split in "" "2" "3"; do 9 | cp ${1}_trainsplit${split} ${1}_${NEW_IDX}_trainsplit${split} 10 | sed -i s/${1}_trainsplit${split}/${1}_${NEW_IDX}_trainsplit${split}/g ${1}_${NEW_IDX}_trainsplit${split} 11 | done 12 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/config/duplicate_trainsplit_config3.sh: -------------------------------------------------------------------------------- 1 | if [ "$#" -ne 1 ]; then 2 | echo "Usage ${0} logfile (without trainsplit)" 3 | exit 4 | fi 5 | 6 | NEW_IDX=3 7 | 8 | for split in "" "2" "3"; do 9 | cp ${1}_trainsplit${split} ${1}_${NEW_IDX}_trainsplit${split} 10 | sed -i s/${1}_trainsplit${split}/${1}_${NEW_IDX}_trainsplit${split}/g ${1}_${NEW_IDX}_trainsplit${split} 11 | done 12 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/eval/eval.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from scipy.ndimage import imread 4 | import pickle 5 | import numpy 6 | import glob 7 | import os 8 | from joblib import Parallel, delayed 9 | import sys 10 | 11 | from ReID_net.datasets.DAVIS.DAVIS_iterative import get_bounding_box 12 | 13 | imgs_path = "/work/mahadevan/data/DAVIS/JPEGImages/480p/" 14 | annots_path = "/work/mahadevan/data/DAVIS/Annotations/480p/" 15 | preds_path_prefix = "/home/mahadevan/vision/savitar/forwarded/" 16 | 17 | 18 | def convert_path(inp): 19 | sp = inp.split("/") 20 | fwd_idx = sp.index("forwarded") 21 | 22 | seq = sp[fwd_idx + 3] 23 | fn = sp[-1] 24 | im_path = imgs_path + seq + "/" + fn.replace(".pickle", ".jpg") 25 | gt_path = annots_path + seq + "/" + fn.replace(".pickle", ".png") 26 | 27 | sp[fwd_idx + 1] += "_crf" 28 | sp[-1] = sp[-1].replace(".pickle", ".png") 29 | return im_path, gt_path 30 | 31 | 32 | def mkdir_p(d): 33 | try: 34 | os.makedirs(d) 35 | except OSError as err: 36 | if err.errno != 17: 37 | raise 38 | 39 | 40 | def do_seq(seq, model): 41 | preds_path = preds_path_prefix + model + "/valid/" 42 | files = sorted(glob.glob(preds_path + seq + "/*.pickle")) 43 | ious = [] 44 | for f in files: 45 | pred_path = f 46 | im_path, gt_path = convert_path(f) 47 | pred = pickle.load(open(pred_path)) 48 | res = numpy.argmax(pred, axis=2) * 255 49 | 50 | #compute iou as well 51 | groundtruth = imread(gt_path) 52 | bbox = get_bounding_box(groundtruth, 255) 53 | res = numpy.logical_and(res / 255, bbox) * 255 54 | I = numpy.logical_and(res == 255, groundtruth == 255).sum() 55 | U = numpy.logical_or(res == 255, groundtruth == 255).sum() 56 | IOU = float(I) / U 57 | ious.append(IOU) 58 | 59 | print(im_path, "IOU", IOU) 60 | 61 | # plt.imshow(before) 62 | # plt.figure() 63 | # plt.imshow(res) 64 | # plt.show() 65 | return numpy.mean(ious[1:-1]) 66 | 67 | 68 | def main(): 69 | #seqs = ["blackswan", "bmx-trees", "breakdance", "camel", "car-roundabout", "car-shadow", "cows", "dance-twirl", 70 | # "dog", "drift-chicane", "drift-straight", "goat", "horsejump-high", "kite-surf", "libby", "motocross-jump", 71 | # "paragliding-launch", "parkour", "scooter-black", "soapbox"] 72 | seqs = ["dance-twirl"] 73 | assert len(sys.argv) == 2 74 | model = sys.argv[1] 75 | 76 | #ious = [] 77 | #for seq in seqs: 78 | # iou = do_seq(seq, model) 79 | # print iou 80 | # ious.append(iou) 81 | 82 | ious = Parallel(n_jobs=20)(delayed(do_seq)(seq, model) for seq in seqs) 83 | 84 | print(ious) 85 | print(numpy.mean(ious)) 86 | 87 | 88 | if __name__ == "__main__": 89 | main() 90 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/eval/eval_KITTI_det.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$#" -ne 1 ]; then 4 | echo "Usage ${0} model" 5 | exit 6 | fi 7 | MODEL=$1 8 | 9 | ( 10 | cd "/home/${USER}/vision/savitar/forwarded/${MODEL}" 11 | det_jsons=$(ls det_*.json) 12 | for det_json in ${det_jsons}; do 13 | cd "/home/${USER}/vision/savitar/" 14 | det=$(echo ${det_json} | sed "s/.json//g") 15 | echo "converting ${det}..." 16 | ./scripts/postproc/convert_coco_detections_to_kitti_format.py ${MODEL} ${det} | grep -v "warning, ignoring detection with unknown class" 17 | cd "/home/$USER/vision/KITTI_object_devkit" 18 | echo "evaluating ${det}..." 19 | ./cpp/evaluate_object ${MODEL} 20 | cd "/home/${USER}/vision/savitar/forwarded/${MODEL}" 21 | echo "==========" 22 | mv stats_car_detection.txt ${det}_car.txt 23 | mv stats_pedestrian_detection.txt ${det}_pedestrian.txt 24 | mv stats_cyclist_detection.txt ${det}_cyclist.txt 25 | done 26 | ) 27 | 28 | 29 | #now collect all the results 30 | cd "/home/${USER}/vision/savitar/forwarded/${MODEL}" 31 | ( 32 | for class in car pedestrian cyclist; do 33 | epochs=$(ls det*_${class}.txt | sed s/det_//g | sed s/_${class}.txt//g | sort -n) 34 | for epoch in $epochs; do 35 | file=$(echo det_${epoch}_${class}.txt) 36 | echo $file | sed "s/det_//g" | sed "s/_${class}//g" | sed "s/.txt//g" 37 | done | tr '\n' ' ' 38 | #result for easy 39 | printf "\n${class} easy\n" 40 | for epoch in $epochs; do 41 | file=$(echo det_${epoch}_${class}.txt) 42 | sed '2q;d' $file | sed "s/ap //g" 43 | done | tr '\n' ' ' 44 | #result for medium 45 | printf "\n${class} medium\n" 46 | for epoch in $epochs; do 47 | file=$(echo det_${epoch}_${class}.txt) 48 | sed '4q;d' $file | sed "s/ap //g" 49 | done | tr '\n' ' ' 50 | #result for hard 51 | printf "\n${class} hard\n" 52 | for epoch in $epochs; do 53 | file=$(echo det_${epoch}_${class}.txt) 54 | sed '6q;d' $file | sed "s/ap //g" 55 | done | tr '\n' ' ' 56 | echo 57 | done 58 | ) | tee results.txt 59 | 60 | cp results.txt /home/voigtlaender/pub/cvpr2018/results/detector/eval/${MODEL} 61 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/eval/eval_coco.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import pycocotools.coco as coco 5 | import pycocotools.cocoeval as cocoeval 6 | 7 | if len(sys.argv) < 2 or len(sys.argv) > 3: 8 | print("usage: {} det_file.json [gt_file.json]".format(sys.argv[0])) 9 | sys.exit(1) 10 | det_file = sys.argv[1] 11 | if len(sys.argv) == 3: 12 | gt_file = sys.argv[2] 13 | else: 14 | gt_file = "/fastwork/voigtlaender/mywork/data/coco/annotations/instances_valid.json" 15 | 16 | coco_gt = coco.COCO(gt_file) 17 | coco_det = coco_gt.loadRes(det_file) 18 | e = cocoeval.COCOeval(coco_gt, coco_det, "bbox") 19 | e.evaluate() 20 | e.accumulate() 21 | e.summarize() 22 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/eval/eval_youtube.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import glob 3 | import sys 4 | import numpy 5 | from scipy.ndimage import imread 6 | 7 | YOUTUBE_PATH = "/data/corpora/youtube-objects/youtube_masks/" 8 | FORWARDED_PATH = "/home/voigtlaender/vision/savitar/forwarded/" 9 | 10 | 11 | def compute_iou_for_binary_segmentation(y_argmax, target): 12 | I = numpy.logical_and(y_argmax == 1, target == 1).sum() 13 | U = numpy.logical_or(y_argmax == 1, target == 1).sum() 14 | if U == 0: 15 | IOU = 1.0 16 | else: 17 | IOU = float(I) / U 18 | return IOU 19 | 20 | 21 | def eval_sequence(gt_folder, recog_folder): 22 | seq = gt_folder.split("/")[-7] + "_" + gt_folder.split("/")[-5] 23 | gt_files = sorted(glob.glob(gt_folder + "/*.jpg")) 24 | 25 | gt_files = gt_files[1:] 26 | recog_folder_seq = recog_folder + seq + "/" 27 | 28 | #for full dataset 29 | recog_files = [] 30 | for gt_file in gt_files: 31 | idx = int(gt_file.split("/")[-1].replace(".jpg", "")) 32 | ending = "frame%04d.png" % idx 33 | recog_file = recog_folder_seq + ending 34 | recog_files.append(recog_file) 35 | 36 | ious = [] 37 | for gt_file, recog_file in zip(gt_files, recog_files): 38 | gt = imread(gt_file) / 255 39 | recog = imread(recog_file) / 255 40 | iou = compute_iou_for_binary_segmentation(recog, gt) 41 | ious.append(iou) 42 | return numpy.mean(ious) 43 | 44 | 45 | def main(): 46 | assert len(sys.argv) == 2 47 | 48 | pattern = YOUTUBE_PATH + "*/data/*/*/*/labels/" 49 | folders = glob.glob(pattern) 50 | 51 | # filter out the 2 sequences, which only have a single annotated frame 52 | folders = [f for f in folders if "motorbike/data/0007/shots/001" not in f and "cow/data/0019/shots/001" not in f] 53 | 54 | ious = {} 55 | for folder in folders: 56 | tag = folder.split("/")[-7] 57 | recog_folder = FORWARDED_PATH + sys.argv[1] + "/valid/" 58 | iou = eval_sequence(folder, recog_folder) 59 | #print iou 60 | if tag in ious: 61 | ious[tag].append(iou) 62 | else: 63 | ious[tag] = [iou] 64 | 65 | class_ious = [] 66 | for k, v in list(ious.items()): 67 | print(k, numpy.mean(v)) 68 | class_ious.append(numpy.mean(v)) 69 | print("-----") 70 | print("total", len(class_ious), numpy.mean(class_ious)) 71 | 72 | if __name__ == "__main__": 73 | main() 74 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/eval/eval_youtube_nonfull.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import glob 3 | import sys 4 | import numpy 5 | from scipy.ndimage import imread 6 | 7 | YOUTUBE_PATH = "/data/corpora/youtube-objects/youtube_masks/" 8 | FORWARDED_PATH = "/home/voigtlaender/vision/savitar/forwarded/" 9 | 10 | 11 | def compute_iou_for_binary_segmentation(y_argmax, target): 12 | I = numpy.logical_and(y_argmax == 1, target == 1).sum() 13 | U = numpy.logical_or(y_argmax == 1, target == 1).sum() 14 | if U == 0: 15 | IOU = 1.0 16 | else: 17 | IOU = float(I) / U 18 | return IOU 19 | 20 | 21 | def eval_sequence(gt_folder, recog_folder): 22 | seq = gt_folder.split("/")[-7] + "_" + gt_folder.split("/")[-5] 23 | gt_files = sorted(glob.glob(gt_folder + "/*.jpg")) 24 | 25 | #checks 26 | #if not gt_files[0].endswith("00001.jpg"): 27 | # print "does not start with 00001.jpg!", gt_files[0] 28 | #indices = [int(f.split("/")[-1][:-4]) for f in gt_files] 29 | #if not (numpy.diff(indices) == 10).all(): 30 | # print "no spacing of 10:", gt_files 31 | 32 | gt_files = gt_files[1:] 33 | recog_folder_seq = recog_folder + seq + "/" 34 | print(recog_folder_seq, end=' ') 35 | recog_files = [gt_file.replace(gt_folder, recog_folder_seq).replace(".jpg", ".png") for gt_file in gt_files] 36 | 37 | ious = [] 38 | for gt_file, recog_file in zip(gt_files, recog_files): 39 | gt = imread(gt_file) / 255 40 | recog = imread(recog_file) / 255 41 | iou = compute_iou_for_binary_segmentation(recog, gt) 42 | ious.append(iou) 43 | return numpy.mean(ious) 44 | 45 | 46 | def main(): 47 | assert len(sys.argv) == 2 48 | 49 | pattern = YOUTUBE_PATH + "*/data/*/*/*/labels/" 50 | folders = glob.glob(pattern) 51 | 52 | # filter out the 2 sequences, which only have a single annotated frame 53 | folders = [f for f in folders if "motorbike/data/0007/shots/001" not in f and "cow/data/0019/shots/001" not in f] 54 | 55 | ious = {} 56 | for folder in folders: 57 | tag = folder.split("/")[-7] 58 | recog_folder = FORWARDED_PATH + sys.argv[1] + "/valid/" 59 | iou = eval_sequence(folder, recog_folder) 60 | print(iou) 61 | if tag in ious: 62 | ious[tag].append(iou) 63 | else: 64 | ious[tag] = [iou] 65 | 66 | class_ious = [] 67 | for k, v in list(ious.items()): 68 | print(k, numpy.mean(v)) 69 | class_ious.append(numpy.mean(v)) 70 | print("-----") 71 | print("total", len(class_ious), numpy.mean(class_ious)) 72 | 73 | if __name__ == "__main__": 74 | main() 75 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/eval_pascal_instance/convert_voc_to_sharpmask.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import glob 3 | import os 4 | import numpy as np 5 | from scipy import misc 6 | import scipy.io 7 | from ReID_net.datasets.Util import Util as dataUtil 8 | 9 | VOC_PATH = "/work/" + dataUtil.username() + "/data/PascalVOC/benchmark_RELEASE/dataset/" 10 | OUT_FN = "list.txt" 11 | VOID_LABEL = 255 12 | 13 | def main(): 14 | files = glob.glob( VOC_PATH + "dets/*.pickle") 15 | out_file = open(OUT_FN, 'w') 16 | for file in files: 17 | filename_without_ext = file.split('/')[-1].split('.')[-2] 18 | boxes = pickle.load( open(file) ) 19 | 20 | for box in boxes: 21 | if box.shape[0] != 0: 22 | [x_min, y_min, x_max, y_max] = box[0][:4].astype(int) 23 | data = VOC_PATH + "JPEGImages/" + filename_without_ext + ' ' + repr(x_min) + ' ' + repr(y_min) \ 24 | + ' ' + repr(x_max - x_min) + ' ' + repr(y_max - y_min) + "\n" 25 | out_file.write(data) 26 | 27 | 28 | def gt_to_sharpmask(): 29 | out_file = open(OUT_FN, 'w') 30 | val_images = "datasets/PascalVOC/val.txt" 31 | for im in open(val_images): 32 | instance_segm=None 33 | file_name_without_ext = im.split('/')[-1].split('.')[-2] 34 | inst_path = VOC_PATH + "inst/" + file_name_without_ext + ".mat" 35 | 36 | if os.path.exists(inst_path): 37 | instance_segm = scipy.io.loadmat(inst_path)['GTinst']['Segmentation'][0][0] 38 | else: 39 | inst_path = VOC_PATH + "/SegmentationObject/" + file_name_without_ext + ".png" 40 | if os.path.exists(inst_path): 41 | instance_segm = misc.imread(inst_path) 42 | else: 43 | print("File: " + im + " does not have any instance annotations.") 44 | 45 | if instance_segm is not None: 46 | inst_labels = np.unique(instance_segm) 47 | inst_labels = np.setdiff1d(inst_labels, 48 | [0, VOID_LABEL]) 49 | for inst in inst_labels: 50 | # Create bounding box from segmentation mask. 51 | rows = np.where(instance_segm == inst)[0] 52 | cols = np.where(instance_segm == inst)[1] 53 | rmin = rows.min() 54 | rmax = rows.max() 55 | cmin = cols.min() 56 | cmax = cols.max() 57 | area = (rmax - rmin) * (cmax - cmin) 58 | [x_min, y_min, x_max, y_max] = [cmin, rmin, cmax, rmax] 59 | if area > 200: 60 | data = VOC_PATH + "JPEGImages/" + file_name_without_ext + ' ' + repr(x_min) + ' ' + repr(y_min) \ 61 | + ' ' + repr(x_max - x_min) + ' ' + repr(y_max - y_min) + "\n" 62 | out_file.write(data) 63 | 64 | if __name__ == '__main__': 65 | gt_to_sharpmask() 66 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/plot_learn_curve2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import matplotlib.pyplot as plt 3 | import sys 4 | 5 | 6 | def doit(fn, col1, col2, tag): 7 | train = [] 8 | val = [] 9 | with open(fn) as f: 10 | for l in f: 11 | if "finished" in l: 12 | sp = l.split() 13 | #clip to 5 14 | # tr = min(float(sp[col1]), 5.0) 15 | # va = min(float(sp[col2]), 5.0) 16 | tr = float(sp[col1]) 17 | va = float(sp[col2]) 18 | # if tr>1: tr = 1/tr 19 | # if va>1: va = 1/va 20 | 21 | train.append(tr) 22 | val.append(va) 23 | plt.plot(train, label="train") 24 | plt.hold(True) 25 | plt.plot(val, label="val") 26 | plt.legend() 27 | plt.title(fn + " " + tag) 28 | #plt.show() 29 | 30 | assert len(sys.argv) == 2 31 | #error plot 32 | doit(sys.argv[1], 6, 8, "rank1") #For plotting for extended 33 | # doit(sys.argv[1], 21, 17, "rank1") #For plotting for triplet 34 | #score plot 35 | # plt.figure() 36 | # doit(sys.argv[1], 22, 18, "rank5") 37 | plt.show() 38 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/postproc/combine_single_object_predictions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import glob 4 | import os 5 | import os.path 6 | import pickle 7 | import numpy 8 | from joblib import Parallel, delayed 9 | from ReID_net.datasets.Util.pascal_colormap import save_with_pascal_colormap 10 | 11 | BASE_DIR = "/home/voigtlaender/vision/savitar/forwarded/" 12 | #PARALLEL = False 13 | PARALLEL = True 14 | 15 | 16 | def mkdir_p(d): 17 | try: 18 | os.makedirs(d) 19 | except OSError as err: 20 | if err.errno != 17: 21 | raise 22 | 23 | 24 | def do_seq(model, seq): 25 | pattern = BASE_DIR + model + "/valid/" + seq + "/*/" 26 | out_folder = BASE_DIR + model + "_merged/valid/" + seq + "/" 27 | mkdir_p(out_folder) 28 | object_folders = sorted(glob.glob(pattern)) 29 | filenames = sorted(map(os.path.basename, glob.glob(object_folders[0] + "/*.pickle"))) 30 | for fn in filenames: 31 | posteriors = [pickle.load(open(obj_folder + "/" + fn)) for obj_folder in object_folders] 32 | background = numpy.stack([x[..., 0] for x in posteriors], axis=2).min(axis=2, keepdims=True) 33 | rest = numpy.stack([x[..., 1] for x in posteriors], axis=2) 34 | combined = numpy.concatenate([background, rest], axis=2) 35 | result = combined.argmax(axis=2) 36 | out_fn = out_folder + fn.replace(".pickle", ".png") 37 | print(out_fn) 38 | save_with_pascal_colormap(out_fn, result) 39 | 40 | 41 | def main(): 42 | assert len(sys.argv) == 2 43 | model = sys.argv[1].replace("forwarded/", "") 44 | if model.endswith("/"): 45 | model = model[:-1] 46 | 47 | #split = "val" 48 | split = "test-dev" 49 | #split = "test-challenge" 50 | seqs = list(map(str.strip, open("/fastwork/voigtlaender/mywork/data/DAVIS2017/ImageSets/2017/" + split + ".txt").readlines())) 51 | 52 | if PARALLEL: 53 | Parallel(n_jobs=20)(delayed(do_seq)(model, seq) for seq in seqs) 54 | else: 55 | for seq in seqs: 56 | print(seq) 57 | do_seq(model, seq) 58 | 59 | if __name__ == "__main__": 60 | main() 61 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/postproc/convert_coco_detections_to_kitti_format.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import sys 5 | import json 6 | 7 | if __name__ == "__main__": 8 | assert len(sys.argv) in (2, 3) 9 | folder = "forwarded/" + sys.argv[1] + "/" 10 | if len(sys.argv) == 3: 11 | json_txt = open(folder + sys.argv[2] + ".json").read() 12 | else: 13 | json_txt = open(folder + "forwarded.json").read() 14 | dets = json.loads(json_txt) 15 | 16 | try: 17 | os.mkdir(folder + "/data/") 18 | except: 19 | pass 20 | opened = None 21 | for det in dets: 22 | filename = folder + "/data/" + det["image_id"].split("/")[-1].replace(".png", ".txt") 23 | if opened is None: 24 | opened = (filename, open(filename, "w")) 25 | else: 26 | if filename != opened[0]: 27 | opened[1].close() 28 | opened = (filename, open(filename, "w")) 29 | class_ = det["category_id"] 30 | if class_ == 1: 31 | class_str = "Pedestrian" 32 | elif class_ == 2: 33 | class_str = "Cyclist" 34 | elif class_ == 3: 35 | class_str = "Car" 36 | else: 37 | print("warning, ignoring detection with unknown class id", class_) 38 | continue 39 | bbox = det["bbox"] 40 | score = det["score"] 41 | print(class_str, 0, 0, -10, bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3], 0, 0, 0, 0, 0, \ 42 | 0, 0, score, file=opened[1]) 43 | if opened is not None: 44 | opened[1].close() 45 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/postproc/ensemble.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import glob 3 | import os 4 | import pickle 5 | import numpy 6 | from scipy.misc import imsave 7 | from joblib import Parallel, delayed 8 | 9 | 10 | def mkdir_p(d): 11 | try: 12 | os.makedirs(d) 13 | except OSError as err: 14 | if err.errno != 17: 15 | raise 16 | 17 | 18 | def do_file(fn): 19 | fns_models = [fn.replace(model1, model) for model in models_in] 20 | posteriors_models = [pickle.load(open(f, "rb")) for f in fns_models] 21 | posteriors_stacked = numpy.stack(posteriors_models, axis=0) 22 | posteriors = numpy.mean(posteriors_stacked, axis=0) 23 | f_out = fn.replace(model1, model_out) 24 | print(f_out) 25 | mkdir_p(os.path.dirname(f_out)) 26 | pickle.dump(posteriors, open(f_out, "wb"), pickle.HIGHEST_PROTOCOL) 27 | recog = numpy.argmax(posteriors, axis=2) 28 | imsave(f_out.replace("pickle", "png"), recog) 29 | 30 | model1 = "paper_VOC2_noDAVIS_oneshot6_2" 31 | model2 = "iterative_forward_no_underscore" 32 | 33 | models_in = [model1, model2] 34 | model_out = "ensemble" 35 | 36 | fwd_path = "/home/mahadevan/vision/savitar/forwarded/" 37 | 38 | assert not os.path.isdir(fwd_path + model_out), fwd_path + model_out 39 | 40 | files = glob.glob(fwd_path + model1 + "/valid/*/*.pickle") 41 | print(fwd_path + model1 + "/valid/*/*.pickle") 42 | 43 | #for f in files: 44 | # do_file(f) 45 | Parallel(n_jobs=20)(delayed(do_file)(f) for f in files) 46 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/postproc/ensemble2017.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import glob 3 | import os 4 | import pickle 5 | import numpy 6 | from scipy.misc import imsave 7 | from joblib import Parallel, delayed 8 | 9 | 10 | def mkdir_p(d): 11 | try: 12 | os.makedirs(d) 13 | except OSError as err: 14 | if err.errno != 17: 15 | raise 16 | 17 | 18 | def do_file(fn): 19 | fns_models = [fn.replace(model1, model) for model in models_in] 20 | posteriors_models = [pickle.load(open(f, "rb")) for f in fns_models] 21 | posteriors_stacked = numpy.stack(posteriors_models, axis=0) 22 | posteriors = numpy.mean(posteriors_stacked, axis=0) 23 | f_out = fn.replace(model1, model_out) 24 | print(f_out) 25 | mkdir_p(os.path.dirname(f_out)) 26 | pickle.dump(posteriors, open(f_out, "wb"), pickle.HIGHEST_PROTOCOL) 27 | recog = numpy.argmax(posteriors, axis=2) 28 | imsave(f_out.replace("pickle", "png"), recog) 29 | 30 | model1 = "workshop_online_up_lucid_1" 31 | model2 = "workshop_online_up_lucid_2" 32 | model3 = "workshop_online_up_lucid_3" 33 | model4 = "workshop_online_up_lucid_4" 34 | #model5 = "VOC_DAVIS17_online1_multi_dev" 35 | 36 | models_in = [model1, model2, model3, model4] 37 | model_out = "workshop_online_up_lucid_ensemble" 38 | 39 | fwd_path = "/home/voigtlaender/vision/savitar/forwarded/" 40 | 41 | assert not os.path.isdir(fwd_path + model_out), fwd_path + model_out 42 | 43 | files = glob.glob(fwd_path + model1 + "/valid/*/*/*.pickle") 44 | 45 | #for f in files: 46 | # do_file(f) 47 | Parallel(n_jobs=20)(delayed(do_file)(f) for f in files) 48 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/rename_weights.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #Adapted from https://gist.github.com/batzner/7c24802dd9c5e15870b4b56e22135c96 3 | 4 | import tensorflow as tf 5 | 6 | def rename(checkpoint_dir, replace_from, replace_to, add_prefix, dry_run=False): 7 | checkpoint = tf.train.get_checkpoint_state(checkpoint_dir) 8 | with tf.Session() as sess: 9 | for var_name, _ in tf.contrib.framework.list_variables(checkpoint_dir): 10 | # Load the variable 11 | var = tf.contrib.framework.load_variable(checkpoint_dir, var_name) 12 | 13 | # Set the new name 14 | new_name = var_name 15 | if None not in [replace_from, replace_to]: 16 | new_name = new_name.replace(replace_from, replace_to) 17 | if add_prefix: 18 | new_name = add_prefix + new_name 19 | 20 | if dry_run: 21 | print(('%s would be renamed to %s.' % (var_name, new_name))) 22 | else: 23 | print(('Renaming %s to %s.' % (var_name, new_name))) 24 | # Rename the variable 25 | var = tf.Variable(var, name=new_name) 26 | 27 | if not dry_run: 28 | # Save the variables 29 | saver = tf.train.Saver() 30 | sess.run(tf.global_variables_initializer()) 31 | # saver.save(sess, checkpoint.model_checkpoint_path) 32 | saver.save(sess, checkpoint_dir) 33 | 34 | #checkpoint_dir="/home/dlrc/dlrc17-gdk/ext/OnAVOS/models/small_lego2-00012453" 35 | #checkpoint_dir="models/COCO_SPN_exp2_with_prefix/COCO_SPN_exp2-00000151" 36 | checkpoint_dir="models/SPN_exp_with_prefix/SPN_exp-00000200" 37 | replace_from=None 38 | replace_to=None 39 | add_prefix="SPN/" 40 | 41 | rename(checkpoint_dir,replace_from,replace_to,add_prefix) 42 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/rename_weights.py~: -------------------------------------------------------------------------------- 1 | #Adapted from https://gist.github.com/batzner/7c24802dd9c5e15870b4b56e22135c96 2 | 3 | import tensorflow as tf 4 | 5 | def rename(checkpoint_dir, replace_from, replace_to, add_prefix, dry_run=False): 6 | checkpoint = tf.train.get_checkpoint_state(checkpoint_dir) 7 | with tf.Session() as sess: 8 | for var_name, _ in tf.contrib.framework.list_variables(checkpoint_dir): 9 | # Load the variable 10 | var = tf.contrib.framework.load_variable(checkpoint_dir, var_name) 11 | 12 | # Set the new name 13 | new_name = var_name 14 | if None not in [replace_from, replace_to]: 15 | new_name = new_name.replace(replace_from, replace_to) 16 | if add_prefix: 17 | new_name = add_prefix + new_name 18 | 19 | if dry_run: 20 | print('%s would be renamed to %s.' % (var_name, new_name)) 21 | else: 22 | print('Renaming %s to %s.' % (var_name, new_name)) 23 | # Rename the variable 24 | var = tf.Variable(var, name=new_name) 25 | 26 | if not dry_run: 27 | # Save the variables 28 | saver = tf.train.Saver() 29 | sess.run(tf.global_variables_initializer()) 30 | # saver.save(sess, checkpoint.model_checkpoint_path) 31 | saver.save(sess, checkpoint_dir) 32 | 33 | checkpoint_dir="/home/dlrc/dlrc17-gdk/ext/OnAVOS/models/small_lego2-00012453" 34 | replace_from=None 35 | replace_to=None 36 | add_prefix="latent/" 37 | 38 | rename(checkpoint_dir,replace_from,replace_to,add_prefix) 39 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/stats/DAVIS_iou.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import numpy 5 | 6 | assert len(sys.argv) == 2 7 | fn = sys.argv[1] 8 | 9 | ious = [None] * 20 10 | 11 | with open(fn) as f: 12 | for l in f: 13 | if "sequence" in l: 14 | sp = l.split() 15 | idx = int(sp[1]) - 1 16 | iou = float(sp[-1]) 17 | assert 0 <= iou <= 1 18 | ious[idx] = iou 19 | 20 | for i in range(10): 21 | assert ious[i] is not None, i 22 | 23 | if ious[10] is None: 24 | for i in range(10, 20): 25 | assert ious[i] is None, i 26 | ious = ious[:10] 27 | else: 28 | for i in range(10, 20): 29 | assert ious[i] is not None, i 30 | 31 | print(len(ious), numpy.mean(ious)) 32 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/stats/compute_iou.sh: -------------------------------------------------------------------------------- 1 | if [ "$#" -lt 1 ]; then 2 | echo "Usage ${0} logfile(s)" 3 | exit 4 | fi 5 | 6 | #grep iou $1 | grep sequence | cut -d"'" -f5| cut -d" " -f2 | cut -d"," -f1 | ../scripts/stats/print_stats.py 7 | grep -h iou $@ | grep sequence | cut -d"'" -f5| cut -d" " -f2 | cut -d"," -f1 | ../scripts/stats/print_stats.py 8 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/stats/compute_iou_cv.sh: -------------------------------------------------------------------------------- 1 | if [ "$#" -ne 1 ]; then 2 | echo "Usage ${0} logfile" 3 | exit 4 | fi 5 | 6 | F1=$1 7 | F2=$(echo $1 | sed s/trainsplit/trainsplit2/g) 8 | F3=$(echo $1 | sed s/trainsplit/trainsplit3/g) 9 | 10 | grep -h iou $F1 $F2 $F3 | grep sequence | cut -d"'" -f5| cut -d" " -f2 | cut -d"," -f1 | ../scripts/print_stats.py 11 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/stats/find_best_err.sh: -------------------------------------------------------------------------------- 1 | if [ "$#" -ne 1 ]; then 2 | echo "Usage ${0} logfile" 3 | exit 4 | fi 5 | 6 | grep $(grep finish $1 | cut -d" " -f13 | sort -g | tail -1) $1 7 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/stats/find_best_score.sh: -------------------------------------------------------------------------------- 1 | if [ "$#" -ne 1 ]; then 2 | echo "Usage ${0} logfile" 3 | exit 4 | fi 5 | 6 | grep $(grep finish $1 | cut -d" " -f10 | sort -gr | tail -1) $1 7 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/stats/per_sequence.sh: -------------------------------------------------------------------------------- 1 | if [ "$#" -ne 1 ]; then 2 | echo "Usage ${0} model" 3 | exit 4 | fi 5 | 6 | for seq in blackswan bmx-trees breakdance camel car-roundabout car-shadow cows dance-twirl dog drift-chicane drift-straight goat horsejump-high kite-surf libby motocross-jump paragliding-launch parkour scooter-black soapbox; do 7 | echo $seq 8 | RES1=$(grep $seq $1.log | grep iou | cut -d"'" -f5| cut -d" " -f2 | cut -d"," -f1) 9 | RES2=$(grep $seq $1_2.log | grep iou | cut -d"'" -f5| cut -d" " -f2 | cut -d"," -f1) 10 | RES3=$(grep $seq $1_3.log | grep iou | cut -d"'" -f5| cut -d" " -f2 | cut -d"," -f1) 11 | echo "$RES1 $RES2 $RES3" | ../scripts/print_mean_std_ddof_pretty.py 12 | done 13 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/stats/print_mean_std_ddof.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import numpy 4 | 5 | #vals = [] 6 | #for l in sys.stdin: 7 | # vals.append(float(l)) 8 | vals = numpy.loadtxt(sys.stdin) 9 | 10 | print(vals.size, "mean", numpy.mean(vals), "std", numpy.std(vals, ddof=1)) 11 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/stats/print_mean_std_ddof_pretty.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import numpy 4 | 5 | #vals = [] 6 | #for l in sys.stdin: 7 | # vals.append(float(l)) 8 | vals = numpy.loadtxt(sys.stdin) 9 | 10 | assert vals.size == 3 11 | print("%.2f +- %.2f" % (numpy.mean(vals) * 100, numpy.std(vals, ddof=1) * 100)) 12 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/stats/print_stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import numpy 4 | 5 | vals = [] 6 | for l in sys.stdin: 7 | vals.append(float(l)) 8 | 9 | print(len(vals), "mean", numpy.mean(vals), "std", numpy.std(vals), "min", numpy.min(vals), "max", numpy.max(vals)) 10 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/stats/show_ious.sh: -------------------------------------------------------------------------------- 1 | if [ "$#" -ne 1 ]; then 2 | echo "Usage ${0} logfile" 3 | exit 4 | fi 5 | 6 | grep iou $1 | grep sequence | cut -d"'" -f5| cut -d" " -f2 | cut -d"," -f1 7 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/stats/show_ious_cv.sh: -------------------------------------------------------------------------------- 1 | if [ "$#" -ne 1 ]; then 2 | echo "Usage ${0} logfile" 3 | exit 4 | fi 5 | 6 | F1=$1 7 | F2=$(echo $1 | sed s/trainsplit/trainsplit2/g) 8 | F3=$(echo $1 | sed s/trainsplit/trainsplit3/g) 9 | 10 | grep -h iou $F1 $F2 $F3 | grep sequence | cut -d"'" -f5| cut -d" " -f2 | cut -d"," -f1 11 | -------------------------------------------------------------------------------- /code/ReID_net/scripts/test_load_pairs.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | def main(): 5 | n_pers = 1454 6 | 7 | image_list = np.genfromtxt('/fastwork/luiten/mywork/data/CUHK03/number_of_images.csv', delimiter=',') 8 | image_list = image_list.astype(np.int32) 9 | num_images = tf.constant(image_list) 10 | 11 | rand = tf.random_uniform([7], maxval=tf.int32.max, dtype=tf.int32) 12 | sample_same_person = rand[0] % 2 13 | pers_id_1 = ((rand[1]-1) % n_pers) + 1 14 | cam_id_1 = ((rand[2]-1) % 2) + 1 15 | pers_1_n_imgs = num_images[n_pers * (cam_id_1 - 1) + pers_id_1][2] 16 | img_id_1 = ((rand[3] - 1) % pers_1_n_imgs) + 1 17 | 18 | def if_same_person(): 19 | pers_id_2 = pers_id_1 20 | cam_id_2 = cam_id_1 21 | img_id_2 = ((rand[4] - 1) % (pers_1_n_imgs - 1)) + 1 22 | img_id_2 = tf.cond(img_id_2 >= img_id_1, lambda: img_id_2 + 1, lambda: img_id_2) 23 | return pers_id_2, cam_id_2, img_id_2 24 | 25 | def if_not_same_person(): 26 | pers_id_2 = ((rand[4]-1) % (n_pers-1)) + 1 27 | pers_id_2 = tf.cond(pers_id_2 >= pers_id_1, lambda: pers_id_2 + 1, lambda: pers_id_2) 28 | cam_id_2 = ((rand[5]-1) % 2) + 1 29 | pers_2_n_imgs = num_images[n_pers * (cam_id_2 - 1) + pers_id_2][2] 30 | img_id_2 = ((rand[6] - 1) % (pers_2_n_imgs -1)) + 1 31 | return pers_id_2, cam_id_2, img_id_2 32 | 33 | pers_id_2, cam_id_2, img_id_2 = tf.cond(tf.cast(sample_same_person,tf.bool),if_same_person,if_not_same_person) 34 | 35 | # pair = tf.stack([sample_same_person, pers_id_1,cam_id_1,img_id_1,pers_id_2,cam_id_2,img_id_2]) 36 | 37 | img1 = tf.as_string(pers_id_1) + "_" + tf.as_string(cam_id_1) + "_" + tf.as_string(img_id_1) + ".png" 38 | img2 = tf.as_string(pers_id_2) + "_" + tf.as_string(cam_id_2) + "_" + tf.as_string(img_id_2) + ".png" 39 | 40 | pair = tf.stack([img1, img2, tf.as_string(sample_same_person)]) 41 | 42 | 43 | # image_reader = tf.WholeFileReader() 44 | # _, image_file1 = image_reader.read(img1) 45 | # _, image_file2 = image_reader.read(img2) 46 | 47 | print(image_list.shape) 48 | batch = tf.train.batch([pair], batch_size=3) 49 | sess = tf.Session() 50 | sess.run(tf.global_variables_initializer()) 51 | tf.train.start_queue_runners(sess) 52 | for i in range(1): 53 | x = sess.run([batch]) 54 | print(x) 55 | 56 | if __name__ == "__main__": 57 | main() -------------------------------------------------------------------------------- /code/lucid_data_dreaming/readme.txt: -------------------------------------------------------------------------------- 1 | This is not necessary for the simple version of PReMVOS but is used for the ACCV paper version and the CVPR challenge winning version (but not the ECCV challenge winning version). 2 | 3 | To use this, the original Lucid Data Dreaming code needs to be downloaded from here: 4 | 5 | https://github.com/ankhoreva/LucidDataDreaming 6 | 7 | And then it can be run with my script (run.m) to generate image augmentations per image sequence. 8 | 9 | Warning: This code is very slow, it could be sped up immensely with some optimisations, we however just simply ran 1000 processes in parallel, on 1000 different CPUs in order to get this to finish quick enough. 10 | -------------------------------------------------------------------------------- /code/lucid_data_dreaming/run.m: -------------------------------------------------------------------------------- 1 | function [] = run(folder) 2 | 3 | curr_files = dir ([folder '/*.jpg']); 4 | num_files = numel (curr_files); 5 | rng('shuffle') 6 | 7 | if num_files == 2500 8 | return 9 | end 10 | 11 | addpath(genpath('patch-inpainting')); 12 | addpath(genpath('patch-inpainting/CSH_code')); 13 | addpath(genpath('PoissonEdiitng')); 14 | addpath(genpath('PoissonEdiitng/PoissonEdiitng20151105')); 15 | addpath(genpath('PoissonEdiitng/PoissonEdiitng20151105/src')); 16 | addpath(genpath('src')); 17 | 18 | img =imread([folder '/0.jpg']); 19 | [gt,map] = imread([folder '/0.png']); 20 | outpath = folder; 21 | 22 | for i = 1:2499 23 | t = tic; 24 | [im1, gt1, ~]= lucid_dream(img,gt,0); 25 | imwrite(im1,sprintf('%s%d.jpg',outpath,i)) 26 | imwrite(gt1,map,sprintf('%s%d.png',outpath,i)) 27 | disp(i) 28 | disp(toc(t)) 29 | end 30 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/__init__.py -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | from torch.utils.ffi import create_extension 4 | 5 | 6 | sources = ['correlation_package/src/corr.c'] 7 | headers = ['correlation_package/src/corr.h'] 8 | 9 | sources += ['correlation_package/src/corr1d.c'] 10 | headers += ['correlation_package/src/corr1d.h'] 11 | 12 | defines = [] 13 | with_cuda = False 14 | 15 | if torch.cuda.is_available(): 16 | print('Including CUDA code.') 17 | sources += ['correlation_package/src/corr_cuda.c'] 18 | headers += ['correlation_package/src/corr_cuda.h'] 19 | 20 | sources += ['correlation_package/src/corr1d_cuda.c'] 21 | headers += ['correlation_package/src/corr1d_cuda.h'] 22 | 23 | defines += [('WITH_CUDA', None)] 24 | with_cuda = True 25 | 26 | this_file = os.path.dirname(os.path.realpath(__file__)) 27 | extra_objects = ['correlation_package/src/corr_cuda_kernel.cu.o'] 28 | extra_objects += ['correlation_package/src/corr1d_cuda_kernel.cu.o'] 29 | extra_objects = [os.path.join(this_file, fname) for fname in extra_objects] 30 | 31 | ffi = create_extension( 32 | 'correlation_package._ext.corr', 33 | package=True, 34 | headers=headers, 35 | sources=sources, 36 | define_macros=defines, 37 | relative_to=__file__, 38 | with_cuda=with_cuda, 39 | extra_objects=extra_objects, 40 | ) 41 | 42 | if __name__ == '__main__': 43 | ffi.build() 44 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/lib.linux-x86_64-3.6/correlation_package/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/lib.linux-x86_64-3.6/correlation_package/__init__.py -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/lib.linux-x86_64-3.6/correlation_package/_ext/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/lib.linux-x86_64-3.6/correlation_package/_ext/__init__.py -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/lib.linux-x86_64-3.6/correlation_package/_ext/corr/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from torch.utils.ffi import _wrap_function 3 | from ._corr import lib as _lib, ffi as _ffi 4 | 5 | __all__ = [] 6 | def _import_symbols(locals): 7 | for symbol in dir(_lib): 8 | fn = getattr(_lib, symbol) 9 | locals[symbol] = _wrap_function(fn, _ffi) 10 | __all__.append(symbol) 11 | 12 | _import_symbols(locals()) 13 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/lib.linux-x86_64-3.6/correlation_package/_ext/corr/_corr.abi3.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/lib.linux-x86_64-3.6/correlation_package/_ext/corr/_corr.abi3.so -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/lib.linux-x86_64-3.6/correlation_package/functions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/lib.linux-x86_64-3.6/correlation_package/functions/__init__.py -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/lib.linux-x86_64-3.6/correlation_package/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/lib.linux-x86_64-3.6/correlation_package/modules/__init__.py -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/lib.linux-x86_64-3.6/correlation_package/modules/corr.py: -------------------------------------------------------------------------------- 1 | from torch.nn.modules.module import Module 2 | from ..functions.corr import correlation, correlation1d 3 | 4 | class Correlation(Module): 5 | 6 | def __init__(self, pad_size=None, kernel_size=None, max_displacement=None, 7 | stride1=None, stride2=None, corr_multiply=None): 8 | super(Correlation, self).__init__() 9 | self.pad_size = pad_size 10 | self.kernel_size = kernel_size 11 | self.max_displacement = max_displacement 12 | self.stride1 = stride1 13 | self.stride2 = stride2 14 | self.corr_multiply = corr_multiply 15 | 16 | def reset_params(self): 17 | return 18 | 19 | def forward(self, input1, input2): 20 | return correlation(self.pad_size, self.kernel_size, self.max_displacement, self.stride1, self.stride2, self.corr_multiply)(input1, input2) 21 | 22 | def __repr__(self): 23 | return self.__class__.__name__ 24 | 25 | 26 | #----- correlation in 1D (for disparity) Jinwei Gu ----- 27 | 28 | class Correlation1d(Module): 29 | 30 | def __init__(self, pad_size=None, kernel_size=None, max_displacement=None, 31 | stride1=None, stride2=None, corr_multiply=None): 32 | super(Correlation1d, self).__init__() 33 | self.pad_size = pad_size 34 | self.kernel_size = kernel_size 35 | self.max_displacement = max_displacement 36 | self.stride1 = stride1 37 | self.stride2 = stride2 38 | self.corr_multiply = corr_multiply 39 | 40 | def reset_params(self): 41 | return 42 | 43 | def forward(self, input1, input2): 44 | return correlation1d(self.pad_size, self.kernel_size, self.max_displacement, self.stride1, self.stride2, self.corr_multiply)(input1, input2) 45 | 46 | def __repr__(self): 47 | return self.__class__.__name__ 48 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/temp.linux-x86_64-3.6/build/temp.linux-x86_64-3.6/correlation_package._ext.corr._corr.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/temp.linux-x86_64-3.6/build/temp.linux-x86_64-3.6/correlation_package._ext.corr._corr.o -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/temp.linux-x86_64-3.6/home/luiten/vision/youtubevos/PWC-net/PWC-Net-master/PyTorch/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/temp.linux-x86_64-3.6/home/luiten/vision/youtubevos/PWC-net/PWC-Net-master/PyTorch/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr.o -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/temp.linux-x86_64-3.6/home/luiten/vision/youtubevos/PWC-net/PWC-Net-master/PyTorch/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr1d.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/temp.linux-x86_64-3.6/home/luiten/vision/youtubevos/PWC-net/PWC-Net-master/PyTorch/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr1d.o -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/temp.linux-x86_64-3.6/home/luiten/vision/youtubevos/PWC-net/PWC-Net-master/PyTorch/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr1d_cuda.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/temp.linux-x86_64-3.6/home/luiten/vision/youtubevos/PWC-net/PWC-Net-master/PyTorch/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr1d_cuda.o -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/temp.linux-x86_64-3.6/home/luiten/vision/youtubevos/PWC-net/PWC-Net-master/PyTorch/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr_cuda.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/build/temp.linux-x86_64-3.6/home/luiten/vision/youtubevos/PWC-net/PWC-Net-master/PyTorch/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr_cuda.o -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 1.0 2 | Name: correlation-package 3 | Version: 0.1 4 | Summary: Correlation layer from FlowNetC 5 | Home-page: https://github.com/jbarker-nvidia/pytorch-correlation 6 | Author: Jon Barker 7 | Author-email: jbarker@nvidia.com 8 | License: UNKNOWN 9 | Description: UNKNOWN 10 | Platform: UNKNOWN 11 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | setup.py 2 | /home/luiten/vision/youtubevos/PWC-net/PWC-Net-master/PyTorch/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr.c 3 | /home/luiten/vision/youtubevos/PWC-net/PWC-Net-master/PyTorch/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr1d.c 4 | /home/luiten/vision/youtubevos/PWC-net/PWC-Net-master/PyTorch/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr1d_cuda.c 5 | /home/luiten/vision/youtubevos/PWC-net/PWC-Net-master/PyTorch/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr_cuda.c 6 | correlation_package/__init__.py 7 | correlation_package.egg-info/PKG-INFO 8 | correlation_package.egg-info/SOURCES.txt 9 | correlation_package.egg-info/dependency_links.txt 10 | correlation_package.egg-info/requires.txt 11 | correlation_package.egg-info/top_level.txt 12 | correlation_package/_ext/__init__.py 13 | correlation_package/_ext/corr/__init__.py 14 | correlation_package/functions/__init__.py 15 | correlation_package/functions/corr.py 16 | correlation_package/modules/__init__.py 17 | correlation_package/modules/corr.py 18 | test/test.py -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package.egg-info/requires.txt: -------------------------------------------------------------------------------- 1 | cffi>=1.0.0 2 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | correlation_package 2 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/__init__.py -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/_ext/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/_ext/__init__.py -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/_ext/corr/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from torch.utils.ffi import _wrap_function 3 | from ._corr import lib as _lib, ffi as _ffi 4 | 5 | __all__ = [] 6 | def _import_symbols(locals): 7 | for symbol in dir(_lib): 8 | fn = getattr(_lib, symbol) 9 | locals[symbol] = _wrap_function(fn, _ffi) 10 | __all__.append(symbol) 11 | 12 | _import_symbols(locals()) 13 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/functions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/functions/__init__.py -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/modules/__init__.py -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/modules/corr.py: -------------------------------------------------------------------------------- 1 | from torch.nn.modules.module import Module 2 | from ..functions.corr import correlation, correlation1d 3 | 4 | class Correlation(Module): 5 | 6 | def __init__(self, pad_size=None, kernel_size=None, max_displacement=None, 7 | stride1=None, stride2=None, corr_multiply=None): 8 | super(Correlation, self).__init__() 9 | self.pad_size = pad_size 10 | self.kernel_size = kernel_size 11 | self.max_displacement = max_displacement 12 | self.stride1 = stride1 13 | self.stride2 = stride2 14 | self.corr_multiply = corr_multiply 15 | 16 | def reset_params(self): 17 | return 18 | 19 | def forward(self, input1, input2): 20 | return correlation(self.pad_size, self.kernel_size, self.max_displacement, self.stride1, self.stride2, self.corr_multiply)(input1, input2) 21 | 22 | def __repr__(self): 23 | return self.__class__.__name__ 24 | 25 | 26 | #----- correlation in 1D (for disparity) Jinwei Gu ----- 27 | 28 | class Correlation1d(Module): 29 | 30 | def __init__(self, pad_size=None, kernel_size=None, max_displacement=None, 31 | stride1=None, stride2=None, corr_multiply=None): 32 | super(Correlation1d, self).__init__() 33 | self.pad_size = pad_size 34 | self.kernel_size = kernel_size 35 | self.max_displacement = max_displacement 36 | self.stride1 = stride1 37 | self.stride2 = stride2 38 | self.corr_multiply = corr_multiply 39 | 40 | def reset_params(self): 41 | return 42 | 43 | def forward(self, input1, input2): 44 | return correlation1d(self.pad_size, self.kernel_size, self.max_displacement, self.stride1, self.stride2, self.corr_multiply)(input1, input2) 45 | 46 | def __repr__(self): 47 | return self.__class__.__name__ 48 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int corr_cpu_forward(THFloatTensor *input1, 4 | THFloatTensor *input2, 5 | THFloatTensor *rbot1, 6 | THFloatTensor *rbot2, 7 | THFloatTensor *output, 8 | int pad_size, 9 | int kernel_size, 10 | int max_displacement, 11 | int stride1, 12 | int stride2, 13 | int corr_type_multiply) 14 | { 15 | return 1; 16 | } 17 | 18 | int corr_cpu_backward(THFloatTensor *input1, 19 | THFloatTensor *input2, 20 | THFloatTensor *rbot1, 21 | THFloatTensor *rbot2, 22 | THFloatTensor *gradOutput, 23 | THFloatTensor *gradInput1, 24 | THFloatTensor *gradInput2, 25 | int pad_size, 26 | int kernel_size, 27 | int max_displacement, 28 | int stride1, 29 | int stride2, 30 | int corr_type_multiply) 31 | { 32 | return 1; 33 | } 34 | 35 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr.h: -------------------------------------------------------------------------------- 1 | int corr_cpu_forward(THFloatTensor *input1, 2 | THFloatTensor *input2, 3 | THFloatTensor *rbot1, 4 | THFloatTensor *rbot2, 5 | THFloatTensor *output, 6 | int pad_size, 7 | int kernel_size, 8 | int max_displacement, 9 | int stride1, 10 | int stride2, 11 | int corr_type_multiply); 12 | 13 | int corr_cpu_backward(THFloatTensor *input1, 14 | THFloatTensor *input2, 15 | THFloatTensor *rbot1, 16 | THFloatTensor *rbot2, 17 | THFloatTensor *gradOutput, 18 | THFloatTensor *gradInput1, 19 | THFloatTensor *gradInput2, 20 | int pad_size, 21 | int kernel_size, 22 | int max_displacement, 23 | int stride1, 24 | int stride2, 25 | int corr_type_multiply); 26 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr1d.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int corr1d_cpu_forward(THFloatTensor *input1, 4 | THFloatTensor *input2, 5 | THFloatTensor *rbot1, 6 | THFloatTensor *rbot2, 7 | THFloatTensor *output, 8 | int pad_size, 9 | int kernel_size, 10 | int max_displacement, 11 | int stride1, 12 | int stride2, 13 | int corr_type_multiply) 14 | { 15 | return 1; 16 | } 17 | 18 | int corr1d_cpu_backward(THFloatTensor *input1, 19 | THFloatTensor *input2, 20 | THFloatTensor *rbot1, 21 | THFloatTensor *rbot2, 22 | THFloatTensor *gradOutput, 23 | THFloatTensor *gradInput1, 24 | THFloatTensor *gradInput2, 25 | int pad_size, 26 | int kernel_size, 27 | int max_displacement, 28 | int stride1, 29 | int stride2, 30 | int corr_type_multiply) 31 | { 32 | return 1; 33 | } 34 | 35 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr1d.h: -------------------------------------------------------------------------------- 1 | int corr1d_cpu_forward(THFloatTensor *input1, 2 | THFloatTensor *input2, 3 | THFloatTensor *rbot1, 4 | THFloatTensor *rbot2, 5 | THFloatTensor *output, 6 | int pad_size, 7 | int kernel_size, 8 | int max_displacement, 9 | int stride1, 10 | int stride2, 11 | int corr_type_multiply); 12 | 13 | int corr1d_cpu_backward(THFloatTensor *input1, 14 | THFloatTensor *input2, 15 | THFloatTensor *rbot1, 16 | THFloatTensor *rbot2, 17 | THFloatTensor *gradOutput, 18 | THFloatTensor *gradInput1, 19 | THFloatTensor *gradInput2, 20 | int pad_size, 21 | int kernel_size, 22 | int max_displacement, 23 | int stride1, 24 | int stride2, 25 | int corr_type_multiply); 26 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr1d_cuda.h: -------------------------------------------------------------------------------- 1 | int corr1d_cuda_forward(THCudaTensor *input1, 2 | THCudaTensor *input2, 3 | THCudaTensor *rbot1, 4 | THCudaTensor *rbot2, 5 | THCudaTensor *output, 6 | int pad_size, 7 | int kernel_size, 8 | int max_displacement, 9 | int stride1, 10 | int stride2, 11 | int corr_type_multiply); 12 | 13 | int corr1d_cuda_backward(THCudaTensor *input1, 14 | THCudaTensor *input2, 15 | THCudaTensor *rbot1, 16 | THCudaTensor *rbot2, 17 | THCudaTensor *grad_output, 18 | THCudaTensor *grad_input1, 19 | THCudaTensor *grad_input2, 20 | int pad_size, 21 | int kernel_size, 22 | int max_displacement, 23 | int stride1, 24 | int stride2, 25 | int corr_type_multiply); 26 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr1d_cuda_kernel.cu.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr1d_cuda_kernel.cu.o -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr1d_cuda_kernel.h: -------------------------------------------------------------------------------- 1 | #ifndef _CORR_CUDA_KERNEL 2 | #define _CORR_CUDA_KERNEL 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | void blob_rearrange_ongpu_1d(const float *in, float *out, int num, int channels, int width, int height, int widthheight, int padding, int pwidthheight, cudaStream_t stream); 9 | 10 | void CorrelateData_ongpu_1d(const float *rbot1, const float *rbot2, float *output, int batchSize, int nOutputCols, int nOutputRows, int nOutputPlane, int max_displacement, int x_shift, int neighborhood_grid_width_, int kernel_radius_, int kernel_size, int stride1, int stride2, int paddedbottomwidth, int paddedbottomheight, int nInputPlane, int corr_type_multiply, cudaStream_t stream); 11 | 12 | void CorrelateDataBackward_ongpu_1d(const float *rbot1, const float *rbot2, const float *gradOutput, float *gradInput1, float *gradInput2, int batchSize, int nOutputCols, int nOutputRows, int nOutputPlane, int max_displacement, int x_shift, int neighborhood_grid_width_, int kernel_radius_, int stride1, int stride2, int nInputCols, int nInputRows, int paddedbottomwidth, int paddedbottomheight, int nInputPlane, int pad_size, int corr_type_multiply, cudaStream_t stream); 13 | 14 | #ifdef __cplusplus 15 | } 16 | #endif 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr_cuda.h: -------------------------------------------------------------------------------- 1 | int corr_cuda_forward(THCudaTensor *input1, 2 | THCudaTensor *input2, 3 | THCudaTensor *rbot1, 4 | THCudaTensor *rbot2, 5 | THCudaTensor *output, 6 | int pad_size, 7 | int kernel_size, 8 | int max_displacement, 9 | int stride1, 10 | int stride2, 11 | int corr_type_multiply); 12 | 13 | int corr_cuda_backward(THCudaTensor *input1, 14 | THCudaTensor *input2, 15 | THCudaTensor *rbot1, 16 | THCudaTensor *rbot2, 17 | THCudaTensor *grad_output, 18 | THCudaTensor *grad_input1, 19 | THCudaTensor *grad_input2, 20 | int pad_size, 21 | int kernel_size, 22 | int max_displacement, 23 | int stride1, 24 | int stride2, 25 | int corr_type_multiply); 26 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr_cuda_kernel.cu.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr_cuda_kernel.cu.o -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/correlation_package/src/corr_cuda_kernel.h: -------------------------------------------------------------------------------- 1 | #ifndef _CORR_CUDA_KERNEL 2 | #define _CORR_CUDA_KERNEL 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | void blob_rearrange_ongpu(const float *in, float *out, int num, int channels, int width, int height, int widthheight, int padding, int pwidthheight, cudaStream_t stream); 9 | 10 | void CorrelateData_ongpu(const float *rbot1, const float *rbot2, float *output, int batchSize, int nOutputCols, int nOutputRows, int nOutputPlane, int max_displacement, int neighborhood_grid_radius_, int neighborhood_grid_width_, int kernel_radius_, int kernel_size, int stride1, int stride2, int paddedbottomwidth, int paddedbottomheight, int nInputPlane, int corr_type_multiply, cudaStream_t stream); 11 | 12 | void CorrelateDataBackward_ongpu(const float *rbot1, const float *rbot2, const float *gradOutput, float *gradInput1, float *gradInput2, int batchSize, int nOutputCols, int nOutputRows, int nOutputPlane, int max_displacement, int neighborhood_grid_radius_, int neighborhood_grid_width_, int kernel_radius_, int stride1, int stride2, int nInputCols, int nInputRows, int paddedbottomwidth, int paddedbottomheight, int nInputPlane, int pad_size, int corr_type_multiply, cudaStream_t stream); 13 | 14 | #ifdef __cplusplus 15 | } 16 | #endif 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/dist/correlation_package-0.1-py3.6-linux-x86_64.egg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/dist/correlation_package-0.1-py3.6-linux-x86_64.egg -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | 5 | from setuptools import setup, find_packages 6 | 7 | this_file = os.path.dirname(__file__) 8 | 9 | setup( 10 | name="correlation_package", 11 | version="0.1", 12 | description="Correlation layer from FlowNetC", 13 | url="https://github.com/jbarker-nvidia/pytorch-correlation", 14 | author="Jon Barker", 15 | author_email="jbarker@nvidia.com", 16 | # Require cffi 17 | install_requires=["cffi>=1.0.0"], 18 | setup_requires=["cffi>=1.0.0"], 19 | # Exclude the build files. 20 | packages=find_packages(exclude=["build"]), 21 | # Package where to put the extensions. Has to be a prefix of build.py 22 | ext_package="", 23 | # Extensions to compile 24 | cffi_modules=[ 25 | os.path.join(this_file, "build.py:ffi") 26 | ], 27 | ) 28 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/correlation-pytorch/test/test.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import torch\n", 12 | "from torch.autograd import Variable\n", 13 | "from correlation_package.modules.corr import Correlation" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 2, 19 | "metadata": { 20 | "collapsed": true 21 | }, 22 | "outputs": [], 23 | "source": [ 24 | "a=Variable(torch.rand(1,128,100,100),requires_grad=True)\n", 25 | "b=Variable(torch.rand(1,128,100,100),requires_grad=True)\n", 26 | "m=Correlation(40,1,40,1,1,1)" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": null, 32 | "metadata": { 33 | "collapsed": false 34 | }, 35 | "outputs": [], 36 | "source": [ 37 | "a=a.cuda()\n", 38 | "b=b.cuda()" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "metadata": { 45 | "collapsed": true 46 | }, 47 | "outputs": [], 48 | "source": [ 49 | "y=m(a,b)" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": null, 55 | "metadata": { 56 | "collapsed": false 57 | }, 58 | "outputs": [], 59 | "source": [ 60 | "y.size()" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": null, 66 | "metadata": { 67 | "collapsed": true 68 | }, 69 | "outputs": [], 70 | "source": [] 71 | } 72 | ], 73 | "metadata": { 74 | "kernelspec": { 75 | "display_name": "Python 2", 76 | "language": "python", 77 | "name": "python2" 78 | }, 79 | "language_info": { 80 | "codemirror_mode": { 81 | "name": "ipython", 82 | "version": 2 83 | }, 84 | "file_extension": ".py", 85 | "mimetype": "text/x-python", 86 | "name": "python", 87 | "nbconvert_exporter": "python", 88 | "pygments_lexer": "ipython2", 89 | "version": "2.7.12" 90 | } 91 | }, 92 | "nbformat": 4, 93 | "nbformat_minor": 0 94 | } 95 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/make_cuda.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | CUDA_PATH=/usr/local/cuda-8.0 4 | 5 | cd correlation-pytorch/correlation_package/src 6 | echo "Compiling correlation layer kernels by nvcc..." 7 | 8 | # TODO (JEB): Check which arches we need 9 | /usr/local/cuda-8.0/bin/nvcc -c -o corr_cuda_kernel.cu.o corr_cuda_kernel.cu -x cu -Xcompiler -fPIC -arch=sm_52 10 | /usr/local/cuda-8.0/bin/nvcc -c -o corr1d_cuda_kernel.cu.o corr1d_cuda_kernel.cu -x cu -Xcompiler -fPIC -arch=sm_52 11 | 12 | cd ../../ 13 | python setup.py build install 14 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/external_packages/correlation-pytorch-master/readme.MD: -------------------------------------------------------------------------------- 1 | This repository contains a custom pytorch package that adds a module and functional interface for the correlation layer described in "FlowNet: Learning Optical Flow with Convolutional Networks" (https://arxiv.org/abs/1504.06852) 2 | 3 | To install: 4 | 5 | 1. Run `pip install cffi` 6 | 7 | 2. Run `make_cuda.sh` 8 | 9 | 3. Run `python setup.py build install` 10 | 11 | 4. (optional) Run `python test/test.py` 12 | 13 | 14 | #### Acknowledgement 15 | - Thanks to Dr. Fitsum Reda for providing the wrapper to the correlation code 16 | 17 | If you find this implementation useful in your work, please acknowledge it appropriately and cite the paper: 18 | ``` 19 | @misc{flownet2-pytorch, 20 | author = {Fitsum Reda and Robert Pottorff and Jon Barker and Bryan Catanzaro}, 21 | title = {flownet2-pytorch: Pytorch implementation of FlowNet 2.0: Evolution of Optical Flow Estimation with Deep Networks}, 22 | year = {2017}, 23 | publisher = {GitHub}, 24 | journal = {GitHub repository}, 25 | howpublished = {\url{https://github.com/NVIDIA/flownet2-pytorch}} 26 | } 27 | ``` 28 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .PWCNet import * 2 | -------------------------------------------------------------------------------- /code/optical_flow_net-PWC-Net/script_pwc.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import cv2 3 | import torch 4 | import numpy as np 5 | from math import ceil 6 | from torch.autograd import Variable 7 | from scipy.ndimage import imread 8 | import models 9 | """ 10 | Contact: Deqing Sun (deqings@nvidia.com); Zhile Ren (jrenzhile@gmail.com) 11 | """ 12 | def writeFlowFile(filename,uv): 13 | """ 14 | According to the matlab code of Deqing Sun and c++ source code of Daniel Scharstein 15 | Contact: dqsun@cs.brown.edu 16 | Contact: schar@middlebury.edu 17 | """ 18 | TAG_STRING = np.array(202021.25, dtype=np.float32) 19 | if uv.shape[2] != 2: 20 | sys.exit("writeFlowFile: flow must have two bands!"); 21 | H = np.array(uv.shape[0], dtype=np.int32) 22 | W = np.array(uv.shape[1], dtype=np.int32) 23 | with open(filename, 'wb') as f: 24 | f.write(TAG_STRING.tobytes()) 25 | f.write(W.tobytes()) 26 | f.write(H.tobytes()) 27 | f.write(uv.tobytes()) 28 | 29 | 30 | im1_fn = 'data/frame_0010.png'; 31 | im2_fn = 'data/frame_0011.png'; 32 | flow_fn = './tmp/frame_0010.flo'; 33 | 34 | if len(sys.argv) > 1: 35 | im1_fn = sys.argv[1] 36 | if len(sys.argv) > 2: 37 | im2_fn = sys.argv[2] 38 | if len(sys.argv) > 3: 39 | flow_fn = sys.argv[3] 40 | 41 | pwc_model_fn = './pwc_net.pth.tar'; 42 | 43 | im_all = [imread(img) for img in [im1_fn, im2_fn]] 44 | im_all = [im[:, :, :3] for im in im_all] 45 | 46 | # rescale the image size to be multiples of 64 47 | divisor = 64. 48 | H = im_all[0].shape[0] 49 | W = im_all[0].shape[1] 50 | 51 | H_ = int(ceil(H/divisor) * divisor) 52 | W_ = int(ceil(W/divisor) * divisor) 53 | for i in range(len(im_all)): 54 | im_all[i] = cv2.resize(im_all[i], (W_, H_)) 55 | 56 | for _i, _inputs in enumerate(im_all): 57 | im_all[_i] = im_all[_i][:, :, ::-1] 58 | im_all[_i] = 1.0 * im_all[_i]/255.0 59 | 60 | im_all[_i] = np.transpose(im_all[_i], (2, 0, 1)) 61 | im_all[_i] = torch.from_numpy(im_all[_i]) 62 | im_all[_i] = im_all[_i].expand(1, im_all[_i].size()[0], im_all[_i].size()[1], im_all[_i].size()[2]) 63 | im_all[_i] = im_all[_i].float() 64 | 65 | im_all = torch.autograd.Variable(torch.cat(im_all,1).cuda(), volatile=True) 66 | 67 | net = models.pwc_dc_net(pwc_model_fn) 68 | net = net.cuda() 69 | net.eval() 70 | 71 | flo = net(im_all) 72 | flo = flo[0] * 20.0 73 | flo = flo.cpu().data.numpy() 74 | 75 | # scale the flow back to the input size 76 | flo = np.swapaxes(np.swapaxes(flo, 0, 1), 1, 2) # 77 | u_ = cv2.resize(flo[:,:,0],(W,H)) 78 | v_ = cv2.resize(flo[:,:,1],(W,H)) 79 | u_ *= W/ float(W_) 80 | v_ *= H/ float(H_) 81 | flo = np.dstack((u_,v_)) 82 | 83 | writeFlowFile(flow_fn, flo) 84 | -------------------------------------------------------------------------------- /code/proposal_net/README: -------------------------------------------------------------------------------- 1 | This implementation is based on the FasterRCNN example of tensorpack 2 | 3 | Installation instruction 4 | You need to clone tensorpack: 5 | git clone https://github.com/ppwwyyxx/tensorpack /home/${USER}/vision/tensorpack/ 6 | 7 | To make sure, your tensorpack is compatible, better checkout the commit on which we base this implementation: 8 | cd /home/${USER}/vision/tensorpack/ 9 | git checkout 6fdde15deac4f23eb31ffef9e8d405d9153a51cd 10 | 11 | install some dependencies (using conda install / pip install / ...): 12 | termcolor,tqdm,tabulate,opencv-python,msgpack_numpy 13 | maybe a few more, maybe scipy, pillow 14 | pycocotools (you can copy from /home/voigtlaender/vision/coco3, also need to add to PYTHONPATH) 15 | 16 | Add tensorpack and pycocotools to your PTHONPATH: 17 | export PYTHONPATH=$PYTHONPATH:/home/${USER}/vision/tensorpack/:/home/${USER}/vision/coco3/PythonAPI/ 18 | 19 | Create a folder on your local disk on which models and logs will be stored and link it here under the name train_log: 20 | mkdir -p /work/${USER}/data/tensorpack_models/ 21 | ln -s /work/${USER}/data/tensorpack_models/ train_log 22 | 23 | Now you should be good to go! 24 | 25 | -- 26 | 27 | The goal of our modifications is to produce class-agnostic proposals 28 | -------------------------------------------------------------------------------- /code/proposal_net/combine_general_and_specific.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import time 4 | import glob 5 | import json 6 | import os 7 | 8 | root_dir = "./output/intermediate/" 9 | one_dir = 'general_proposals/' 10 | two_dir = 'specific_proposals/' 11 | out_dir = 'combined_proposals/' 12 | 13 | t = time.time() 14 | files = sorted(glob.glob(root_dir + one_dir + '*/*.json')) 15 | translated_files = [file.replace(one_dir,two_dir) for file in files] 16 | second_files = sorted(glob.glob(root_dir + two_dir + '*/*.json')) 17 | to_add_files = [file for file in second_files if file not in translated_files] 18 | if to_add_files: 19 | print(to_add_files) 20 | files += to_add_files 21 | print(time.time()-t) 22 | for file in files: 23 | two_file = file.replace(one_dir,two_dir) 24 | out_file = file.replace(one_dir,out_dir) 25 | try: 26 | with open(file, 'r') as f: 27 | proposals1 = json.load(f) 28 | except: 29 | proposals1 = [] 30 | try: 31 | with open(two_file, 'r') as f: 32 | proposals2 = json.load(f) 33 | except: 34 | proposals2 = [] 35 | fin_proposals = proposals1+proposals2 36 | outfolder = '/'.join(out_file.split('/')[:-1]) 37 | if not os.path.exists(outfolder): 38 | os.makedirs(outfolder) 39 | with open(out_file, "w") as f: 40 | json.dump(fin_proposals,f) 41 | 42 | print(time.time()-t) -------------------------------------------------------------------------------- /code/proposal_net/forward_proto.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import cv2 4 | 5 | from hypotheses_pb2 import HypothesisSet 6 | import config 7 | from eval import detect_one_image 8 | 9 | 10 | def forward_protobuf(pred_func, output_folder, forward_dataset, generic_images_folder, generic_images_pattern): 11 | tf.gfile.MakeDirs(output_folder) 12 | hypo_set = HypothesisSet() 13 | with open("/work/merkelbach/datasets/dashcam_mining/tracking_results_raw/vid_01/eval/12-Mar-2018--19-25-00vayV/hypos_protobuf/hypotheses_500frames_1860hyp___work__pv182253__data__dashcam__dashcam_videos_frames__vid_01__frames_cropped__clip_005.hypset", "rb") as f: 14 | hypo_set.ParseFromString(f.read()) 15 | for hyp in hypo_set.hypotheses: 16 | boxes = hyp.bounding_boxes_2D_with_timestamps 17 | for t, box in boxes.items(): 18 | img_filename = "/work/merkelbach/datasets/dashcam_mining/videos/vid_01/frames_cropped/all_frames/video_0001_frames" + "%09d" % t + ".png" 19 | print(img_filename, box) 20 | img_val = cv2.imread(img_filename, cv2.IMREAD_COLOR) 21 | assert config.PROVIDE_BOXES_AS_INPUT 22 | input_boxes = np.array([[box.x0, box.y0, box.x0 + box.w, box.y0 + box.h]], dtype=np.float32) 23 | results = detect_one_image(img_val, pred_func, input_boxes) 24 | print(results) 25 | -------------------------------------------------------------------------------- /code/proposal_net/utils/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Some third-party helper functions 3 | 4 | + generate_anchors.py: copied from [py-faster-rcnn](https://github.com/rbgirshick/py-faster-rcnn/blob/master/lib/rpn/generate_anchors.py). 5 | + box_ops.py: modified from [TF object detection API](https://github.com/tensorflow/models/blob/master/research/object_detection/core/box_list_ops.py). 6 | 7 | -------------------------------------------------------------------------------- /code/proposal_net/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/proposal_net/utils/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/README: -------------------------------------------------------------------------------- 1 | SAVITAR 2 codebase: 2 | 3 | Code base contributed to by: 4 | Paul Voigtlaender, Jonathon Luiten, Sabarinath Mahadevan, Michael Krause. 5 | 6 | MIT license. 7 | 8 | 9 | -------------------------------------------------------------------------------- /code/refinement_net/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/configs/live: -------------------------------------------------------------------------------- 1 | { 2 | "model": "live_model", 3 | "write_summaries": false, 4 | 5 | "model_dir": "/home/luiten/vision/PReMVOS/refinement_finetuning/", 6 | "load": "../weights/PReMVOS_weights/refinement_net/specific_weights/refinement_specific_weights", 7 | 8 | "need_train": false, 9 | "log_dir":"/home/luiten/vision/youtubevos/refinement_net/logs", 10 | "task": "few_shot_segmentation", 11 | "dataset": "jono_vos_feed", 12 | "gpus": 1, 13 | "batch_size": 1, 14 | "batch_size_eval": 1, 15 | "log_verbosity": 5, 16 | "optimizer": "adam", 17 | "min_size": 100, 18 | "learning_rates": "{1: 0.00001}", 19 | "freeze_batchnorm": true, 20 | "save": true, 21 | "num_epochs": 5, 22 | 23 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 24 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 25 | "bbox_jitter_factor": 0.05, 26 | "input_size_train": [385, 385], 27 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 28 | "use_bbox_guidance": true, 29 | 30 | "network": { 31 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 32 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 33 | "from": ["deeplab"]} 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/mapillary_quarter_jitter005_deeplab: -------------------------------------------------------------------------------- 1 | { 2 | "model": "test-challenge/bike-trial", 3 | 4 | "model_dir": "/home/luiten/vision/PReMVOS/refinement_finetuning/", 5 | 6 | "task": "few_shot_segmentation", 7 | "dataset": "jono_davis_fewshot", 8 | "gpus": 2, 9 | "batch_size": 1, 10 | "batch_size_eval": 1, 11 | "log_verbosity": 5, 12 | "optimizer": "adam", 13 | "min_size": 100, 14 | "learning_rates": "{1: 0.00001}", 15 | "freeze_batchnorm": true, 16 | "save": true, 17 | "num_epochs": 5, 18 | 19 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 20 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 21 | "bbox_jitter_factor": 0.05, 22 | "input_size_train": [385, 385], 23 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 24 | "use_bbox_guidance": true, 25 | 26 | "network": { 27 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 28 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 29 | "from": ["deeplab"]} 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/mapillary_quarter_jitter005_deeplab_FINETUNE: -------------------------------------------------------------------------------- 1 | { 2 | "model": "mapillary_quarter_jitter005_deeplab", 3 | "model_dir": "/home/luiten/vision/savitar2/models", 4 | "task": "few_shot_segmentation", 5 | "dataset": "jono_davis_fewshot", 6 | "n_finetune_steps": 12500, 7 | "gpus": 1, 8 | "batch_size": 6, 9 | "batch_size_eval": 1, 10 | "log_verbosity": 5, 11 | "optimizer": "adam", 12 | "min_size": 100, 13 | "learning_rates": "{1: 0.00001, 10: 0.000001, 15: 0.0000003}", 14 | "freeze_batchnorm": true, 15 | "save": true, 16 | "num_epochs": 20, 17 | "load_init": ["/home/voigtlaender/pretrained_models/deeplabv3plus/model.ckpt","DeepLabRGB:/home/voigtlaender/pretrained_models/deeplabv3plus/model.ckpt"], 18 | 19 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 20 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 21 | "bbox_jitter_factor": 0.05, 22 | "input_size_train": [385, 385], 23 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 24 | "use_bbox_guidance": true, 25 | 26 | "epoch_length_train": 100000, 27 | "validation_set_size": 10000, 28 | #"use_summaries": true, 29 | "use_summaries": false, 30 | 31 | 32 | "network": { 33 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 34 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 35 | "from": ["deeplab"]} 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/mapillary_quarter_jitter005_deeplab_TRAIN: -------------------------------------------------------------------------------- 1 | { 2 | "model": "lucid1", 3 | "model_dir": 4 | "task": "train_no_val", 5 | "dataset": "davis_lucid", 6 | "DAVIS_sequence": "test-challenge/bike-trial", 7 | "gpus": 1, 8 | "batch_size": 8, 9 | "batch_size_eval": 1, 10 | "log_verbosity": 5, 11 | "optimizer": "adam", 12 | "min_size": 100, 13 | "learning_rates": "{1: 0.00001, 10: 0.000001, 15: 0.0000003}", 14 | "freeze_batchnorm": true, 15 | "save": true, 16 | "num_epochs": 5, 17 | "load": "models/mapillary_quarter_jitter005_deeplab/mapillary_quarter_jitter005_deeplab-00000020", 18 | "load_init": ["/home/voigtlaender/pretrained_models/deeplabv3plus/model.ckpt","DeepLabRGB:/home/voigtlaender/pretrained_models/deeplabv3plus/model.ckpt"], 19 | 20 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 21 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 22 | "bbox_jitter_factor": 0.05, 23 | "input_size_train": [385, 385], 24 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 25 | "use_bbox_guidance": true, 26 | 27 | #"epoch_length_train": 100000, 28 | #"epoch_length_train": 2207, 29 | #"validation_set_size": 10000, 30 | #"use_summaries": true, 31 | "use_summaries": false, 32 | 33 | 34 | "network": { 35 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 36 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 37 | "from": ["deeplab"]} 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/forward: -------------------------------------------------------------------------------- 1 | { 2 | "image_input_dir":"/home/luiten/vision/youtubevos/%s", 3 | "bb_input_dir":"/home/luiten/vision/youtubevos/%s", 4 | "output_dir":"/home/luiten/vision/youtubevos/%s", 5 | "load": "/home/luiten/vision/youtubevos/refinement_net/models/%s", 6 | 7 | "model": "forward", 8 | "write_summaries": false, 9 | "need_train": false, 10 | "log_dir":"/home/luiten/vision/youtubevos/refinement_net/logs", 11 | "task": "few_shot_segmentation", 12 | "dataset": "jono_davis_fewshot", 13 | "gpus": 1, 14 | "batch_size": 1, 15 | "batch_size_eval": 1, 16 | "log_verbosity": 5, 17 | "optimizer": "adam", 18 | "min_size": 100, 19 | "learning_rates": "{1: 0.00001}", 20 | "freeze_batchnorm": true, 21 | "save": true, 22 | "num_epochs": 5, 23 | 24 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 25 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 26 | "bbox_jitter_factor": 0.05, 27 | "input_size_train": [385, 385], 28 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 29 | "use_bbox_guidance": true, 30 | 31 | "network": { 32 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 33 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 34 | "from": ["deeplab"]} 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/forward_temp: -------------------------------------------------------------------------------- 1 | { 2 | "image_input_dir":"/home/luiten/vision/youtubevos/DAVIS_new/val-tog/images/", 3 | "bb_input_dir":"/home/luiten/vision/youtubevos/DAVIS_new/val-tog/coarse-props/", 4 | "output_dir":"/home/luiten/vision/youtubevos/DAVIS_new/val-tog/refined-props/", 5 | "load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/together_davis/together_davis-00000050", 6 | 7 | "model": "forward", 8 | "write_summaries": false, 9 | "need_train": false, 10 | "log_dir":"/home/luiten/vision/youtubevos/refinement_net/logs", 11 | "task": "few_shot_segmentation", 12 | "dataset": "jono_davis_fewshot", 13 | "gpus": 1, 14 | "batch_size": 1, 15 | "batch_size_eval": 1, 16 | "log_verbosity": 5, 17 | "optimizer": "adam", 18 | "min_size": 100, 19 | "learning_rates": "{1: 0.00001}", 20 | "freeze_batchnorm": true, 21 | "save": true, 22 | "num_epochs": 5, 23 | 24 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 25 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 26 | "bbox_jitter_factor": 0.05, 27 | "input_size_train": [385, 385], 28 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 29 | "use_bbox_guidance": true, 30 | 31 | "network": { 32 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 33 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 34 | "from": ["deeplab"]} 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/train: -------------------------------------------------------------------------------- 1 | { 2 | "model": "coco_trained", 3 | 4 | "model_dir": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/", 5 | "load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/init_weights", 6 | 7 | "task": "train_no_val", 8 | "dataset": "davis_lucid", 9 | "gpus": 1, 10 | "batch_size": 8, 11 | "batch_size_eval": 1, 12 | "log_verbosity": 5, 13 | "optimizer": "adam", 14 | "min_size": 100, 15 | "learning_rates": "{1: 0.00001}", 16 | "freeze_batchnorm": true, 17 | "save": true, 18 | "num_epochs": 5, 19 | 20 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 21 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 22 | "bbox_jitter_factor": 0.05, 23 | "input_size_train": [385, 385], 24 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 25 | "use_bbox_guidance": true, 26 | 27 | "network": { 28 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 29 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 30 | "from": ["deeplab"]} 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/train_coco: -------------------------------------------------------------------------------- 1 | { 2 | "model": "coco_trained", 3 | 4 | "model_dir": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/", 5 | #"load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/init_weights", 6 | 7 | "task": "train_no_val", 8 | "dataset": "COCO_for_davis", 9 | 10 | "gpus": 4, 11 | "batch_size": 40, 12 | 13 | #"gpus": 2, 14 | #"batch_size": 16, 15 | 16 | #"gpus": 1, 17 | #"batch_size": 8, 18 | 19 | # One real epoch is 544738, we do 50 * 1/10 epochs, approx 19 hours per real epoch for 1 gpu. 20 | "num_epochs": 500, 21 | "epoch_length_train": 54474, 22 | #"epoch_length_train": 1, 23 | 24 | "log_dir": "refinement_net/logs/", 25 | 26 | "batch_size_eval": 1, 27 | "log_verbosity": 5, 28 | "optimizer": "adam", 29 | "min_size": 100, 30 | "learning_rates": "{1: 0.00001}", 31 | "freeze_batchnorm": true, 32 | "save": true, 33 | 34 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 35 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 36 | "bbox_jitter_factor": 0.05, 37 | "input_size_train": [385, 385], 38 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 39 | "use_bbox_guidance": true, 40 | 41 | "network": { 42 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 43 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 44 | "from": ["deeplab"]} 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/train_davis_with_old_ldd: -------------------------------------------------------------------------------- 1 | { 2 | "model": "davis_with_old_ldd", 3 | 4 | "model_dir": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/", 5 | "load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/coco_trained-93", 6 | 7 | "task": "train_no_val", 8 | "dataset": "davis_lucid", 9 | 10 | "gpus": 4, 11 | "batch_size": 40, 12 | 13 | #"gpus": 1, 14 | #"batch_size": 8, 15 | 16 | # One real epoch is 225000, we do 25 * 1/5 epochs. 17 | #"num_epochs": 25, 18 | "num_epochs": 500, 19 | "epoch_length_train": 45000, 20 | 21 | "log_dir": "refinement_net/logs/", 22 | 23 | "batch_size_eval": 1, 24 | "log_verbosity": 5, 25 | "optimizer": "adam", 26 | "min_size": 100, 27 | "learning_rates": "{1: 0.00001}", 28 | "freeze_batchnorm": true, 29 | "save": true, 30 | 31 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 32 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 33 | "bbox_jitter_factor": 0.05, 34 | "input_size_train": [385, 385], 35 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 36 | "use_bbox_guidance": true, 37 | 38 | "network": { 39 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 40 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 41 | "from": ["deeplab"]} 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/train_together: -------------------------------------------------------------------------------- 1 | { 2 | "model": "together", 3 | 4 | "model_dir": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/", 5 | "load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/ytvos/ytvos-00000100", 6 | 7 | "task": "train_no_val", 8 | "dataset": "davis_lucid", 9 | 10 | #"gpus": 4, 11 | #"batch_size": 40, 12 | 13 | "gpus": 8, 14 | "batch_size": 64, 15 | 16 | #"gpus": 1, 17 | #"batch_size": 8, 18 | 19 | # One real epoch is 414,752, we do 1000 * 1/10 epochs. 20 | # In reality we want to 5 real epochs = 50 here epochs = approx 11 hours 21 | #"num_epochs": 100, 22 | "num_epochs": 1000, 23 | "epoch_length_train": 41475, 24 | 25 | # Idea behind lr schedule (decrease every real epoch to 3 levels accuracy) 26 | "learning_rates": "{1: 0.00001,11: 0.000005,21: 0.000001,31: 0.0000005,41: 0.0000001}", 27 | #"learning_rates": "{1: 0.00001}", 28 | #"learning_rates": "{1: 0.000001}", 29 | #"learning_rates": "{1: 0.0000001}", 30 | 31 | 32 | "log_dir": "refinement_net/logs/", 33 | 34 | "batch_size_eval": 1, 35 | "log_verbosity": 5, 36 | "optimizer": "adam", 37 | "min_size": 100, 38 | "freeze_batchnorm": true, 39 | "save": true, 40 | 41 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 42 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 43 | "bbox_jitter_factor": 0.05, 44 | "input_size_train": [385, 385], 45 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 46 | "use_bbox_guidance": true, 47 | 48 | "network": { 49 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 50 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 51 | "from": ["deeplab"]} 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/train_together_davis: -------------------------------------------------------------------------------- 1 | { 2 | #sbatch -c 24 --mem=32G --time=7-00:00:00 --gres=gpu:1080ti:4 --partition=lopri run_now.sh 3 | #python /home/luiten/vision/youtubevos/refinement_net/main.py /home/luiten/vision/youtubevos/refinement_net/configs/train_together_davis 4 | 5 | 6 | "model": "together_davis", 7 | 8 | "model_dir": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/", 9 | "load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/ytvos/ytvos-00000100", 10 | 11 | "task": "train_no_val", 12 | "dataset": "davis_lucid", 13 | 14 | "gpus": 4, 15 | "batch_size": 32, 16 | 17 | #"gpus": 8, 18 | #"batch_size": 64, 19 | 20 | #"gpus": 1, 21 | #"batch_size": 8, 22 | 23 | # One real epoch is 918,112, we do 50 * 1/10 epochs. 24 | # In reality we want to 5 real epochs = 50 here epochs = approx 11 hours 25 | "num_epochs": 50, 26 | "epoch_length_train": 91812, 27 | 28 | # LR seems to work best without decreasing 29 | "learning_rates": "{1: 0.00001}", 30 | 31 | # Idea behind lr schedule (decrease every real epoch to 3 levels accuracy) 32 | #"learning_rates": "{1: 0.00001,11: 0.000005,21: 0.000001,31: 0.0000005,41: 0.0000001}", 33 | #"learning_rates": "{1: 0.00001}", 34 | #"learning_rates": "{1: 0.000001}", 35 | #"learning_rates": "{1: 0.0000001}", 36 | 37 | 38 | "log_dir": "refinement_net/logs/", 39 | 40 | "batch_size_eval": 1, 41 | "log_verbosity": 5, 42 | "optimizer": "adam", 43 | "min_size": 100, 44 | "freeze_batchnorm": true, 45 | "save": true, 46 | 47 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 48 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 49 | "bbox_jitter_factor": 0.05, 50 | "input_size_train": [385, 385], 51 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 52 | "use_bbox_guidance": true, 53 | 54 | "network": { 55 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 56 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 57 | "from": ["deeplab"]} 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/train_ytvos: -------------------------------------------------------------------------------- 1 | { 2 | "model": "ytvos", 3 | 4 | "model_dir": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/", 5 | #"load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/coco_trained-93", 6 | 7 | "task": "train_no_val", 8 | "dataset": "davis_lucid", 9 | 10 | #"gpus": 4, 11 | #"batch_size": 40, 12 | 13 | "gpus": 8, 14 | "batch_size": 64, 15 | 16 | #"gpus": 1, 17 | #"batch_size": 8, 18 | 19 | # One real epoch is 147499, we do 100 * 1/5 epochs. 20 | #"num_epochs": 100, 21 | "num_epochs": 1000, 22 | "epoch_length_train": 29500, 23 | 24 | # Learning rate reduced after 34 epochs and 82 25 | #"learning_rates": "{1: 0.00001}", 26 | #"learning_rates": "{1: 0.000001}", 27 | "learning_rates": "{1: 0.0000001}", 28 | 29 | 30 | "log_dir": "refinement_net/logs/", 31 | 32 | "batch_size_eval": 1, 33 | "log_verbosity": 5, 34 | "optimizer": "adam", 35 | "min_size": 100, 36 | "freeze_batchnorm": true, 37 | "save": true, 38 | 39 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 40 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 41 | "bbox_jitter_factor": 0.05, 42 | "input_size_train": [385, 385], 43 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 44 | "use_bbox_guidance": true, 45 | 46 | "network": { 47 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 48 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 49 | "from": ["deeplab"]} 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/train_ytvos_orignal: -------------------------------------------------------------------------------- 1 | { 2 | "model": "ytvos", 3 | 4 | "model_dir": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/", 5 | #"load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/coco_trained-93", 6 | 7 | "task": "train_no_val", 8 | "dataset": "davis_lucid", 9 | 10 | "gpus": 4, 11 | "batch_size": 40, 12 | 13 | #"gpus": 1, 14 | #"batch_size": 8, 15 | 16 | # One real epoch is 147499, we do 100 * 1/5 epochs. 17 | "num_epochs": 100, 18 | "epoch_length_train": 29500, 19 | 20 | # Learning rate reduced after 34 epochs and 82 21 | #"learning_rates": "{1: 0.00001}", 22 | #"learning_rates": "{1: 0.000001}", 23 | "learning_rates": "{1: 0.0000001}", 24 | 25 | 26 | "log_dir": "refinement_net/logs/", 27 | 28 | "batch_size_eval": 1, 29 | "log_verbosity": 5, 30 | "optimizer": "adam", 31 | "min_size": 100, 32 | "freeze_batchnorm": true, 33 | "save": true, 34 | 35 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 36 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 37 | "bbox_jitter_factor": 0.05, 38 | "input_size_train": [385, 385], 39 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 40 | "use_bbox_guidance": true, 41 | 42 | "network": { 43 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 44 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 45 | "from": ["deeplab"]} 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/ytvos1_davis: -------------------------------------------------------------------------------- 1 | { 2 | "image_input_dir":"/home/luiten/vision/youtubevos/DAVIS/val17/", 3 | 4 | 5 | 6 | 7 | "bb_input_dir":"/home/luiten/vision/youtubevos/DAVIS/coarse-props/", 8 | "output_dir":"/home/luiten/vision/youtubevos/DAVIS/refined_props/", 9 | 10 | "model": "davis_model", 11 | "write_summaries": false, 12 | 13 | #"model_dir": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/", 14 | #"load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/init_weights", 15 | "model_dir": "/home/luiten/vision/youtubevos/refinement_net/models/coco_trained/", 16 | "load": "/home/luiten/vision/youtubevos/refinement_net/models/coco_trained/coco_trained-25", 17 | 18 | "need_train": false, 19 | "log_dir":"/home/luiten/vision/youtubevos/refinement_net/logs", 20 | "task": "few_shot_segmentation", 21 | "dataset": "jono_davis_fewshot", 22 | "gpus": 1, 23 | "batch_size": 1, 24 | "batch_size_eval": 1, 25 | "log_verbosity": 5, 26 | "optimizer": "adam", 27 | "min_size": 100, 28 | "learning_rates": "{1: 0.00001}", 29 | "freeze_batchnorm": true, 30 | "save": true, 31 | "num_epochs": 5, 32 | 33 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 34 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 35 | "bbox_jitter_factor": 0.05, 36 | "input_size_train": [385, 385], 37 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 38 | "use_bbox_guidance": true, 39 | 40 | "network": { 41 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 42 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 43 | "from": ["deeplab"]} 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/ytvos1_davis-with-ldd-weights: -------------------------------------------------------------------------------- 1 | { 2 | "image_input_dir":"/home/luiten/vision/youtubevos/DAVIS/val17/", 3 | #"bb_input_dir":"/home/luiten/vision/youtubevos/DAVIS/coarse-props-with-tracks/", 4 | "bb_input_dir":"/home/luiten/vision/youtubevos/DAVIS/coarse-props/", 5 | "output_dir":"/home/luiten/vision/youtubevos/DAVIS/refined_props-davisldd-weights/", 6 | "model": "davis_model-lddweights", 7 | "write_summaries": false, 8 | 9 | #"model_dir": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/", 10 | #"load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/init_weights", 11 | "model_dir": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/davis_with_old_ldd/", 12 | "load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/davis_with_old_ldd/davis_with_old_ldd-00000025", 13 | 14 | "need_train": false, 15 | "log_dir":"/home/luiten/vision/youtubevos/refinement_net/logs", 16 | "task": "few_shot_segmentation", 17 | "dataset": "jono_davis_fewshot", 18 | "gpus": 1, 19 | "batch_size": 1, 20 | "batch_size_eval": 1, 21 | "log_verbosity": 5, 22 | "optimizer": "adam", 23 | "min_size": 100, 24 | "learning_rates": "{1: 0.00001}", 25 | "freeze_batchnorm": true, 26 | "save": true, 27 | "num_epochs": 5, 28 | 29 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 30 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 31 | "bbox_jitter_factor": 0.05, 32 | "input_size_train": [385, 385], 33 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 34 | "use_bbox_guidance": true, 35 | 36 | "network": { 37 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 38 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 39 | "from": ["deeplab"]} 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/ytvos1_davis-with-ytvos-weights: -------------------------------------------------------------------------------- 1 | { 2 | "image_input_dir":"/home/luiten/vision/youtubevos/DAVIS/val17/", 3 | 4 | #"bb_input_dir":"/home/luiten/vision/youtubevos/DAVIS/coarse-props-with-tracks/", 5 | #"output_dir":"/home/luiten/vision/youtubevos/DAVIS/refined_props-ytvos-weights/", 6 | 7 | "bb_input_dir":"/home/luiten/vision/youtubevos/DAVIS/coarse-props/", 8 | "output_dir":"/home/luiten/vision/youtubevos/DAVIS/refined_props-ytvos-weights-no-tracks/", 9 | 10 | "model": "davis_model-ytvosweights", 11 | "write_summaries": false, 12 | 13 | #"model_dir": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/", 14 | #"load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/init_weights", 15 | "model_dir": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/ytvos/", 16 | "load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/ytvos/ytvos-00000100", 17 | 18 | "need_train": false, 19 | "log_dir":"/home/luiten/vision/youtubevos/refinement_net/logs", 20 | "task": "few_shot_segmentation", 21 | "dataset": "jono_davis_fewshot", 22 | "gpus": 1, 23 | "batch_size": 1, 24 | "batch_size_eval": 1, 25 | "log_verbosity": 5, 26 | "optimizer": "adam", 27 | "min_size": 100, 28 | "learning_rates": "{1: 0.00001}", 29 | "freeze_batchnorm": true, 30 | "save": true, 31 | "num_epochs": 5, 32 | 33 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 34 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 35 | "bbox_jitter_factor": 0.05, 36 | "input_size_train": [385, 385], 37 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 38 | "use_bbox_guidance": true, 39 | 40 | "network": { 41 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 42 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 43 | "from": ["deeplab"]} 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/ytvos1_davis_test: -------------------------------------------------------------------------------- 1 | { 2 | "image_input_dir":"/home/luiten/vision/youtubevos/DAVIS/test/images/", 3 | "bb_input_dir":"/home/luiten/vision/youtubevos/DAVIS/test/props/", 4 | "output_dir":"/home/luiten/vision/youtubevos/DAVIS/test/output/", 5 | "model": "bmx-trees", 6 | "write_summaries": false, 7 | 8 | "model_dir": "/home/luiten/vision/PReMVOS/refinement_finetuning/", 9 | "load": "/home/luiten/vision/PReMVOS/refinement_finetuning/init_weights", 10 | 11 | "need_train": false, 12 | "log_dir":"/home/luiten/vision/youtubevos/refinement_net/logs", 13 | "task": "few_shot_segmentation", 14 | "dataset": "jono_davis_fewshot", 15 | "gpus": 1, 16 | "batch_size": 1, 17 | "batch_size_eval": 1, 18 | "log_verbosity": 5, 19 | "optimizer": "adam", 20 | "min_size": 100, 21 | "learning_rates": "{1: 0.00001}", 22 | "freeze_batchnorm": true, 23 | "save": true, 24 | "num_epochs": 5, 25 | 26 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 27 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 28 | "bbox_jitter_factor": 0.05, 29 | "input_size_train": [385, 385], 30 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 31 | "use_bbox_guidance": true, 32 | 33 | "network": { 34 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 35 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 36 | "from": ["deeplab"]} 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/ytvos1_final: -------------------------------------------------------------------------------- 1 | { 2 | #"image_input_dir":"/home/luiten/vision/youtubevos/DAVIS/val17/", 3 | #"bb_input_dir":"/home/luiten/vision/youtubevos/DAVIS/coarse-props-final/", 4 | #"output_dir":"/home/luiten/vision/youtubevos/DAVIS/refined_props-final/", 5 | 6 | #"image_input_dir":"/home/luiten/vision/youtubevos/ytvos_data/valid/JPEGImages/", 7 | #"bb_input_dir":"/home/luiten/vision/youtubevos/ytvos_data/valid/coarse-props-final/", 8 | #"output_dir":"/home/luiten/vision/youtubevos/ytvos_data/valid/refined_props-final/", 9 | 10 | "image_input_dir":"/home/luiten/vision/youtubevos/ytvos_data/test_all_frames/JPEGImages/", 11 | "bb_input_dir":"/home/luiten/vision/youtubevos/ytvos_data/test_all_frames/coarse-props-final/", 12 | "output_dir":"/home/luiten/vision/youtubevos/ytvos_data/test_all_frames/refined_props-final/", 13 | 14 | "model": "final", 15 | "write_summaries": false, 16 | 17 | "model_dir": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/", 18 | "load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/final-66", 19 | 20 | "need_train": false, 21 | "log_dir":"/home/luiten/vision/youtubevos/refinement_net/logs", 22 | "task": "few_shot_segmentation", 23 | "dataset": "jono_davis_fewshot", 24 | "gpus": 1, 25 | "batch_size": 1, 26 | "batch_size_eval": 1, 27 | "log_verbosity": 5, 28 | "optimizer": "adam", 29 | "min_size": 100, 30 | "learning_rates": "{1: 0.00001}", 31 | "freeze_batchnorm": true, 32 | "save": true, 33 | "num_epochs": 5, 34 | 35 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 36 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 37 | "bbox_jitter_factor": 0.05, 38 | "input_size_train": [385, 385], 39 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 40 | "use_bbox_guidance": true, 41 | 42 | "network": { 43 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 44 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 45 | "from": ["deeplab"]} 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/ytvos1_live: -------------------------------------------------------------------------------- 1 | { 2 | "model": "live_model", 3 | "write_summaries": false, 4 | 5 | "model_dir": "/home/luiten/vision/PReMVOS/refinement_finetuning/", 6 | #"load": "/home/luiten/vision/PReMVOS/refinement_finetuning/init_weights", 7 | #"load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/ytvos/ytvos-00000100", 8 | "load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/final-66", 9 | 10 | "need_train": false, 11 | "log_dir":"/home/luiten/vision/youtubevos/refinement_net/logs", 12 | "task": "few_shot_segmentation", 13 | "dataset": "jono_vos_feed", 14 | "gpus": 1, 15 | "batch_size": 1, 16 | "batch_size_eval": 1, 17 | "log_verbosity": 5, 18 | "optimizer": "adam", 19 | "min_size": 100, 20 | "learning_rates": "{1: 0.00001}", 21 | "freeze_batchnorm": true, 22 | "save": true, 23 | "num_epochs": 5, 24 | 25 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 26 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 27 | "bbox_jitter_factor": 0.05, 28 | "input_size_train": [385, 385], 29 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 30 | "use_bbox_guidance": true, 31 | 32 | "network": { 33 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 34 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 35 | "from": ["deeplab"]} 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/ytvos1_test: -------------------------------------------------------------------------------- 1 | { 2 | "image_input_dir":"/home/luiten/vision/youtubevos/ytvos_data/test_all_frames/JPEGImages/", 3 | "bb_input_dir":"/home/luiten/vision/youtubevos/ytvos_data/test_all_frames/coarse-props/", 4 | "output_dir":"/home/luiten/vision/youtubevos/ytvos_data/test_all_frames/refined_props/", 5 | 6 | "model": "davis_model-ytvosweights", 7 | "write_summaries": false, 8 | 9 | "model_dir": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/ytvos/", 10 | "load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/ytvos/ytvos-00000100", 11 | 12 | "need_train": false, 13 | "log_dir":"/home/luiten/vision/youtubevos/refinement_net/logs", 14 | "task": "few_shot_segmentation", 15 | "dataset": "jono_davis_fewshot", 16 | "gpus": 1, 17 | "batch_size": 1, 18 | "batch_size_eval": 1, 19 | "log_verbosity": 5, 20 | "optimizer": "adam", 21 | "min_size": 100, 22 | "learning_rates": "{1: 0.00001}", 23 | "freeze_batchnorm": true, 24 | "save": true, 25 | "num_epochs": 5, 26 | 27 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 28 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 29 | "bbox_jitter_factor": 0.05, 30 | "input_size_train": [385, 385], 31 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 32 | "use_bbox_guidance": true, 33 | 34 | "network": { 35 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 36 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 37 | "from": ["deeplab"]} 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/old_reference_configs/ytvos1_val: -------------------------------------------------------------------------------- 1 | { 2 | "image_input_dir":"/home/luiten/vision/youtubevos/ytvos_data/valid/JPEGImages/", 3 | 4 | "bb_input_dir":"/home/luiten/vision/youtubevos/ytvos_data/valid/coarse-props/", 5 | "output_dir":"/home/luiten/vision/youtubevos/ytvos_data/valid/refined_props/", 6 | 7 | 8 | "model": "davis_model-ytvosweights", 9 | "write_summaries": false, 10 | 11 | "model_dir": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/ytvos/", 12 | "load": "/home/luiten/vision/youtubevos/refinement_net/models/initial_weights/ytvos/ytvos-00000100", 13 | 14 | "need_train": false, 15 | "log_dir":"/home/luiten/vision/youtubevos/refinement_net/logs", 16 | "task": "few_shot_segmentation", 17 | "dataset": "jono_davis_fewshot", 18 | "gpus": 1, 19 | "batch_size": 1, 20 | "batch_size_eval": 1, 21 | "log_verbosity": 5, 22 | "optimizer": "adam", 23 | "min_size": 100, 24 | "learning_rates": "{1: 0.00001}", 25 | "freeze_batchnorm": true, 26 | "save": true, 27 | "num_epochs": 5, 28 | 29 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 30 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 31 | "bbox_jitter_factor": 0.05, 32 | "input_size_train": [385, 385], 33 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 34 | "use_bbox_guidance": true, 35 | 36 | "network": { 37 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 38 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 39 | "from": ["deeplab"]} 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/refinement_network_FORWARD_ITERATIVE_template: -------------------------------------------------------------------------------- 1 | { 2 | "model": "val/india", 3 | 4 | "model_dir": "/home/luiten/vision/PReMVOS/refinement_finetuning/", 5 | 6 | "task": "davis_iterative_few_shot_segmentation", 7 | "dataset": "jono_davis_fewshot", 8 | "gpus": 1, 9 | "batch_size": 1, 10 | "batch_size_eval": 1, 11 | "log_verbosity": 5, 12 | "optimizer": "adam", 13 | "min_size": 100, 14 | "learning_rates": "{1: 0.00001}", 15 | "freeze_batchnorm": true, 16 | "save": true, 17 | "num_epochs": 5, 18 | 19 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 20 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 21 | "bbox_jitter_factor": 0.05, 22 | "input_size_train": [385, 385], 23 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 24 | "use_bbox_guidance": true, 25 | 26 | "network": { 27 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 28 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 29 | "from": ["deeplab"]} 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/refinement_network_FORWARD_template: -------------------------------------------------------------------------------- 1 | { 2 | "model": "val/india", 3 | 4 | "model_dir": "/home/luiten/vision/PReMVOS/refinement_finetuning/", 5 | 6 | "task": "few_shot_segmentation", 7 | "dataset": "jono_davis_fewshot", 8 | "gpus": 1, 9 | "batch_size": 1, 10 | "batch_size_eval": 1, 11 | "log_verbosity": 5, 12 | "optimizer": "adam", 13 | "min_size": 100, 14 | "learning_rates": "{1: 0.00001}", 15 | "freeze_batchnorm": true, 16 | "save": true, 17 | "num_epochs": 5, 18 | 19 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 20 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 21 | "bbox_jitter_factor": 0.05, 22 | "input_size_train": [385, 385], 23 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 24 | "use_bbox_guidance": true, 25 | 26 | "network": { 27 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 28 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 29 | "from": ["deeplab"]} 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /code/refinement_net/configs/old_configs/refinement_network_TRAIN_template: -------------------------------------------------------------------------------- 1 | { 2 | "model": "test-challenge/bike-trial", 3 | 4 | "model_dir": "/home/luiten/vision/PReMVOS/refinement_finetuning/", 5 | "load": "/home/luiten/vision/PReMVOS/refinement_finetuning/init_weights", 6 | 7 | "task": "train_no_val", 8 | "dataset": "davis_lucid", 9 | "gpus": 1, 10 | "batch_size": 8, 11 | "batch_size_eval": 1, 12 | "log_verbosity": 5, 13 | "optimizer": "adam", 14 | "min_size": 100, 15 | "learning_rates": "{1: 0.00001}", 16 | "freeze_batchnorm": true, 17 | "save": true, 18 | "num_epochs": 5, 19 | 20 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 21 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 22 | "bbox_jitter_factor": 0.05, 23 | "input_size_train": [385, 385], 24 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 25 | "use_bbox_guidance": true, 26 | 27 | "network": { 28 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 29 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 30 | "from": ["deeplab"]} 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /code/refinement_net/configs/run: -------------------------------------------------------------------------------- 1 | { 2 | "image_input_dir":"../data/DAVIS/JPEGImages/480p/", 3 | "bb_input_dir":"../output/intermediate/combined_proposals/", 4 | "output_dir":"../output/intermediate/refined_proposals/", 5 | 6 | "model": "final", 7 | "write_summaries": false, 8 | "model_dir": "../weights/PReMVOS_weights/refinement_net/specific_weights", 9 | "load": "../weights/PReMVOS_weights/refinement_net/specific_weights/refinement_specific_weights", 10 | 11 | "need_train": false, 12 | "log_dir":"../output/logs/refinement_net", 13 | "task": "few_shot_segmentation", 14 | "dataset": "jono_davis_fewshot", 15 | "gpus": 1, 16 | "batch_size": 1, 17 | "batch_size_eval": 1, 18 | "log_verbosity": 5, 19 | "optimizer": "adam", 20 | "min_size": 100, 21 | "learning_rates": "{1: 0.00001}", 22 | "freeze_batchnorm": true, 23 | "save": true, 24 | "num_epochs": 5, 25 | 26 | "resize_mode_train": "bbox_crop_and_resize_fixed_size", 27 | "augmentors_train": ["gamma", "flip", "bbox_jitter"], 28 | "bbox_jitter_factor": 0.05, 29 | "input_size_train": [385, 385], 30 | "resize_mode_val": "bbox_crop_and_resize_fixed_size", 31 | "use_bbox_guidance": true, 32 | 33 | "network": { 34 | "deeplab": {"class": "DeepLabV3Plus", "n_features": 2}, 35 | "output": {"class": "SegmentationSoftmax", "loss": "bootstrapped_ce", "fraction": 0.25, "resize_logits": true, 36 | "from": ["deeplab"]} 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /code/refinement_net/core/Extractions.py: -------------------------------------------------------------------------------- 1 | 2 | EXTRACTIONS = "extractions" 3 | 4 | SEGMENTATION_POSTERIORS = "segmentation_posteriors" 5 | SEGMENTATION_POSTERIORS_ORIGINAL_SIZE = "segmentation_posteriors_original_size" 6 | SEGMENTATION_MASK_ORIGINAL_SIZE = "segmentation_mask_original_size" 7 | SEGMENTATION_MASK_INPUT_SIZE = "segmentation_mask_input_size" 8 | 9 | 10 | def accumulate_extractions(extractions_accumulator, *new_extractions): 11 | if len(new_extractions) == 0: 12 | return 13 | 14 | if len(extractions_accumulator) == 0: 15 | extractions_accumulator.update(new_extractions[0]) 16 | new_extractions = new_extractions[1:] 17 | 18 | # each extraction will actually be a list, so we can just sum up the lists (extend the accumulator list with the next) 19 | for k, v in extractions_accumulator.items(): 20 | for ext in new_extractions: 21 | extractions_accumulator[k] += ext[k] 22 | 23 | return extractions_accumulator 24 | -------------------------------------------------------------------------------- /code/refinement_net/core/Timer.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | 4 | class Timer: 5 | def __init__(self, message="", stream=None): 6 | if stream is None: 7 | from refinement_net.core.Log import log 8 | stream = log.v4 9 | self.stream = stream 10 | self.start_time = time.time() 11 | self.message = message 12 | 13 | def __enter__(self): 14 | self.start_time = time.time() 15 | 16 | def __exit__(self, exc_type, exc_val, exc_tb): 17 | if self.message is not None: 18 | # print(self.message, "elapsed", self.elapsed(), file=self.stream) 19 | print(self.message, "elapsed", self.elapsed()) 20 | 21 | def elapsed(self): 22 | end = time.time() 23 | start = self.start_time 24 | elapsed = end - start 25 | return elapsed 26 | -------------------------------------------------------------------------------- /code/refinement_net/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/core/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/datasets/COCO/COCO_instance.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | from refinement_net.core.Log import log 5 | from refinement_net.datasets.Loader import register_dataset 6 | from refinement_net.datasets import DataKeys 7 | from refinement_net.datasets.COCO.COCO import COCODataset 8 | 9 | NAME = "COCO_instance" 10 | 11 | 12 | @register_dataset(NAME) 13 | class COCOInstanceDataset(COCODataset): 14 | def __init__(self, config, subset, name=NAME): 15 | super(COCOInstanceDataset, self).__init__(config, subset, name=name, num_classes=2) 16 | 17 | def build_filename_to_anns_dict(self): 18 | for ann in self.anns: 19 | ann_id = ann['id'] 20 | img_id = ann['image_id'] 21 | img = self.coco.loadImgs(img_id) 22 | file_name = img[0]['file_name'] 23 | 24 | file_name = file_name + ":" + repr(img_id) + ":" + repr(ann_id) 25 | if file_name in self.filename_to_anns: 26 | print("Ignoring instance as an instance with the same id exists in filename_to_anns.", log.v1) 27 | else: 28 | self.filename_to_anns[file_name] = [ann] 29 | 30 | self.filter_anns() 31 | 32 | def filter_anns(self): 33 | self.filter_crowd_images = True 34 | super().filter_anns() 35 | 36 | def load_image(self, img_filename): 37 | path = tf.string_split([img_filename], ':').values[0] 38 | path = tf.string_split([path], '/').values[-1] 39 | img_dir = tf.cond(tf.equal(tf.string_split([path], '_').values[1], tf.constant("train2014")), 40 | lambda: '%s/%s/' % (self.data_dir, "train2014"), 41 | lambda: '%s/%s/' % (self.data_dir, "val2014")) 42 | path = img_dir + path 43 | return super().load_image(path) 44 | 45 | def load_annotation(self, img, img_filename, annotation_filename): 46 | label, raw_label = tf.py_func(self._get_mask, [img_filename], [tf.uint8, tf.uint8], name="load_mask") 47 | label.set_shape((None, None, 1)) 48 | raw_label.set_shape((None, None, 1)) 49 | return {DataKeys.SEGMENTATION_LABELS: label, DataKeys.RAW_SEGMENTATION_LABELS: raw_label, 50 | DataKeys.IMAGE_FILENAMES: img_filename} 51 | 52 | def _get_mask(self, img_filename): 53 | img_filename = img_filename.decode("UTF-8") 54 | label = super()._get_mask(img_filename) 55 | img_id = img_filename.split(":")[1] 56 | ann_ids = self.coco.getAnnIds([int(img_id)]) 57 | anns = self.coco.loadAnns(ann_ids) 58 | raw_label = self.coco.annToMask(anns[0]) 59 | 60 | for ann in anns[1:]: 61 | if not ann['iscrowd']: 62 | raw_label = np.logical_or(raw_label, self.coco.annToMask(ann)) 63 | 64 | raw_label = np.expand_dims(raw_label, axis=2) 65 | return label.astype(np.uint8), raw_label.astype(np.uint8) 66 | -------------------------------------------------------------------------------- /code/refinement_net/datasets/COCO/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/datasets/COCO/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/datasets/DAVIS/DAVIS_lucid.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from PIL import Image 3 | import numpy as np 4 | 5 | from refinement_net.datasets.Loader import register_dataset 6 | from refinement_net.datasets.Mapillary.MapillaryLike_instance import MapillaryLikeInstanceDataset 7 | from refinement_net.datasets.util.Util import username 8 | 9 | # DEFAULT_PATH = "/home/luiten/vision/PReMVOS/data/" 10 | # LIST_PATH_ROOT = "/home/luiten/vision/PReMVOS/refinement_finetuning/" 11 | # DEFAULT_PATH = "/home/luiten/vision/youtubevos/ytvos_data/paris/ldd/" 12 | # LIST_PATH_ROOT = "/home/luiten/vision/youtubevos/ytvos_data/paris/ref_weights/" 13 | # DEFAULT_PATH = "/home/luiten/vision/PReMVOS/home_data/" 14 | # LIST_PATH_ROOT = "/home/luiten/vision/youtubevos/DAVIS/" 15 | # DEFAULT_PATH = "/home/luiten/vision/youtubevos/ytvos_data/train/" 16 | # LIST_PATH_ROOT = "/home/luiten/vision/youtubevos/ytvos_data/train/" 17 | 18 | # DEFAULT_PATH = "/home/luiten/vision/youtubevos/ytvos_data/together/generated/" 19 | # LIST_PATH_ROOT = "/home/luiten/vision/youtubevos/ytvos_data/together/generated/" 20 | 21 | DEFAULT_PATH = "/home/luiten/vision/youtubevos/DAVIS/davis_together/" 22 | LIST_PATH_ROOT = "/home/luiten/vision/youtubevos/DAVIS/davis_together/" 23 | 24 | NAME = "davis_lucid" 25 | 26 | @register_dataset(NAME) 27 | class DAVISLucidDataset(MapillaryLikeInstanceDataset): 28 | def __init__(self, config, subset): 29 | davis_sequence = config.string("model", '') 30 | data_list_path = LIST_PATH_ROOT + davis_sequence + '/' 31 | super().__init__(config, subset, NAME, DEFAULT_PATH, data_list_path, 100, cat_ids_to_use=None) 32 | 33 | def load_annotation(self, img, img_filename, annotation_filename): 34 | annotation_filename_without_id = tf.string_split([annotation_filename], ':').values[0] 35 | #ann_data = tf.read_file(annotation_filename_without_id) 36 | #ann = tf.image.decode_png(ann_data, dtype=tf.uint16, channels=1) 37 | ann, = tf.py_func(load_ann_with_colormap, [annotation_filename_without_id], [tf.uint8]) 38 | ann.set_shape(img.get_shape().as_list()[:-1] + [1]) 39 | ann = self.postproc_annotation(annotation_filename, ann) 40 | return ann 41 | 42 | 43 | def load_ann_with_colormap(filename): 44 | filename = filename.decode("utf-8") 45 | return np.array(Image.open(filename))[..., np.newaxis] 46 | -------------------------------------------------------------------------------- /code/refinement_net/datasets/DAVIS/DAVISjono.py: -------------------------------------------------------------------------------- 1 | import glob 2 | 3 | import tensorflow as tf 4 | 5 | from refinement_net.datasets.Loader import register_dataset 6 | from refinement_net.datasets.Dataset import FileListDataset 7 | from refinement_net.datasets.util.Util import username 8 | 9 | NUM_CLASSES = 2 10 | VOID_LABEL = 255 # for translation augmentation 11 | DAVIS_DEFAULT_IMAGES_PATH = "/work2/" + username() + "/data/DAVIS/train-val/val/" 12 | DAVIS_DEFAULT_GT_PATH = "/work2/" + username() + "/data/DAVIS/train-val/val-gt/" 13 | DAVIS_DEFAULT_PROPOSALS_PATH = "/home/" + username() + "/vision/maskrcnn_tensorpack/train_log/thesis1/notrain-val/" 14 | DAVIS_IMAGE_SIZE = (480, 854) 15 | 16 | @register_dataset("davisjono") 17 | class DAVISjonoDataset(FileListDataset): 18 | def __init__(self, config, subset, name="davisjono"): 19 | self.image_dir = config.string("DAVIS_images_dir", DAVIS_DEFAULT_IMAGES_PATH) 20 | self.gt_dir = config.string("DAVIS_gt_dir", DAVIS_DEFAULT_GT_PATH) 21 | self.proposals_dir = config.string("DAVIS_proposals_dir", DAVIS_DEFAULT_PROPOSALS_PATH) 22 | 23 | super().__init__(config, name, subset, DAVIS_DEFAULT_IMAGES_PATH, 2) 24 | 25 | def read_inputfile_lists(self): 26 | img_filenames = glob.glob(self.image_dir + "*/*.jpg") 27 | seq_tags = [f.split("/")[-2] for f in img_filenames] 28 | framenum = [f.split('/')[-1].split('.jpg')[0] for f in img_filenames] 29 | label_filenames = [self.gt_dir + s + '/' + f + '.png' for s,f in zip(seq_tags,framenum)] 30 | return img_filenames, label_filenames 31 | 32 | def load_annotation(self, img, img_filename, annotation_filename): 33 | seq = img_filename.split("/")[-2] 34 | framenum = img_filename.split('/')[-1].split('.jpg')[0] 35 | proposal_filename = self.proposals_dir + seq + '/' + framenum + '.json' 36 | def _load_ann(img_, ann_filename): 37 | json = ann_filename.decode("utf-8") 38 | seq = json.split("/")[-3] 39 | t = int(json.split("/")[-1].replace(".json", "")) 40 | id_ = int(json.split("/")[-2]) 41 | label_ = load_mask_from_json(img_, json) 42 | bbox_ = self._tracking_gt[seq][id_][t].bbox_x0y0x1y1.astype("float32")[[1, 0, 3, 2]] 43 | return label_, bbox_ 44 | label, bbox = tf.py_func(_load_ann, [img, annotation_filename], [tf.uint8, tf.float32]) 45 | label.set_shape((None, None, 1)) 46 | bbox.set_shape((4,)) 47 | return {DataKeys.SEGMENTATION_LABELS: label, DataKeys.BBOXES_y0x0y1x1: bbox} -------------------------------------------------------------------------------- /code/refinement_net/datasets/DAVIS/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/datasets/DAVIS/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/datasets/DataKeys.py: -------------------------------------------------------------------------------- 1 | # we cannot use an enum here as tensorflow needs ordered keys 2 | 3 | SEGMENTATION_LABELS = "segmentation_labels" 4 | SEGMENTATION_LABELS_ORIGINAL_SIZE = "segmentation_labels_original_size" 5 | BBOX_GUIDANCE = "bbox_guidance" 6 | UNSIGNED_DISTANCE_TRANSFORM_GUIDANCE = "unsigned_distance_transform_guidance" 7 | SIGNED_DISTANCE_TRANSFORM_GUIDANCE = "signed_distance_transform_guidance" 8 | LASER_GUIDANCE = "laser_guidance" 9 | IMAGES = "images" 10 | RAW_IMAGES = "raw_images" 11 | INPUTS = "inputs" 12 | IMAGE_FILENAMES = "image_filenames" 13 | BBOXES_y0x0y1x1 = "bboxes_y0x0y1x1" # Order like in the tensorflow object detection API 14 | BBOXES_x0y0x1y1 = "bboxes_x0y0x1y1" # Alternative order, used for example in the frcnn implementation 15 | CLASSES = "classes" 16 | IDS = "ids" # bounding box number 1,2,3... or 0 if no bounding box 17 | CROP_BOXES_y0x0y1x1 = "crop_boxes_y0x0y1x1" 18 | DT_METHOD = "dt_method" 19 | RAW_SEGMENTATION_LABELS = "raw_segmentation_labels" 20 | CLICK_MAPS = "clicks_maps" 21 | NEG_CLICKS = "neg_clicks" 22 | POS_CLICKS = "pos_clicks" 23 | OBJ_TAGS = "obj_tags" 24 | FEATUREMAP_LABELS = "featuremap_labels" 25 | FEATUREMAP_BOXES = "featuremap_boxes" 26 | IMAGE_ID = "image_id" 27 | IS_CROWD = "is_crowd" 28 | SEGMENTATION_MASK = "segmentation_mask" 29 | SKIP_EXAMPLE = "skip_example" # If true, filter the corresponding example from the datastream. Useful, when an example 30 | # turns out to be bad during preprocessing. 31 | -------------------------------------------------------------------------------- /code/refinement_net/datasets/Loader.py: -------------------------------------------------------------------------------- 1 | from refinement_net.core.Util import import_submodules 2 | 3 | _registered_datasets = {} 4 | 5 | 6 | def register_dataset(name, **args): 7 | name = name.lower() 8 | 9 | def _register(dataset): 10 | _registered_datasets[name] = (dataset, args) 11 | return dataset 12 | return _register 13 | 14 | 15 | def load_dataset(config, subset, session, name): 16 | if not hasattr(load_dataset, "_imported"): 17 | load_dataset._imported = True 18 | import_submodules("refinement_net.datasets") 19 | # import_submodules("datasets") 20 | name = name.lower() 21 | if name not in _registered_datasets: 22 | raise ValueError("dataset " + name + " not registered.") 23 | dataset, args = _registered_datasets[name] 24 | return dataset(config=config, subset=subset, **args) 25 | -------------------------------------------------------------------------------- /code/refinement_net/datasets/Mapillary/Mapillary_instance.py: -------------------------------------------------------------------------------- 1 | from refinement_net.datasets.Loader import register_dataset 2 | from refinement_net.datasets.Mapillary.MapillaryLike_instance import MapillaryLikeInstanceDataset 3 | from refinement_net.datasets.util.Util import username 4 | 5 | DEFAULT_PATH = "/fastwork/" + username() + "/mywork/data/mapillary/" 6 | NAME = "mapillary_instance" 7 | 8 | 9 | @register_dataset("mapillary_instance_full", resolution="full") 10 | @register_dataset("mapillary_instance_half", resolution="half") 11 | @register_dataset("mapillary_instance_quarter", resolution="quarter") 12 | class MapillaryInstanceDataset(MapillaryLikeInstanceDataset): 13 | def __init__(self, config, subset, resolution): 14 | assert resolution in ("quarter", "half", "full"), resolution 15 | if resolution == "full": 16 | default_path = DEFAULT_PATH 17 | else: 18 | default_path = DEFAULT_PATH.replace("/mapillary/", "/mapillary_{}/".format(resolution)) 19 | 20 | # there are 37 classes with instances in total 21 | 22 | # we excluded the following: 23 | # 8: construction--flat--crosswalk-plain -> doesn't really look like a useful object category 24 | # 34: object--bike-rack -> holes* 25 | # 45: object--support--pole -> very large and thin -> bounding box does not capture it well 26 | # 46: object--support--traffic-sign-frame -> holes* 27 | # 47: object--support--utility-pole -> holes* 28 | 29 | # further candidate for exclusion: 30 | # 0: animal--bird -> usually very small 31 | 32 | # *: holes means that there are large "holes" in the object which usually are still annotated as part of the object 33 | # this will not work well together with laser, so we exclude them 34 | vehicle_ids = [52, 53, 54, 55, 56, 57, 59, 60, 61, 62] 35 | human_ids = [19, 20, 21, 22] 36 | animal_ids = [0, 1] 37 | object_ids = [32, 33, 35, 36, 37, 38, 39, 40, 41, 42, 44, 48, 49, 50, 51] 38 | crosswalk_zebra_id = [23] 39 | cat_ids_to_use = vehicle_ids + human_ids + animal_ids + object_ids + crosswalk_zebra_id 40 | 41 | super().__init__(config, subset, NAME, default_path, "datasets/Mapillary/", 256, cat_ids_to_use) 42 | -------------------------------------------------------------------------------- /code/refinement_net/datasets/Mapillary/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/datasets/Mapillary/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/datasets/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/datasets/few_shot_segmentation/FewShotFeedSegmentationDataset.py: -------------------------------------------------------------------------------- 1 | from abc import abstractmethod 2 | from random import shuffle 3 | import numpy as np 4 | 5 | from refinement_net.datasets.Dataset import DataKeys 6 | from refinement_net.datasets.FeedDataset import FeedDataset 7 | from refinement_net.datasets.util.Util import username 8 | from refinement_net.core.Log import log 9 | from refinement_net.datasets.Loader import register_dataset 10 | 11 | NUM_CLASSES = 2 12 | DEFAULT_DATA_KEYS = (DataKeys.IMAGES, DataKeys.SEGMENTATION_LABELS) 13 | DATA_KEYS_TO_USE = (DataKeys.IMAGES, DataKeys.SEGMENTATION_LABELS, DataKeys.BBOXES_y0x0y1x1, DataKeys.IMAGE_FILENAMES, DataKeys.OBJ_TAGS) 14 | 15 | 16 | @register_dataset("jono_vos_feed") 17 | class FewShotFeedSegmentationDataset(FeedDataset): 18 | def __init__(self, config, subset): 19 | data_keys_to_use = DATA_KEYS_TO_USE 20 | num_classes = 2 21 | super().__init__(config, subset, data_keys_to_use, num_classes) 22 | 23 | def get_feed_dict_for_next_step(self,image_data,bbox_idx): 24 | feed_dict = {} 25 | for idx in range(self._batch_size): 26 | example = image_data[bbox_idx] 27 | for data_key in self._data_keys_to_use: 28 | assert data_key in example, data_key 29 | feed_dict[self._placeholders[data_key][idx]] = example[data_key] 30 | return feed_dict 31 | 32 | def get_placeholders(self, key): 33 | return self._placeholders[key] 34 | 35 | def set_up_data_for_image(self,image,boxes): 36 | obj_data = {} 37 | image = image/255 38 | label = np.zeros(image.shape[:2] + (1,), dtype=np.uint8) 39 | for box_id,box in enumerate(boxes): 40 | x0, y0, x1, y1 = box 41 | x1 = x1 + x0 42 | y1 = y1 + y0 43 | bbox = [y0, x0, y1, x1] 44 | obj_data[box_id] = {DataKeys.IMAGES: image, 45 | DataKeys.SEGMENTATION_LABELS: label, 46 | DataKeys.IMAGE_FILENAMES: "", 47 | DataKeys.BBOXES_y0x0y1x1: bbox, 48 | DataKeys.OBJ_TAGS: str(box_id)} 49 | if len(obj_data) > 0: 50 | return obj_data 51 | else: 52 | return None -------------------------------------------------------------------------------- /code/refinement_net/datasets/few_shot_segmentation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/datasets/few_shot_segmentation/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/datasets/util/BoundingBox.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | 5 | def get_bbox_from_segmentation_mask(mask): 6 | object_locations = tf.cast(tf.where(tf.equal(mask, 1))[:, :2], tf.int32) 7 | y0 = tf.reduce_min(object_locations[:, 0]) 8 | x0 = tf.reduce_min(object_locations[:, 1]) 9 | y1 = tf.reduce_max(object_locations[:, 0]) + 1 10 | x1 = tf.reduce_max(object_locations[:, 1]) + 1 11 | bbox = tf.stack([y0, x0, y1, x1]) 12 | return bbox 13 | 14 | 15 | def encode_bbox_as_mask_np(bbox, shape): 16 | encoded = np.zeros(tuple(shape[:2]) + (1,), np.uint8) 17 | y0, x0, y1, x1 = np.round(bbox).astype(np.int) 18 | encoded[y0:y1, x0:x1] = 1 19 | return encoded 20 | 21 | 22 | def encode_bbox_as_mask(bbox, shape): 23 | encoded = tf.py_func(encode_bbox_as_mask_np, [bbox, shape], tf.uint8, name="encode_bbox_as_mask") 24 | encoded.set_shape((None, None, 1)) 25 | return encoded 26 | -------------------------------------------------------------------------------- /code/refinement_net/datasets/util/DistanceTransform.py: -------------------------------------------------------------------------------- 1 | import pdb 2 | 3 | import numpy as np 4 | import tensorflow as tf 5 | from PIL import Image 6 | from scipy.ndimage import distance_transform_edt 7 | from scipy.stats import norm 8 | 9 | 10 | def get_distance_transform(pts, label): 11 | dt = np.ones_like(label) 12 | if len(pts) > 0: 13 | for y, x in pts: 14 | dt[y, x] = 0 15 | dt = distance_transform_edt(dt) 16 | return dt 17 | else: 18 | # This is important since we divide it by 255 while normalizing the inputs. 19 | return dt * 255 20 | 21 | 22 | def encode_extreme_points(mask, use_gaussian): 23 | nz = mask.nonzero() 24 | assert nz[0].size > 0, "mask cannot be empty" 25 | ymin_idx = nz[0].argmin() 26 | ymax_idx = nz[0].argmax() 27 | xmin_idx = nz[1].argmin() 28 | xmax_idx = nz[1].argmax() 29 | ymin = (nz[0][ymin_idx], nz[1][ymin_idx]) 30 | ymax = (nz[0][ymax_idx], nz[1][ymax_idx]) 31 | xmin = (nz[0][xmin_idx], nz[1][xmin_idx]) 32 | xmax = (nz[0][xmax_idx], nz[1][xmax_idx]) 33 | pts = (ymin, ymax, xmin, xmax) 34 | distance_transform_extreme_pts = get_distance_transform(pts, mask) 35 | distance_transform_extreme_pts[distance_transform_extreme_pts > 20] = 20 36 | if use_gaussian: 37 | distance_transform_extreme_pts = norm.pdf(distance_transform_extreme_pts, loc=0, scale=10) * 25 38 | else: 39 | distance_transform_extreme_pts /= 20.0 40 | return distance_transform_extreme_pts.astype(np.float32) 41 | 42 | 43 | def signed_distance_transform(mask): 44 | sdt = tf.py_func(signed_distance_transform_np, [mask], tf.float32) 45 | sdt.set_shape(mask.get_shape()) 46 | return sdt 47 | 48 | 49 | def signed_distance_transform_np(mask): 50 | # Handle empty mask 51 | if not np.any(mask): 52 | dt_pos = np.ones_like(mask) * 128 53 | else: 54 | dt_pos = distance_transform_edt(np.logical_not(mask)) 55 | dt_neg = distance_transform_edt(mask) 56 | sdt = dt_pos - dt_neg 57 | sdt = np.clip(sdt, -128, 128) 58 | sdt /= 128 59 | return sdt.astype("float32") 60 | 61 | 62 | def unsigned_distance_transform(mask): 63 | sdt = signed_distance_transform(mask) 64 | udt = tf.maximum(sdt, 0) 65 | return udt 66 | 67 | 68 | def test_signed_distance_transform_np(): 69 | import matplotlib.pyplot as plt 70 | # mask_filename = "/work/voigtlaender/data/LASER/KITTI/KITTI_laser_v1/000000.png" 71 | # mask = np.array(Image.open(mask_filename)) > 0 72 | mask = np.zeros([250,250]) 73 | mask[20:50, 20:50] = 1 74 | sdt = signed_distance_transform_np(mask) 75 | plt.imshow(sdt) 76 | plt.show() 77 | 78 | 79 | if __name__ == "__main__": 80 | test_signed_distance_transform_np() 81 | -------------------------------------------------------------------------------- /code/refinement_net/datasets/util/Normalization.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | # IMAGENET_RGB_MEAN = numpy.array((124.0, 117.0, 104.0), dtype=numpy.float32) / 255.0 4 | # values from https://github.com/itijyou/ademxapp/blob/0239e6cf53c081b3803ccad109a7beb56e3a386f/iclass/ilsvrc.py 5 | IMAGENET_RGB_MEAN = np.array([0.485, 0.456, 0.406], dtype="float32") 6 | IMAGENET_RGB_STD = np.array([0.229, 0.224, 0.225], dtype="float32") 7 | 8 | 9 | def normalize(img, img_mean=IMAGENET_RGB_MEAN, img_std=IMAGENET_RGB_STD): 10 | if hasattr(img, "get_shape"): 11 | l = img.get_shape()[-1] 12 | if img_mean is not None and l != img_mean.size: 13 | img_mean = np.concatenate([img_mean, np.zeros(l - img_mean.size, dtype="float32")], axis=0) 14 | if img_std is not None and l != img_std.size: 15 | img_std = np.concatenate([img_std, np.ones(l - img_std.size, dtype="float32")], axis=0) 16 | 17 | if img_mean is not None: 18 | img -= img_mean 19 | if img_std is not None: 20 | img /= img_std 21 | return img 22 | 23 | 24 | def unnormalize(img, img_mean=IMAGENET_RGB_MEAN, img_std=IMAGENET_RGB_STD): 25 | if hasattr(img, "get_shape"): 26 | l = img.get_shape()[-1] 27 | if img_mean is not None and l != img_mean.size: 28 | img_mean = np.concatenate([img_mean, np.zeros(l - img_mean.size, dtype="float32")], axis=0) 29 | if img_std is not None and l != img_std.size: 30 | img_std = np.concatenate([img_std, np.ones(l - img_std.size, dtype="float32")], axis=0) 31 | 32 | if img_std is not None: 33 | img *= img_std 34 | if img_mean is not None: 35 | img += img_mean 36 | return img 37 | -------------------------------------------------------------------------------- /code/refinement_net/datasets/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/datasets/util/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/forwarding/Forwarder.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class Forwarder(ABC): 5 | def __init__(self, engine): 6 | self.engine = engine 7 | self.config = engine.config 8 | self.session = engine.session 9 | self.val_data = self.engine.valid_data 10 | self.train_data = self.engine.train_data 11 | self.trainer = self.engine.trainer 12 | self.saver = self.engine.saver 13 | 14 | @abstractmethod 15 | def forward(self): 16 | pass 17 | -------------------------------------------------------------------------------- /code/refinement_net/forwarding/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/forwarding/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | import os 4 | 5 | from refinement_net.core.Engine import Engine 6 | from refinement_net.core.Config import Config 7 | from refinement_net.core.Log import log 8 | import tensorflow as tf 9 | 10 | 11 | def init_log(config): 12 | log_dir = config.dir("log_dir", "logs") 13 | model = config.string("model") 14 | filename = log_dir + model + ".log" 15 | verbosity = config.int("log_verbosity", 3) 16 | log.initialize([filename], [verbosity], []) 17 | 18 | 19 | def main(_): 20 | assert len(sys.argv) == 2, "usage: main.py " 21 | config_path = sys.argv[1] 22 | assert os.path.exists(config_path), config_path 23 | try: 24 | config = Config(config_path) 25 | except ValueError as e: 26 | print("Malformed config file:", e) 27 | return -1 28 | init_log(config) 29 | # dump the config into the log 30 | print(open(config_path).read(), file=log.v4) 31 | engine = Engine(config) 32 | engine.run() 33 | 34 | 35 | if __name__ == '__main__': 36 | print("starting") 37 | tf.app.run(main) 38 | -------------------------------------------------------------------------------- /code/refinement_net/network/FullyConnected.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from refinement_net.network.Layer import Layer 4 | from refinement_net.network.Util import get_activation, prepare_input, apply_dropout 5 | 6 | 7 | # See also savitar1 8 | class FullyConnected(Layer): 9 | def __init__(self, name, inputs, n_features, tower_setup, activation="relu", dropout=0.0, batch_norm=False, 10 | batch_norm_decay=Layer.BATCH_NORM_DECAY_DEFAULT, l2=Layer.L2_DEFAULT): 11 | super(FullyConnected, self).__init__() 12 | inp, n_features_inp = prepare_input(inputs) 13 | 14 | with tf.variable_scope(name): 15 | inp = apply_dropout(inp, dropout) 16 | if batch_norm: 17 | inp = tf.expand_dims(inp, axis=0) 18 | inp = tf.expand_dims(inp, axis=0) 19 | inp = self.create_and_apply_batch_norm(inp, n_features_inp, batch_norm_decay, tower_setup) 20 | inp = tf.squeeze(inp, axis=[0, 1]) 21 | W = self.create_weight_variable("W", [n_features_inp, n_features], l2, tower_setup) 22 | b = self.create_bias_variable("b", [n_features], tower_setup) 23 | z = tf.matmul(inp, W) + b 24 | h = get_activation(activation)(z) 25 | self.outputs = [h] 26 | self.n_features = n_features -------------------------------------------------------------------------------- /code/refinement_net/network/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/network/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/network/deeplab/DeepLabV3Plus.py: -------------------------------------------------------------------------------- 1 | from refinement_net.network.Layer import Layer 2 | from . import model, common 3 | from refinement_net.datasets.Dataset import unnormalize 4 | 5 | 6 | class DeepLabV3Plus(Layer): 7 | def __init__(self, name, inputs, n_features, tower_setup, l2=Layer.L2_DEFAULT): 8 | super().__init__() 9 | assert len(inputs) == 1, len(inputs) 10 | inputs = inputs[0] 11 | # revert normalization since DeepLab will do it's own normalization 12 | # also multiply by 255 since DeepLab expects values in the range 0..255 13 | inputs = unnormalize(inputs) * 255 14 | if tower_setup.is_training: 15 | crop_size = [int(inputs.get_shape()[1]), int(inputs.get_shape()[2])] 16 | else: 17 | crop_size = None 18 | # for choosing options and a pretrained model to load, see here: https://github.com/tensorflow/models/blob/master/research/deeplab/g3doc/model_zoo.md 19 | model_options = common.ModelOptions( 20 | outputs_to_num_classes={"features": n_features}, 21 | crop_size=crop_size, 22 | atrous_rates=[6, 12, 18], 23 | output_stride=16, 24 | merge_method="max", 25 | add_image_level_feature=True, 26 | aspp_with_batch_norm=True, 27 | aspp_with_separable_conv=True, 28 | multi_grid=None, 29 | decoder_output_stride=4, 30 | decoder_use_separable_conv=True, 31 | logits_kernel_size=1, 32 | model_variant="xception_65") 33 | outputs_to_scales_to_logits = model.multi_scale_logits(inputs, 34 | model_options=model_options, 35 | image_pyramid=None, 36 | weight_decay=l2, 37 | is_training=tower_setup.is_training, 38 | fine_tune_batch_norm=not tower_setup.freeze_batchnorm) 39 | self.outputs = [outputs_to_scales_to_logits["features"]["merged_logits"]] 40 | -------------------------------------------------------------------------------- /code/refinement_net/network/deeplab/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/network/deeplab/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/network/deeplab/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/network/deeplab/core/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/scripts/DAVIS/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/scripts/DAVIS/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/scripts/DAVIS/create_DAVIS_instance_list.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import glob 3 | from PIL import Image 4 | import numpy as np 5 | import tqdm 6 | 7 | from refinement_net.datasets.util.Util import username 8 | import datasets.DAVIS 9 | 10 | PATH = "/home/luiten/vision/PReMVOS/data/first/bike-trial/" 11 | DAVIS_CODE_PATH = datasets.DAVIS.__file__.replace("__init__.py", "") 12 | 13 | for subset, outfile in [("train", "training.txt"), ("val", "validation.txt")]: 14 | with open(DAVIS_CODE_PATH + outfile, "w") as f: 15 | instances = glob.glob(PATH + "lucid_data_dreaming/*.png") 16 | for inst in tqdm.tqdm(instances): 17 | x = np.array(Image.open(inst)) 18 | ids = np.unique(x) 19 | im = inst.replace(".png", ".jpg") 20 | print(im.replace(PATH, ""), inst.replace(PATH, ""), file=f, sep=" ", end="") 21 | for id_ in ids[1:]: 22 | n = (x == id_).sum() 23 | print(" ", id_, ":", n, file=f, sep="", end="") 24 | print(file=f) 25 | -------------------------------------------------------------------------------- /code/refinement_net/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/scripts/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/scripts/eval/Datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/scripts/eval/Datasets/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/scripts/eval/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/scripts/eval/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/scripts/eval/analyze_few_shot_results.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | 7 | assert len(sys.argv) == 2 8 | COMBINE_INSTANCE_RESULTS_OVER_TIME = False 9 | PRINT_LARGEST_DECREASE = False 10 | RECALL_PERCENTAGES = [50, 70, 80, 90] 11 | VISUALIZE_DIFFERENCES = True 12 | 13 | log = sys.argv[1] 14 | step_ious = {} 15 | lines = [] 16 | with open(log) as f: 17 | for l in f: 18 | if COMBINE_INSTANCE_RESULTS_OVER_TIME and l.startswith("results of"): 19 | step = int(l.split("after step ")[1].split(" {")[0].strip()) 20 | elif not COMBINE_INSTANCE_RESULTS_OVER_TIME and l.startswith("forwarded/"): 21 | step = int(l.split("step_")[1].split("/")[0]) 22 | else: 23 | continue 24 | #iou = float(l.split("'IoU': '")[1].split("'")[0].strip()) 25 | iou = float(l.split("IoU: ")[1].split(",")[0].strip()) 26 | if step not in step_ious: 27 | step_ious[step] = [] 28 | step_ious[step].append(iou) 29 | if step == 0: 30 | lines.append(l) 31 | print(step, iou) 32 | 33 | print("----") 34 | for step in sorted(step_ious.keys()): 35 | s = str(step) + " " + str(len(step_ious[step])) + " " + str(np.mean(step_ious[step])) 36 | for pct in RECALL_PERCENTAGES: 37 | s += " recall@{}%: {}".format(pct, np.mean(np.array(step_ious[step]) > pct / 100)) 38 | print(s) 39 | 40 | ious = [np.array(step_ious[k]) for k in sorted(step_ious.keys())] 41 | minlen = min([x.size for x in ious]) 42 | shortened = [x[:minlen] for x in ious] 43 | maxed = np.maximum.reduce(shortened) 44 | s = "oracle " + str(maxed.size) + " " + str(maxed.mean()) 45 | for pct in RECALL_PERCENTAGES: 46 | s += " recall@{}%: {}".format(pct, np.mean(np.array(maxed) > pct / 100)) 47 | print(s) 48 | 49 | if PRINT_LARGEST_DECREASE: 50 | print("----") 51 | print("largest decrease in performance by fine-tuning") 52 | 53 | # find largest decrease in performance 54 | x = np.stack(ious) 55 | diff = x[1] - x[0] 56 | indices = np.argsort(diff) 57 | for idx in indices[:30]: 58 | #print(diff[idx], lines[idx], end="") 59 | fn = lines[idx].split(" ")[0] 60 | print(fn.split("/")[-3] + "/" + fn.split("/")[-1], diff[idx]) 61 | 62 | if VISUALIZE_DIFFERENCES: 63 | minlen = min([len(i) for i in ious]) 64 | ious = [i[:minlen] for i in ious] 65 | x = np.array(ious) 66 | print(x.shape) 67 | diff = x[-1] - x[0] 68 | plt.hist(diff, bins=30) 69 | plt.show() 70 | -------------------------------------------------------------------------------- /code/refinement_net/scripts/eval/create_KITTI_eval_config.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then 4 | echo "Usage ${0} config epoch [config_folder]" 5 | exit 6 | fi 7 | 8 | config_folder=eval_KITTI 9 | if [ "$#" -eq 3 ]; then 10 | config_folder=$3 11 | fi 12 | 13 | config=$1 14 | epoch=$2 15 | savitar_path=/home/${USER}/vision/savitar2/ 16 | 17 | model=$(basename ${config}) 18 | model_path=${savitar_path}/models/${model} 19 | epoch_str=$(printf %08d ${epoch}) 20 | load=models/${model}/${model}-${epoch_str} 21 | model_file=${load}.data-00000-of-00001 22 | config_out=${savitar_path}/configs/${config_folder}/${model}_${epoch_str} 23 | 24 | if [ ! -f ${config} ]; then 25 | echo "config not found: ${config}" 26 | exit 1 27 | fi 28 | 29 | if [ ! -f ${model_file} ]; then 30 | echo "model file not found: ${model_file}" 31 | exit 1 32 | fi 33 | 34 | firstline=$(head -1 ${config}) 35 | if [ ${firstline} != "{" ]; then 36 | echo 'config does not start with "{"' 37 | exit 1 38 | fi 39 | 40 | 41 | echo "{" > ${config_out} 42 | echo \"model\": \"${model}_${epoch_str}_eval_KITTI\", >> ${config_out} 43 | echo '"task": "few_shot_segmentation",' >> ${config_out} 44 | echo '"dataset": "laser_as_clicks_guided_segmentation",' >> ${config_out} 45 | echo \"load\": \"${load}\", >> ${config_out} 46 | echo '"print_per_object_stats": false,' >> ${config_out} 47 | echo '"n_finetune_steps": 0,' >> ${config_out} 48 | echo '"forward_initial": true,' >> ${config_out} 49 | 50 | 51 | # tail -n +2 will remove the first line 52 | tail -n +2 ${config} | grep -v '"model"' | grep -v '"task"' | grep -v '"dataset"' | grep -v '"load"' | grep -v '"n_finetune_steps"' | grep -v '"forward_initial"' >> ${config_out} 53 | 54 | echo ${config_out} 55 | -------------------------------------------------------------------------------- /code/refinement_net/scripts/eval/create_PASCAL_eval_config.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "$#" -ne 3 ]; then 4 | echo "Usage ${0} config epoch bbox_jitter_factor" 5 | exit 6 | fi 7 | 8 | config=$1 9 | epoch=$2 10 | bbox_jitter_factor=$3 11 | savitar_path=/home/${USER}/vision/savitar2/ 12 | 13 | model=$(basename ${config}) 14 | model_path=${savitar_path}/models/${model} 15 | epoch_str=$(printf %08d ${epoch}) 16 | load=models/${model}/${model}-${epoch_str} 17 | model_file=${load}.data-00000-of-00001 18 | config_out=${savitar_path}/configs/eval_PASCAL/${model}_${epoch_str}_jitter_${bbox_jitter_factor} 19 | 20 | if [ ! -f ${config} ]; then 21 | echo "config not found: ${config}" 22 | exit 1 23 | fi 24 | 25 | if [ ! -f ${model_file} ]; then 26 | echo "model file not found: ${model_file}" 27 | exit 1 28 | fi 29 | 30 | firstline=$(head -1 ${config}) 31 | if [ ${firstline} != "{" ]; then 32 | echo 'config does not start with "{"' 33 | exit 1 34 | fi 35 | 36 | 37 | echo "{" > ${config_out} 38 | echo \"model\": \"${model}_${epoch_str}_eval_PASCAL_jitter_${bbox_jitter_factor}\", >> ${config_out} 39 | echo '"task": "eval",' >> ${config_out} 40 | echo '"dataset": "pascalVOC_instance",' >> ${config_out} 41 | echo \"load\": \"${load}\", >> ${config_out} 42 | echo \"bbox_jitter_factor\": ${bbox_jitter_factor}, >> ${config_out} 43 | echo '"augmentors_val": ["bbox_jitter"],' >> ${config_out} 44 | 45 | 46 | # tail -n +2 will remove the first line 47 | tail -n +2 ${config} | grep -v '"model"' | grep -v '"task"' | grep -v '"dataset"' | grep -v '"load"' | grep -v '"bbox_jitter_factor"' | grep -v '"augmentors_val"' >> ${config_out} 48 | 49 | echo ${config_out} 50 | -------------------------------------------------------------------------------- /code/refinement_net/scripts/eval/eval_KITTI_all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "$#" -ne 1 ]; then 4 | echo "Usage ${0} config" 5 | exit 6 | fi 7 | 8 | config=$1 9 | 10 | grep forwarded $config | my_cut.py 2 | sed s/,//g | print_mean.py 11 | -------------------------------------------------------------------------------- /code/refinement_net/scripts/eval/eval_KITTI_testset.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "$#" -ne 1 ]; then 4 | echo "Usage ${0} config" 5 | exit 6 | fi 7 | 8 | config=$1 9 | 10 | count=$(grep forwarded $config | wc -l) 11 | if [ $count -ne 934 ]; then 12 | echo "incorrect number of ious" 13 | exit 14 | fi 15 | 16 | grep forwarded $config | my_cut.py 2 | sed s/,//g | head -651 | print_mean.py 17 | -------------------------------------------------------------------------------- /code/refinement_net/scripts/eval/eval_KITTI_tuning.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "$#" -ne 1 ]; then 4 | echo "Usage ${0} config" 5 | exit 6 | fi 7 | 8 | config=$1 9 | 10 | count=$(grep forwarded $config | wc -l) 11 | if [ $count -ne 934 ]; then 12 | echo "incorrect number of ious" 13 | exit 14 | fi 15 | 16 | grep forwarded $config | my_cut.py 2 | sed s/,//g | tail -283 | print_mean.py 17 | -------------------------------------------------------------------------------- /code/refinement_net/scripts/eval/eval_coco_detection.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pycocotools.coco as coco 3 | from pycocotools.cocoeval import COCOeval 4 | from refinement_net.datasets.util.Util import username 5 | 6 | coco_data_folder = "/fastwork/" + username() + "/mywork/data/coco/" 7 | minival_gt_file = coco_data_folder + "annotations/instances_val2014.json" 8 | minival_det_file = "/home/krause/vision/savitar2/forwarded/frcnn_test/frcnn_test-1-detections.json" 9 | 10 | 11 | def evaluate_coco(): 12 | c = coco.COCO(minival_gt_file) 13 | cocoDt = c.loadRes(minival_det_file) 14 | cocoEval = COCOeval(c, cocoDt, 'bbox') 15 | cocoEval.evaluate() 16 | cocoEval.accumulate() 17 | cocoEval.summarize() 18 | print(cocoEval.stats[0]) 19 | 20 | 21 | def adjust_detections(): 22 | c = coco.COCO(minival_gt_file) 23 | keys = list(c.cats.keys()) 24 | detections_list = json.load(open(minival_det_file)) 25 | for det in detections_list: 26 | det['category_id'] = keys[det['category_id']] 27 | json.dump(detections_list, open("/home/krause/vision/savitar2/forwarded/temp_edited.json", 'w')) 28 | 29 | 30 | if __name__ == '__main__': 31 | evaluate_coco() 32 | -------------------------------------------------------------------------------- /code/refinement_net/scripts/flownet_config_maker.py: -------------------------------------------------------------------------------- 1 | import glob 2 | # IMAGE_DIR = "/work2/luiten/data/DAVIS/train-val/val17/" 3 | # FLOW_DIR = "/home/luiten/vision/flownet2/" 4 | # FLOW_OUT_DIR = "/work2/luiten/data/DAVIS/train-val/flow/" 5 | 6 | # IMAGE_DIR = "/work2/luiten/data/DAVIS/test-challenge/JPEGImages/480p/" 7 | # FLOW_DIR = "/home/luiten/vision/flownet2/" 8 | # FLOW_OUT_DIR = "/work2/luiten/data/DAVIS/test-challenge/flow/" 9 | 10 | # IMAGE_DIR = "/work2/luiten/data/DAVIS/test-dev/JPEGImages/480p/" 11 | # FLOW_DIR = "/home/luiten/vision/flownet2/" 12 | # FLOW_OUT_DIR = "/work2/luiten/data/DAVIS/test-dev/flow/" 13 | 14 | IMAGE_DIR = "/work2/luiten/data/DAVIS/2016/DAVIS-data/DAVIS/train/" 15 | FLOW_DIR = "/home/luiten/vision/flownet2/" 16 | FLOW_OUT_DIR = "/work2/luiten/data/DAVIS/train/flow/" 17 | 18 | new_file= open(FLOW_DIR + "filelist.txt","w+") 19 | folders = glob.glob(IMAGE_DIR+"*/") 20 | for folder in folders: 21 | folder_solo = folder.split('/')[-2] 22 | files = glob.glob(folder+"/*.jpg") 23 | nums = [int(f.split('/')[-1].split('.jpg')[0]) for f in files] 24 | nums.sort() 25 | firsts = nums[:-1] 26 | lasts = nums[1:] 27 | first_fs = [IMAGE_DIR+folder_solo+'/%05d.jpg'%f for f in firsts] 28 | last_fs = [IMAGE_DIR+folder_solo+'/%05d.jpg'%l for l in lasts] 29 | to_saves_1 = [FLOW_OUT_DIR + folder_solo+'-%05d-%05d.flo'%(f,l) for f,l in zip(firsts,lasts)] 30 | to_saves_2 = [FLOW_OUT_DIR + folder_solo + '-%05d-%05d.flo' % (l, f) for f, l in zip(firsts, lasts)] 31 | texts_1 = [f + ' ' + ' ' + l + ' ' + t+'\n' for f, l, t in zip(first_fs, last_fs, to_saves_1)] 32 | texts_2 = [l + ' ' + ' ' + f + ' ' + t+'\n' for f, l, t in zip(first_fs, last_fs, to_saves_2)] 33 | for t in texts_1: 34 | new_file.write(t) 35 | for t in texts_2: 36 | new_file.write(t) 37 | -------------------------------------------------------------------------------- /code/refinement_net/scripts/prepare_tensorpack_checkpoints.py: -------------------------------------------------------------------------------- 1 | from refinement_net.scripts.rename_weights import rename 2 | import tensorflow as tf 3 | import numpy as np 4 | 5 | 6 | def switch_initial_conv_order(checkpoint_dir, prefix): 7 | checkpoint = tf.train.get_checkpoint_state(checkpoint_dir) 8 | with tf.Session() as sess: 9 | for var_name, _ in tf.contrib.framework.list_variables(checkpoint_dir): 10 | # Load the variable 11 | var = tf.contrib.framework.load_variable(checkpoint_dir, var_name) 12 | 13 | if var_name == prefix + "/conv0/W": 14 | var = np.flip(var, 2) 15 | 16 | var = tf.Variable(var, name=var_name) 17 | 18 | # Save the variables 19 | saver = tf.train.Saver() 20 | sess.run(tf.global_variables_initializer()) 21 | saver.save(sess, checkpoint_dir) 22 | 23 | 24 | def prepare_resnet_checkpoint(): 25 | rename("models/resnet_test/resnet_test-1", 26 | [("bn/mean/EMA", "bn/mean_ema"), ("bn/variance/EMA", "bn/var_ema")], "resnet/", 27 | ["global_step", "learning_rate"], convert_global_step_to_int32=True, dry_run=False) 28 | 29 | 30 | def prepare_frcnn_checkpoint(): 31 | rename("models/frcnn_test/frcnn_test-1", 32 | [("bn/mean/EMA", "bn/mean_ema"), ("bn/variance/EMA", "bn/var_ema")], "frcnn/", 33 | ["global_step", "learning_rate"], convert_global_step_to_int32=True, dry_run=False) 34 | switch_initial_conv_order("models/frcnn_test/frcnn_test-1", "frcnn") 35 | 36 | 37 | if __name__ == '__main__': 38 | prepare_frcnn_checkpoint() 39 | -------------------------------------------------------------------------------- /code/refinement_net/scripts/preproc/KITTI/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/scripts/preproc/KITTI/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/scripts/preproc/KITTI/create_KITTI_instance_list.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import glob 3 | from PIL import Image 4 | import numpy as np 5 | import tqdm 6 | 7 | from refinement_net.datasets.util.Util import username 8 | import datasets.KITTI 9 | 10 | PATH = "/home/" + username() + "/data/KITTI_instance/" 11 | KITTI_CODE_PATH = datasets.KITTI.__file__.replace("__init__.py", "") 12 | 13 | with open(KITTI_CODE_PATH + "training.txt", "w") as f: 14 | instances = glob.glob(PATH + "instance/*.png") 15 | for inst in tqdm.tqdm(instances): 16 | x = np.array(Image.open(inst)) 17 | ids = np.unique(x) 18 | im = inst.replace("/instance/", "/image_2/") 19 | print(im.replace(PATH, ""), inst.replace(PATH, ""), file=f, sep=" ", end="") 20 | for id_ in ids: 21 | n = (x == id_).sum() 22 | print(" ", id_, ":", n, file=f, sep="", end="") 23 | print(file=f) 24 | -------------------------------------------------------------------------------- /code/refinement_net/scripts/preproc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/scripts/preproc/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/scripts/preproc/cityscapes/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/scripts/preproc/cityscapes/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/scripts/preproc/cityscapes/create_cityscapes_instance_list.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import glob 3 | from PIL import Image 4 | import numpy as np 5 | import tqdm 6 | 7 | from refinement_net.datasets.util.Util import username 8 | import datasets.cityscapes 9 | 10 | PATH = "/fastwork/" + username() + "/mywork/data/cityscapes/" 11 | CITYSCAPES_CODE_PATH = datasets.cityscapes.__file__.replace("__init__.py", "") 12 | 13 | for subset, outfile in [("train", "training.txt"), ("val", "validation.txt")]: 14 | with open(CITYSCAPES_CODE_PATH + outfile, "w") as f: 15 | instances = glob.glob(PATH + "gtFine/" + subset + "/*/*_instanceIds.png") 16 | for inst in tqdm.tqdm(instances): 17 | x = np.array(Image.open(inst)) 18 | ids = np.unique(x) 19 | im = inst.replace("/gtFine/", "/leftImg8bit/").replace("_gtFine_instanceIds", "_leftImg8bit") 20 | print(im.replace(PATH, ""), inst.replace(PATH, ""), file=f, sep=" ", end="") 21 | for id_ in ids: 22 | n = (x == id_).sum() 23 | print(" ", id_, ":", n, file=f, sep="", end="") 24 | print(file=f) 25 | -------------------------------------------------------------------------------- /code/refinement_net/scripts/preproc/mapillary/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/code/refinement_net/scripts/preproc/mapillary/__init__.py -------------------------------------------------------------------------------- /code/refinement_net/scripts/preproc/mapillary/create_mapillary_instance_list.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import glob 3 | from PIL import Image 4 | import numpy as np 5 | 6 | from refinement_net.datasets.util.Util import username 7 | import datasets.Mapillary 8 | 9 | PATH = "/fastwork/" + username() + "/mywork/data/mapillary_quarter/" 10 | MAPILLARY_CODE_PATH = datasets.Mapillary.__file__.replace("__init__.py", "") 11 | 12 | for subset in ["training", "validation"]: 13 | with open(MAPILLARY_CODE_PATH + subset, "w") as f: 14 | instances = glob.glob(PATH + subset + "/instances/*.png") 15 | for inst in instances: 16 | x = np.array(Image.open(inst)) 17 | ids = np.unique(x) 18 | im = inst.replace("/instances/", "/images/").replace(".png", ".jpg") 19 | print(im.replace(PATH, ""), inst.replace(PATH, ""), file=f, sep=" ", end="") 20 | for id_ in ids: 21 | n = (x == id_).sum() 22 | print(" ", id_, ":", n, file=f, sep="", end="") 23 | print(file=f) 24 | -------------------------------------------------------------------------------- /code/refinement_net/scripts/preproc/mapillary/remove_dashes_at_beginning.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import glob 4 | import os 5 | 6 | from refinement_net.datasets.util.Util import username 7 | 8 | PATH = "/fastwork/" + username() + "/mywork/data/mapillary/" 9 | files = glob.glob(PATH + "training/*/-*.png") + glob.glob(PATH + "training/*/-*.jpg") + \ 10 | glob.glob(PATH + "validation/*/-*.png") + glob.glob(PATH + "validation/*/-*.jpg") 11 | with open(PATH + "renamed.txt", "w") as f_out: 12 | for f in files: 13 | idx_begin = f.rfind("/-") 14 | idx_end = idx_begin + 1 15 | while f[idx_end] == "-": 16 | idx_end += 1 17 | f_new = f[:idx_begin+1] + f[idx_end:] 18 | print(f, "->", f_new) 19 | os.rename(f, f_new) 20 | print(f, f_new, file=f_out) 21 | -------------------------------------------------------------------------------- /code/refinement_net/scripts/rename_weights.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #Adapted from https://gist.github.com/batzner/7c24802dd9c5e15870b4b56e22135c96 3 | 4 | import tensorflow as tf 5 | 6 | 7 | def rename(checkpoint_dir, replace_pairs, add_prefix, exclude, convert_global_step_to_int32=False, dry_run=False): 8 | checkpoint = tf.train.get_checkpoint_state(checkpoint_dir) 9 | with tf.Session() as sess: 10 | for var_name, _ in tf.contrib.framework.list_variables(checkpoint_dir): 11 | # Load the variable 12 | var = tf.contrib.framework.load_variable(checkpoint_dir, var_name) 13 | 14 | if var_name == "global_step" and convert_global_step_to_int32: 15 | var = tf.cast(var, dtype=tf.int32) 16 | 17 | new_name = var_name 18 | if exclude and var_name not in exclude: 19 | # Set the new name 20 | if replace_pairs: 21 | for replace_pair in replace_pairs: 22 | new_name = new_name.replace(replace_pair[0], replace_pair[1]) 23 | if add_prefix: 24 | new_name = add_prefix + new_name 25 | 26 | if dry_run: 27 | print('%s would be renamed to %s.' % (var_name, new_name)) 28 | else: 29 | print('Renaming %s to %s.' % (var_name, new_name)) 30 | # Rename the variable 31 | var = tf.Variable(var, name=new_name) 32 | 33 | if not dry_run: 34 | # Save the variables 35 | saver = tf.train.Saver() 36 | sess.run(tf.global_variables_initializer()) 37 | # saver.save(sess, checkpoint.model_checkpoint_path) 38 | saver.save(sess, checkpoint_dir) 39 | -------------------------------------------------------------------------------- /code/refinement_net/scripts/visualization/plot_learn_curve.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import matplotlib.pyplot as plt 3 | import sys 4 | 5 | 6 | def doit(fn, tag): 7 | train = [] 8 | val = [] 9 | with open(fn) as f: 10 | for l in f: 11 | if "finished" in l and "epoch" in l: 12 | sp = [x.replace("{", "").replace("}", "").replace(",", "") for x in l.split()] 13 | indices = [i + 1 for i, x in enumerate(sp) if x == tag + ":"] 14 | assert len(indices) == 2, sp 15 | tr = float(sp[indices[0]]) 16 | va = float(sp[indices[1]]) 17 | train.append(tr) 18 | val.append(va) 19 | print(train, val) 20 | x_axis = range(1, len(train) + 1) 21 | plt.plot(x_axis, train, label="train") 22 | plt.plot(x_axis, val, label="val") 23 | plt.legend() 24 | plt.title(fn + " " + tag) 25 | 26 | assert len(sys.argv) == 2 27 | doit(sys.argv[1], "loss") 28 | plt.figure() 29 | doit(sys.argv[1], "IoU") 30 | plt.show() 31 | -------------------------------------------------------------------------------- /code/refinement_net/scripts/visualization/visualize_coco_detections.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pycocotools.coco as coco 3 | import matplotlib.pyplot as plt 4 | import matplotlib.patches as patches 5 | import matplotlib.image as mpimg 6 | from refinement_net.datasets.util.Util import username 7 | 8 | coco_data_folder = "/fastwork/" + username() + "/mywork/data/coco/" 9 | minival_gt_file = coco_data_folder + "annotations/instances_val2014.json" 10 | minival_det_file = "/home/krause/vision/tensorpack/examples/FasterRCNN/TensorpackModelzooResnet50MinivalDetections.json" 11 | 12 | coco = coco.COCO(minival_gt_file) 13 | detections_list = json.load(open(minival_det_file)) 14 | detections_by_imgid = {} 15 | for det in detections_list: 16 | img_id = det['image_id'] 17 | if img_id in detections_by_imgid: 18 | detections_by_imgid[img_id].append(det) 19 | else: 20 | detections_by_imgid[img_id] = [det] 21 | 22 | 23 | def visualize(img_id): 24 | img_descriptor = coco.loadImgs(img_id) 25 | file_name = coco_data_folder + "val/" + img_descriptor[0]['file_name'] 26 | 27 | fig, ax = plt.subplots(1) 28 | img = mpimg.imread(file_name) 29 | ax.imshow(img) 30 | 31 | gt_ann_ids = coco.getAnnIds(imgIds=[img_id]) 32 | gt_anns = coco.loadAnns(gt_ann_ids) 33 | dets = detections_by_imgid[img_id] 34 | print("Image", img_id, "Dets", len(dets), "GT", len(gt_anns)) 35 | 36 | for gt in gt_anns: 37 | draw_box(ax, gt['bbox'], 'r', gt['category_id'], 1.0) 38 | for det in dets: 39 | draw_box(ax, det['bbox'], 'b', det['category_id'], det['score']) 40 | 41 | plt.show() 42 | 43 | 44 | def draw_box(ax, bbox, color, cat_id, alpha): 45 | rect = patches.Rectangle((bbox[0], bbox[1]), bbox[2], bbox[3], linewidth=1, 46 | edgecolor=color, facecolor='none', alpha=alpha) 47 | ax.add_patch(rect) 48 | ax.annotate(str(cat_id), (bbox[0] + 0.5 * bbox[2], bbox[1] + 0.5 * bbox[3]), color=color, weight='bold', 49 | fontsize=10, ha='center', va='center', alpha=1.0) 50 | 51 | 52 | if __name__ == '__main__': 53 | img_ids = coco.getImgIds() 54 | for img_id in img_ids: 55 | visualize(img_id) 56 | -------------------------------------------------------------------------------- /data/README.txt: -------------------------------------------------------------------------------- 1 | In this folder, the video data that PReMVOS runs on can be stored. 2 | 3 | -------------------------------------------------------------------------------- /data/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JonathonLuiten/PReMVOS/2666acaae0f1b4bd7d88a0cfb9b7cf5c5de12a4b/data/teaser.png -------------------------------------------------------------------------------- /output/logs/ReID_net/README.txt: -------------------------------------------------------------------------------- 1 | Log folder for the ReID net 2 | -------------------------------------------------------------------------------- /output/logs/refinement_net/README.txt: -------------------------------------------------------------------------------- 1 | Log folder for the refinement net 2 | -------------------------------------------------------------------------------- /seq_to_run.txt: -------------------------------------------------------------------------------- 1 | data/DAVIS/JPEGImages/480p/bike-packing/ 2 | data/DAVIS/JPEGImages/480p/blackswan/ 3 | data/DAVIS/JPEGImages/480p/bmx-trees/ 4 | data/DAVIS/JPEGImages/480p/breakdance/ 5 | data/DAVIS/JPEGImages/480p/camel/ 6 | data/DAVIS/JPEGImages/480p/car-roundabout/ 7 | data/DAVIS/JPEGImages/480p/car-shadow/ 8 | data/DAVIS/JPEGImages/480p/cows/ 9 | data/DAVIS/JPEGImages/480p/dance-twirl/ 10 | data/DAVIS/JPEGImages/480p/dog/ 11 | data/DAVIS/JPEGImages/480p/dogs-jump/ 12 | data/DAVIS/JPEGImages/480p/drift-chicane/ 13 | data/DAVIS/JPEGImages/480p/drift-straight/ 14 | data/DAVIS/JPEGImages/480p/goat/ 15 | data/DAVIS/JPEGImages/480p/gold-fish/ 16 | data/DAVIS/JPEGImages/480p/horsejump-high/ 17 | data/DAVIS/JPEGImages/480p/india/ 18 | data/DAVIS/JPEGImages/480p/judo/ 19 | data/DAVIS/JPEGImages/480p/kite-surf/ 20 | data/DAVIS/JPEGImages/480p/lab-coat/ 21 | data/DAVIS/JPEGImages/480p/libby/ 22 | data/DAVIS/JPEGImages/480p/loading/ 23 | data/DAVIS/JPEGImages/480p/mbike-trick/ 24 | data/DAVIS/JPEGImages/480p/motocross-jump/ 25 | data/DAVIS/JPEGImages/480p/paragliding-launch/ 26 | data/DAVIS/JPEGImages/480p/parkour/ 27 | data/DAVIS/JPEGImages/480p/pigs/ 28 | data/DAVIS/JPEGImages/480p/scooter-black/ 29 | data/DAVIS/JPEGImages/480p/shooting/ 30 | data/DAVIS/JPEGImages/480p/soapbox/ 31 | -------------------------------------------------------------------------------- /weights/README.txt: -------------------------------------------------------------------------------- 1 | In this folder, the network weights for PReMVOS can be stored. 2 | 3 | --------------------------------------------------------------------------------