├── .idea
├── encodings.xml
├── misc.xml
├── modules.xml
├── vRGV.iml
├── vcs.xml
└── workspace.xml
├── README.md
├── cfgs
├── res101.yml
├── res101_ls.yml
├── res50.yml
└── vgg16.yml
├── dataloader
├── __init__.py
├── build_vocab.py
├── data_preparation.py
├── detect_frame_loader.py
├── ground_loader.py
└── util.py
├── dataset
└── vidvrd
│ ├── .gitignore
│ ├── gt_relation_frame.json
│ ├── static_relations.txt
│ ├── test.txt
│ ├── trainval.txt
│ ├── val_list.txt
│ ├── videos.txt
│ ├── vocab.pkl
│ ├── vrelation_train.json
│ ├── vrelation_val.json
│ └── vrelation_zero.json
├── detect_frame.py
├── detection.py
├── detection.sh
├── eval_ground.py
├── evaluations
├── common.py
└── util.py
├── generate_track_link.py
├── ground.py
├── ground.sh
├── ground_relation.py
├── introduction.png
├── lib
├── datasets
│ ├── VOCdevkit-matlab-wrapper
│ │ ├── get_voc_opts.m
│ │ ├── voc_eval.m
│ │ └── xVOCap.m
│ ├── __init__.py
│ ├── __pycache__
│ │ ├── __init__.cpython-36.pyc
│ │ ├── coco.cpython-36.pyc
│ │ ├── ds_utils.cpython-36.pyc
│ │ ├── factory.cpython-36.pyc
│ │ ├── imagenet.cpython-36.pyc
│ │ ├── imdb.cpython-36.pyc
│ │ ├── pascal_voc.cpython-36.pyc
│ │ ├── vg.cpython-36.pyc
│ │ ├── vg_eval.cpython-36.pyc
│ │ └── voc_eval.cpython-36.pyc
│ ├── coco.py
│ ├── ds_utils.py
│ ├── factory.py
│ ├── imagenet.py
│ ├── imdb.py
│ ├── pascal_voc.py
│ ├── pascal_voc_rbg.py
│ ├── tools
│ │ └── mcg_munge.py
│ ├── vg.py
│ ├── vg_eval.py
│ ├── voc_eval.py
│ └── voc_eval_bak.py
├── make.sh
├── model
│ ├── __init__.py
│ ├── __pycache__
│ │ └── __init__.cpython-36.pyc
│ ├── faster_rcnn
│ │ ├── __init__.py
│ │ ├── __pycache__
│ │ │ ├── __init__.cpython-36.pyc
│ │ │ ├── faster_rcnn.cpython-36.pyc
│ │ │ ├── resnet.cpython-36.pyc
│ │ │ └── vgg16.cpython-36.pyc
│ │ ├── faster_rcnn.py
│ │ ├── resnet.py
│ │ └── vgg16.py
│ ├── nms
│ │ ├── .gitignore
│ │ ├── __init__.py
│ │ ├── __pycache__
│ │ │ ├── __init__.cpython-36.pyc
│ │ │ ├── nms_cpu.cpython-36.pyc
│ │ │ ├── nms_gpu.cpython-36.pyc
│ │ │ └── nms_wrapper.cpython-36.pyc
│ │ ├── _ext
│ │ │ ├── __init__.py
│ │ │ ├── __pycache__
│ │ │ │ └── __init__.cpython-36.pyc
│ │ │ └── nms
│ │ │ │ ├── __init__.py
│ │ │ │ └── __pycache__
│ │ │ │ └── __init__.cpython-36.pyc
│ │ ├── build.py
│ │ ├── make.sh
│ │ ├── nms_cpu.py
│ │ ├── nms_gpu.py
│ │ ├── nms_kernel.cu
│ │ ├── nms_wrapper.py
│ │ └── src
│ │ │ ├── nms_cuda.c
│ │ │ ├── nms_cuda.h
│ │ │ ├── nms_cuda_kernel.cu
│ │ │ ├── nms_cuda_kernel.cu.o
│ │ │ └── nms_cuda_kernel.h
│ ├── roi_align
│ │ ├── __init__.py
│ │ ├── __pycache__
│ │ │ └── __init__.cpython-36.pyc
│ │ ├── _ext
│ │ │ ├── __init__.py
│ │ │ ├── __pycache__
│ │ │ │ └── __init__.cpython-36.pyc
│ │ │ └── roi_align
│ │ │ │ ├── __init__.py
│ │ │ │ ├── __pycache__
│ │ │ │ └── __init__.cpython-36.pyc
│ │ │ │ └── _roi_align.so
│ │ ├── build.py
│ │ ├── functions
│ │ │ ├── __init__.py
│ │ │ ├── __pycache__
│ │ │ │ ├── __init__.cpython-36.pyc
│ │ │ │ └── roi_align.cpython-36.pyc
│ │ │ └── roi_align.py
│ │ ├── make.sh
│ │ ├── modules
│ │ │ ├── __init__.py
│ │ │ ├── __pycache__
│ │ │ │ ├── __init__.cpython-36.pyc
│ │ │ │ └── roi_align.cpython-36.pyc
│ │ │ └── roi_align.py
│ │ └── src
│ │ │ ├── roi_align.c
│ │ │ ├── roi_align.h
│ │ │ ├── roi_align_cuda.c
│ │ │ ├── roi_align_cuda.h
│ │ │ ├── roi_align_kernel.cu
│ │ │ ├── roi_align_kernel.cu.o
│ │ │ └── roi_align_kernel.h
│ ├── roi_crop
│ │ ├── __init__.py
│ │ ├── __pycache__
│ │ │ └── __init__.cpython-36.pyc
│ │ ├── _ext
│ │ │ ├── __init__.py
│ │ │ ├── __pycache__
│ │ │ │ └── __init__.cpython-36.pyc
│ │ │ ├── crop_resize
│ │ │ │ ├── __init__.py
│ │ │ │ └── _crop_resize.so
│ │ │ └── roi_crop
│ │ │ │ ├── __init__.py
│ │ │ │ ├── __pycache__
│ │ │ │ └── __init__.cpython-36.pyc
│ │ │ │ └── _roi_crop.so
│ │ ├── build.py
│ │ ├── functions
│ │ │ ├── __init__.py
│ │ │ ├── __pycache__
│ │ │ │ ├── __init__.cpython-36.pyc
│ │ │ │ └── roi_crop.cpython-36.pyc
│ │ │ ├── crop_resize.py
│ │ │ ├── gridgen.py
│ │ │ └── roi_crop.py
│ │ ├── make.sh
│ │ ├── modules
│ │ │ ├── __init__.py
│ │ │ ├── __pycache__
│ │ │ │ ├── __init__.cpython-36.pyc
│ │ │ │ └── roi_crop.cpython-36.pyc
│ │ │ ├── gridgen.py
│ │ │ └── roi_crop.py
│ │ └── src
│ │ │ ├── roi_crop.c
│ │ │ ├── roi_crop.h
│ │ │ ├── roi_crop_cuda.c
│ │ │ ├── roi_crop_cuda.h
│ │ │ ├── roi_crop_cuda_kernel.cu
│ │ │ ├── roi_crop_cuda_kernel.cu.o
│ │ │ └── roi_crop_cuda_kernel.h
│ ├── roi_pooling
│ │ ├── __init__.py
│ │ ├── __pycache__
│ │ │ └── __init__.cpython-36.pyc
│ │ ├── _ext
│ │ │ ├── __init__.py
│ │ │ ├── __pycache__
│ │ │ │ └── __init__.cpython-36.pyc
│ │ │ └── roi_pooling
│ │ │ │ ├── __init__.py
│ │ │ │ ├── __pycache__
│ │ │ │ └── __init__.cpython-36.pyc
│ │ │ │ └── _roi_pooling.so
│ │ ├── build.py
│ │ ├── functions
│ │ │ ├── __init__.py
│ │ │ ├── __pycache__
│ │ │ │ ├── __init__.cpython-36.pyc
│ │ │ │ └── roi_pool.cpython-36.pyc
│ │ │ └── roi_pool.py
│ │ ├── modules
│ │ │ ├── __init__.py
│ │ │ ├── __pycache__
│ │ │ │ ├── __init__.cpython-36.pyc
│ │ │ │ └── roi_pool.cpython-36.pyc
│ │ │ └── roi_pool.py
│ │ └── src
│ │ │ ├── roi_pooling.c
│ │ │ ├── roi_pooling.cu.o
│ │ │ ├── roi_pooling.h
│ │ │ ├── roi_pooling_cuda.c
│ │ │ ├── roi_pooling_cuda.h
│ │ │ ├── roi_pooling_kernel.cu
│ │ │ └── roi_pooling_kernel.h
│ ├── rpn
│ │ ├── __init__.py
│ │ ├── __pycache__
│ │ │ ├── __init__.cpython-36.pyc
│ │ │ ├── anchor_target_layer.cpython-36.pyc
│ │ │ ├── bbox_transform.cpython-36.pyc
│ │ │ ├── generate_anchors.cpython-36.pyc
│ │ │ ├── proposal_layer.cpython-36.pyc
│ │ │ ├── proposal_target_layer_cascade.cpython-36.pyc
│ │ │ └── rpn.cpython-36.pyc
│ │ ├── anchor_target_layer.py
│ │ ├── bbox_transform.py
│ │ ├── generate_anchors.py
│ │ ├── proposal_layer.py
│ │ ├── proposal_target_layer_cascade.py
│ │ └── rpn.py
│ └── utils
│ │ ├── .gitignore
│ │ ├── __init__.py
│ │ ├── __pycache__
│ │ ├── __init__.cpython-36.pyc
│ │ ├── blob.cpython-36.pyc
│ │ ├── config.cpython-36.pyc
│ │ └── net_utils.cpython-36.pyc
│ │ ├── bbox.pyx
│ │ ├── blob.py
│ │ ├── config.py
│ │ ├── logger.py
│ │ └── net_utils.py
├── pycocotools
│ ├── UPSTREAM_REV
│ ├── __init__.py
│ ├── __pycache__
│ │ ├── __init__.cpython-36.pyc
│ │ ├── coco.cpython-36.pyc
│ │ ├── cocoeval.cpython-36.pyc
│ │ └── mask.cpython-36.pyc
│ ├── _mask.c
│ ├── _mask.cpython-36m-x86_64-linux-gnu.so
│ ├── _mask.pyx
│ ├── coco.py
│ ├── cocoeval.py
│ ├── license.txt
│ ├── mask.py
│ ├── maskApi.c
│ └── maskApi.h
├── roi_data_layer
│ ├── __init__.py
│ ├── __pycache__
│ │ ├── __init__.cpython-36.pyc
│ │ ├── minibatch.cpython-36.pyc
│ │ ├── roibatchLoader.cpython-36.pyc
│ │ └── roidb.cpython-36.pyc
│ ├── minibatch.py
│ ├── roibatchLoader.py
│ └── roidb.py
└── setup.py
├── model.png
├── models
├── .gitignore
├── pretrained_models
└── vidvrd
│ ├── visual_bbox_trans_temp2-ground-6.ckpt
│ └── visual_bbox_trans_temp2-reconstruct-6.ckpt
├── networks
├── basic.py
└── relation2relation.py
├── requirements.txt
├── tools
├── sample_video_feature.py
├── util.py
└── word_feature.py
├── tube.py
└── utils.py
/.idea/encodings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/vRGV.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
2 | #include
3 |
4 | int roi_pooling_forward(int pooled_height, int pooled_width, float spatial_scale,
5 | THFloatTensor * features, THFloatTensor * rois, THFloatTensor * output)
6 | {
7 | // Grab the input tensor
8 | float * data_flat = THFloatTensor_data(features);
9 | float * rois_flat = THFloatTensor_data(rois);
10 |
11 | float * output_flat = THFloatTensor_data(output);
12 |
13 | // Number of ROIs
14 | int num_rois = THFloatTensor_size(rois, 0);
15 | int size_rois = THFloatTensor_size(rois, 1);
16 | // batch size
17 | int batch_size = THFloatTensor_size(features, 0);
18 | if(batch_size != 1)
19 | {
20 | return 0;
21 | }
22 | // data height
23 | int data_height = THFloatTensor_size(features, 1);
24 | // data width
25 | int data_width = THFloatTensor_size(features, 2);
26 | // Number of channels
27 | int num_channels = THFloatTensor_size(features, 3);
28 |
29 | // Set all element of the output tensor to -inf.
30 | THFloatStorage_fill(THFloatTensor_storage(output), -1);
31 |
32 | // For each ROI R = [batch_index x1 y1 x2 y2]: max pool over R
33 | int index_roi = 0;
34 | int index_output = 0;
35 | int n;
36 | for (n = 0; n < num_rois; ++n)
37 | {
38 | int roi_batch_ind = rois_flat[index_roi + 0];
39 | int roi_start_w = round(rois_flat[index_roi + 1] * spatial_scale);
40 | int roi_start_h = round(rois_flat[index_roi + 2] * spatial_scale);
41 | int roi_end_w = round(rois_flat[index_roi + 3] * spatial_scale);
42 | int roi_end_h = round(rois_flat[index_roi + 4] * spatial_scale);
43 | // CHECK_GE(roi_batch_ind, 0);
44 | // CHECK_LT(roi_batch_ind, batch_size);
45 |
46 | int roi_height = fmaxf(roi_end_h - roi_start_h + 1, 1);
47 | int roi_width = fmaxf(roi_end_w - roi_start_w + 1, 1);
48 | float bin_size_h = (float)(roi_height) / (float)(pooled_height);
49 | float bin_size_w = (float)(roi_width) / (float)(pooled_width);
50 |
51 | int index_data = roi_batch_ind * data_height * data_width * num_channels;
52 | const int output_area = pooled_width * pooled_height;
53 |
54 | int c, ph, pw;
55 | for (ph = 0; ph < pooled_height; ++ph)
56 | {
57 | for (pw = 0; pw < pooled_width; ++pw)
58 | {
59 | int hstart = (floor((float)(ph) * bin_size_h));
60 | int wstart = (floor((float)(pw) * bin_size_w));
61 | int hend = (ceil((float)(ph + 1) * bin_size_h));
62 | int wend = (ceil((float)(pw + 1) * bin_size_w));
63 |
64 | hstart = fminf(fmaxf(hstart + roi_start_h, 0), data_height);
65 | hend = fminf(fmaxf(hend + roi_start_h, 0), data_height);
66 | wstart = fminf(fmaxf(wstart + roi_start_w, 0), data_width);
67 | wend = fminf(fmaxf(wend + roi_start_w, 0), data_width);
68 |
69 | const int pool_index = index_output + (ph * pooled_width + pw);
70 | int is_empty = (hend <= hstart) || (wend <= wstart);
71 | if (is_empty)
72 | {
73 | for (c = 0; c < num_channels * output_area; c += output_area)
74 | {
75 | output_flat[pool_index + c] = 0;
76 | }
77 | }
78 | else
79 | {
80 | int h, w, c;
81 | for (h = hstart; h < hend; ++h)
82 | {
83 | for (w = wstart; w < wend; ++w)
84 | {
85 | for (c = 0; c < num_channels; ++c)
86 | {
87 | const int index = (h * data_width + w) * num_channels + c;
88 | if (data_flat[index_data + index] > output_flat[pool_index + c * output_area])
89 | {
90 | output_flat[pool_index + c * output_area] = data_flat[index_data + index];
91 | }
92 | }
93 | }
94 | }
95 | }
96 | }
97 | }
98 |
99 | // Increment ROI index
100 | index_roi += size_rois;
101 | index_output += pooled_height * pooled_width * num_channels;
102 | }
103 | return 1;
104 | }
--------------------------------------------------------------------------------
/lib/model/roi_pooling/src/roi_pooling.cu.o:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/model/roi_pooling/src/roi_pooling.cu.o
--------------------------------------------------------------------------------
/lib/model/roi_pooling/src/roi_pooling.h:
--------------------------------------------------------------------------------
1 | int roi_pooling_forward(int pooled_height, int pooled_width, float spatial_scale,
2 | THFloatTensor * features, THFloatTensor * rois, THFloatTensor * output);
--------------------------------------------------------------------------------
/lib/model/roi_pooling/src/roi_pooling_cuda.c:
--------------------------------------------------------------------------------
1 | #include
2 | #include
3 | #include "roi_pooling_kernel.h"
4 |
5 | extern THCState *state;
6 |
7 | int roi_pooling_forward_cuda(int pooled_height, int pooled_width, float spatial_scale,
8 | THCudaTensor * features, THCudaTensor * rois, THCudaTensor * output, THCudaIntTensor * argmax)
9 | {
10 | // Grab the input tensor
11 | float * data_flat = THCudaTensor_data(state, features);
12 | float * rois_flat = THCudaTensor_data(state, rois);
13 |
14 | float * output_flat = THCudaTensor_data(state, output);
15 | int * argmax_flat = THCudaIntTensor_data(state, argmax);
16 |
17 | // Number of ROIs
18 | int num_rois = THCudaTensor_size(state, rois, 0);
19 | int size_rois = THCudaTensor_size(state, rois, 1);
20 | if (size_rois != 5)
21 | {
22 | return 0;
23 | }
24 |
25 | // batch size
26 | // int batch_size = THCudaTensor_size(state, features, 0);
27 | // if (batch_size != 1)
28 | // {
29 | // return 0;
30 | // }
31 | // data height
32 | int data_height = THCudaTensor_size(state, features, 2);
33 | // data width
34 | int data_width = THCudaTensor_size(state, features, 3);
35 | // Number of channels
36 | int num_channels = THCudaTensor_size(state, features, 1);
37 |
38 | cudaStream_t stream = THCState_getCurrentStream(state);
39 |
40 | ROIPoolForwardLaucher(
41 | data_flat, spatial_scale, num_rois, data_height,
42 | data_width, num_channels, pooled_height,
43 | pooled_width, rois_flat,
44 | output_flat, argmax_flat, stream);
45 |
46 | return 1;
47 | }
48 |
49 | int roi_pooling_backward_cuda(int pooled_height, int pooled_width, float spatial_scale,
50 | THCudaTensor * top_grad, THCudaTensor * rois, THCudaTensor * bottom_grad, THCudaIntTensor * argmax)
51 | {
52 | // Grab the input tensor
53 | float * top_grad_flat = THCudaTensor_data(state, top_grad);
54 | float * rois_flat = THCudaTensor_data(state, rois);
55 |
56 | float * bottom_grad_flat = THCudaTensor_data(state, bottom_grad);
57 | int * argmax_flat = THCudaIntTensor_data(state, argmax);
58 |
59 | // Number of ROIs
60 | int num_rois = THCudaTensor_size(state, rois, 0);
61 | int size_rois = THCudaTensor_size(state, rois, 1);
62 | if (size_rois != 5)
63 | {
64 | return 0;
65 | }
66 |
67 | // batch size
68 | int batch_size = THCudaTensor_size(state, bottom_grad, 0);
69 | // if (batch_size != 1)
70 | // {
71 | // return 0;
72 | // }
73 | // data height
74 | int data_height = THCudaTensor_size(state, bottom_grad, 2);
75 | // data width
76 | int data_width = THCudaTensor_size(state, bottom_grad, 3);
77 | // Number of channels
78 | int num_channels = THCudaTensor_size(state, bottom_grad, 1);
79 |
80 | cudaStream_t stream = THCState_getCurrentStream(state);
81 | ROIPoolBackwardLaucher(
82 | top_grad_flat, spatial_scale, batch_size, num_rois, data_height,
83 | data_width, num_channels, pooled_height,
84 | pooled_width, rois_flat,
85 | bottom_grad_flat, argmax_flat, stream);
86 |
87 | return 1;
88 | }
89 |
--------------------------------------------------------------------------------
/lib/model/roi_pooling/src/roi_pooling_cuda.h:
--------------------------------------------------------------------------------
1 | int roi_pooling_forward_cuda(int pooled_height, int pooled_width, float spatial_scale,
2 | THCudaTensor * features, THCudaTensor * rois, THCudaTensor * output, THCudaIntTensor * argmax);
3 |
4 | int roi_pooling_backward_cuda(int pooled_height, int pooled_width, float spatial_scale,
5 | THCudaTensor * top_grad, THCudaTensor * rois, THCudaTensor * bottom_grad, THCudaIntTensor * argmax);
--------------------------------------------------------------------------------
/lib/model/roi_pooling/src/roi_pooling_kernel.h:
--------------------------------------------------------------------------------
1 | #ifndef _ROI_POOLING_KERNEL
2 | #define _ROI_POOLING_KERNEL
3 |
4 | #ifdef __cplusplus
5 | extern "C" {
6 | #endif
7 |
8 | int ROIPoolForwardLaucher(
9 | const float* bottom_data, const float spatial_scale, const int num_rois, const int height,
10 | const int width, const int channels, const int pooled_height,
11 | const int pooled_width, const float* bottom_rois,
12 | float* top_data, int* argmax_data, cudaStream_t stream);
13 |
14 |
15 | int ROIPoolBackwardLaucher(const float* top_diff, const float spatial_scale, const int batch_size, const int num_rois,
16 | const int height, const int width, const int channels, const int pooled_height,
17 | const int pooled_width, const float* bottom_rois,
18 | float* bottom_diff, const int* argmax_data, cudaStream_t stream);
19 |
20 | #ifdef __cplusplus
21 | }
22 | #endif
23 |
24 | #endif
25 |
26 |
--------------------------------------------------------------------------------
/lib/model/rpn/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/model/rpn/__init__.py
--------------------------------------------------------------------------------
/lib/model/rpn/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/model/rpn/__pycache__/__init__.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/model/rpn/__pycache__/anchor_target_layer.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/model/rpn/__pycache__/anchor_target_layer.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/model/rpn/__pycache__/bbox_transform.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/model/rpn/__pycache__/bbox_transform.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/model/rpn/__pycache__/generate_anchors.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/model/rpn/__pycache__/generate_anchors.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/model/rpn/__pycache__/proposal_layer.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/model/rpn/__pycache__/proposal_layer.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/model/rpn/__pycache__/proposal_target_layer_cascade.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/model/rpn/__pycache__/proposal_target_layer_cascade.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/model/rpn/__pycache__/rpn.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/model/rpn/__pycache__/rpn.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/model/rpn/generate_anchors.py:
--------------------------------------------------------------------------------
1 | from __future__ import print_function
2 | # --------------------------------------------------------
3 | # Faster R-CNN
4 | # Copyright (c) 2015 Microsoft
5 | # Licensed under The MIT License [see LICENSE for details]
6 | # Written by Ross Girshick and Sean Bell
7 | # --------------------------------------------------------
8 |
9 | import numpy as np
10 | import pdb
11 |
12 | # Verify that we compute the same anchors as Shaoqing's matlab implementation:
13 | #
14 | # >> load output/rpn_cachedir/faster_rcnn_VOC2007_ZF_stage1_rpn/anchors.mat
15 | # >> anchors
16 | #
17 | # anchors =
18 | #
19 | # -83 -39 100 56
20 | # -175 -87 192 104
21 | # -359 -183 376 200
22 | # -55 -55 72 72
23 | # -119 -119 136 136
24 | # -247 -247 264 264
25 | # -35 -79 52 96
26 | # -79 -167 96 184
27 | # -167 -343 184 360
28 |
29 | #array([[ -83., -39., 100., 56.],
30 | # [-175., -87., 192., 104.],
31 | # [-359., -183., 376., 200.],
32 | # [ -55., -55., 72., 72.],
33 | # [-119., -119., 136., 136.],
34 | # [-247., -247., 264., 264.],
35 | # [ -35., -79., 52., 96.],
36 | # [ -79., -167., 96., 184.],
37 | # [-167., -343., 184., 360.]])
38 |
39 | try:
40 | xrange # Python 2
41 | except NameError:
42 | xrange = range # Python 3
43 |
44 |
45 | def generate_anchors(base_size=16, ratios=[0.5, 1, 2],
46 | scales=2**np.arange(3, 6)):
47 | """
48 | Generate anchor (reference) windows by enumerating aspect ratios X
49 | scales wrt a reference (0, 0, 15, 15) window.
50 | """
51 |
52 | base_anchor = np.array([1, 1, base_size, base_size]) - 1
53 | ratio_anchors = _ratio_enum(base_anchor, ratios)
54 | anchors = np.vstack([_scale_enum(ratio_anchors[i, :], scales)
55 | for i in xrange(ratio_anchors.shape[0])])
56 | return anchors
57 |
58 | def _whctrs(anchor):
59 | """
60 | Return width, height, x center, and y center for an anchor (window).
61 | """
62 |
63 | w = anchor[2] - anchor[0] + 1
64 | h = anchor[3] - anchor[1] + 1
65 | x_ctr = anchor[0] + 0.5 * (w - 1)
66 | y_ctr = anchor[1] + 0.5 * (h - 1)
67 | return w, h, x_ctr, y_ctr
68 |
69 | def _mkanchors(ws, hs, x_ctr, y_ctr):
70 | """
71 | Given a vector of widths (ws) and heights (hs) around a center
72 | (x_ctr, y_ctr), output a set of anchors (windows).
73 | """
74 |
75 | ws = ws[:, np.newaxis]
76 | hs = hs[:, np.newaxis]
77 | anchors = np.hstack((x_ctr - 0.5 * (ws - 1),
78 | y_ctr - 0.5 * (hs - 1),
79 | x_ctr + 0.5 * (ws - 1),
80 | y_ctr + 0.5 * (hs - 1)))
81 | return anchors
82 |
83 | def _ratio_enum(anchor, ratios):
84 | """
85 | Enumerate a set of anchors for each aspect ratio wrt an anchor.
86 | """
87 |
88 | w, h, x_ctr, y_ctr = _whctrs(anchor)
89 | size = w * h
90 | size_ratios = size / ratios
91 | ws = np.round(np.sqrt(size_ratios))
92 | hs = np.round(ws * ratios)
93 | anchors = _mkanchors(ws, hs, x_ctr, y_ctr)
94 | return anchors
95 |
96 | def _scale_enum(anchor, scales):
97 | """
98 | Enumerate a set of anchors for each scale wrt an anchor.
99 | """
100 |
101 | w, h, x_ctr, y_ctr = _whctrs(anchor)
102 | ws = w * scales
103 | hs = h * scales
104 | anchors = _mkanchors(ws, hs, x_ctr, y_ctr)
105 | return anchors
106 |
107 | if __name__ == '__main__':
108 | import time
109 | t = time.time()
110 | a = generate_anchors()
111 | print(time.time() - t)
112 | print(a)
113 | from IPython import embed; embed()
114 |
--------------------------------------------------------------------------------
/lib/model/rpn/rpn.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | import torch
3 | import torch.nn as nn
4 | import torch.nn.functional as F
5 | from torch.autograd import Variable
6 |
7 | from model.utils.config import cfg
8 | from .proposal_layer import _ProposalLayer
9 | from .anchor_target_layer import _AnchorTargetLayer
10 | from model.utils.net_utils import _smooth_l1_loss
11 |
12 | import numpy as np
13 | import math
14 | import pdb
15 | import time
16 |
17 | class _RPN(nn.Module):
18 | """ region proposal network """
19 | def __init__(self, din):
20 | super(_RPN, self).__init__()
21 |
22 | self.din = din # get depth of input feature map, e.g., 512
23 | self.anchor_scales = cfg.ANCHOR_SCALES
24 | self.anchor_ratios = cfg.ANCHOR_RATIOS
25 | self.feat_stride = cfg.FEAT_STRIDE[0]
26 |
27 | # define the convrelu layers processing input feature map
28 | self.RPN_Conv = nn.Conv2d(self.din, 512, 3, 1, 1, bias=True)
29 |
30 | # define bg/fg classifcation score layer
31 | self.nc_score_out = len(self.anchor_scales) * len(self.anchor_ratios) * 2 # 2(bg/fg) * 9 (anchors)
32 | self.RPN_cls_score = nn.Conv2d(512, self.nc_score_out, 1, 1, 0)
33 |
34 | # define anchor box offset prediction layer
35 | self.nc_bbox_out = len(self.anchor_scales) * len(self.anchor_ratios) * 4 # 4(coords) * 9 (anchors)
36 | self.RPN_bbox_pred = nn.Conv2d(512, self.nc_bbox_out, 1, 1, 0)
37 |
38 | # define proposal layer
39 | self.RPN_proposal = _ProposalLayer(self.feat_stride, self.anchor_scales, self.anchor_ratios)
40 |
41 | # define anchor target layer
42 | self.RPN_anchor_target = _AnchorTargetLayer(self.feat_stride, self.anchor_scales, self.anchor_ratios)
43 |
44 | self.rpn_loss_cls = 0
45 | self.rpn_loss_box = 0
46 |
47 | @staticmethod
48 | def reshape(x, d):
49 | input_shape = x.size()
50 | x = x.view(
51 | input_shape[0],
52 | int(d),
53 | int(float(input_shape[1] * input_shape[2]) / float(d)),
54 | input_shape[3]
55 | )
56 | return x
57 |
58 | def forward(self, base_feat, im_info, gt_boxes, num_boxes):
59 |
60 | batch_size = base_feat.size(0)
61 |
62 | # return feature map after convrelu layer
63 | rpn_conv1 = F.relu(self.RPN_Conv(base_feat), inplace=True)
64 | # get rpn classification score
65 | rpn_cls_score = self.RPN_cls_score(rpn_conv1)
66 |
67 | rpn_cls_score_reshape = self.reshape(rpn_cls_score, 2)
68 | rpn_cls_prob_reshape = F.softmax(rpn_cls_score_reshape, 1)
69 | rpn_cls_prob = self.reshape(rpn_cls_prob_reshape, self.nc_score_out)
70 |
71 | # get rpn offsets to the anchor boxes
72 | rpn_bbox_pred = self.RPN_bbox_pred(rpn_conv1)
73 |
74 | # proposal layer
75 | cfg_key = 'TRAIN' if self.training else 'TEST'
76 |
77 | rois = self.RPN_proposal((rpn_cls_prob.data, rpn_bbox_pred.data,
78 | im_info, cfg_key))
79 |
80 | self.rpn_loss_cls = 0
81 | self.rpn_loss_box = 0
82 |
83 | # generating training labels and build the rpn loss
84 | if self.training:
85 | assert gt_boxes is not None
86 |
87 | rpn_data = self.RPN_anchor_target((rpn_cls_score.data, gt_boxes, im_info, num_boxes))
88 |
89 | # compute classification loss
90 | rpn_cls_score = rpn_cls_score_reshape.permute(0, 2, 3, 1).contiguous().view(batch_size, -1, 2)
91 | rpn_label = rpn_data[0].view(batch_size, -1)
92 |
93 | rpn_keep = Variable(rpn_label.view(-1).ne(-1).nonzero().view(-1))
94 | rpn_cls_score = torch.index_select(rpn_cls_score.view(-1,2), 0, rpn_keep)
95 | rpn_label = torch.index_select(rpn_label.view(-1), 0, rpn_keep.data)
96 | rpn_label = Variable(rpn_label.long())
97 | self.rpn_loss_cls = F.cross_entropy(rpn_cls_score, rpn_label)
98 | fg_cnt = torch.sum(rpn_label.data.ne(0))
99 |
100 | rpn_bbox_targets, rpn_bbox_inside_weights, rpn_bbox_outside_weights = rpn_data[1:]
101 |
102 | # compute bbox regression loss
103 | rpn_bbox_inside_weights = Variable(rpn_bbox_inside_weights)
104 | rpn_bbox_outside_weights = Variable(rpn_bbox_outside_weights)
105 | rpn_bbox_targets = Variable(rpn_bbox_targets)
106 |
107 | self.rpn_loss_box = _smooth_l1_loss(rpn_bbox_pred, rpn_bbox_targets, rpn_bbox_inside_weights,
108 | rpn_bbox_outside_weights, sigma=3, dim=[1,2,3])
109 |
110 | return rois, self.rpn_loss_cls, self.rpn_loss_box
111 |
--------------------------------------------------------------------------------
/lib/model/utils/.gitignore:
--------------------------------------------------------------------------------
1 | *.c
2 | *.cpp
3 | *.so
4 |
--------------------------------------------------------------------------------
/lib/model/utils/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/model/utils/__init__.py
--------------------------------------------------------------------------------
/lib/model/utils/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/model/utils/__pycache__/__init__.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/model/utils/__pycache__/blob.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/model/utils/__pycache__/blob.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/model/utils/__pycache__/config.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/model/utils/__pycache__/config.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/model/utils/__pycache__/net_utils.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/model/utils/__pycache__/net_utils.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/model/utils/bbox.pyx:
--------------------------------------------------------------------------------
1 | # --------------------------------------------------------
2 | # Fast R-CNN
3 | # Copyright (c) 2015 Microsoft
4 | # Licensed under The MIT License [see LICENSE for details]
5 | # Written by Sergey Karayev
6 | # --------------------------------------------------------
7 |
8 | cimport cython
9 | import numpy as np
10 | cimport numpy as np
11 |
12 | DTYPE = np.float
13 | ctypedef np.float_t DTYPE_t
14 |
15 | def bbox_overlaps(np.ndarray[DTYPE_t, ndim=2] boxes,
16 | np.ndarray[DTYPE_t, ndim=2] query_boxes):
17 | return bbox_overlaps_c(boxes, query_boxes)
18 |
19 | cdef np.ndarray[DTYPE_t, ndim=2] bbox_overlaps_c(
20 | np.ndarray[DTYPE_t, ndim=2] boxes,
21 | np.ndarray[DTYPE_t, ndim=2] query_boxes):
22 | """
23 | Parameters
24 | ----------
25 | boxes: (N, 4) ndarray of float
26 | query_boxes: (K, 4) ndarray of float
27 | Returns
28 | -------
29 | overlaps: (N, K) ndarray of overlap between boxes and query_boxes
30 | """
31 | cdef unsigned int N = boxes.shape[0]
32 | cdef unsigned int K = query_boxes.shape[0]
33 | cdef np.ndarray[DTYPE_t, ndim=2] overlaps = np.zeros((N, K), dtype=DTYPE)
34 | cdef DTYPE_t iw, ih, box_area
35 | cdef DTYPE_t ua
36 | cdef unsigned int k, n
37 | for k in range(K):
38 | box_area = (
39 | (query_boxes[k, 2] - query_boxes[k, 0] + 1) *
40 | (query_boxes[k, 3] - query_boxes[k, 1] + 1)
41 | )
42 | for n in range(N):
43 | iw = (
44 | min(boxes[n, 2], query_boxes[k, 2]) -
45 | max(boxes[n, 0], query_boxes[k, 0]) + 1
46 | )
47 | if iw > 0:
48 | ih = (
49 | min(boxes[n, 3], query_boxes[k, 3]) -
50 | max(boxes[n, 1], query_boxes[k, 1]) + 1
51 | )
52 | if ih > 0:
53 | ua = float(
54 | (boxes[n, 2] - boxes[n, 0] + 1) *
55 | (boxes[n, 3] - boxes[n, 1] + 1) +
56 | box_area - iw * ih
57 | )
58 | overlaps[n, k] = iw * ih / ua
59 | return overlaps
60 |
61 |
62 | def bbox_intersections(
63 | np.ndarray[DTYPE_t, ndim=2] boxes,
64 | np.ndarray[DTYPE_t, ndim=2] query_boxes):
65 | return bbox_intersections_c(boxes, query_boxes)
66 |
67 |
68 | cdef np.ndarray[DTYPE_t, ndim=2] bbox_intersections_c(
69 | np.ndarray[DTYPE_t, ndim=2] boxes,
70 | np.ndarray[DTYPE_t, ndim=2] query_boxes):
71 | """
72 | For each query box compute the intersection ratio covered by boxes
73 | ----------
74 | Parameters
75 | ----------
76 | boxes: (N, 4) ndarray of float
77 | query_boxes: (K, 4) ndarray of float
78 | Returns
79 | -------
80 | overlaps: (N, K) ndarray of intersec between boxes and query_boxes
81 | """
82 | cdef unsigned int N = boxes.shape[0]
83 | cdef unsigned int K = query_boxes.shape[0]
84 | cdef np.ndarray[DTYPE_t, ndim=2] intersec = np.zeros((N, K), dtype=DTYPE)
85 | cdef DTYPE_t iw, ih, box_area
86 | cdef DTYPE_t ua
87 | cdef unsigned int k, n
88 | for k in range(K):
89 | box_area = (
90 | (query_boxes[k, 2] - query_boxes[k, 0] + 1) *
91 | (query_boxes[k, 3] - query_boxes[k, 1] + 1)
92 | )
93 | for n in range(N):
94 | iw = (
95 | min(boxes[n, 2], query_boxes[k, 2]) -
96 | max(boxes[n, 0], query_boxes[k, 0]) + 1
97 | )
98 | if iw > 0:
99 | ih = (
100 | min(boxes[n, 3], query_boxes[k, 3]) -
101 | max(boxes[n, 1], query_boxes[k, 1]) + 1
102 | )
103 | if ih > 0:
104 | intersec[n, k] = iw * ih / box_area
105 | return intersec
--------------------------------------------------------------------------------
/lib/model/utils/blob.py:
--------------------------------------------------------------------------------
1 | # --------------------------------------------------------
2 | # Fast R-CNN
3 | # Copyright (c) 2015 Microsoft
4 | # Licensed under The MIT License [see LICENSE for details]
5 | # Written by Ross Girshick
6 | # --------------------------------------------------------
7 |
8 | """Blob helper functions."""
9 |
10 | import numpy as np
11 | # from scipy.misc import imread, imresize
12 | import cv2
13 |
14 | try:
15 | xrange # Python 2
16 | except NameError:
17 | xrange = range # Python 3
18 |
19 |
20 | def im_list_to_blob(ims):
21 | """Convert a list of images into a network input.
22 |
23 | Assumes images are already prepared (means subtracted, BGR order, ...).
24 | """
25 | max_shape = np.array([im.shape for im in ims]).max(axis=0)
26 | num_images = len(ims)
27 | blob = np.zeros((num_images, max_shape[0], max_shape[1], 3),
28 | dtype=np.float32)
29 | for i in xrange(num_images):
30 | im = ims[i]
31 | blob[i, 0:im.shape[0], 0:im.shape[1], :] = im
32 |
33 | return blob
34 |
35 | def prep_im_for_blob(im, pixel_means, target_size, max_size):
36 | """Mean subtract and scale an image for use in a blob."""
37 |
38 | im = im.astype(np.float32, copy=False)
39 | im -= pixel_means
40 | # im = im[:, :, ::-1]
41 | im_shape = im.shape
42 | im_size_min = np.min(im_shape[0:2])
43 | im_size_max = np.max(im_shape[0:2])
44 | im_scale = float(target_size) / float(im_size_min)
45 | # Prevent the biggest axis from being more than MAX_SIZE
46 | # if np.round(im_scale * im_size_max) > max_size:
47 | # im_scale = float(max_size) / float(im_size_max)
48 | # im = imresize(im, im_scale)
49 | im = cv2.resize(im, None, None, fx=im_scale, fy=im_scale,
50 | interpolation=cv2.INTER_LINEAR)
51 |
52 | return im, im_scale
53 |
--------------------------------------------------------------------------------
/lib/model/utils/logger.py:
--------------------------------------------------------------------------------
1 | # Code referenced from https://gist.github.com/gyglim/1f8dfb1b5c82627ae3efcfbbadb9f514
2 | import tensorflow as tf
3 | import numpy as np
4 | import scipy.misc
5 | try:
6 | from StringIO import StringIO # Python 2.7
7 | except ImportError:
8 | from io import BytesIO # Python 3.x
9 |
10 |
11 | class Logger(object):
12 |
13 | def __init__(self, log_dir):
14 | """Create a summary writer logging to log_dir."""
15 | self.writer = tf.summary.FileWriter(log_dir)
16 |
17 | def scalar_summary(self, tag, value, step):
18 | """Log a scalar variable."""
19 | summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=value)])
20 | self.writer.add_summary(summary, step)
21 |
22 | def image_summary(self, tag, images, step):
23 | """Log a list of images."""
24 |
25 | img_summaries = []
26 | for i, img in enumerate(images):
27 | # Write the image to a string
28 | try:
29 | s = StringIO()
30 | except:
31 | s = BytesIO()
32 | scipy.misc.toimage(img).save(s, format="png")
33 |
34 | # Create an Image object
35 | img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(),
36 | height=img.shape[0],
37 | width=img.shape[1])
38 | # Create a Summary value
39 | img_summaries.append(tf.Summary.Value(tag='%s/%d' % (tag, i), image=img_sum))
40 |
41 | # Create and write Summary
42 | summary = tf.Summary(value=img_summaries)
43 | self.writer.add_summary(summary, step)
44 |
45 | def histo_summary(self, tag, values, step, bins=1000):
46 | """Log a histogram of the tensor of values."""
47 |
48 | # Create a histogram using numpy
49 | counts, bin_edges = np.histogram(values, bins=bins)
50 |
51 | # Fill the fields of the histogram proto
52 | hist = tf.HistogramProto()
53 | hist.min = float(np.min(values))
54 | hist.max = float(np.max(values))
55 | hist.num = int(np.prod(values.shape))
56 | hist.sum = float(np.sum(values))
57 | hist.sum_squares = float(np.sum(values**2))
58 |
59 | # Drop the start of the first bin
60 | bin_edges = bin_edges[1:]
61 |
62 | # Add bin edges and counts
63 | for edge in bin_edges:
64 | hist.bucket_limit.append(edge)
65 | for c in counts:
66 | hist.bucket.append(c)
67 |
68 | # Create and write Summary
69 | summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)])
70 | self.writer.add_summary(summary, step)
71 | self.writer.flush()
72 |
--------------------------------------------------------------------------------
/lib/pycocotools/UPSTREAM_REV:
--------------------------------------------------------------------------------
1 | https://github.com/pdollar/coco/commit/3ac47c77ebd5a1ed4254a98b7fbf2ef4765a3574
2 |
--------------------------------------------------------------------------------
/lib/pycocotools/__init__.py:
--------------------------------------------------------------------------------
1 | __author__ = 'tylin'
2 |
--------------------------------------------------------------------------------
/lib/pycocotools/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/pycocotools/__pycache__/__init__.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/pycocotools/__pycache__/coco.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/pycocotools/__pycache__/coco.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/pycocotools/__pycache__/cocoeval.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/pycocotools/__pycache__/cocoeval.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/pycocotools/__pycache__/mask.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/pycocotools/__pycache__/mask.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/pycocotools/_mask.cpython-36m-x86_64-linux-gnu.so:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/pycocotools/_mask.cpython-36m-x86_64-linux-gnu.so
--------------------------------------------------------------------------------
/lib/pycocotools/license.txt:
--------------------------------------------------------------------------------
1 | Copyright (c) 2014, Piotr Dollar and Tsung-Yi Lin
2 | All rights reserved.
3 |
4 | Redistribution and use in source and binary forms, with or without
5 | modification, are permitted provided that the following conditions are met:
6 |
7 | 1. Redistributions of source code must retain the above copyright notice, this
8 | list of conditions and the following disclaimer.
9 | 2. Redistributions in binary form must reproduce the above copyright notice,
10 | this list of conditions and the following disclaimer in the documentation
11 | and/or other materials provided with the distribution.
12 |
13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
17 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
20 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23 |
24 | The views and conclusions contained in the software and documentation are those
25 | of the authors and should not be interpreted as representing official policies,
26 | either expressed or implied, of the FreeBSD Project.
27 |
--------------------------------------------------------------------------------
/lib/pycocotools/mask.py:
--------------------------------------------------------------------------------
1 | __author__ = 'tsungyi'
2 |
3 | from . import _mask
4 |
5 | # Interface for manipulating masks stored in RLE format.
6 | #
7 | # RLE is a simple yet efficient format for storing binary masks. RLE
8 | # first divides a vector (or vectorized image) into a series of piecewise
9 | # constant regions and then for each piece simply stores the length of
10 | # that piece. For example, given M=[0 0 1 1 1 0 1] the RLE counts would
11 | # be [2 3 1 1], or for M=[1 1 1 1 1 1 0] the counts would be [0 6 1]
12 | # (note that the odd counts are always the numbers of zeros). Instead of
13 | # storing the counts directly, additional compression is achieved with a
14 | # variable bitrate representation based on a common scheme called LEB128.
15 | #
16 | # Compression is greatest given large piecewise constant regions.
17 | # Specifically, the size of the RLE is proportional to the number of
18 | # *boundaries* in M (or for an image the number of boundaries in the y
19 | # direction). Assuming fairly simple shapes, the RLE representation is
20 | # O(sqrt(n)) where n is number of pixels in the object. Hence space usage
21 | # is substantially lower, especially for large simple objects (large n).
22 | #
23 | # Many common operations on masks can be computed directly using the RLE
24 | # (without need for decoding). This includes computations such as area,
25 | # union, intersection, etc. All of these operations are linear in the
26 | # size of the RLE, in other words they are O(sqrt(n)) where n is the area
27 | # of the object. Computing these operations on the original mask is O(n).
28 | # Thus, using the RLE can result in substantial computational savings.
29 | #
30 | # The following API functions are defined:
31 | # encode - Encode binary masks using RLE.
32 | # decode - Decode binary masks encoded via RLE.
33 | # merge - Compute union or intersection of encoded masks.
34 | # iou - Compute intersection over union between masks.
35 | # area - Compute area of encoded masks.
36 | # toBbox - Get bounding boxes surrounding encoded masks.
37 | # frPyObjects - Convert polygon, bbox, and uncompressed RLE to encoded RLE mask.
38 | #
39 | # Usage:
40 | # Rs = encode( masks )
41 | # masks = decode( Rs )
42 | # R = merge( Rs, intersect=false )
43 | # o = iou( dt, gt, iscrowd )
44 | # a = area( Rs )
45 | # bbs = toBbox( Rs )
46 | # Rs = frPyObjects( [pyObjects], h, w )
47 | #
48 | # In the API the following formats are used:
49 | # Rs - [dict] Run-length encoding of binary masks
50 | # R - dict Run-length encoding of binary mask
51 | # masks - [hxwxn] Binary mask(s) (must have type np.ndarray(dtype=uint8) in column-major order)
52 | # iscrowd - [nx1] list of np.ndarray. 1 indicates corresponding gt image has crowd region to ignore
53 | # bbs - [nx4] Bounding box(es) stored as [x y w h]
54 | # poly - Polygon stored as [[x1 y1 x2 y2...],[x1 y1 ...],...] (2D list)
55 | # dt,gt - May be either bounding boxes or encoded masks
56 | # Both poly and bbs are 0-indexed (bbox=[0 0 1 1] encloses first pixel).
57 | #
58 | # Finally, a note about the intersection over union (iou) computation.
59 | # The standard iou of a ground truth (gt) and detected (dt) object is
60 | # iou(gt,dt) = area(intersect(gt,dt)) / area(union(gt,dt))
61 | # For "crowd" regions, we use a modified criteria. If a gt object is
62 | # marked as "iscrowd", we allow a dt to match any subregion of the gt.
63 | # Choosing gt' in the crowd gt that best matches the dt can be done using
64 | # gt'=intersect(dt,gt). Since by definition union(gt',dt)=dt, computing
65 | # iou(gt,dt,iscrowd) = iou(gt',dt) = area(intersect(gt,dt)) / area(dt)
66 | # For crowd gt regions we use this modified criteria above for the iou.
67 | #
68 | # To compile run "python setup.py build_ext --inplace"
69 | # Please do not contact us for help with compiling.
70 | #
71 | # Microsoft COCO Toolbox. version 2.0
72 | # Data, paper, and tutorials available at: http://mscoco.org/
73 | # Code written by Piotr Dollar and Tsung-Yi Lin, 2015.
74 | # Licensed under the Simplified BSD License [see coco/license.txt]
75 |
76 | encode = _mask.encode
77 | decode = _mask.decode
78 | iou = _mask.iou
79 | merge = _mask.merge
80 | area = _mask.area
81 | toBbox = _mask.toBbox
82 | frPyObjects = _mask.frPyObjects
--------------------------------------------------------------------------------
/lib/pycocotools/maskApi.h:
--------------------------------------------------------------------------------
1 | /**************************************************************************
2 | * Microsoft COCO Toolbox. version 2.0
3 | * Data, paper, and tutorials available at: http://mscoco.org/
4 | * Code written by Piotr Dollar and Tsung-Yi Lin, 2015.
5 | * Licensed under the Simplified BSD License [see coco/license.txt]
6 | **************************************************************************/
7 | #pragma once
8 | #include
9 |
10 | typedef unsigned int uint;
11 | typedef unsigned long siz;
12 | typedef unsigned char byte;
13 | typedef double* BB;
14 | typedef struct { siz h, w, m; uint *cnts; } RLE;
15 |
16 | // Initialize/destroy RLE.
17 | void rleInit( RLE *R, siz h, siz w, siz m, uint *cnts );
18 | void rleFree( RLE *R );
19 |
20 | // Initialize/destroy RLE array.
21 | void rlesInit( RLE **R, siz n );
22 | void rlesFree( RLE **R, siz n );
23 |
24 | // Encode binary masks using RLE.
25 | void rleEncode( RLE *R, const byte *mask, siz h, siz w, siz n );
26 |
27 | // Decode binary masks encoded via RLE.
28 | void rleDecode( const RLE *R, byte *mask, siz n );
29 |
30 | // Compute union or intersection of encoded masks.
31 | void rleMerge( const RLE *R, RLE *M, siz n, bool intersect );
32 |
33 | // Compute area of encoded masks.
34 | void rleArea( const RLE *R, siz n, uint *a );
35 |
36 | // Compute intersection over union between masks.
37 | void rleIou( RLE *dt, RLE *gt, siz m, siz n, byte *iscrowd, double *o );
38 |
39 | // Compute intersection over union between bounding boxes.
40 | void bbIou( BB dt, BB gt, siz m, siz n, byte *iscrowd, double *o );
41 |
42 | // Get bounding boxes surrounding encoded masks.
43 | void rleToBbox( const RLE *R, BB bb, siz n );
44 |
45 | // Convert bounding boxes to encoded masks.
46 | void rleFrBbox( RLE *R, const BB bb, siz h, siz w, siz n );
47 |
48 | // Convert polygon to encoded mask.
49 | void rleFrPoly( RLE *R, const double *xy, siz k, siz h, siz w );
50 |
51 | // Get compressed string representation of encoded mask.
52 | char* rleToString( const RLE *R );
53 |
54 | // Convert from compressed string representation of encoded mask.
55 | void rleFrString( RLE *R, char *s, siz h, siz w );
56 |
--------------------------------------------------------------------------------
/lib/roi_data_layer/__init__.py:
--------------------------------------------------------------------------------
1 | # --------------------------------------------------------
2 | # Fast R-CNN
3 | # Copyright (c) 2015 Microsoft
4 | # Licensed under The MIT License [see LICENSE for details]
5 | # Written by Ross Girshick
6 | # --------------------------------------------------------
7 |
--------------------------------------------------------------------------------
/lib/roi_data_layer/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/roi_data_layer/__pycache__/__init__.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/roi_data_layer/__pycache__/minibatch.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/roi_data_layer/__pycache__/minibatch.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/roi_data_layer/__pycache__/roibatchLoader.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/roi_data_layer/__pycache__/roibatchLoader.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/roi_data_layer/__pycache__/roidb.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/lib/roi_data_layer/__pycache__/roidb.cpython-36.pyc
--------------------------------------------------------------------------------
/lib/roi_data_layer/minibatch.py:
--------------------------------------------------------------------------------
1 | # --------------------------------------------------------
2 | # Fast R-CNN
3 | # Copyright (c) 2015 Microsoft
4 | # Licensed under The MIT License [see LICENSE for details]
5 | # Written by Ross Girshick and Xinlei Chen
6 | # --------------------------------------------------------
7 |
8 | """Compute minibatch blobs for training a Fast R-CNN network."""
9 | from __future__ import absolute_import
10 | from __future__ import division
11 | from __future__ import print_function
12 |
13 | import numpy as np
14 | import numpy.random as npr
15 | from scipy.misc import imread
16 | from model.utils.config import cfg
17 | from model.utils.blob import prep_im_for_blob, im_list_to_blob
18 | import pdb
19 | def get_minibatch(roidb, num_classes):
20 | """Given a roidb, construct a minibatch sampled from it."""
21 | num_images = len(roidb)
22 | # Sample random scales to use for each image in this batch
23 | random_scale_inds = npr.randint(0, high=len(cfg.TRAIN.SCALES),
24 | size=num_images)
25 | assert(cfg.TRAIN.BATCH_SIZE % num_images == 0), \
26 | 'num_images ({}) must divide BATCH_SIZE ({})'. \
27 | format(num_images, cfg.TRAIN.BATCH_SIZE)
28 |
29 | # Get the input image blob, formatted for caffe
30 | im_blob, im_scales, im_names = _get_image_blob(roidb, random_scale_inds)
31 |
32 | blobs = {'data': im_blob}
33 |
34 | assert len(im_scales) == 1, "Single batch only"
35 | assert len(roidb) == 1, "Single batch only"
36 |
37 | # gt boxes: (x1, y1, x2, y2, cls)
38 | if cfg.TRAIN.USE_ALL_GT:
39 | # Include all ground truth boxes
40 | gt_inds = np.where(roidb[0]['gt_classes'] != 0)[0]
41 | else:
42 | # For the COCO ground truth boxes, exclude the ones that are ''iscrowd''
43 | gt_inds = np.where((roidb[0]['gt_classes'] != 0) & np.all(roidb[0]['gt_overlaps'].toarray() > -1.0, axis=1))[0]
44 | gt_boxes = np.empty((len(gt_inds), 5), dtype=np.float32)
45 | gt_boxes[:, 0:4] = roidb[0]['boxes'][gt_inds, :] * im_scales[0]
46 | gt_boxes[:, 4] = roidb[0]['gt_classes'][gt_inds]
47 | blobs['gt_boxes'] = gt_boxes
48 | blobs['im_info'] = np.array(
49 | [[im_blob.shape[1], im_blob.shape[2], im_scales[0]]],
50 | dtype=np.float32)
51 |
52 | blobs['img_id'] = roidb[0]['img_id']
53 | blobs['img_name'] = im_names
54 |
55 | return blobs
56 |
57 | def _get_image_blob(roidb, scale_inds):
58 | """Builds an input blob from the images in the roidb at the specified
59 | scales.
60 | """
61 | num_images = len(roidb)
62 |
63 | processed_ims = []
64 | im_scales = []
65 | im_names = []
66 | for i in range(num_images):
67 | #im = cv2.imread(roidb[i]['image'])
68 | im_name = roidb[i]['image']
69 | im = imread(im_name)
70 |
71 | if len(im.shape) == 2:
72 | im = im[:,:,np.newaxis]
73 | im = np.concatenate((im,im,im), axis=2)
74 | # flip the channel, since the original one using cv2
75 | # rgb -> bgr
76 | im = im[:,:,::-1]
77 |
78 | if roidb[i]['flipped']:
79 | im = im[:, ::-1, :]
80 | target_size = cfg.TRAIN.SCALES[scale_inds[i]]
81 | im, im_scale = prep_im_for_blob(im, cfg.PIXEL_MEANS, target_size,
82 | cfg.TRAIN.MAX_SIZE)
83 | im_scales.append(im_scale)
84 | processed_ims.append(im)
85 | im_names.append(im_name)
86 |
87 | # Create a blob to hold the input images
88 | blob = im_list_to_blob(processed_ims)
89 |
90 | return blob, im_scales, im_names
91 |
--------------------------------------------------------------------------------
/lib/roi_data_layer/roidb.py:
--------------------------------------------------------------------------------
1 | """Transform a roidb into a trainable roidb by adding a bunch of metadata."""
2 | from __future__ import absolute_import
3 | from __future__ import division
4 | from __future__ import print_function
5 |
6 | import datasets
7 | import numpy as np
8 | from model.utils.config import cfg
9 | from datasets.factory import get_imdb
10 | import PIL
11 | import pdb
12 |
13 | def prepare_roidb(imdb):
14 | """Enrich the imdb's roidb by adding some derived quantities that
15 | are useful for training. This function precomputes the maximum
16 | overlap, taken over ground-truth boxes, between each ROI and
17 | each ground-truth box. The class with maximum overlap is also
18 | recorded.
19 | """
20 |
21 | roidb = imdb.roidb
22 | if not (imdb.name.startswith('coco')):
23 | sizes = [PIL.Image.open(imdb.image_path_at(i)).size
24 | for i in range(imdb.num_images)]
25 |
26 | for i in range(len(imdb.image_index)):
27 | roidb[i]['img_id'] = imdb.image_id_at(i)
28 | roidb[i]['image'] = imdb.image_path_at(i)
29 | if not (imdb.name.startswith('coco')):
30 | roidb[i]['width'] = sizes[i][0]
31 | roidb[i]['height'] = sizes[i][1]
32 | # need gt_overlaps as a dense array for argmax
33 | gt_overlaps = roidb[i]['gt_overlaps'].toarray()
34 | # max overlap with gt over classes (columns)
35 | max_overlaps = gt_overlaps.max(axis=1)
36 | # gt class that had the max overlap
37 | max_classes = gt_overlaps.argmax(axis=1)
38 | roidb[i]['max_classes'] = max_classes
39 | roidb[i]['max_overlaps'] = max_overlaps
40 | # sanity checks
41 | # max overlap of 0 => class should be zero (background)
42 | zero_inds = np.where(max_overlaps == 0)[0]
43 | assert all(max_classes[zero_inds] == 0)
44 | # max overlap > 0 => class should not be zero (must be a fg class)
45 | nonzero_inds = np.where(max_overlaps > 0)[0]
46 | assert all(max_classes[nonzero_inds] != 0)
47 |
48 |
49 | def rank_roidb_ratio(roidb):
50 | # rank roidb based on the ratio between width and height.
51 | ratio_large = 2 # largest ratio to preserve.
52 | ratio_small = 0.5 # smallest ratio to preserve.
53 |
54 | ratio_list = []
55 | for i in range(len(roidb)):
56 | width = roidb[i]['width']
57 | height = roidb[i]['height']
58 | ratio = width / float(height)
59 |
60 | if ratio > ratio_large:
61 | roidb[i]['need_crop'] = 1
62 | ratio = ratio_large
63 | elif ratio < ratio_small:
64 | roidb[i]['need_crop'] = 1
65 | ratio = ratio_small
66 | else:
67 | roidb[i]['need_crop'] = 0
68 |
69 | ratio_list.append(ratio)
70 |
71 | ratio_list = np.array(ratio_list)
72 | ratio_index = np.argsort(ratio_list)
73 | return ratio_list[ratio_index], ratio_index
74 |
75 | def filter_roidb(roidb):
76 | # filter the image without bounding box.
77 | print('before filtering, there are %d images...' % (len(roidb)))
78 | i = 0
79 | while i < len(roidb):
80 | if len(roidb[i]['boxes']) == 0:
81 | del roidb[i]
82 | i -= 1
83 | i += 1
84 |
85 | print('after filtering, there are %d images...' % (len(roidb)))
86 | return roidb
87 |
88 | def combined_roidb(imdb_names, training=True):
89 | """
90 | Combine multiple roidbs
91 | """
92 |
93 | def get_training_roidb(imdb):
94 | """Returns a roidb (Region of Interest database) for use in training."""
95 | if cfg.TRAIN.USE_FLIPPED:
96 | print('Appending horizontally-flipped training examples...')
97 | imdb.append_flipped_images()
98 | print('done')
99 |
100 | print('Preparing data...')
101 |
102 | prepare_roidb(imdb)
103 | #ratio_index = rank_roidb_ratio(imdb)
104 | print('done')
105 |
106 | return imdb.roidb
107 |
108 | def get_roidb(imdb_name):
109 | imdb = get_imdb(imdb_name)
110 | print('Loaded dataset `{:s}` for training'.format(imdb.name))
111 | imdb.set_proposal_method(cfg.TRAIN.PROPOSAL_METHOD)
112 | print('Set proposal method: {:s}'.format(cfg.TRAIN.PROPOSAL_METHOD))
113 | roidb = get_training_roidb(imdb)
114 | return roidb
115 |
116 | roidbs = [get_roidb(s) for s in imdb_names.split('+')]
117 | roidb = roidbs[0]
118 |
119 | if len(roidbs) > 1:
120 | for r in roidbs[1:]:
121 | roidb.extend(r)
122 | tmp = get_imdb(imdb_names.split('+')[1])
123 | imdb = datasets.imdb.imdb(imdb_names, tmp.classes)
124 | else:
125 | imdb = get_imdb(imdb_names)
126 |
127 | if training:
128 | roidb = filter_roidb(roidb)
129 |
130 | ratio_list, ratio_index = rank_roidb_ratio(roidb)
131 |
132 | return imdb, roidb, ratio_list, ratio_index
133 |
--------------------------------------------------------------------------------
/lib/setup.py:
--------------------------------------------------------------------------------
1 | from __future__ import print_function
2 | # --------------------------------------------------------
3 | # Fast R-CNN
4 | # Copyright (c) 2015 Microsoft
5 | # Licensed under The MIT License [see LICENSE for details]
6 | # Written by Ross Girshick
7 | # --------------------------------------------------------
8 |
9 | import os
10 | from os.path import join as pjoin
11 | import numpy as np
12 | from distutils.core import setup
13 | from distutils.extension import Extension
14 | from Cython.Distutils import build_ext
15 |
16 |
17 | def find_in_path(name, path):
18 | "Find a file in a search path"
19 | # adapted fom http://code.activestate.com/recipes/52224-find-a-file-given-a-search-path/
20 | for dir in path.split(os.pathsep):
21 | binpath = pjoin(dir, name)
22 | if os.path.exists(binpath):
23 | return os.path.abspath(binpath)
24 | return None
25 |
26 |
27 | # def locate_cuda():
28 | # """Locate the CUDA environment on the system
29 | #
30 | # Returns a dict with keys 'home', 'nvcc', 'include', and 'lib64'
31 | # and values giving the absolute path to each directory.
32 | #
33 | # Starts by looking for the CUDAHOME env variable. If not found, everything
34 | # is based on finding 'nvcc' in the PATH.
35 | # """
36 | #
37 | # # first check if the CUDAHOME env variable is in use
38 | # if 'CUDAHOME' in os.environ:
39 | # home = os.environ['CUDAHOME']
40 | # nvcc = pjoin(home, 'bin', 'nvcc')
41 | # else:
42 | # # otherwise, search the PATH for NVCC
43 | # default_path = pjoin(os.sep, 'usr', 'local', 'cuda', 'bin')
44 | # nvcc = find_in_path('nvcc', os.environ['PATH'] + os.pathsep + default_path)
45 | # if nvcc is None:
46 | # raise EnvironmentError('The nvcc binary could not be '
47 | # 'located in your $PATH. Either add it to your path, or set $CUDAHOME')
48 | # home = os.path.dirname(os.path.dirname(nvcc))
49 | #
50 | # cudaconfig = {'home': home, 'nvcc': nvcc,
51 | # 'include': pjoin(home, 'include'),
52 | # 'lib64': pjoin(home, 'lib64')}
53 | # for k, v in cudaconfig.iteritems():
54 | # if not os.path.exists(v):
55 | # raise EnvironmentError('The CUDA %s path could not be located in %s' % (k, v))
56 | #
57 | # return cudaconfig
58 |
59 |
60 | # CUDA = locate_cuda()
61 |
62 | # Obtain the numpy include directory. This logic works across numpy versions.
63 | try:
64 | numpy_include = np.get_include()
65 | except AttributeError:
66 | numpy_include = np.get_numpy_include()
67 |
68 |
69 | def customize_compiler_for_nvcc(self):
70 | """inject deep into distutils to customize how the dispatch
71 | to gcc/nvcc works.
72 |
73 | If you subclass UnixCCompiler, it's not trivial to get your subclass
74 | injected in, and still have the right customizations (i.e.
75 | distutils.sysconfig.customize_compiler) run on it. So instead of going
76 | the OO route, I have this. Note, it's kindof like a wierd functional
77 | subclassing going on."""
78 |
79 | # tell the compiler it can processes .cu
80 | self.src_extensions.append('.cu')
81 |
82 | # save references to the default compiler_so and _comple methods
83 | default_compiler_so = self.compiler_so
84 | super = self._compile
85 |
86 | # now redefine the _compile method. This gets executed for each
87 | # object but distutils doesn't have the ability to change compilers
88 | # based on source extension: we add it.
89 | def _compile(obj, src, ext, cc_args, extra_postargs, pp_opts):
90 | print(extra_postargs)
91 | if os.path.splitext(src)[1] == '.cu':
92 | # use the cuda for .cu files
93 | self.set_executable('compiler_so', CUDA['nvcc'])
94 | # use only a subset of the extra_postargs, which are 1-1 translated
95 | # from the extra_compile_args in the Extension class
96 | postargs = extra_postargs['nvcc']
97 | else:
98 | postargs = extra_postargs['gcc']
99 |
100 | super(obj, src, ext, cc_args, postargs, pp_opts)
101 | # reset the default compiler_so, which we might have changed for cuda
102 | self.compiler_so = default_compiler_so
103 |
104 | # inject our redefined _compile method into the class
105 | self._compile = _compile
106 |
107 |
108 | # run the customize_compiler
109 | class custom_build_ext(build_ext):
110 | def build_extensions(self):
111 | customize_compiler_for_nvcc(self.compiler)
112 | build_ext.build_extensions(self)
113 |
114 |
115 | ext_modules = [
116 | Extension(
117 | "model.utils.cython_bbox",
118 | ["model/utils/bbox.pyx"],
119 | extra_compile_args={'gcc': ["-Wno-cpp", "-Wno-unused-function"]},
120 | include_dirs=[numpy_include]
121 | ),
122 | Extension(
123 | 'pycocotools._mask',
124 | sources=['pycocotools/maskApi.c', 'pycocotools/_mask.pyx'],
125 | include_dirs=[numpy_include, 'pycocotools'],
126 | extra_compile_args={
127 | 'gcc': ['-Wno-cpp', '-Wno-unused-function', '-std=c99']},
128 | ),
129 | ]
130 |
131 | setup(
132 | name='faster_rcnn',
133 | ext_modules=ext_modules,
134 | # inject our custom trigger
135 | cmdclass={'build_ext': custom_build_ext},
136 | )
137 |
--------------------------------------------------------------------------------
/model.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/model.png
--------------------------------------------------------------------------------
/models/.gitignore:
--------------------------------------------------------------------------------
1 | # ignore all except .gitignore file
2 | *
3 | !.gitignore
4 |
--------------------------------------------------------------------------------
/models/pretrained_models:
--------------------------------------------------------------------------------
1 | /storage/jbxiao/workspace/vrelation/models/pretrained_models
--------------------------------------------------------------------------------
/models/vidvrd/visual_bbox_trans_temp2-ground-6.ckpt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/models/vidvrd/visual_bbox_trans_temp2-ground-6.ckpt
--------------------------------------------------------------------------------
/models/vidvrd/visual_bbox_trans_temp2-reconstruct-6.ckpt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/doc-doc/vRGV/805cf996054e5bb295ca3833bcfe34442556382f/models/vidvrd/visual_bbox_trans_temp2-reconstruct-6.ckpt
--------------------------------------------------------------------------------
/networks/basic.py:
--------------------------------------------------------------------------------
1 | # ====================================================
2 | # @Author : Xiao Junbin
3 | # @Email : junbin@comp.nus.edu.sg
4 | # @File : basic.py
5 | # ====================================================
6 | import torch
7 | import torch.nn as nn
8 | from torch.autograd import Variable
9 | import torch.nn.functional as F
10 | from torch.nn.utils.rnn import pack_padded_sequence
11 | import numpy as np
12 | import pickle as pkl
13 |
14 |
15 | class AttHierarchicalGround(nn.Module):
16 |
17 | def __init__(self, input_size, hidden_size, visual_dim, word_dim, num_layers=1):
18 | super(AttHierarchicalGround, self).__init__()
19 |
20 | self.input_size = input_size
21 | self.hidden_size = hidden_size
22 | self.embed_dim = hidden_size // 2
23 |
24 |
25 | self.num_layers = num_layers
26 | self.word_dim = word_dim
27 |
28 | self.max_seg_len = 12
29 | dropout = 0.2
30 |
31 | self.embedding_word = nn.Sequential(nn.Linear(word_dim, self.embed_dim),
32 | nn.ReLU(),
33 | nn.Dropout(dropout))
34 |
35 | self.embedding_visual = nn.Sequential(nn.Linear(visual_dim, hidden_size),
36 | nn.ReLU(),
37 | nn.Dropout(dropout))
38 |
39 | # affine transformation for lstm hidden state
40 | self.linear1 = nn.Linear(hidden_size*2, hidden_size)
41 |
42 | # affine transformation for context
43 | self.linear2 = nn.Linear(hidden_size, 1, bias=False)
44 |
45 | self.transform_visual = nn.Sequential(nn.Linear(hidden_size, hidden_size),
46 | nn.ReLU(),
47 | nn.Dropout(dropout))
48 |
49 | self.relu = nn.ReLU()
50 | self.softmax = nn.Softmax(dim=1)
51 |
52 | self.within_seg_rnn = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)
53 | self.seg_rnn = nn.LSTM(hidden_size, hidden_size, num_layers, batch_first=True)
54 |
55 |
56 | def soft_attention(self, input, context):
57 | """
58 | compute temporal self-attention
59 | :param input: (batch_size, seq_len, feat_dim)
60 | :param context: (batch_size, feat_dim)
61 | :return: vfeat: (batch_size, feat_dim), beta
62 | """
63 | batch_size, seq_len, feat_dim = input.size()
64 | context = context.unsqueeze(1).repeat(1, seq_len, 1)
65 | inputs = torch.cat((input, context), 2).view(-1, feat_dim*2)
66 |
67 | o = self.linear2(torch.tanh(self.linear1(inputs)))
68 | e = o.view(batch_size, seq_len)
69 | beta = self.softmax(e)
70 | vfeat = torch.bmm(beta.unsqueeze(1), input).squeeze(1)
71 |
72 | return vfeat, beta
73 |
74 |
75 | def forward(self, videos, relation_text, mode='train'):
76 | """
77 | Without participation of textual relation, to warm-up the decoder only
78 | """
79 |
80 | frame_count = videos.shape[1]
81 |
82 | max_seg_num = int(frame_count / self.max_seg_len)
83 |
84 | ori_x = self.embedding_visual(videos).sum(dim=2).squeeze()
85 |
86 | x_trans = self.transform_visual(ori_x)
87 |
88 | within_seg_rnn_out, _ = self.within_seg_rnn(x_trans)
89 | self.within_seg_rnn.flatten_parameters()
90 |
91 | idx = np.round(np.linspace(self.max_seg_len-1, frame_count-1, max_seg_num)).astype('int')
92 |
93 | seg_rnn_input = within_seg_rnn_out[:,idx,:]
94 |
95 | seg_out, hidden = self.seg_rnn(seg_rnn_input)
96 | self.seg_rnn.flatten_parameters()
97 |
98 | output, _ = self.soft_attention(within_seg_rnn_out, hidden[0].squeeze(0))
99 |
100 | return output, hidden
101 |
102 |
103 | class DecoderRNN(nn.Module):
104 | def __init__(self, embed_size, hidden_size, vocab_size, num_layers, max_seq_length=10):
105 | """Set the hyper-parameters and build the layers."""
106 | super(DecoderRNN, self).__init__()
107 | self.embed = nn.Embedding(vocab_size, embed_size)
108 | self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True)
109 | self.linear = nn.Linear(hidden_size, vocab_size)
110 | self.max_seq_length = max_seq_length
111 |
112 | def forward(self, video_out, video_hidden, relations, lengths):
113 | """
114 | Decode relation attended video feature and reconstruct the relation.
115 | :param video_out: (batch, seq_len, dim_hidden * num_directions)
116 | :param video_hidden: (num_layers * num_directions, batch_size, dim_hidden)
117 | :param relations:
118 | :param lengths:
119 | :return:
120 | """
121 | embeddings = self.embed(relations)
122 | batch_size, seq_len, _ = embeddings.size()
123 |
124 | embeddings = torch.cat((video_out.unsqueeze(1), embeddings), 1)
125 |
126 | packed = pack_padded_sequence(embeddings, lengths, batch_first=True)
127 |
128 | hiddens, _ = self.lstm(packed, video_hidden)
129 | outputs = self.linear(hiddens[0])
130 |
131 | # print(outputs.shape)
132 | return outputs
133 |
134 | def sample(self, video_out, states=None):
135 | """reconstruct the relation using greedy search"""
136 | sampled_ids = []
137 | inputs = video_out.unsqueeze(1)
138 | for i in range(self.max_seq_length):
139 | hiddens, states = self.lstm(inputs, states)
140 | outputs = self.linear(hiddens.squeeze(1))
141 | _, predicted = outputs.max(1)
142 | sampled_ids.append(predicted)
143 | inputs = self.embed(predicted)
144 | inputs = inputs.unsqueeze(1)
145 |
146 | sampled_ids = torch.stack(sampled_ids, 1)
147 | return sampled_ids
148 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | cython
2 | cffi
3 | certifi==2020.11.8
4 | click==7.1.2
5 | easydict==1.9
6 | joblib==0.17.0
7 | nltk==3.5
8 | numpy==1.19.4
9 | opencv-python==4.4.0.46
10 | Pillow==8.0.1
11 | regex==2020.11.13
12 | torch==0.4.1
13 | tqdm==4.54.0
14 |
--------------------------------------------------------------------------------
/tools/sample_video_feature.py:
--------------------------------------------------------------------------------
1 | import sys
2 | sys.path.insert(0, '../')
3 | from dataloader.util import *
4 |
5 | def sample_video_feature(src, dst, sample_list_file):
6 | nframes, nbbox, feat_dim = 120, 40, 2048+5
7 | samples = load_file(sample_list_file)
8 | sp_num = len(samples)
9 | for it, item in enumerate(samples):
10 | # if it < 24000: continue
11 | # if it >= 24000: break
12 | video_name, frame_count, width, height, relation = item
13 | dst_file = osp.join(dst, video_name)
14 | src_dir = osp.join(src, video_name)
15 | if osp.exists(dst_file+'.npy'):
16 | print('exist {}.npy'.format(dst_file))
17 | continue
18 | get_video_feature(src_dir, dst_file, frame_count, width, height, nbbox, nframes, feat_dim)
19 | if it % 200 == 0:
20 | print(it, sp_num)
21 |
22 |
23 | def main():
24 | dataset = 'vidvrd/'
25 | root_dir = '/path/to/workspace/' #this directory includes two folders: ground_data and vRGV
26 | video_feature_path = osp.join(root_dir, 'ground_data/{}/frame_feature/'.format(dataset))
27 | video_feature_cache = osp.join(root_dir, 'ground_data/{}/video_feature/'.format(dataset))
28 | dset_dir = '../dataset/'+dataset+'/'
29 | train_list = osp.join(dset_dir, 'vrelation_train.json')
30 | val_list = osp.join(dset_dir, 'vrelation_val.json')
31 | sample_video_feature(video_feature_path, video_feature_cache, val_list)
32 | # sample_video_feature(video_feature_path, video_feature_cache, train_list)
33 |
34 |
35 | if __name__ == "__main__":
36 | main()
37 |
--------------------------------------------------------------------------------
/tools/util.py:
--------------------------------------------------------------------------------
1 | # ====================================================
2 | # @Author : Xiao Junbin
3 | # @Email : junbin@comp.nus.edu.sg
4 | # @File : util.py
5 | # ====================================================
6 | import json
7 | import os.path as osp
8 | import os
9 | import shutil
10 | import subprocess
11 |
12 |
13 | def load_file(file_name):
14 |
15 | with open(file_name, 'r') as fp:
16 | if osp.splitext(file_name)[1]== '.txt':
17 | annos = fp.readlines()
18 | annos = [line.rstrip() for line in annos]
19 | if osp.splitext(file_name)[1] == '.json':
20 | annos = json.load(fp)
21 |
22 | return annos
23 |
24 | def set_gpu_devices(gpu_id):
25 | gpu = ''
26 | if gpu_id != -1:
27 | gpu = str(gpu_id)
28 | os.environ['CUDA_VISIBLE_DEVICES'] = gpu
29 |
30 |
31 | def extract_frames(video, dst):
32 | with open(os.devnull, "w") as ffmpeg_log:
33 | if osp.exists(dst):
34 | # print(" cleanup: " + dst + "/")
35 | shutil.rmtree(dst)
36 | os.makedirs(dst)
37 | video_to_frames_command = ["ffmpeg",
38 | # (optional) overwrite output file if it exists
39 | '-y',
40 | '-i', video, # input file
41 | '-qscale:v', "2", # quality for JPEG
42 | '{0}/%06d.jpg'.format(dst)]
43 | subprocess.call(video_to_frames_command,
44 | stdout=ffmpeg_log, stderr=ffmpeg_log)
45 |
--------------------------------------------------------------------------------
/tools/word_feature.py:
--------------------------------------------------------------------------------
1 | # ====================================================
2 | # @Author : Xiao Junbin
3 | # @Email : junbin@comp.nus.edu.sg
4 | # @File : word_feature.py
5 | # ====================================================
6 | import numpy as np
7 | import json
8 | import os.path as osp
9 | import pickle as pkl
10 | from util import load_file
11 |
12 |
13 |
14 |
15 | def word_selection(word_file, feature_file, out_file):
16 | """
17 |
18 | :param word_file:
19 | :param feature_file:
20 | :param out_file:
21 | :return:
22 | """
23 | with open(word_file, 'r') as fp:
24 | word_idx = json.load(fp)
25 |
26 | keys = []
27 | for key, item in word_idx.items():
28 | key = key.split('_')
29 | for k in key:
30 | keys.append(k)
31 | keys = set(keys)
32 | keys = sorted(list(keys))
33 | print(len(keys))
34 | print(keys)
35 |
36 | with open(feature_file, 'r') as fp:
37 | word_glove = fp.readlines()
38 |
39 | word_feature = {}
40 | for wg in word_glove:
41 | wg = wg.rstrip('\n').split(' ')
42 | if wg[0] in keys:
43 | word = wg[0]
44 | feature = np.asarray(wg[1:])
45 | print(feature.size)
46 | word_feature[word] = feature
47 |
48 | with open(out_file, 'wb') as fp:
49 | pkl.dump(word_feature, fp)
50 |
51 | print('finished')
52 |
53 | def main():
54 |
55 | root_dir = '/storage/jbxiao/workspace/'
56 | word_file = osp.join(root_dir, 'ground_code/dataset/vidvrd/word_idx.json')
57 |
58 | feature_file = osp.join(root_dir, 'ground_data/glove/glove.6B.300d.txt')
59 |
60 | out_file = osp.join(root_dir, 'ground_data/glove/vidvrd_word_glove.pkl')
61 |
62 | word_selection(word_file, feature_file, out_file)
63 |
64 |
65 | if __name__ == "__main__":
66 | main()
67 |
--------------------------------------------------------------------------------
/utils.py:
--------------------------------------------------------------------------------
1 | # ====================================================
2 | # @Author : Xiao Junbin
3 | # @Email : junbin@comp.nus.edu.sg
4 | # @File : utils.py
5 | # ====================================================
6 | import json
7 | import os
8 | import os.path as osp
9 | import shutil
10 | import numpy as np
11 | import pickle as pkl
12 |
13 |
14 | def clip_gradient(optimizer, grad_clip):
15 | """
16 | Clips gradients computed during backpropagation to avoid explosion of gradients.
17 | :param optimizer: optimizer with the gradients to be clipped
18 | :param grad_clip: clip value
19 | """
20 | for group in optimizer.param_groups:
21 | for param in group['params']:
22 | if param.grad is not None:
23 | param.grad.data.clamp_(-grad_clip, grad_clip)
24 |
25 |
26 | def save_results(save_name, data):
27 |
28 | print('Save to {}'.format(save_name))
29 |
30 | path = osp.dirname(save_name)
31 | if not osp.exists(path):
32 | os.makedirs(path)
33 |
34 | with open(save_name, 'w') as fp:
35 | json.dump(data, fp)
36 |
37 |
38 | def delete(vname):
39 | if vname != '':
40 | frame_dir = '../ground_data/vidor/frames/'
41 | print('Clean up {}'.format(vname))
42 | shutil.rmtree(osp.join(frame_dir, vname))
43 |
44 | def sort_bbox(bboxes, width, height):
45 | """
46 | sort bbox according to the top-left to bottom-right order
47 | :param bboxes:
48 | :return:
49 | """
50 | x_c = (bboxes[:, 2] - bboxes[:, 0]) / 2
51 | y_c = (bboxes[:, 3] - bboxes[:, 1]) / 2
52 |
53 | points = []
54 | for x, y in zip(x_c, y_c):
55 | points.append((y-1)*width+x)
56 |
57 | index = np.argsort(points)
58 |
59 | return index
60 |
61 |
62 | def pkload(file):
63 | data = None
64 | if osp.exists(file) and osp.getsize(file) > 0:
65 | with open(file, 'rb') as fp:
66 | data = pkl.load(fp)
67 | # print('{} does not exist'.format(file))
68 | return data
69 |
70 |
71 | def pkdump(data, file):
72 | dirname = osp.dirname(file)
73 | if not osp.exists(dirname):
74 | os.makedirs(dirname)
75 | with open(file, 'wb') as fp:
76 | pkl.dump(data, fp)
77 |
78 |
79 |
80 |
81 |
--------------------------------------------------------------------------------
|