├── .gitignore ├── LICENSE ├── README.md ├── REFERENCES.md ├── SETUP.md ├── app ├── app.py ├── assets │ └── default.min.css └── model.py ├── assets ├── CCCar_org_seq_inp_vert.gif ├── CCperson_org_seq_inp_vert.gif ├── Docker-config.png └── Evaluation-Flow.png ├── data └── .gitkeep ├── detect ├── docker │ ├── Dockerfile │ ├── README.md │ ├── build_detector.sh │ ├── run_detector.sh │ └── set_X11.sh └── scripts │ ├── ObjectDetection │ ├── __init__.py │ ├── detect.py │ ├── imutils.py │ └── inpaintRemote.py │ ├── demo.py │ ├── run_detection.py │ ├── test_detect.sh │ ├── test_inpaint_remote.sh │ └── test_sshparamiko.py ├── inpaint ├── docker │ ├── Dockerfile │ ├── build_inpaint.sh │ ├── run_inpainting.sh │ └── set_X11.sh ├── pretrained_models │ └── .gitkeep └── scripts │ ├── test_inpaint_ex_container.sh │ └── test_inpaint_in_container.sh ├── setup ├── _download_googledrive.sh ├── download_inpaint_models.sh ├── setup_network.sh ├── setup_venv.sh └── ssh │ └── sshd_config └── tools ├── convert_frames2video.py ├── convert_video2frames.py └── play_video.py /.gitignore: -------------------------------------------------------------------------------- 1 | # force explicit directory keeping 2 | data/** 3 | !data/.gitkeep 4 | 5 | # vscode 6 | **.vscode 7 | 8 | # general files 9 | .DS_Store 10 | app/static/** 11 | app/upload/** 12 | app/_cache/** 13 | *.ipynb 14 | *.pyc 15 | 16 | # detectron2 files 17 | 18 | # inpainting files 19 | inpaint/pretrained_models/*.pth 20 | inpaint/pretrained_models/*.tar 21 | inpaint/pretrained_models/DAVIS_model/** 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 RexBarker 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # VideoObjectRemoval 2 | ### *Video inpainting with automatic object detection* 3 | --- 4 | 5 | ### Quick Links ### 6 | 7 | - [Setup](./SETUP.md) 8 | 9 | - [Background](./DESCRIPTION.md) 10 | 11 | - [References](./REFERENCES.md) 12 | 13 | ### Overview 14 | In this project, a prototype video editing system based on “inpainting” is demonstrated. Inpainting is an image editing method for replacement of masked regions of an image with a suitable background. The resulting video is thus free from the selected objects and more readily adaptable for further simulation. The system utilizes a three-step approach to simulation: (1) detection, (2) mask grouping, and (3) inpainting. The detection step involves the identification of objects within the video based upon a given object class definition, and the production of pixel level masks. Next, the object masks are grouped and tracked through the frame sequence to determine persistence and allow correction of classified results. Finally, the grouped masks are used to target specific objects instances in the video for inpainting removal. 15 | 16 | The end result of this project is a video editing platform in the context of locomotive route simulation. The final video output demonstrates the system’s ability to automatically remove moving pedestrians in a video sequence, which commonly occur in most street tram simulations. This work also addresses the limitations of the system, in particular, the inability to remove quasi-stationary objects. The overall outcome of the project is a video editing system with automation capabilities rivaling commercial inpainting software. 17 | 18 | ### Project Video 19 | 20 | Video Object Removal Project 23 | 24 | 25 | ### Project Results 26 | 27 | | Result | Description | 28 | | ------ |:------------ | 29 | | | **Single object removal**
- A single vehichle is removed using a conforming mask
- elliptical dilation mask of 21 pixels used
Source: YouTube video 15:16-15:17| 30 | 31 | --- 32 | 33 | | Result | Description | 34 | | ------ |:------------ | 35 | | | **Multiple object removal**
- pedestrians are removed using bounding-box shaped masks
- elliptical dilation mask of 21 pixels used
Source: YouTube video 15:26-15:27 | 36 | 37 | ### Project Setup 38 | 39 | 1. Install NVIDIA docker, if not already installed (see [setup](./SETUP.md) ) 40 | 41 | 2. Follow the instructions from the YouTube video for the specific project setup: 42 | 43 | IMAGE ALT TEXT HERE 46 | 47 | 48 | -------------------------------------------------------------------------------- /REFERENCES.md: -------------------------------------------------------------------------------- 1 | # References 2 | 3 | The following resources were utilized in this project: 4 | 5 | - [1] nbei (R. Xu), “Deep-Flow-Guided-Video-Inpainting”, Git hub repository: https://github.com/nbei/Deep-Flow-Guided-Video-Inpainting 6 | 7 | - [2] nbei (R. Xu), "Deep-Flow-Guided-Video-Inpainting", Model training weights: https://drive.google.com/drive/folders/1a2FrHIQGExJTHXxSIibZOGMukNrypr_g 8 | 9 | - [3] Facebookresearch, Y. Wu, A. Kirillov, F. Massa, W-Y Lo, R. Girshick, “Detectron2”, Git hub repository: https://github.com/facebookresearch/detectron2 10 | 11 | - [4] Facebookresearch, (various authors), “Detectron2 Model Zoo and Baselines”, Git hub repository: https://github.com/facebookresearch/detectron2/blob/master/MODEL_ZOO.md 12 | -------------------------------------------------------------------------------- /SETUP.md: -------------------------------------------------------------------------------- 1 | # Setup 2 | --- 3 | 4 | ### Docker Setup 5 | (based on the [NVIDIA Docker installation guide](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker)) 6 | 7 | Here, the NVIDIA Docker installation for Ubuntu is described – specifically Ubuntu 18.04 LTS and 20.04 LTS. At the time of this report, NVIDIA Docker platform is only supported on Linux. 8 | 9 | **Remove existing older Docker Installations:** 10 | 11 | If there an existing Docker installation, this should upgraded as necessary. This project basis was constructed with Docker v19.03.
12 | `$ sudo apt-get remove docker docker-engine docker.io containerd runc` 13 | 14 | **Install latest Docker engine:** 15 | 16 | The following script can be used to install docker all repositories as required: 17 | ``` 18 | $ curl -fsSL https://get.docker.com -o get-docker.sh 19 | $ sudo sh get-docker.sh 20 | $ sudo sudo systemctl start docker && sudo systemctl enable docker 21 | ``` 22 | 23 | Add user name to docker run group:
24 | `$ sudo usermod -aG docker your-user` 25 | 26 | **Install NVIDIA Docker:** 27 | 28 | These installation instructions are based on the NVIDIA Docker documentation [6]. First, ensure that the appropriate NVIDIA driver is installed on the host system. This can be tested with the following command. This should produce a listing of the current GPU state, along with the current version of the driver: 29 | `$ nvidia-smi` 30 | 31 | If there is an existing earlier version of the NVIDIA Docker system (<=1.0), this must be first uninstalled:
32 | `$ docker volume ls -q -f driver=nvidia-docker | xargs -r -I{} -n1 docker ps -q -a -f volume={} | xargs -r docker rm -f` 33 | 34 | `$ sudo apt-get purge nvidia-docker` 35 | 36 | Set the distribution package RPMs: 37 | ``` 38 | $ distribution=$(. /etc/os-release;echo $ID$VERSION_ID) 39 | $ curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - 40 | $ curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list 41 | ``` 42 | 43 | **Install the nvidia-docker2 package:** 44 | ``` 45 | $ sudo apt-get update 46 | $ sudo apt-get install -y nvidia-docker2 47 | $ sudo systemctl restart docker 48 | ``` 49 | 50 | Test the NVIDIA Docker installation by executing nvidia-smi from within a container:
51 | `$ sudo docker run --rm --gpus all nvidia/cuda:11.0-base nvidia-smi` 52 | 53 | --- 54 | # Project Setup 55 | 56 | Follow the instructions from the YouTube video for the specific project setup: 57 | 58 | IMAGE ALT TEXT HERE 61 | 62 | -------------------------------------------------------------------------------- /app/app.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import base64 4 | from glob import glob 5 | from shutil import rmtree 6 | from io import BytesIO 7 | import time 8 | from datetime import datetime 9 | 10 | #import flask 11 | #from flask import send_file, make_response 12 | #from flask import send_from_directory 13 | from flask_caching import Cache 14 | import dash 15 | import dash_player 16 | import dash_bootstrap_components as dbc 17 | from dash.dependencies import Input, Output, State 18 | import dash_core_components as dcc 19 | import dash_html_components as html 20 | import plotly.graph_objects as go 21 | from PIL import Image 22 | 23 | from model import detect_scores_bboxes_classes, \ 24 | detr, createNullVideo 25 | from model import CLASSES, DEVICE 26 | from model import inpaint, testContainerWrite, performInpainting 27 | 28 | libpath = "/home/appuser/scripts/" # to keep the dev repo in place, w/o linking 29 | sys.path.insert(1,libpath) 30 | import ObjectDetection.imutils as imu 31 | 32 | 33 | basedir = os.getcwd() 34 | uploaddir = os.path.join(basedir,"upload") 35 | print(f"ObjectDetect loaded, using DEVICE={DEVICE}") 36 | print(f"Basedirectory: {basedir}") 37 | 38 | # cleaup old uploads 39 | if os.path.exists(uploaddir): 40 | rmtree(uploaddir) 41 | 42 | os.mkdir(uploaddir) 43 | 44 | # cleanup old runs 45 | if os.path.exists(os.path.join(basedir,"static")): 46 | for f in [*glob(os.path.join(basedir,'static/sequence_*.mp4')), 47 | *glob(os.path.join(basedir,'static/inpaint_*.mp4'))]: 48 | os.remove(f) 49 | else: 50 | os.mkdir(os.path.join(basedir,"static")) 51 | 52 | # remove and create dummy video for place holder 53 | tempfile = os.path.join(basedir,'static/result.mp4') 54 | if not os.path.exists(tempfile): 55 | createNullVideo(tempfile) 56 | 57 | 58 | # ---------- 59 | # Helper functions 60 | def getImageFileNames(dirPath): 61 | if not os.path.isdir(dirPath): 62 | return go.Figure().update_layout(title='Incorrect Directory Path!') 63 | 64 | for filetype in ('*.png', '*.jpg'): 65 | fnames = sorted(glob(os.path.join(dirPath,filetype))) 66 | if len(fnames) > 0: break 67 | 68 | if not fnames: 69 | go.Figure().update_layout(title='No files found!') 70 | return None 71 | else: 72 | return fnames 73 | 74 | 75 | # ---------- 76 | # Dash component wrappers 77 | def Row(children=None, **kwargs): 78 | return html.Div(children, className="row", **kwargs) 79 | 80 | 81 | def Column(children=None, width=1, **kwargs): 82 | nb_map = { 83 | 1: 'one', 2: 'two', 3: 'three', 4: 'four', 5: 'five', 6: 'six', 84 | 7: 'seven', 8: 'eight', 9: 'nine', 10: 'ten', 11: 'eleven', 12: 'twelve'} 85 | 86 | return html.Div(children, className=f"{nb_map[width]} columns", **kwargs) 87 | 88 | # ---------- 89 | # plotly.py helper functions 90 | 91 | # upload files 92 | def save_file(name, content): 93 | """Decode and store a file uploaded with Plotly Dash.""" 94 | filepath = os.path.join(uploaddir, name) 95 | data = content.encode("utf8").split(b";base64,")[1] 96 | with open(filepath, "wb") as fp: 97 | fp.write(base64.decodebytes(data)) 98 | 99 | return filepath 100 | 101 | def vfile_to_frames(fname): 102 | assert os.path.exists(fname), "Could not determine path to video file" 103 | dirPath = os.path.join(uploaddir,fname.split(".")[-2]) 104 | nfiles = imu.videofileToFramesDirectory(videofile=fname,dirPath=dirPath, 105 | padlength=5, imgtype='png', cleanDirectory=True) 106 | return dirPath 107 | 108 | # PIL images 109 | def pil_to_b64(im, enc="png"): 110 | io_buf = BytesIO() 111 | im.save(io_buf, format=enc) 112 | encoded = base64.b64encode(io_buf.getvalue()).decode("utf-8") 113 | return f"data:img/{enc};base64, " + encoded 114 | 115 | 116 | def pil_to_fig(im, showlegend=False, title=None): 117 | img_width, img_height = im.size 118 | fig = go.Figure() 119 | # This trace is added to help the autoresize logic work. 120 | fig.add_trace(go.Scatter( 121 | x=[img_width * 0.05, img_width * 0.95], 122 | y=[img_height * 0.95, img_height * 0.05], 123 | showlegend=False, mode="markers", marker_opacity=0, 124 | hoverinfo="none", legendgroup='Image')) 125 | 126 | fig.add_layout_image(dict( 127 | source=pil_to_b64(im), sizing="stretch", opacity=1, layer="below", 128 | x=0, y=0, xref="x", yref="y", sizex=img_width, sizey=img_height,)) 129 | 130 | # Adapt axes to the right width and height, lock aspect ratio 131 | fig.update_xaxes( 132 | showgrid=False, visible=False, constrain="domain", range=[0, img_width]) 133 | 134 | fig.update_yaxes( 135 | showgrid=False, visible=False, 136 | scaleanchor="x", scaleratio=1, 137 | range=[img_height, 0]) 138 | 139 | fig.update_layout(title=title, showlegend=showlegend) 140 | 141 | return fig 142 | 143 | # graph 144 | def add_bbox(fig, x0, y0, x1, y1, 145 | showlegend=True, name=None, color=None, 146 | opacity=0.5, group=None, text=None): 147 | fig.add_trace(go.Scatter( 148 | x=[x0, x1, x1, x0, x0], 149 | y=[y0, y0, y1, y1, y0], 150 | mode="lines", 151 | fill="toself", 152 | opacity=opacity, 153 | marker_color=color, 154 | hoveron="fills", 155 | name=name, 156 | hoverlabel_namelength=0, 157 | text=text, 158 | legendgroup=group, 159 | showlegend=showlegend, 160 | )) 161 | 162 | def get_index_by_bbox(objGroupingDict,class_name,bbox_in): 163 | # gets the order of the object within the groupingSequence 164 | # by bounding box overlap 165 | maxIoU = 0.0 166 | maxIdx = 0 167 | for instIdx, objInst in enumerate(objGroupingDict[class_name]): 168 | # only search for object found in seq_i= 0 169 | # there must exist objects at seq_i=0 since we predicted them on first frame 170 | bbx,_,seq_i = objInst[0] 171 | 172 | if seq_i != 0: continue 173 | IoU = imu.bboxIoU(bbx, bbox_in) 174 | if IoU > maxIoU: 175 | maxIoU = IoU 176 | maxIdx = instIdx 177 | 178 | return maxIdx 179 | 180 | 181 | def get_selected_objects(figure,objGroupingDict): 182 | # recovers the selected items from the figure 183 | # infer the object location by the bbox IoU (more stable than order) 184 | # returns a dict of selected indexed items 185 | 186 | obsObjects = {} 187 | for d in figure['data']: 188 | objInst = d.get('legendgroup') 189 | wasVisible = False if d.get('visible') and d['visible'] == 'legendonly' else True 190 | if objInst is not None and ":" in objInst and wasVisible: 191 | 192 | classname = objInst.split(":")[0] 193 | x = d['x'] 194 | y = d['y'] 195 | bbox = [x[0], y[0], x[2], y[2]] 196 | inst = get_index_by_bbox(objGroupingDict,classname,bbox) 197 | if obsObjects.get(classname,None) is None: 198 | obsObjects[classname] = [int(inst)] 199 | else: 200 | obsObjects[classname].append(int(inst)) 201 | 202 | return obsObjects 203 | 204 | 205 | # colors for visualization 206 | COLORS = ['#fe938c','#86e7b8','#f9ebe0','#208aae','#fe4a49', 207 | '#291711', '#5f4b66', '#b98b82', '#87f5fb', '#63326e'] * 50 208 | 209 | # Start Dash 210 | app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP]) 211 | server = app.server # Expose the server variable for deployments 212 | cache = Cache() 213 | CACHE_CONFIG = { 214 | 'CACHE_TYPE': 'filesystem', 215 | 'CACHE_DIR': os.path.join(basedir,'_cache') 216 | } 217 | cache.init_app(server,config=CACHE_CONFIG) 218 | 219 | # ---------- 220 | # layout 221 | app.layout = html.Div(className='container', children=[ 222 | Row(html.H1("Video Object Removal App")), 223 | 224 | # Input 225 | Row(html.P("Input Directory Path:")), 226 | Row([ 227 | Column(width=6, children=[ 228 | dcc.Upload( 229 | id="upload-file", 230 | children=html.Div( 231 | ["Drag and drop or click to select a file to upload."] 232 | ), 233 | style={ 234 | "width": "100%", 235 | "height": "60px", 236 | "lineHeight": "60px", 237 | "borderWidth": "1px", 238 | "borderStyle": "dashed", 239 | "borderRadius": "5px", 240 | "textAlign": "center", 241 | "margin": "10px", 242 | }, 243 | multiple=False, 244 | ) 245 | ]), 246 | ]), 247 | 248 | Row([ 249 | Column(width=6, children=[ 250 | html.H3('Upload file',id='input-dirpath-display', style={'width': '100%'}) 251 | ]), 252 | Column(width=2,children=[ 253 | html.Button("Run Single", id='button-single', n_clicks=0) 254 | ]), 255 | ]), 256 | 257 | html.Hr(), 258 | 259 | # frame number selection 260 | Row([ 261 | Column(width=2, children=[ html.P('Frame range:')]), 262 | Column(width=10, children=[ 263 | dcc.Input(id='input-framenmin',type='number',value=0), 264 | dcc.RangeSlider( 265 | id='slider-framenums', min=0, max=100, step=1, value=[0,100], 266 | marks={0: '0', 100: '100'}, allowCross=False), 267 | dcc.Input(id='input-framenmax',type='number',value=100) 268 | ], style={"display": "grid", "grid-template-columns": "10% 70% 10%"}), 269 | ],style={'width':"100%"}), 270 | 271 | html.Hr(), 272 | 273 | Row([ 274 | Column(width=5, children=[ 275 | html.P('Confidence Threshold:'), 276 | dcc.Slider( 277 | id='slider-confidence', min=0, max=1, step=0.05, value=0.7, 278 | marks={0: '0', 1: '1'}) 279 | ]), 280 | Column(width=7, children=[ 281 | html.P('Object selection:'), 282 | Row([ 283 | Column(width=3, children=dcc.Checklist( 284 | id='cb-person', 285 | options=[ 286 | {'label': ' person', 'value': 'person'}, 287 | {'label': ' handbag', 'value': 'handbag'}, 288 | {'label': ' backpack', 'value': 'backpack'}, 289 | {'label': ' suitcase', 'value': 'suitcase'}, 290 | ], 291 | value = ['person', 'handbag','backpack','suitcase']) 292 | ), 293 | Column(width=3, children=dcc.Checklist( 294 | id='cb-vehicle', 295 | options=[ 296 | {'label': ' car', 'value': 'car'}, 297 | {'label': ' truck', 'value': 'truck'}, 298 | {'label': ' motorcycle', 'value': 'motorcycle'}, 299 | {'label': ' bus', 'value': 'bus'}, 300 | ], 301 | value = ['car', 'truck', 'motorcycle']) 302 | ), 303 | Column(width=3, children=dcc.Checklist( 304 | id='cb-environment', 305 | options=[ 306 | {'label': ' traffic light', 'value': 'traffic light'}, 307 | {'label': ' stop sign', 'value': 'stop sign'}, 308 | {'label': ' bench', 'value': 'bench'}, 309 | ], 310 | value = []) 311 | ) 312 | ]) 313 | ]), 314 | ]), 315 | 316 | # prediction output graph 317 | Row(dcc.Graph(id='model-output', style={"height": "70vh"})), 318 | 319 | # processing options 320 | html.Hr(), 321 | Row([ 322 | Column(width=7, children=[ 323 | html.P('Processing options:'), 324 | Row([ 325 | Column(width=3, children=dcc.Checklist( 326 | id='cb-options', 327 | options=[ 328 | {'label': ' accept all', 'value': 'acceptall'}, 329 | {'label': ' fill sequence', 'value': 'fillSequence'}, 330 | {'label': ' use BBmask', 'value': 'useBBmasks'}, 331 | ], 332 | value=['fillSequence']) 333 | ), 334 | Column(width=3, children= [ 335 | html.P('dilation half-width (no dilation=0):'), 336 | dcc.Input( 337 | id='input-dilationhwidth', 338 | type='number', 339 | value=0) 340 | ]), 341 | Column(width=3, children=[ 342 | html.P('minimum seq. length'), 343 | dcc.Input( 344 | id='input-minseqlength', 345 | type='number', 346 | value=0) 347 | ]) 348 | ]) 349 | ]), 350 | ]), 351 | 352 | # Sequence Video 353 | html.Hr(), 354 | Row([ 355 | Column(width=2,children=[ 356 | html.Button("Run Sequence", id='button-sequence', n_clicks=0), 357 | dcc.Loading(id='loading-sequence-bobble', 358 | type='circle', 359 | children=html.Div(id='loading-sequence')) 360 | ]), 361 | Column(width=2,children=[]), # place holder 362 | Row([ 363 | Column(width=4, children=[ 364 | html.P("Start Frame:"), 365 | html.Label("0",id='sequence-startframe') 366 | ]), 367 | Column(width=4, children=[]), 368 | Column(width=4, children=[ 369 | html.P("End Frame:"), 370 | html.Label("0",id='sequence-endframe') 371 | ]) 372 | ]), 373 | ]), 374 | html.P("Sequence Output"), 375 | html.Div([ dash_player.DashPlayer( 376 | id='sequence-output', 377 | url='static/result.mp4', 378 | controls=True, 379 | style={"height": "70vh"}) ]), 380 | 381 | # Inpainting Video 382 | html.Hr(), 383 | Row([ 384 | Column(width=2,children=[ 385 | html.Button("Run Inpaint", id='button-inpaint', n_clicks=0), 386 | dcc.Loading(id='loading-inpaint-bobble', 387 | type='circle', 388 | children=html.Div(id='loading-inpaint')) 389 | ]), 390 | Column(width=2,children=[]), # place holder 391 | Row([ 392 | Column(width=4, children=[ 393 | html.P("Start Frame:"), 394 | html.Label("0",id='inpaint-startframe') 395 | ]), 396 | Column(width=4, children=[]), 397 | Column(width=4, children=[ 398 | html.P("End Frame:"), 399 | html.Label("0",id='inpaint-endframe') 400 | ]) 401 | ]), 402 | ]), 403 | html.P("Inpainting Output"), 404 | html.Div([ dash_player.DashPlayer( 405 | id='inpaint-output', 406 | url='static/result.mp4', 407 | controls=True, 408 | style={"height": "70vh"}) ]), 409 | 410 | # hidden signal value 411 | html.Div(id='signal-sequence', style={'display': 'none'}), 412 | html.Div(id='signal-inpaint', style={'display': 'none'}), 413 | html.Div(id='input-dirpath', style={'display': 'none'}), 414 | 415 | ]) 416 | 417 | 418 | # ---------- 419 | # callbacks 420 | 421 | # upload file 422 | @app.callback( 423 | Output("input-dirpath", "children"), 424 | [Input("upload-file", "filename"), Input("upload-file", "contents")], 425 | ) 426 | def update_output(uploaded_filename, uploaded_file_content): 427 | # Save uploaded files and regenerate the file list. 428 | 429 | if uploaded_filename is not None and uploaded_file_content is not None and \ 430 | uploaded_filename.split(".")[-1] in ('mp4', 'mov', 'avi') : 431 | dirPath = vfile_to_frames(save_file(uploaded_filename, uploaded_file_content)) 432 | return dirPath 433 | else: 434 | return "(none)" 435 | 436 | # update_framenum_minmax() 437 | # purpose: to update min/max boxes of the slider 438 | @app.callback( 439 | [Output('input-framenmin','value'), 440 | Output('input-framenmax','value'), 441 | Output('sequence-startframe','children'), 442 | Output('sequence-endframe','children'), 443 | Output('inpaint-startframe','children'), 444 | Output('inpaint-endframe','children') 445 | ], 446 | [Input('slider-framenums','value')] 447 | ) 448 | def update_framenum_minmax(framenumrange): 449 | return framenumrange[0], framenumrange[1], \ 450 | str(framenumrange[0]), str(framenumrange[1]), \ 451 | str(framenumrange[0]), str(framenumrange[1]) 452 | 453 | 454 | @app.callback( 455 | [Output('slider-framenums','max'), 456 | Output('slider-framenums','marks'), 457 | Output('slider-framenums','value'), 458 | Output('input-dirpath-display','children')], 459 | [Input('button-single','n_clicks'), 460 | Input('button-sequence','n_clicks'), 461 | Input('button-inpaint', 'n_clicks'), 462 | Input('input-dirpath', 'children')], 463 | [State('upload-file', 'filename'), 464 | State('slider-framenums','max'), 465 | State('slider-framenums','marks'), 466 | State('slider-framenums','value') ] 467 | ) 468 | def update_dirpath(nc_single, nc_sequence, nc_inpaint, s_dirpath, 469 | vfilename, s_fnmax, s_fnmarks, s_fnvalue): 470 | if s_dirpath is None or s_dirpath == "(none)": 471 | #s_dirpath = "/home/appuser/data/Colomar/frames" #temporary fix 472 | return 100, {'0':'0', '100':'100'}, [0,100], '(none)' 473 | 474 | dirpath = s_dirpath 475 | fnames = getImageFileNames(s_dirpath) 476 | if fnames: 477 | fnmax = len(fnames)-1 478 | if fnmax != s_fnmax: 479 | fnmarks = {0: '0', fnmax: f"{fnmax}"} 480 | fnvalue = [0, fnmax] 481 | else: 482 | fnmarks = s_fnmarks 483 | fnvalue = s_fnvalue 484 | else: 485 | fnmax = s_fnmax 486 | fnmarks = s_fnmarks 487 | fnvalue = s_fnvalue 488 | 489 | return fnmax, fnmarks, fnvalue, vfilename 490 | 491 | # *************** 492 | # * run_single 493 | # *************** 494 | # create single prediction at first frame 495 | @app.callback( 496 | [Output('model-output', 'figure')], 497 | [Input('button-single', 'n_clicks')], 498 | [State('input-dirpath', 'children'), 499 | State('slider-framenums','value'), 500 | State('slider-confidence', 'value'), 501 | State('cb-person','value'), 502 | State('cb-vehicle','value'), 503 | State('cb-environment','value') 504 | ], 505 | ) 506 | def run_single(n_clicks, dirpath, framerange, confidence, 507 | cb_person, cb_vehicle, cb_environment): 508 | 509 | if dirpath is not None and os.path.isdir(dirpath): 510 | fnames = getImageFileNames(dirpath) 511 | imgfile = fnames[framerange[0]] 512 | im = Image.open(imgfile) 513 | else: 514 | go.Figure().update_layout(title='Incorrect dirpath') 515 | im = Image.new('RGB',(640,480)) 516 | fig = pil_to_fig(im, showlegend=True, title='No Image') 517 | return fig, 518 | 519 | theseObjects = [ *cb_person, *cb_vehicle, *cb_environment] 520 | wasConfidence = detr.cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST 521 | 522 | if sorted(theseObjects) != sorted(detr.selObjNames) or \ 523 | abs(confidence - wasConfidence) > 0.00001: 524 | detr.__init__(selectObjectNames=theseObjects, score_threshold=confidence) 525 | 526 | tstart = time.time() 527 | scores, boxes, selClasses = detect_scores_bboxes_classes(imgfile, detr) 528 | tend = time.time() 529 | 530 | fig = pil_to_fig(im, showlegend=True, title=f'DETR Predictions ({tend-tstart:.2f}s)') 531 | 532 | seenClassIndex = {} 533 | insertOrder = [] 534 | for i,class_id in enumerate(selClasses): 535 | classname = CLASSES[class_id] 536 | 537 | if seenClassIndex.get(classname,None) is not None: 538 | seenClassIndex[classname] += 1 539 | else: 540 | seenClassIndex[classname] = 0 541 | 542 | insertOrder.append([classname + ":" + str(seenClassIndex[classname]),i]) 543 | 544 | insertOrder = sorted(insertOrder, \ 545 | key=lambda x: (x[0].split(":")[0], int(x[0].split(":")[1]))) 546 | 547 | for label,i in insertOrder: 548 | 549 | confidence = scores[i] 550 | x0, y0, x1, y1 = boxes[i] 551 | class_id = selClasses[i] 552 | 553 | # only display legend when it's not in the existing classes 554 | text = f"class={label}
confidence={confidence:.3f}" 555 | 556 | add_bbox( 557 | fig, x0, y0, x1, y1, 558 | opacity=0.7, group=label, name=label, color=COLORS[class_id], 559 | showlegend=True, text=text, 560 | ) 561 | 562 | return fig, 563 | 564 | 565 | # *************** 566 | # run_sequence 567 | # *************** 568 | # Produce sequence prediction with grouping 569 | @app.callback( 570 | [Output('loading-sequence', 'children'), 571 | Output('signal-sequence','children')], 572 | [Input('button-sequence', 'n_clicks')], 573 | [State('input-dirpath', 'children'), 574 | State('slider-framenums','value'), 575 | State('slider-confidence', 'value'), 576 | State('model-output','figure'), 577 | State('cb-person','value'), 578 | State('cb-vehicle','value'), 579 | State('cb-environment','value'), 580 | State('cb-options','value'), 581 | State('input-dilationhwidth','value'), 582 | State('input-minseqlength','value')] 583 | ) 584 | def run_sequence(n_clicks, dirpath, framerange, confidence,figure, 585 | cb_person, cb_vehicle, cb_environment, 586 | cb_options, dilationhwidth, minsequencelength): 587 | 588 | if dirpath is not None and os.path.isdir(dirpath): 589 | fnames = getImageFileNames(dirpath) 590 | else: 591 | return "", "Null:None" 592 | 593 | selectObjects = [ *cb_person, *cb_vehicle, *cb_environment] 594 | acceptall = 'acceptall' in cb_options 595 | fillSequence = 'fillSequence' in cb_options 596 | useBBmasks = 'useBBmasks' in cb_options 597 | 598 | fmin, fmax = framerange 599 | fnames = fnames[fmin:fmax] 600 | 601 | # was this a repeat? 602 | if len(detr.imglist) != 0: 603 | if fnames == detr.selectFiles: 604 | return "", "Null:None" 605 | 606 | detr.__init__(score_threshold=confidence,selectObjectNames=selectObjects) 607 | 608 | vfile = compute_sequence(fnames,framerange,confidence, 609 | figure, selectObjects, 610 | acceptall, fillSequence, useBBmasks, 611 | dilationhwidth, minsequencelength) 612 | 613 | return "", f'sequencevid:{vfile}' 614 | 615 | 616 | @cache.memoize() 617 | def compute_sequence(fnames,framerange,confidence, 618 | figure,selectObjectNames, 619 | acceptall, fillSequence, useBBmasks, 620 | dilationhwidth, minsequencelength): 621 | detr.selectFiles = fnames 622 | 623 | staticdir = os.path.join(os.getcwd(),"static") 624 | detr.load_images(filelist=fnames) 625 | detr.predict_sequence(useBBmasks=useBBmasks,selObjectNames=selectObjectNames) 626 | detr.groupObjBBMaskSequence() 627 | 628 | obsObjectsDict = get_selected_objects(figure=figure, 629 | objGroupingDict=detr.objBBMaskSeqGrpDict) \ 630 | if not acceptall else None 631 | 632 | # filtered by object class, instance, and length 633 | detr.filter_ObjBBMaskSeq(allowObjNameInstances= obsObjectsDict, 634 | minCount=minsequencelength) 635 | 636 | # fill sequence 637 | if fillSequence: 638 | detr.fill_ObjBBMaskSequence(specificObjectNameInstances=obsObjectsDict) 639 | 640 | # use dilation 641 | if dilationhwidth > 0: 642 | detr.combine_MaskSequence() 643 | detr.dilateErode_MaskSequence(kernelShape='el', 644 | maskHalfWidth=dilationhwidth) 645 | 646 | vfile = 'sequence_' + datetime.now().strftime("%Y%m%d_%H%M%S") + ".mp4" 647 | if not os.environ.get("VSCODE_DEBUG"): 648 | detr.create_animationObject(framerange=framerange, 649 | useMasks=True, 650 | toHTML=False, 651 | figsize=(20,15), 652 | interval=30, 653 | MPEGfile=os.path.join(staticdir,vfile), 654 | useFFMPEGdirect=True) 655 | return vfile 656 | 657 | 658 | @app.callback(Output('sequence-output','url'), 659 | [Input('signal-sequence','children')], 660 | [State('sequence-output','url')]) 661 | def serve_sequence_video(signal,currurl): 662 | if currurl is None: 663 | return 'static/result.mp4' 664 | else: 665 | sigtype,vfile = signal.split(":") 666 | if vfile is not None and \ 667 | isinstance(vfile,str) and \ 668 | sigtype == 'sequencevid' and \ 669 | os.path.exists(f"./static/{vfile}"): 670 | 671 | # remove old file 672 | if os.path.exists(currurl): 673 | os.remove(currurl) 674 | 675 | # serve new file 676 | return f"static/{vfile}" 677 | else: 678 | return currurl 679 | 680 | 681 | # *************** 682 | # run_inpaint 683 | # *************** 684 | # Produce inpainting results 685 | 686 | @app.callback( 687 | [Output('loading-inpaint', 'children'), 688 | Output('signal-inpaint','children')], 689 | [Input('button-inpaint', 'n_clicks')] 690 | ) 691 | def run_inpaint(n_clicks): 692 | if not n_clicks: 693 | return "", "Null:None" 694 | 695 | assert testContainerWrite(inpaintObj=inpaint, 696 | workDir="../data", 697 | hardFail=False) , "Errors connecting with write access in containers" 698 | 699 | staticdir = os.path.join(os.getcwd(),"static") 700 | vfile = 'inpaint_' + datetime.now().strftime("%Y%m%d_%H%M%S") + ".mp4" 701 | performInpainting(detrObj=detr, 702 | inpaintObj=inpaint, 703 | workDir = "../data", 704 | outputVideo=os.path.join(staticdir,vfile)) 705 | 706 | return "", f"inpaintvid:{vfile}" 707 | 708 | 709 | @app.callback(Output('inpaint-output','url'), 710 | [Input('signal-inpaint','children')], 711 | [State('inpaint-output','url')]) 712 | def serve_inpaint_video(signal,currurl): 713 | if currurl is None: 714 | return 'static/result.mp4' 715 | else: 716 | sigtype,vfile = signal.split(":") 717 | if vfile is not None and \ 718 | isinstance(vfile,str) and \ 719 | sigtype == 'inpaintvid' and \ 720 | os.path.exists(f"./static/{vfile}"): 721 | 722 | # remove old file 723 | if os.path.exists(currurl): 724 | os.remove(currurl) 725 | 726 | # serve new file 727 | return f"static/{vfile}" 728 | else: 729 | return currurl 730 | 731 | 732 | # --------------------------------------------------------------------- 733 | if __name__ == '__main__': 734 | 735 | app.run_server(debug=True,host='0.0.0.0',processes=1,threaded=True) 736 | -------------------------------------------------------------------------------- /app/assets/default.min.css: -------------------------------------------------------------------------------- 1 | /* Table of contents 2 | –––––––––––––––––––––––––––––––––––––––––––––––––– 3 | - Plotly.js 4 | - Grid 5 | - Base Styles 6 | - Typography 7 | - Links 8 | - Buttons 9 | - Forms 10 | - Lists 11 | - Code 12 | - Tables 13 | - Spacing 14 | - Utilities 15 | - Clearing 16 | - Media Queries 17 | */ 18 | 19 | /* PLotly.js 20 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 21 | /* plotly.js's modebar's z-index is 1001 by default 22 | * https://github.com/plotly/plotly.js/blob/7e4d8ab164258f6bd48be56589dacd9bdd7fded2/src/css/_modebar.scss#L5 23 | * In case a dropdown is above the graph, the dropdown's options 24 | * will be rendered below the modebar 25 | * Increase the select option's z-index 26 | */ 27 | 28 | /* This was actually not quite right - 29 | dropdowns were overlapping each other (edited October 26) 30 | 31 | .Select { 32 | z-index: 1002; 33 | }*/ 34 | 35 | 36 | /* Grid 37 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 38 | .container { 39 | position: relative; 40 | width: 100%; 41 | max-width: 960px; 42 | margin: 0 auto; 43 | padding: 0 20px; 44 | box-sizing: border-box; } 45 | .column, 46 | .columns { 47 | width: 100%; 48 | float: left; 49 | box-sizing: border-box; } 50 | 51 | /* For devices larger than 400px */ 52 | @media (min-width: 400px) { 53 | .container { 54 | width: 85%; 55 | padding: 0; } 56 | } 57 | 58 | /* For devices larger than 550px */ 59 | @media (min-width: 550px) { 60 | .container { 61 | width: 80%; } 62 | .column, 63 | .columns { 64 | margin-left: 4%; } 65 | .column:first-child, 66 | .columns:first-child { 67 | margin-left: 0; } 68 | 69 | .one.column, 70 | .one.columns { width: 4.66666666667%; } 71 | .two.columns { width: 13.3333333333%; } 72 | .three.columns { width: 22%; } 73 | .four.columns { width: 30.6666666667%; } 74 | .five.columns { width: 39.3333333333%; } 75 | .six.columns { width: 48%; } 76 | .seven.columns { width: 56.6666666667%; } 77 | .eight.columns { width: 65.3333333333%; } 78 | .nine.columns { width: 74.0%; } 79 | .ten.columns { width: 82.6666666667%; } 80 | .eleven.columns { width: 91.3333333333%; } 81 | .twelve.columns { width: 100%; margin-left: 0; } 82 | 83 | .one-third.column { width: 30.6666666667%; } 84 | .two-thirds.column { width: 65.3333333333%; } 85 | 86 | .one-half.column { width: 48%; } 87 | 88 | /* Offsets */ 89 | .offset-by-one.column, 90 | .offset-by-one.columns { margin-left: 8.66666666667%; } 91 | .offset-by-two.column, 92 | .offset-by-two.columns { margin-left: 17.3333333333%; } 93 | .offset-by-three.column, 94 | .offset-by-three.columns { margin-left: 26%; } 95 | .offset-by-four.column, 96 | .offset-by-four.columns { margin-left: 34.6666666667%; } 97 | .offset-by-five.column, 98 | .offset-by-five.columns { margin-left: 43.3333333333%; } 99 | .offset-by-six.column, 100 | .offset-by-six.columns { margin-left: 52%; } 101 | .offset-by-seven.column, 102 | .offset-by-seven.columns { margin-left: 60.6666666667%; } 103 | .offset-by-eight.column, 104 | .offset-by-eight.columns { margin-left: 69.3333333333%; } 105 | .offset-by-nine.column, 106 | .offset-by-nine.columns { margin-left: 78.0%; } 107 | .offset-by-ten.column, 108 | .offset-by-ten.columns { margin-left: 86.6666666667%; } 109 | .offset-by-eleven.column, 110 | .offset-by-eleven.columns { margin-left: 95.3333333333%; } 111 | 112 | .offset-by-one-third.column, 113 | .offset-by-one-third.columns { margin-left: 34.6666666667%; } 114 | .offset-by-two-thirds.column, 115 | .offset-by-two-thirds.columns { margin-left: 69.3333333333%; } 116 | 117 | .offset-by-one-half.column, 118 | .offset-by-one-half.columns { margin-left: 52%; } 119 | 120 | } 121 | 122 | 123 | /* Base Styles 124 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 125 | /* NOTE 126 | html is set to 62.5% so that all the REM measurements throughout Skeleton 127 | are based on 10px sizing. So basically 1.5rem = 15px :) */ 128 | html { 129 | font-size: 62.5%; } 130 | body { 131 | font-size: 1.5em; /* currently ems cause chrome bug misinterpreting rems on body element */ 132 | line-height: 1.6; 133 | font-weight: 400; 134 | font-family: "Open Sans", "HelveticaNeue", "Helvetica Neue", Helvetica, Arial, sans-serif; 135 | color: rgb(50, 50, 50); } 136 | 137 | 138 | /* Typography 139 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 140 | h1, h2, h3, h4, h5, h6 { 141 | margin-top: 0; 142 | margin-bottom: 0; 143 | font-weight: 300; } 144 | h1 { font-size: 4.5rem; line-height: 1.2; letter-spacing: -.1rem; margin-bottom: 2rem; } 145 | h2 { font-size: 3.6rem; line-height: 1.25; letter-spacing: -.1rem; margin-bottom: 1.8rem; margin-top: 1.8rem;} 146 | h3 { font-size: 3.0rem; line-height: 1.3; letter-spacing: -.1rem; margin-bottom: 1.5rem; margin-top: 1.5rem;} 147 | h4 { font-size: 2.6rem; line-height: 1.35; letter-spacing: -.08rem; margin-bottom: 1.2rem; margin-top: 1.2rem;} 148 | h5 { font-size: 2.2rem; line-height: 1.5; letter-spacing: -.05rem; margin-bottom: 0.6rem; margin-top: 0.6rem;} 149 | h6 { font-size: 2.0rem; line-height: 1.6; letter-spacing: 0; margin-bottom: 0.75rem; margin-top: 0.75rem;} 150 | 151 | p { 152 | margin-top: 0; } 153 | 154 | 155 | /* Blockquotes 156 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 157 | blockquote { 158 | border-left: 4px lightgrey solid; 159 | padding-left: 1rem; 160 | margin-top: 2rem; 161 | margin-bottom: 2rem; 162 | margin-left: 0rem; 163 | } 164 | 165 | 166 | /* Links 167 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 168 | a { 169 | color: #1EAEDB; 170 | text-decoration: underline; 171 | cursor: pointer;} 172 | a:hover { 173 | color: #0FA0CE; } 174 | 175 | 176 | /* Buttons 177 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 178 | .button, 179 | button, 180 | input[type="submit"], 181 | input[type="reset"], 182 | input[type="button"] { 183 | display: inline-block; 184 | height: 38px; 185 | padding: 0 30px; 186 | color: #555; 187 | text-align: center; 188 | font-size: 11px; 189 | font-weight: 600; 190 | line-height: 38px; 191 | letter-spacing: .1rem; 192 | text-transform: uppercase; 193 | text-decoration: none; 194 | white-space: nowrap; 195 | background-color: transparent; 196 | border-radius: 4px; 197 | border: 1px solid #bbb; 198 | cursor: pointer; 199 | box-sizing: border-box; } 200 | .button:hover, 201 | button:hover, 202 | input[type="submit"]:hover, 203 | input[type="reset"]:hover, 204 | input[type="button"]:hover, 205 | .button:focus, 206 | button:focus, 207 | input[type="submit"]:focus, 208 | input[type="reset"]:focus, 209 | input[type="button"]:focus { 210 | color: #333; 211 | border-color: #888; 212 | outline: 0; } 213 | .button.button-primary, 214 | button.button-primary, 215 | input[type="submit"].button-primary, 216 | input[type="reset"].button-primary, 217 | input[type="button"].button-primary { 218 | color: #FFF; 219 | background-color: #33C3F0; 220 | border-color: #33C3F0; } 221 | .button.button-primary:hover, 222 | button.button-primary:hover, 223 | input[type="submit"].button-primary:hover, 224 | input[type="reset"].button-primary:hover, 225 | input[type="button"].button-primary:hover, 226 | .button.button-primary:focus, 227 | button.button-primary:focus, 228 | input[type="submit"].button-primary:focus, 229 | input[type="reset"].button-primary:focus, 230 | input[type="button"].button-primary:focus { 231 | color: #FFF; 232 | background-color: #1EAEDB; 233 | border-color: #1EAEDB; } 234 | 235 | 236 | /* Forms 237 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 238 | input[type="email"], 239 | input[type="number"], 240 | input[type="search"], 241 | input[type="text"], 242 | input[type="tel"], 243 | input[type="url"], 244 | input[type="password"], 245 | textarea, 246 | select { 247 | height: 38px; 248 | padding: 6px 10px; /* The 6px vertically centers text on FF, ignored by Webkit */ 249 | background-color: #fff; 250 | border: 1px solid #D1D1D1; 251 | border-radius: 4px; 252 | box-shadow: none; 253 | box-sizing: border-box; 254 | font-family: inherit; 255 | font-size: inherit; /*https://stackoverflow.com/questions/6080413/why-doesnt-input-inherit-the-font-from-body*/} 256 | /* Removes awkward default styles on some inputs for iOS */ 257 | input[type="email"], 258 | input[type="number"], 259 | input[type="search"], 260 | input[type="text"], 261 | input[type="tel"], 262 | input[type="url"], 263 | input[type="password"], 264 | textarea { 265 | -webkit-appearance: none; 266 | -moz-appearance: none; 267 | appearance: none; } 268 | textarea { 269 | min-height: 65px; 270 | padding-top: 6px; 271 | padding-bottom: 6px; } 272 | input[type="email"]:focus, 273 | input[type="number"]:focus, 274 | input[type="search"]:focus, 275 | input[type="text"]:focus, 276 | input[type="tel"]:focus, 277 | input[type="url"]:focus, 278 | input[type="password"]:focus, 279 | textarea:focus, 280 | select:focus { 281 | border: 1px solid #33C3F0; 282 | outline: 0; } 283 | label, 284 | legend { 285 | display: block; 286 | margin-bottom: 0px; } 287 | fieldset { 288 | padding: 0; 289 | border-width: 0; } 290 | input[type="checkbox"], 291 | input[type="radio"] { 292 | display: inline; } 293 | label > .label-body { 294 | display: inline-block; 295 | margin-left: .5rem; 296 | font-weight: normal; } 297 | 298 | 299 | /* Lists 300 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 301 | ul { 302 | list-style: circle inside; } 303 | ol { 304 | list-style: decimal inside; } 305 | ol, ul { 306 | padding-left: 0; 307 | margin-top: 0; } 308 | ul ul, 309 | ul ol, 310 | ol ol, 311 | ol ul { 312 | margin: 1.5rem 0 1.5rem 3rem; 313 | font-size: 90%; } 314 | li { 315 | margin-bottom: 1rem; } 316 | 317 | 318 | /* Tables 319 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 320 | table { 321 | border-collapse: collapse; 322 | } 323 | th:not(.CalendarDay), 324 | td:not(.CalendarDay) { 325 | padding: 12px 15px; 326 | text-align: left; 327 | border-bottom: 1px solid #E1E1E1; } 328 | th:first-child:not(.CalendarDay), 329 | td:first-child:not(.CalendarDay) { 330 | padding-left: 0; } 331 | th:last-child:not(.CalendarDay), 332 | td:last-child:not(.CalendarDay) { 333 | padding-right: 0; } 334 | 335 | 336 | /* Spacing 337 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 338 | button, 339 | .button { 340 | margin-bottom: 0rem; } 341 | input, 342 | textarea, 343 | select, 344 | fieldset { 345 | margin-bottom: 0rem; } 346 | pre, 347 | dl, 348 | figure, 349 | table, 350 | form { 351 | margin-bottom: 0rem; } 352 | p, 353 | ul, 354 | ol { 355 | margin-bottom: 0.75rem; } 356 | 357 | /* Utilities 358 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 359 | .u-full-width { 360 | width: 100%; 361 | box-sizing: border-box; } 362 | .u-max-full-width { 363 | max-width: 100%; 364 | box-sizing: border-box; } 365 | .u-pull-right { 366 | float: right; } 367 | .u-pull-left { 368 | float: left; } 369 | 370 | 371 | /* Misc 372 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 373 | hr { 374 | margin-top: 3rem; 375 | margin-bottom: 3.5rem; 376 | border-width: 0; 377 | border-top: 1px solid #E1E1E1; } 378 | 379 | 380 | /* Clearing 381 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 382 | 383 | /* Self Clearing Goodness */ 384 | .container:after, 385 | .row:after, 386 | .u-cf { 387 | content: ""; 388 | display: table; 389 | clear: both; } 390 | 391 | 392 | /* Media Queries 393 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 394 | /* 395 | Note: The best way to structure the use of media queries is to create the queries 396 | near the relevant code. For example, if you wanted to change the styles for buttons 397 | on small devices, paste the mobile query code up in the buttons section and style it 398 | there. 399 | */ 400 | 401 | 402 | /* Larger than mobile */ 403 | @media (min-width: 400px) {} 404 | 405 | /* Larger than phablet (also point when grid becomes active) */ 406 | @media (min-width: 550px) {} 407 | 408 | /* Larger than tablet */ 409 | @media (min-width: 750px) {} 410 | 411 | /* Larger than desktop */ 412 | @media (min-width: 1000px) {} 413 | 414 | /* Larger than Desktop HD */ 415 | @media (min-width: 1200px) {} -------------------------------------------------------------------------------- /app/model.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import tempfile 4 | import cv2 5 | from threading import Thread 6 | from time import sleep 7 | from glob import glob 8 | 9 | libpath = "/home/appuser/scripts/" # to keep the dev repo in place, w/o linking 10 | sys.path.insert(1,libpath) 11 | import ObjectDetection.imutils as imu 12 | from ObjectDetection.detect import DetectSingle, TrackSequence, GroupSequence 13 | from ObjectDetection.inpaintRemote import InpaintRemote 14 | 15 | # ------------ 16 | # helper functions 17 | 18 | class ThreadWithReturnValue(Thread): 19 | def __init__(self, group=None, target=None, name=None, 20 | args=(), kwargs={}, Verbose=None): 21 | Thread.__init__(self, group, target, name, args, kwargs) 22 | self._return = None 23 | def run(self): 24 | if self._target is not None: 25 | self._return = self._target(*self._args, **self._kwargs) 26 | def join(self, *args): 27 | Thread.join(self, *args) 28 | return self._return 29 | 30 | 31 | # for output bounding box post-processing 32 | #def box_cxcywh_to_xyxy(x): 33 | # x_c, y_c, w, h = x.unbind(1) 34 | # b = [(x_c - 0.5 * w), (y_c - 0.5 * h), 35 | # (x_c + 0.5 * w), (y_c + 0.5 * h)] 36 | # return torch.stack(b, dim=1) 37 | 38 | #def rescale_bboxes(out_bbox, size): 39 | # img_w, img_h = size 40 | # b = box_cxcywh_to_xyxy(out_bbox) 41 | # b = b * torch.tensor([img_w, img_h, img_w, img_h], dtype=torch.float32) 42 | # return b 43 | 44 | 45 | def detect_scores_bboxes_classes(im,model): 46 | detr.predict(im) 47 | return detr.scores, detr.bboxes, detr.selClassList 48 | 49 | 50 | #def filter_boxes(scores, boxes, confidence=0.7, apply_nms=True, iou=0.5): 51 | # keep = scores.max(-1).values > confidence 52 | # scores, boxes = scores[keep], boxes[keep] 53 | # 54 | # if apply_nms: 55 | # top_scores, labels = scores.max(-1) 56 | # keep = batched_nms(boxes, top_scores, labels, iou) 57 | # scores, boxes = scores[keep], boxes[keep] 58 | # 59 | # return scores, boxes 60 | 61 | def createNullVideo(filePath,message="No Images", heightWidth=(100,100)): 62 | return imu.createNullVideo(filePath=filePath, message=message, heightWidth=heightWidth) 63 | 64 | def testContainerWrite(inpaintObj, workDir=None, hardFail=True): 65 | # workDir must be accessible from both containers 66 | if workDir is None: 67 | workDir = os.getcwd() 68 | 69 | workDir = os.path.abspath(workDir) 70 | 71 | hasErrors = False 72 | 73 | # access from detect container 74 | if not os.access(workDir,os.W_OK): 75 | msg = f"Do not have write access to 'detect' container:{workDir}" 76 | if hardFail: 77 | raise Exception(msg) 78 | else: 79 | hasErrors = True 80 | print(msg) 81 | 82 | # access from inpaint container 83 | inpaintObj.connectInpaint() 84 | results = inpaintObj.executeCommandsInpaint( 85 | commands=[f'if [ -w "{workDir}" ]; then echo "OK"; else echo "FAIL"; fi'] 86 | ) 87 | inpaintObj.disconnectInpaint() 88 | 89 | res = [ l.strip() for l in results['stdout'][0]] 90 | 91 | if "FAIL" in res: 92 | msg = f"Do not have write access to 'inpaint' container:{workDir}" + \ 93 | ",".join([l for l in results['stderr'][0]]) 94 | if hardFail: 95 | raise Exception(msg) 96 | else: 97 | hasErrors = True 98 | print(msg) 99 | 100 | return not hasErrors 101 | 102 | 103 | def performInpainting(detrObj,inpaintObj,workDir,outputVideo, useFFMPEGdirect=False): 104 | 105 | # perform inpainting 106 | # (write access tested previously) 107 | workDir = os.path.abspath(workDir) 108 | 109 | with tempfile.TemporaryDirectory(dir=workDir) as tempdir: 110 | 111 | frameDirPath =os.path.join(tempdir,"frames") 112 | maskDirPath = os.path.join(tempdir,"masks") 113 | resultDirPath = os.path.join(os.path.join(tempdir,"Inpaint_Res"),"inpaint_res") 114 | 115 | if detrObj.combinedMaskList is None: 116 | detrObj.combine_MaskSequence() 117 | 118 | detrObj.write_ImageMaskSequence( 119 | writeImagesToDirectory=frameDirPath, 120 | writeMasksToDirectory=maskDirPath) 121 | 122 | inpaintObj.connectInpaint() 123 | 124 | trd1 = ThreadWithReturnValue(target=inpaintObj.runInpaint, 125 | kwargs={'frameDirPath':frameDirPath,'maskDirPath':maskDirPath}) 126 | trd1.start() 127 | 128 | print("working:",end='',flush=True) 129 | while trd1.is_alive(): 130 | print('.',end='',flush=True) 131 | sleep(1) 132 | 133 | print("\nfinished") 134 | inpaintObj.disconnectInpaint() 135 | 136 | stdin,stdout,stderr = trd1.join() 137 | ok = False 138 | for l in stdout: 139 | print(l.strip()) 140 | if "Propagation has been finished" in l: 141 | ok = True 142 | 143 | assert ok, "Could not determine if results were valid!" 144 | 145 | print(f"\n....Writing results to {outputVideo}") 146 | 147 | resultfiles = sorted(glob(os.path.join(resultDirPath,"*.png"))) 148 | imgres = [ cv2.imread(f) for f in resultfiles] 149 | imu.writeFramesToVideo(imgres, filePath=outputVideo, fps=30, useFFMPEGdirect=True) 150 | 151 | return True 152 | 153 | 154 | # *************************** 155 | # Model import 156 | # *************************** 157 | 158 | # Load detection model 159 | detr = GroupSequence() 160 | CLASSES = detr.thing_classes 161 | DEVICE = detr.DEVICE 162 | 163 | # load Inpaint remote 164 | inpaint = InpaintRemote() 165 | 166 | 167 | # The following are imported in app: 168 | # >> detect, filter_boxes, detr, transform, CLASSES, DEVICE 169 | -------------------------------------------------------------------------------- /assets/CCCar_org_seq_inp_vert.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RexBarker/VideoObjectRemoval/26b8648645044389e9b7311f609c04b41f92f0b7/assets/CCCar_org_seq_inp_vert.gif -------------------------------------------------------------------------------- /assets/CCperson_org_seq_inp_vert.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RexBarker/VideoObjectRemoval/26b8648645044389e9b7311f609c04b41f92f0b7/assets/CCperson_org_seq_inp_vert.gif -------------------------------------------------------------------------------- /assets/Docker-config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RexBarker/VideoObjectRemoval/26b8648645044389e9b7311f609c04b41f92f0b7/assets/Docker-config.png -------------------------------------------------------------------------------- /assets/Evaluation-Flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RexBarker/VideoObjectRemoval/26b8648645044389e9b7311f609c04b41f92f0b7/assets/Evaluation-Flow.png -------------------------------------------------------------------------------- /data/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RexBarker/VideoObjectRemoval/26b8648645044389e9b7311f609c04b41f92f0b7/data/.gitkeep -------------------------------------------------------------------------------- /detect/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:10.1-cudnn7-devel 2 | 3 | ENV DEBIAN_FRONTEND noninteractive 4 | RUN apt-get update && apt-get install -y \ 5 | ca-certificates python3-dev python3-virtualenv git wget sudo \ 6 | cmake ninja-build protobuf-compiler libprotobuf-dev openssh-server \ 7 | vim ffmpeg && \ 8 | rm -rf /var/lib/apt/lists/* 9 | RUN ln -sv /usr/bin/python3 /usr/bin/python 10 | 11 | # create a non-root user 12 | ARG USER_ID=1000 13 | RUN useradd -m --no-log-init --system --uid ${USER_ID} appuser -s /bin/bash -g sudo -G root 14 | RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers 15 | RUN echo 'appuser:appuser' | chpasswd 16 | USER appuser 17 | WORKDIR /home/appuser 18 | 19 | # setup ssh 20 | RUN sudo service ssh start 21 | EXPOSE 22 22 | 23 | ENV PATH="/home/appuser/.local/bin:${PATH}" 24 | RUN wget https://bootstrap.pypa.io/get-pip.py && \ 25 | python3 get-pip.py --user && \ 26 | rm get-pip.py 27 | 28 | # install dependencies 29 | # See https://pytorch.org/ for other options if you use a different version of CUDA 30 | RUN pip install --user tensorboard 31 | RUN pip install --user torch==1.5 torchvision==0.6 -f https://download.pytorch.org/whl/cu101/torch_stable.html 32 | 33 | RUN pip install --user 'git+https://github.com/facebookresearch/fvcore' 34 | 35 | # install detectron2 36 | RUN git clone https://github.com/facebookresearch/detectron2 detectron2_repo 37 | 38 | # set FORCE_CUDA because during `docker build` cuda is not accessible 39 | ENV FORCE_CUDA="1" 40 | 41 | # This will by default build detectron2 for all common cuda architectures and take a lot more time, 42 | # because inside `docker build`, there is no way to tell which architecture will be used. 43 | ARG TORCH_CUDA_ARCH_LIST="Kepler;Kepler+Tesla;Maxwell;Maxwell+Tegra;Pascal;Volta;Turing" 44 | ENV TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST}" 45 | 46 | RUN pip install --user -e detectron2_repo 47 | RUN pip install --user jupyter scipy pandas scikit-learn ipykernel opencv-python \ 48 | paramiko dash dash-player dash-bootstrap-components flask_caching pytube3 49 | 50 | # Set a fixed model cache directory. 51 | ENV FVCORE_CACHE="/tmp" 52 | 53 | # working location 54 | ENV LC_ALL=C.UTF-8 55 | ENV LANG=C.UTF-8 56 | 57 | WORKDIR /home/appuser/ 58 | 59 | #CMD ["/usr/sbin/sshd","-D"] 60 | 61 | #ENTRYPOINT ["jupyter", "notebook", "--ip=0.0.0.0", "--no-browser"] 62 | 63 | # run detectron2 under user "appuser": 64 | # wget http://images.cocodataset.org/val2017/000000439715.jpg -O input.jpg 65 | # python3 demo/demo.py \ 66 | #--config-file configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml \ 67 | #--input input.jpg --output outputs/ \ 68 | #--opts MODEL.WEIGHTS detectron2://COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/model_final_f10217.pkl 69 | -------------------------------------------------------------------------------- /detect/docker/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## Use the container (with docker ≥ 19.03) 3 | 4 | ``` 5 | cd docker/ 6 | # Build: 7 | docker build --build-arg USER_ID=$UID -t detectron2:v0 . 8 | # Run: 9 | docker run --gpus all -it \ 10 | --shm-size=8gb --env="DISPLAY" --volume="/tmp/.X11-unix:/tmp/.X11-unix:rw" \ 11 | --name=detectron2 detectron2:v0 12 | 13 | # Grant docker access to host X server to show images 14 | xhost +local:`docker inspect --format='{{ .Config.Hostname }}' detectron2` 15 | ``` 16 | 17 | ## Use the container (with docker < 19.03) 18 | 19 | Install docker-compose and nvidia-docker2, then run: 20 | ``` 21 | cd docker && USER_ID=$UID docker-compose run detectron2 22 | ``` 23 | 24 | #### Using a persistent cache directory 25 | 26 | You can prevent models from being re-downloaded on every run, 27 | by storing them in a cache directory. 28 | 29 | To do this, add `--volume=$HOME/.torch/fvcore_cache:/tmp:rw` in the run command. 30 | 31 | ## Install new dependencies 32 | Add the following to `Dockerfile` to make persistent changes. 33 | ``` 34 | RUN sudo apt-get update && sudo apt-get install -y vim 35 | ``` 36 | Or run them in the container to make temporary changes. 37 | -------------------------------------------------------------------------------- /detect/docker/build_detector.sh: -------------------------------------------------------------------------------- 1 | # Build: 2 | docker build --build-arg USER_ID=$UID -t detector:v0 . 3 | -------------------------------------------------------------------------------- /detect/docker/run_detector.sh: -------------------------------------------------------------------------------- 1 | export PUBLIC_IP=`curl ifconfig.io` 2 | 3 | docker run --gpus all -ti --shm-size=8gb --env="DISPLAY" \ 4 | --volume="/tmp/.X11-unix:/tmp/.X11-unix:rw" \ 5 | --volume=$HOME/.torch/fvcore_cache:/tmp:rw \ 6 | --volume="${PWD}/../../app:/home/appuser/app" \ 7 | --volume="${PWD}/../../setup:/home/appuser/setup" \ 8 | --volume="${PWD}/../../data:/home/appuser/data" \ 9 | --volume="${PWD}/../scripts:/home/appuser/scripts" \ 10 | -p 8889:8889/tcp -p 48171:22 -p 8050:8050 \ 11 | --network detectinpaint \ 12 | --name=detect detector:v0 13 | 14 | -------------------------------------------------------------------------------- /detect/docker/set_X11.sh: -------------------------------------------------------------------------------- 1 | # Grant docker access to host X server to show images 2 | xhost +local:`docker inspect --format='{{ .Config.Hostname }}' detectron2` 3 | -------------------------------------------------------------------------------- /detect/scripts/ObjectDetection/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RexBarker/VideoObjectRemoval/26b8648645044389e9b7311f609c04b41f92f0b7/detect/scripts/ObjectDetection/__init__.py -------------------------------------------------------------------------------- /detect/scripts/ObjectDetection/detect.py: -------------------------------------------------------------------------------- 1 | # Detection to object instances 2 | 3 | # You may need to restart your runtime prior to this, to let your installation take effect 4 | # Some basic setup: 5 | # Setup detectron2 logger 6 | import os 7 | from glob import glob 8 | import detectron2 9 | 10 | # import some common libraries 11 | import numpy as np 12 | import pandas as pd 13 | import cv2 14 | import random 15 | 16 | # import some common detectron2 utilities 17 | from detectron2 import model_zoo 18 | from detectron2.engine import DefaultPredictor 19 | from detectron2.config import get_cfg 20 | from detectron2.utils.visualizer import Visualizer 21 | from detectron2.data import MetadataCatalog 22 | 23 | # plotting utilities 24 | import matplotlib.pyplot as plt 25 | import matplotlib.animation as animation 26 | 27 | # module specific library 28 | import ObjectDetection.imutils as imu 29 | 30 | # --------------------------------------------------------------------- 31 | class DetectSingle: 32 | def __init__(self, 33 | score_threshold = 0.5, 34 | model_zoo_config_path="COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml", 35 | selectObjectNames = None, #default to all 36 | **kwargs): 37 | # engine specific variables 38 | self.outputs = None 39 | self.cfg = None 40 | self.predictor = None 41 | self.things = None 42 | self.DEVICE = None 43 | 44 | # instance specific variables 45 | self.im = None 46 | self.selClassList = [] 47 | self.selObjNames = selectObjectNames 48 | self.selObjIndices = [] 49 | 50 | # initialize engine 51 | self.__initEngine(score_threshold, model_zoo_config_path) 52 | 53 | # annotation configuration 54 | self.fontconfig = { 55 | "fontFace" : cv2.FONT_HERSHEY_SIMPLEX, 56 | "fontScale" : 1, 57 | "color" : (0,255,0), 58 | "lineType" : 3 59 | } 60 | 61 | def __initEngine(self, score_threshold, model_zoo_config_path): 62 | #initialize configuration 63 | self.cfg = get_cfg() 64 | 65 | # add project-specific config (e.g., TensorMask) here if you're not running a model in detectron2's core library 66 | self.cfg.merge_from_file(model_zoo.get_config_file((model_zoo_config_path))) 67 | self.cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = score_threshold # score threshold to consider object of a class 68 | 69 | # Find a model from detectron2's model zoo. You can use the https://dl.fbaipublicfiles... url as well 70 | self.cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(model_zoo_config_path) 71 | 72 | self.predictor = DefaultPredictor(self.cfg) 73 | self.things = MetadataCatalog.get(self.cfg.DATASETS.TRAIN[0]) 74 | self.thing_classes = self.things.thing_classes 75 | self.thing_colors = self.things.thing_colors 76 | self.DEVICE = self.predictor.model.device.type + str(self.predictor.model.device.index) 77 | 78 | if self.selObjNames is None: 79 | self.selObjNames = self.thing_classes 80 | self.selObjIndices = list(range(len(self.selObjNames))) 81 | else: 82 | for n in self.selObjNames: 83 | assert n in self.thing_classes, f"Error finding object class name: {n}" 84 | self.selObjIndices.append(self.selObjNames.index(n)) 85 | 86 | 87 | def __convertToBBmasks(self): 88 | """ 89 | converts masks into mask of BBox Shape, replacing the orignal mask 90 | """ 91 | h,w,_ = self.im.shape 92 | self.masks = [imu.bboxToMask(bbx,(h,w)) for bbx in self.bboxes] 93 | 94 | def set_score_threshold(self,confidence): 95 | self.score_threshold = confidence 96 | 97 | def predict(self,img,selObjectNames=None,useBBmasks=False): 98 | if isinstance(img,str): 99 | # was an image file path 100 | assert os.path.exists(img), f"Specified image file {img} does not exist" 101 | self.im = cv2.imread(img) 102 | elif isinstance(img,np.ndarray): 103 | self.im = img 104 | else: 105 | raise Exception("Could not determine the object instance of 'img'") 106 | 107 | if selObjectNames is None: 108 | selObjectNames = self.selObjNames 109 | 110 | outputs = self.predictor(self.im) 111 | classes = list(outputs['instances'].pred_classes.cpu().numpy()) 112 | scores = list(outputs['instances'].scores.cpu().numpy()) 113 | objects = [self.thing_classes[c] in selObjectNames for c in classes] 114 | 115 | self.outputs = outputs 116 | self.masks = [ outputs['instances'].pred_masks[i].cpu().numpy() for i,o in enumerate(objects) if o ] 117 | self.bboxes = [ imu.bboxToList(outputs['instances'].pred_boxes[i]) for i,o in enumerate(objects) if o ] 118 | self.scores = [ scores[i] for i,o in enumerate(objects) if o ] 119 | 120 | if useBBmasks: 121 | self.__convertToBBmasks() 122 | 123 | self.selClassList = [ classes[i] for i,o in enumerate(objects) if o ] 124 | 125 | 126 | def get_results(self,getImage=True, getMasks=True, getBBoxes=True, getClasses=True): 127 | res = dict() 128 | if getImage: res['im'] = self.im 129 | if getMasks: res['masks'] = self.masks 130 | if getBBoxes: res['bboxes'] = self.bboxes 131 | if getClasses: res['classes'] = self.selClassList 132 | 133 | return res 134 | 135 | def annotate(self, im=None, masks=None, bboxes=None, addIndices=True): 136 | """ 137 | Adds annotation of the selected instances to the image 138 | Indices are added according to the order of prediction 139 | """ 140 | if im is None: 141 | im = self.im 142 | 143 | outim = im.copy() 144 | if masks is None: 145 | masks = self.masks 146 | 147 | if bboxes is None: 148 | bboxes = self.bboxes 149 | 150 | for i,(msk,bbox) in enumerate(zip(masks,bboxes)): 151 | color = self.thing_colors[i] 152 | x,y = [round(c) for c in imu.bboxCenter(bbox)] 153 | outim = imu.maskImage(outim,msk,color) 154 | if addIndices: 155 | cv2.putText(outim,str(i), (x,y), **self.fontconfig) 156 | 157 | return outim 158 | 159 | def visualize_all(self,im=None, scale=1.0): 160 | """ 161 | Adds full annotation to all objects within the image, based 162 | upon the detectron2 specification 163 | """ 164 | if im is None: 165 | im = self.im 166 | 167 | outim = im.copy() 168 | # assume that im is opencv format (BGR), so reverse 169 | vout = Visualizer(outim[:, :, ::-1], MetadataCatalog.get(self.cfg.DATASETS.TRAIN[0]), scale=scale) 170 | vout = vout.draw_instance_predictions(self.outputs["instances"].to("cpu")) 171 | 172 | return vout.get_image()[:, :, ::-1] # reverses channels again 173 | 174 | # --------------------------------------------------------------------- 175 | # Class: TrackSequence 176 | # - creates the basic sequence collection for set of input frames 177 | # - defined by either the filelist or a fileglob (list is better, you do the picking) 178 | # - assumptions are that all frames will be sequenced as XXX.jpg or XXX.png 179 | # - where XXX is the numerical order of the frame in the sequence 180 | class TrackSequence(DetectSingle): 181 | def __init__(self, *args, **kwargs): 182 | super(TrackSequence,self).__init__(*args, **kwargs) 183 | 184 | # sequence tracking variables 185 | self.selectFiles = None 186 | self.imglist = [] 187 | self.masklist = [] 188 | self.bboxlist = [] 189 | self.objclasslist = [] 190 | 191 | 192 | def load_images(self,fileglob=None, filelist=None): 193 | if fileglob is not None: 194 | files = sorted(glob(fileglob)) 195 | elif filelist is not None: 196 | files = filelist 197 | # check if these files really existed 198 | # (this doesn't make sense for globs, since they were checked to make the glob) 199 | checkfiles = [ os.path.exists(f) for f in files] 200 | if not all(checkfiles): 201 | mfiles = ",".join([f for f,e in zip(files,checkfiles) if not e]) 202 | raise Exception("Missing files in list: " + mfiles) 203 | else: 204 | raise Exception("No filelist or fileglob was supplied") 205 | 206 | # load images using OpenCV 207 | for fname in files: 208 | im = cv2.imread(fname) 209 | self.imglist.append(im) 210 | 211 | return len(self.imglist) 212 | 213 | def set_imagelist(self,imglist): 214 | self.imglist = imglist 215 | 216 | def get_images(self): 217 | return self.imglist 218 | 219 | 220 | def predict_sequence(self,fileglob=None, filelist=None, **kwargs): 221 | if len(self.imglist) == 0: 222 | # load images was not called yet 223 | self.load_images(fileglob=fileglob, filelist=filelist, **kwargs) 224 | 225 | for im in self.imglist: 226 | self.predict(im, **kwargs) 227 | self.masklist.append(self.masks) 228 | self.bboxlist.append(self.bboxes) 229 | self.objclasslist.append(self.selClassList) 230 | 231 | return len(self.masklist) 232 | 233 | 234 | def get_annotatedResults(self, fileglob=None, filelist=None, **kwargs): 235 | 236 | if len(self.masklist) == 0: 237 | self.predict_sequence(fileglob=fileglob, filelist=filelist, **kwargs) 238 | 239 | annoImgs = [] 240 | for im, msks, bbxs in zip(self.imglist, self.masklist, self.bboxlist): 241 | annoImgs.append(self.annotate(im,msks,bbxs)) 242 | 243 | return annoImgs 244 | 245 | 246 | def get_sequenceResults(self,getImage=True, getMasks=True, getBBoxes=True, getClasses=True): 247 | """ 248 | Results for sequence prediction, returned as dictionary for objName 249 | (easily pickelable) 250 | """ 251 | res = dict() 252 | if getImage: res['im'] = self.imglist 253 | if getMasks: res['masks'] = self.masklist 254 | if getBBoxes: res['bboxes'] = self.bboxlist 255 | if getClasses: res['classes'] = self.objclasslist 256 | 257 | return res 258 | 259 | 260 | # --------------------------------------------------------------------- 261 | # Class GroupSequence 262 | # - creates the grouping of a sequence by object based on class 263 | # - Object (in this sense): A single person, car, truck, teddy bear, etc. 264 | # - Class (in this sense): The type of object (i.e. persons, cars, trucks, teddy bears, etc.) 265 | # Main output: 266 | # - "objBBMaskSeq": { 267 | # objName1: [ [[seqBB], [seqMask] (obj1)], [[seqBB], [seqMask] (obj2)], etc] 268 | # objName2: [ [[seqBB], [seqMask] (obj1)], [[seqBB], [seqMask] (obj2)], etc] 269 | # } 270 | class GroupSequence(TrackSequence): 271 | def __init__(self, *args, **kwargs): 272 | super(GroupSequence,self).__init__(*args, **kwargs) 273 | 274 | # sequence tracking variables 275 | self.objBBMaskSeqDict = None 276 | self.objBBMaskSeqGrpDict = None 277 | self.combinedMaskList = None 278 | self.orginalSequenceMap = None 279 | self.MPEGconfig = { 280 | 'fps': 50, 281 | 'metadata': {'artist': "appuser"}, 282 | 'bitrate' : 1800 283 | } 284 | 285 | @staticmethod 286 | def __assignBBMaskToGroupByDistIndex(attainedGroups, trialBBs, trialMasks, index=None, widthFactor=2.0): 287 | """ 288 | Assign points to grouping: 289 | - [trialBBs] list of bounding boxs in given frame 290 | - [trialMasks] list of Masks in given frame 291 | - index : the index of the frame in the sequence 292 | - Note: len(trialBBs) == len(trialMasks), if nothing is predicted, an empty list is a place holder 293 | This function is meant to be called recursively, adding to its previous description of attainedGroups 294 | """ 295 | 296 | # trivial case. No points to work on 297 | if trialBBs is None or not len(trialBBs): 298 | return attainedGroups 299 | 300 | # no actual groups assigned yet, so enumerate the existing points into groups 301 | if attainedGroups is None or not len(attainedGroups): 302 | return [ [[bb,msk,index]] for bb,msk in zip(trialBBs,trialMasks)] 303 | 304 | lastGrpBBMsk = [grp[-1][:2] for grp in attainedGroups] 305 | currgrps = { i for i in range(len(attainedGroups))} 306 | 307 | for bbx,msk in zip(trialBBs,trialMasks): 308 | x1,y1,x2,y2 = bbx 309 | w = x2 - x1 310 | h = y2 - y1 311 | xc = (x1+x2)/2 312 | yc = (y1+y2)/2 313 | 314 | dist = [] 315 | for gi,gbbmsk in enumerate(lastGrpBBMsk): 316 | gbb,gmsk = gbbmsk 317 | gx1,gy1,gx2,gy2 = gbb 318 | gxc = (gx1+gx2)/2 319 | gyc = (gy1+gy2)/2 320 | d = np.sqrt((xc - gxc)**2 + (yc - gyc)**2) 321 | dist.append([d,gi]) 322 | 323 | #remove possible groups which were previously found 324 | dist = [[di,gi] for di,gi in dist if gi in currgrps] 325 | 326 | dist0 = dist[0] if dist else None 327 | mdist = dist0[0] if dist0 else None 328 | mdist_idx = dist0[1] if dist0 else None 329 | 330 | if len(dist) > 1: 331 | for d,idx in dist[1:]: 332 | if d < mdist: 333 | mdist = d 334 | mdist_idx = idx 335 | 336 | if mdist is None or mdist > widthFactor * w: 337 | # must be a new group 338 | attainedGroups.append([[bbx,msk,index]]) 339 | else: 340 | # belongs to an existing group 341 | attainedGroups[mdist_idx].append([bbx,msk,index]) 342 | #currgrps.remove(mdist_idx) #--> cleanout 343 | 344 | return attainedGroups 345 | 346 | 347 | def __createObjBBMask(self): 348 | assert self.objclasslist is not None, "No objclass sequences exist" 349 | assert self.imglist is not None, "No image sequences exist" 350 | assert self.bboxlist is not None, "No bbox sequences exist" 351 | assert self.masklist is not None, "No mask sequences exist" 352 | 353 | seenObjects = { o for olist in self.objclasslist for o in olist } 354 | 355 | self.objBBMaskSeqDict = dict() 356 | for objind in list(seenObjects): 357 | objName = self.thing_classes[objind] 358 | bbxlist = [] 359 | msklist = [] 360 | 361 | for bbxl, mskl, indl in zip(self.bboxlist, self.masklist, self.objclasslist): 362 | bbxlist.append([bbx for bbx,ind in zip(bbxl,indl) if ind == objind]) 363 | msklist.append([msk for msk,ind in zip(mskl,indl) if ind == objind]) 364 | 365 | self.objBBMaskSeqDict[objName] = [bbxlist, msklist] 366 | 367 | return len(self.objBBMaskSeqDict) 368 | 369 | def groupObjBBMaskSequence(self,fileglob=None, filelist=None, **kwargs): 370 | 371 | if len(self.imglist) == 0: 372 | # load images was not called yet 373 | self.load_images(fileglob=fileglob, filelist=filelist, **kwargs) 374 | 375 | if len(self.masklist) == 0: 376 | # predict images was not called yet 377 | self.predict_sequence(fileglob=fileglob, filelist=filelist, **kwargs) 378 | 379 | self.__createObjBBMask() 380 | assert self.objBBMaskSeqDict is not None, "BBox and Mask sequences have not been grouped by objectName" 381 | 382 | self.objBBMaskSeqGrpDict = dict() 383 | for objName, bblmskl in self.objBBMaskSeqDict.items(): 384 | bbl,mskl = bblmskl 385 | attGrpBBMsk = [] 386 | for i,bbsmsks in enumerate(zip(bbl,mskl)): 387 | bbxs,msks = bbsmsks 388 | if not bbxs: continue 389 | attGrpBBMsk = self.__assignBBMaskToGroupByDistIndex(attGrpBBMsk,bbxs,msks,i) 390 | 391 | self.objBBMaskSeqGrpDict[objName] = attGrpBBMsk 392 | 393 | 394 | def filter_ObjBBMaskSeq(self,allowObjNameInstances=None,minCount=10,inPlace=True): 395 | """ 396 | Performs filtering for minimum group size 397 | Eventually also for minimum relative object size 398 | """ 399 | if allowObjNameInstances is None: 400 | allowObjNameInstances = { objn:list(range(len(objl))) for objn,objl in self.objBBMaskSeqGrpDict.items()} 401 | elif not isinstance(allowObjNameInstances,dict): 402 | raise Exception("Expected dictionary object for 'objNameListInstDict',but got something else") 403 | 404 | objNameList = list(allowObjNameInstances.keys()) 405 | 406 | assert all([objN in list(self.objBBMaskSeqGrpDict.keys()) for objN in objNameList]), \ 407 | "Invalid list of object names given" 408 | 409 | self.orginalSequenceMap = \ 410 | {objn:{ i:None for i in range(len(self.objBBMaskSeqGrpDict[objn]))} \ 411 | for objn in self.objBBMaskSeqGrpDict.keys() } 412 | 413 | filteredSeq = dict() 414 | for grpName,grpInst in allowObjNameInstances.items(): 415 | inew = 0 416 | for iold, grp in enumerate(self.objBBMaskSeqGrpDict[grpName]): 417 | if ( grpInst is None or iold in grpInst ) and ( len(grp) >= minCount ): 418 | if not filteredSeq.get(grpName): 419 | filteredSeq[grpName] = [grp] 420 | else: 421 | filteredSeq[grpName].append(grp) 422 | 423 | self.orginalSequenceMap[grpName][iold] = inew 424 | inew += 1 425 | 426 | if inPlace: 427 | self.objBBMaskSeqGrpDict = filteredSeq 428 | return True 429 | else: 430 | return filteredSeq 431 | 432 | 433 | def fill_ObjBBMaskSequence(self, specificObjectNameInstances=None): 434 | """ 435 | Purpose: fill in missing masks of a specific object sequence 436 | masks are stretched to interpolated bbox region 437 | 'specificObjectNameInstances' = { {'objectName', [0, 2 , ..]} 438 | Examples: 439 | 'specificObjectNameInstances' = { 'person', [0, 2 ]} # return person objects, instance 0 and 2 440 | 'specificObjectNameInstances' = { 'person', None } # return 'person' objects, perform for all instances 441 | 'specificObjectNameInstances' = None # perform for all object names, for all instances 442 | Note: the indices of the 'specificObjectNameInstances' refer to the orginal order before filtering 443 | """ 444 | def maximizeBBoxCoordinates(bbxlist): 445 | # provides maximum scope of set of bbox inputs 446 | if isinstance(bbxlist[0],(float,int)): 447 | return bbxlist # single bounding box passed, first index is x0 448 | 449 | # multiple bbx 450 | minx,miny,maxx,maxy = bbxlist[0] 451 | 452 | if len(bbxlist) > 1: 453 | for x0,y0,x1,y1 in bbxlist[1:]: 454 | minx = min(minx,x0) 455 | maxx = max(maxx,x1) 456 | miny = min(miny,y0) 457 | maxy = max(maxy,y1) 458 | 459 | return [minx,miny,maxx,maxy] 460 | 461 | allObjNameIndices = { objn: list(range(len(obji))) for objn,obji in self.objBBMaskSeqGrpDict.items() } 462 | allObjNames = list(self.objBBMaskSeqGrpDict.keys()) 463 | 464 | if specificObjectNameInstances is None: # all objects, all instances 465 | objNameIndices = allObjNameIndices 466 | specificObjectNameInstances = allObjNameIndices 467 | objIndexMap = { objn: {i:i for i in range(len(objl))} \ 468 | for objn,objl in self.objBBMaskSeqGrpDict.items() } 469 | elif any([len(v)== 0 for v in specificObjectNameInstances.values()]): 470 | objIndexMap = {} 471 | for sobjn in specificObjectNameInstances.keys(): 472 | objIndexMap[sobjn] = {i:i for i in range(len(self.objBBMaskSeqGrpDict[sobjn])) } 473 | specificObjectNameInstances[sobjn] = [i for i in range(len(self.objBBMaskSeqGrpDict[sobjn]))] 474 | 475 | else: 476 | assert isinstance(specificObjectNameInstances,dict), \ 477 | "Expected a dictionary object for 'specificeeObjectNameInstances'" 478 | 479 | assert all([o in allObjNames for o in specificObjectNameInstances.keys()]), \ 480 | "Object name specified which are not in predicted list" 481 | 482 | if self.orginalSequenceMap is None: 483 | assert all([max(obji) < len(self.objBBMaskSeqGrpDict[objn]) for objn,obji in specificObjectNameInstances.items() ]), \ 484 | "Specified objectName index exceeded number of known instances of objectName from detection" 485 | 486 | objIndexMap = { objn: {i:i for i in range(len(objl))} \ 487 | for objn,objl in self.objBBMaskSeqGrpDict.items() } 488 | else: 489 | for objn,objl in specificObjectNameInstances.items(): 490 | # check that the original object index was mapped after filtering 491 | assert all([self.orginalSequenceMap[objn][i] is not None for i in objl]), \ 492 | f"Specified object '{objn}' index list was invalid due to missing index:[{objl}]" 493 | 494 | objIndexMap = {objn: {i:self.orginalSequenceMap[objn][i] \ 495 | for i in objl} for objn,objl in specificObjectNameInstances.items()} 496 | 497 | 498 | for objn, objindices in specificObjectNameInstances.items(): 499 | for objiold in objindices: 500 | objinew = objIndexMap[objn][objiold] 501 | rseq = self.objBBMaskSeqGrpDict[objn][objinew] 502 | 503 | mskseq = [[] for _ in range(len(self.imglist))] 504 | bbxseq = [[] for _ in range(len(self.imglist))] 505 | 506 | for bbxs, msks, ind in rseq: 507 | bbx = maximizeBBoxCoordinates(bbxs) 508 | msk = imu.combineMasks(msks) 509 | bbxseq[ind]=bbx 510 | mskseq[ind]=msk 511 | 512 | # build dataframe of known bbox coordinates 513 | targetBBDF = pd.DataFrame(bbxseq,columns=['x0','y0','x1','y1']) 514 | 515 | # determine missing indices 516 | missedIndices = [index for index, row in targetBBDF.iterrows() if row.isnull().any()] 517 | goodIndices = [ i for i in range(len(targetBBDF)) if i not in missedIndices] 518 | 519 | minGoodIndex = min(goodIndices) 520 | maxGoodIndex = max(goodIndices) 521 | missedIndices = [ i for i in missedIndices if i > minGoodIndex and i < maxGoodIndex] 522 | 523 | # extrapolate missing bbox coordinates 524 | targetBBDF = targetBBDF.interpolate(limit_direction='both', kind='linear') 525 | 526 | # output bboxes, resequenced 527 | for i in missedIndices: 528 | r = targetBBDF.iloc[i] 529 | bbxseq[i] = [r.x0,r.y0,r.x1,r.y1] 530 | 531 | # create missing masks by stretching or shrinking known masks from i-1 532 | # (relies on prior prediction of missing mask, for sequential missing masks) 533 | for i in missedIndices: 534 | lasti = i-1 # can't have i=0 missing, otherwise there's a corrupt system 535 | lastmsk = mskseq[lasti] 536 | 537 | # masks which were not good are found here 538 | x0o,y0o,x1o,y1o = [round(v) for v in targetBBDF.iloc[lasti]] 539 | x0r,y0r,x1r,y1r = [round(v) for v in targetBBDF.iloc[i]] 540 | 541 | wr = x1r - x0r 542 | hr = y1r - y0r 543 | 544 | msko = mskseq[lasti]*1.0 545 | mskr = np.zeros_like(msko) 546 | 547 | submsko = msko[y0o:y1o,x0o:x1o] 548 | submskr = cv2.resize(submsko,(wr,hr)) 549 | 550 | mskr[y0r:y1r,x0r:x1r] = submskr 551 | mskseq[i] = mskr > 0.0 # returns np.array(dtype=np.bool) 552 | 553 | # recollate into original class object, with the new definitions 554 | outrseq = [ [bbxmsk[0],bbxmsk[1], ind] for ind,bbxmsk in enumerate(zip(bbxseq, mskseq))] 555 | self.objBBMaskSeqGrpDict[objn][objinew] = outrseq 556 | 557 | return True 558 | 559 | 560 | def get_groupedResults(self, getObjNamesOnly=False, getSpecificObjNames=None): 561 | """ 562 | Results for sequence prediction, returned as dictionary for objName 563 | (easily pickelable) 564 | """ 565 | if getObjNamesOnly: 566 | return list(self.objBBMaskSeqGrpDict.keys()) 567 | 568 | if getSpecificObjNames is not None: 569 | if not isinstance(getSpecificObjNames,list): 570 | getSpecificObjNames = [getSpecificObjNames] 571 | 572 | getNames = [ n for n in self.objBBMaskSeqGrpDict.keys() if n in getSpecificObjNames ] 573 | else: 574 | getNames = list(self.objBBMaskSeqGrpDict.keys()) 575 | 576 | res = { k:v for k,v in self.objBBMaskSeqGrpDict.items() if k in getNames} 577 | return res 578 | 579 | 580 | def dilateErode_MaskSequence(self,masklist=None, 581 | actionList=['dilate'], 582 | kernelShape='rect', 583 | maskHalfWidth=4, 584 | inPlace=True): 585 | """ 586 | Purpose: to dilate or erode mask, where the mask list is 587 | either the exisitng mask list or one passed. The mask list must 588 | be a flat list of single masks for one frame index (all masks must have been combined) 589 | """ 590 | if masklist is None: 591 | if self.combinedMaskList is None: 592 | raise Exception("No combined masks were given for dilateErode operation") 593 | else: 594 | masklist = self.combinedMaskList 595 | 596 | maskListOut = [] 597 | 598 | for msk in masklist: 599 | msksout = imu.dilateErodeMask(msk, actionList=actionList, 600 | kernelShape=kernelShape, 601 | maskHalfWidth=maskHalfWidth) 602 | maskListOut.append(msksout) 603 | 604 | if inPlace: 605 | self.combinedMaskList = maskListOut 606 | return True 607 | else: 608 | return maskListOut 609 | 610 | 611 | def combine_MaskSequence(self,objNameList=None, 612 | inPlace=True): 613 | """ 614 | Purpose is to combine all masks at a given time index 615 | to a single mask. Result is stored 616 | """ 617 | if objNameList is None: 618 | objNameList = list(self.objBBMaskSeqGrpDict.keys()) 619 | elif not isinstance(objNameList,list): 620 | objNameList = [objNameList] 621 | 622 | assert all([objN in list(self.objBBMaskSeqGrpDict.keys()) for objN in objNameList]), \ 623 | "Invalid list of object names given" 624 | 625 | n_frames = len(self.imglist) 626 | 627 | # combine and write masks 628 | seqMasks = [ [] for _ in range(n_frames)] 629 | for objName in objNameList: 630 | for objgrp in self.objBBMaskSeqGrpDict[objName]: 631 | for bbx,msk,ind in objgrp: 632 | seqMasks[ind].append(msk) 633 | 634 | combinedMasks = [imu.combineMasks(msks) for msks in seqMasks] 635 | if inPlace: 636 | self.combinedMaskList = combinedMasks 637 | return True 638 | else: 639 | return combinedMasks 640 | 641 | 642 | def write_ImageMaskSequence(self,imagelist=None, 643 | masklist=None, 644 | writeMasksToDirectory=None, 645 | writeImagesToDirectory=None, 646 | cleanDirectory=False): 647 | 648 | if imagelist is None: 649 | if self.imglist is not None: 650 | imagelist = self.imglist 651 | 652 | if masklist is None: 653 | if self.combinedMaskList is not None: 654 | masklist = self.combinedMaskList 655 | 656 | # write images (which are paired with masks) 657 | if (writeImagesToDirectory is not None) and (imagelist is not None): 658 | imu.writeImagesToDirectory(imagelist,writeImagesToDirectory, 659 | minPadLength=5, 660 | cleanDirectory=cleanDirectory) 661 | 662 | # write masks (which are paired with masks) 663 | if (writeMasksToDirectory is not None) and (masklist is not None) : 664 | imu.writeMasksToDirectory(masklist,writeMasksToDirectory, 665 | minPadLength=5, 666 | cleanDirectory=cleanDirectory) 667 | 668 | return True 669 | 670 | 671 | def create_animationObject(self, 672 | getSpecificObjNames=None, 673 | framerange=None, 674 | useMasks=True, 675 | toHTML=False, 676 | MPEGfile=None, 677 | MPEGconfig=None, 678 | figsize=(10,10), 679 | interval=30, 680 | repeat_delay=1000, 681 | useFFMPEGdirect=False, 682 | useFOURCCstr=None): 683 | """ 684 | Purpose: produce an animation object of the masked frames 685 | returns an animation object to be rendered with HTML() 686 | """ 687 | if getSpecificObjNames is not None: 688 | if not isinstance(getSpecificObjNames,list): 689 | getSpecificObjNames = [getSpecificObjNames] 690 | 691 | getNames = [ n for n in self.objBBMaskSeqGrpDict.keys() if n in getSpecificObjNames ] 692 | else: 693 | getNames = list(self.objBBMaskSeqGrpDict.keys()) 694 | 695 | if framerange is None: 696 | framemin,framemax = 0, len(self.imglist)-1 697 | else: 698 | framemin,framemax = framerange 699 | 700 | fig = plt.figure(figsize=figsize) 701 | plt.axis('off') 702 | 703 | # combine and write masks 704 | if useMasks: 705 | if self.combinedMaskList is None: 706 | seqMasks = [ [] for _ in range(framemin,framemax+1)] 707 | for objName in getNames: 708 | for objgrp in self.objBBMaskSeqGrpDict[objName]: 709 | for bbx,msk,ind in objgrp: 710 | seqMasks[ind].append(msk) 711 | else: 712 | seqMasks = self.combinedMaskList 713 | 714 | outims = [] 715 | outrenders = [] 716 | for i,im in enumerate(self.imglist): 717 | if useMasks: 718 | msks = seqMasks[i] 719 | if isinstance(msks,list): 720 | for gi,msk in enumerate(msks): 721 | im = imu.maskImage(im,msk,self.thing_colors[gi]) 722 | else: 723 | im = imu.maskImage(im,msks,self.thing_colors[0]) 724 | 725 | 726 | outims.append(im) 727 | im = im[:,:,::-1] # convert from BGR to RGB 728 | 729 | renderplt = plt.imshow(im,animated=True) 730 | outrenders.append([renderplt]) 731 | 732 | ani = animation.ArtistAnimation(fig, outrenders, interval=interval, blit=True, 733 | repeat_delay=repeat_delay) 734 | 735 | if MPEGfile is not None: 736 | # expecting path to write file 737 | assert os.path.isdir(os.path.dirname(MPEGfile)), f"Could not write to path {os.path.dirname(MPEGfile)}" 738 | 739 | if MPEGconfig is None: 740 | MPEGconfig = self.MPEGconfig 741 | 742 | if useFFMPEGdirect: 743 | imu.writeFramesToVideo(outims,filePath=MPEGfile, fps=interval, 744 | fourccstr=useFOURCCstr,useFFMPEGdirect=True) 745 | else: 746 | mpegWriter = animation.writers['ffmpeg'] 747 | writer = mpegWriter(**MPEGconfig) 748 | ani.save(MPEGfile,writer=writer) 749 | 750 | # return html object, or just animation object 751 | return ani.to_html5_video() if toHTML else ani 752 | 753 | 754 | if __name__ == "__main__": 755 | pass 756 | -------------------------------------------------------------------------------- /detect/scripts/ObjectDetection/imutils.py: -------------------------------------------------------------------------------- 1 | # Basic image utilities 2 | 3 | import os 4 | from glob import glob 5 | import subprocess as sp 6 | 7 | # import some common libraries 8 | import cv2 9 | import numpy as np 10 | from math import log10, ceil 11 | 12 | fontconfig = { 13 | "fontFace" : cv2.FONT_HERSHEY_SIMPLEX, 14 | "fontScale" : 5, 15 | "color" : (0,0,255), 16 | "lineType" : 3 17 | } 18 | 19 | # --------------- 20 | # video editing tools 21 | 22 | def get_fourcc_string(vfile): 23 | if not os.path.isdir(vfile): 24 | cap = cv2.VideoCapture(vfile) 25 | vcodec = cap.get(cv2.CAP_PROP_FOURCC) 26 | vcodecstr = "".join([chr((int(vcodec) >> 8 * i) & 0xFF) for i in range(4)]) 27 | cap.release() 28 | return vcodecstr 29 | else: 30 | return None 31 | 32 | def get_fps(vfile): 33 | if not os.path.isdir(vfile): 34 | cap = cv2.VideoCapture(vfile) 35 | fps = cap.get(cv2.CAP_PROP_FPS) 36 | cap.release() 37 | return fps 38 | else: 39 | return None 40 | 41 | 42 | def get_nframes(vfile): 43 | if not os.path.isdir(vfile): 44 | cap = cv2.VideoCapture(vfile) 45 | n_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) 46 | cap.release() 47 | else: 48 | images = glob(os.path.join(vfile, '*.jp*')) 49 | if not images: 50 | images = glob(os.path.join(vfile, '*.png')) 51 | assert images, f"No image file (*.jpg or *.png) found in {vfile}" 52 | n_frames = len(images) 53 | 54 | return n_frames 55 | 56 | 57 | def get_WidthHeight(vfile): 58 | if not os.path.isdir(vfile): 59 | cap = cv2.VideoCapture(vfile) 60 | width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) 61 | height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) 62 | cap.release() 63 | else: 64 | images = glob(os.path.join(vfile, '*.jp*')) 65 | if not images: 66 | images = glob(os.path.join(vfile, '*.png')) 67 | assert images, f"No image file (*.jpg or *.png) found in {vfile}" 68 | img = cv2.imread(images[0]) 69 | height,width = img.shape[:2] 70 | 71 | return (width, height) 72 | 73 | 74 | def get_frame(vfile, n_frames, startframe=0, finishframe=None): 75 | if os.path.isdir(vfile): 76 | images = glob(os.path.join(vfile, '*.jp*')) 77 | if not images: 78 | images = glob(os.path.join(vfile, '*.png')) 79 | assert images, f"No image file (*.jpg or *.png) found in {vfile}" 80 | 81 | assert len(images) == n_frames, \ 82 | f"Mismatch in number of mask files versus number of frames\n" + \ 83 | f"n_frames={n_frames}, n_masks={len(images)}" 84 | 85 | images = sorted(images, 86 | key=lambda x: int(x.split('/')[-1].split('.')[0])) 87 | 88 | if finishframe is None: 89 | finishframe = n_frames 90 | 91 | images = images[startframe:finishframe] 92 | 93 | for img in images: 94 | frame = cv2.imread(img) 95 | yield frame 96 | 97 | else: 98 | cap = cv2.VideoCapture(vfile) 99 | 100 | # start frame is indexed 101 | # stop frame is set by controlling loop (caller) 102 | if startframe != 0: 103 | cap.set(cv2.CAP_PROP_POS_FRAMES, startframe) 104 | i = startframe 105 | else: 106 | i = 0 107 | 108 | while True: 109 | ret, frame = cap.read() 110 | 111 | if ret and i <= finishframe: 112 | yield frame 113 | else: 114 | cap.release() 115 | break 116 | 117 | i +=1 118 | 119 | 120 | # ------------ 121 | # Bounding box (bbox) and mask utilities 122 | def bboxToList(bboxTensor): 123 | return [float(x) for x in bboxTensor.to('cpu').tensor.numpy().ravel()] 124 | 125 | def bboxCenter(bbox): 126 | """ 127 | Returns (x_c,y_c) of center of bounding box list (x_0,y_0,x_1,y_1) 128 | """ 129 | return [(bbox[0] + bbox[2])/2,(bbox[1] + bbox[3])/2] 130 | 131 | def bboxIoU(boxA, boxB): 132 | """ 133 | Returns Intersection-Over-Union value for bounding bounding boxA and boxB 134 | where boxA and boxB are formed by (x_0,y_0,x_1,y_1) lists 135 | """ 136 | # determine the (x, y)-coordinates of the intersection rectangle 137 | xA = max(boxA[0], boxB[0]) 138 | yA = max(boxA[1], boxB[1]) 139 | xB = min(boxA[2], boxB[2]) 140 | yB = min(boxA[3], boxB[3]) 141 | 142 | # compute the area of intersection rectangle 143 | interArea = max(0, xB - xA + 1) * max(0, yB - yA + 1) 144 | 145 | # compute the area of both the prediction and ground-truthrectangles 146 | boxAArea = (boxA[2] - boxA[0] + 1) * (boxA[3] - boxA[1] + 1) 147 | boxBArea = (boxB[2] - boxB[0] + 1) * (boxB[3] - boxB[1] + 1) 148 | 149 | # compute the intersection over union by taking the intersection 150 | # area and dividing it by the sum of prediction + ground-truth 151 | # areas - the interesection area 152 | iou = interArea / float(boxAArea + boxBArea - interArea) 153 | 154 | # return the intersection over union value 155 | return iou 156 | 157 | def bboxToMask(bbox,maskShape): 158 | """ 159 | Creates a mask(np.bool) based on bbox area for equivalent mask shape 160 | """ 161 | assert isinstance(bbox,list), "bbox must be list" 162 | assert isinstance(maskShape,(tuple, list)), "maskShape must be list or tuple" 163 | x0,y0,x1,y1 = [round(x) for x in bbox] 164 | 165 | bbmask = np.full(maskShape,fill_value=False, dtype=np.bool) 166 | bbmask[y0:y1,x0:x1] = True 167 | return bbmask 168 | 169 | 170 | def combineMasks(maskList): 171 | """ 172 | Combines the list of masks into a single mask 173 | """ 174 | # single mask passed 175 | if not isinstance(maskList,list): 176 | return maskList 177 | elif len(maskList) == 1: 178 | return maskList[0] 179 | 180 | masks = [ m for m in maskList if len(m) ] 181 | maskcomb = masks.pop(0).copy() 182 | for msk in masks: 183 | maskcomb = np.logical_or(maskcomb,msk) 184 | 185 | return maskcomb 186 | 187 | 188 | def maskImage(im, mask, mask_color=(0,0,255), inplace=False): 189 | if inplace: 190 | outim = im 191 | else: 192 | outim = im.copy() 193 | 194 | if not isinstance(mask,list): 195 | for i in range(3): 196 | outim[:,:,i] = (mask > 0) * mask_color[i] + (mask == 0) * outim[:, :, i] 197 | 198 | return outim 199 | 200 | def maskToImg(mask, toThreeChannel=False): 201 | """ 202 | converts a mask(dtype=np.bool) to cv2 compatable image (dytpe=np.uint8) 203 | copies to a 3 channel array if requested 204 | """ 205 | maskout = np.zeros_like(mask,dtype=np.uint8) 206 | if mask.dtype == np.bool: 207 | maskout = np.uint8(255*mask) 208 | else: 209 | maskout = np.uint8((mask > 0) * 255) 210 | 211 | if toThreeChannel and len(maskout.shape) == 2: 212 | mmaskout = np.zeros([*maskout.shape,3],dtype=np.uint8) 213 | mmaskout[:,:,0] = maskout 214 | mmaskout[:,:,1] = maskout 215 | mmaskout[:,:,2] = maskout 216 | return mmaskout 217 | else: 218 | return maskout 219 | 220 | 221 | def dilateErodeMask(mask, actionList=['dilate'], kernelShape='rect', maskHalfWidth=4): 222 | """ 223 | Dilates or Erodes image mask ('mask') by 'kernelShape', based on mask width 224 | 'maskWidth'= 2 * maskHalfWidth + 1 225 | 'actionList' is a list of actions ('dilate' or 'erode') to perform on the mask 226 | """ 227 | for act in actionList: 228 | assert act in ('dilate', 'erode'), "Invalid action specified in actionList" 229 | 230 | if kernelShape.lower().startswith('re'): 231 | krnShape = cv2.MORPH_RECT # rectangular mask 232 | elif kernelShape.lower().startswith('cr'): 233 | krnShape = cv2.MORPH_CROSS # cross shape 234 | elif kernelShape.lower().startswith('el'): 235 | krnShape = cv2.MORPH_ELLIPSE # elliptical shape (or circlular) 236 | else: 237 | raise Exception(f"Unknown kernel mask shape specified: {kernelShape}") 238 | 239 | assert maskHalfWidth > 0, "Error: maskHalfWidth must be > 0" 240 | 241 | maskWasDtype = mask.dtype 242 | maskWidth = 2 * maskHalfWidth + 1 243 | krnElement = cv2.getStructuringElement(krnShape, 244 | (maskWidth,maskWidth), 245 | (maskHalfWidth, maskHalfWidth)) 246 | 247 | maskout = np.uint8(mask.copy()) 248 | for act in actionList: 249 | if act == 'dilate': 250 | maskout = cv2.dilate(maskout,krnElement) 251 | elif act == 'erode': 252 | maskout = cv2.erode(maskout,krnElement) 253 | else: 254 | pass # hmm, shouldn't get here 255 | 256 | maskout.dtype = maskWasDtype 257 | return maskout 258 | 259 | 260 | def videofileToFramesDirectory(videofile,dirPath,padlength=5,imgtype='png',cleanDirectory=True): 261 | """ 262 | writes a video file (.mp4, .avi, or .mov) to frames directory 263 | Here, it is understood that images are an np.array, dtype='uint8' 264 | of shape (w,h,3) 265 | """ 266 | assert imgtype in ('png', 'jpg'), f"Invalid image type '{imgtype}' given" 267 | 268 | if not os.path.isdir(dirPath): 269 | path = '/' if dirPath.startswith("/") else '' 270 | for d in dirPath.split('/'): 271 | if not d: continue 272 | path += d + '/' 273 | if not os.path.isdir(path): 274 | os.mkdir(path) 275 | elif cleanDirectory: 276 | for f in glob(os.path.join(dirPath,"*." + imgtype)): 277 | os.remove(f) # danger Will Robinson 278 | 279 | cap = cv2.VideoCapture(videofile) 280 | n = 0 281 | while True: 282 | ret,frame = cap.read() 283 | 284 | if not ret: 285 | cap.release() 286 | break 287 | fname = str(n).rjust(padlength,'0') + '.' + imgtype 288 | cv2.imwrite(os.path.join(dirPath,fname),frame) 289 | 290 | n += 1 291 | 292 | return n 293 | 294 | 295 | def writeImagesToDirectory(imageList,dirPath,minPadLength=None,imgtype='png',cleanDirectory=False): 296 | """ 297 | writes flat list of image arrays to directory 298 | Here, it is understood that images are an np.array, dtype='uint8' 299 | of shape (w,h,3) 300 | """ 301 | assert imgtype in ('png', 'jpg'), f"Invalid image type '{imgtype}' given" 302 | 303 | if not os.path.isdir(dirPath): 304 | path = '/' if dirPath.startswith("/") else '' 305 | for d in dirPath.split('/'): 306 | if not d: continue 307 | path += d + '/' 308 | if not os.path.isdir(path): 309 | os.mkdir(path) 310 | elif cleanDirectory: 311 | for f in glob(os.path.join(dirPath,"*." + imgtype)): 312 | os.remove(f) # danger Will Robinson 313 | 314 | n_frames = len(imageList) 315 | padlength = ceil(log10(n_frames)) if minPadLength is None else minPadLength 316 | for i,img in enumerate(imageList): 317 | fname = str(i).rjust(padlength,'0') + '.' + imgtype 318 | fname = os.path.join(dirPath,fname) 319 | cv2.imwrite(fname,img) 320 | 321 | return n_frames 322 | 323 | 324 | def writeMasksToDirectory(maskList,dirPath,minPadLength=None,imgtype='png',cleanDirectory=False): 325 | """ 326 | writes flat list of mask arrays to directory 327 | Here, it is understood that mask is an np.array,dtype='bool' 328 | of shape (w,h), will be output to (w,h,3) for compatibility 329 | """ 330 | assert imgtype in ('png', 'jpg'), f"Invalid image type '{imgtype}' given" 331 | 332 | if not os.path.isdir(dirPath): 333 | path = '/' if dirPath.startswith("/") else '' 334 | for d in dirPath.split('/'): 335 | if not d: continue 336 | path += d + '/' 337 | if not os.path.isdir(path): 338 | os.mkdir(path) 339 | elif cleanDirectory: 340 | for f in glob(os.path.join(dirPath,"*." + imgtype)): 341 | os.remove(f) # danger Will Robinson 342 | 343 | n_frames = len(maskList) 344 | padlength = ceil(log10(n_frames)) if minPadLength is None else minPadLength 345 | for i,msk in enumerate(maskList): 346 | fname = str(i).rjust(padlength,'0') + '.' + imgtype 347 | fname = os.path.join(dirPath,fname) 348 | cv2.imwrite(fname,msk * 255) 349 | 350 | return n_frames 351 | 352 | 353 | def writeFramesToVideo(imageList,filePath,fps=30, 354 | fourccstr=None, useFFMPEGdirect=False): 355 | """ 356 | Writes given set of frames to video file (platform specific coding) 357 | format is 'mp4' or 'avi' 358 | """ 359 | assert len(imageList) > 1, "Cannot make video with single frame" 360 | height,width =imageList[0].shape[:2] 361 | 362 | dirPath = os.path.dirname(filePath) 363 | if not os.path.isdir(dirPath): 364 | path = '' 365 | for d in dirPath.split('/'): 366 | if not d: continue 367 | path += d + '/' 368 | if not os.path.isdir(path): 369 | os.mkdir(path) 370 | 371 | if useFFMPEGdirect: 372 | # use ffmpeg installed in container (assuming were in container) 373 | # the ffmpeg, as compiled for Linux, contains the H264 codec 374 | # as available in the libx264 library 375 | assert filePath.endswith(".mp4"), "Cannot use non-mp4 formats with ffmpeg" 376 | 377 | # assume image list is from OpenCV read. Thus reverse the channels for the correct colors 378 | clip = [ im[:, :, ::-1] for im in imageList] 379 | h,w = clip[0].shape[:2] 380 | 381 | clippack = np.stack(clip) 382 | out,err = __ffmpegDirect(clippack,outputfile=filePath,fps=fps, size=[h,w]) 383 | assert os.path.exists(filePath), print(err) 384 | 385 | else: 386 | # use openCV method 387 | # this works, but native 'mp4v' codec is not compatible 388 | # with html.Video(). H264 codec is not availabe with OpenCV 389 | # unless you compile it from source (GPL issues) 390 | if filePath.endswith(".mp4"): 391 | if fourccstr is None: 392 | fourccstr = 'mp4v' 393 | fourcc = cv2.VideoWriter_fourcc(*fourccstr) 394 | elif filePath.endswith(".avi"): 395 | fourcc = cv2.VideoWriter_fourcc(*'XVID') 396 | else: 397 | assert False, f"Could not determine the video output type from {filePath}" 398 | 399 | outvid = cv2.VideoWriter(filePath, fourcc, fps, (width,height) ) 400 | 401 | # write out frames to video 402 | for im in imageList: 403 | outvid.write(im) 404 | 405 | outvid.release() 406 | 407 | return len(imageList) 408 | 409 | 410 | def __ffmpegDirect(clip, outputfile, fps, size=[256, 256]): 411 | 412 | vf = clip.shape[0] 413 | command = ['ffmpeg', 414 | '-y', # overwrite output file if it exists 415 | '-f', 'rawvideo', 416 | '-s', '%dx%d' % (size[1], size[0]), # '256x256', # size of one frame 417 | '-pix_fmt', 'rgb24', 418 | '-r', '25', # frames per second 419 | '-an', # Tells FFMPEG not to expect any audio 420 | '-i', '-', # The input comes from a pipe 421 | '-vcodec', 'libx264', 422 | '-b:v', '1500k', 423 | '-vframes', str(vf), # 5*25 424 | '-s', '%dx%d' % (size[1], size[0]), # '256x256', # size of one frame 425 | outputfile] 426 | 427 | pipe = sp.Popen(command, stdin=sp.PIPE, stderr=sp.PIPE) 428 | out, err = pipe.communicate(clip.tostring()) 429 | pipe.wait() 430 | pipe.terminate() 431 | return out,err 432 | 433 | 434 | def createNullVideo(filePath,message="No Image",heightWidth=(100,100)): 435 | h,w = heightWidth 436 | imgblank = np.zeros((h,w,3),dtype=np.uint8) 437 | if message: 438 | imgblank = cv2.putText(imgblank,message,(h // 2, w // 2),**fontconfig) 439 | 440 | # create blank video with 2 frames 441 | return writeFramesToVideo([imgblank,imgblank],filePath=filePath,fps=1) 442 | 443 | 444 | def maskedItemRelativeHistogram(img, msk,n_bins=10): 445 | im = img.copy() 446 | 447 | # reduce image to masked portion only 448 | for i in range(3): 449 | im[:,:,i] = (msk == 0) * 0 + (msk > 0) * im[:, :, i] 450 | 451 | take_ys= im.sum(axis=2).mean(axis=1) > 0 452 | take_xs= im.sum(axis=2).mean(axis=0) > 0 453 | 454 | imsub=im[take_ys,:,:] 455 | imsub= imsub[:,take_xs,:] 456 | 457 | # determine average vectors for each direction 458 | h_av = np.mean((imsub == 0) * 0 + (imsub > 0) * imsub,axis=1) 459 | v_av = np.mean((imsub == 0) * 0 + (imsub > 0) * imsub,axis=0) 460 | 461 | #h_abs_vec = np.array(range(h_av.shape[0]))/h_av.shape[0] 462 | h_ord_vec = h_av.sum(axis=1)/h_av.sum(axis=1).max() 463 | 464 | #v_abs_vec = np.array(range(v_av.shape[0]))/v_av.shape[0] 465 | v_ord_vec = v_av.sum(axis=1)/v_av.sum(axis=1).max() 466 | 467 | h_hist=np.histogram(h_ord_vec,bins=n_bins) 468 | v_hist=np.histogram(v_ord_vec,bins=n_bins) 469 | 470 | return (h_hist[0]/h_hist[0].sum(), v_hist[0]/v_hist[0].sum()) 471 | 472 | 473 | def drawPoint(im, XY, color=(0,0,255), radius=0, thickness = -1, inplace=False): 474 | """ 475 | draws a points over the top of an image 476 | point : (x,y) 477 | color : (R,G,B) 478 | """ 479 | xy = tuple([round(v) for v in XY]) 480 | if inplace: 481 | outim = im 482 | else: 483 | outim = im.copy() 484 | 485 | outim = cv2.circle(outim, xy, radius=radius, color=color, thickness=thickness) 486 | return outim 487 | 488 | 489 | def drawPointList(im, XY_color_list, radius=0, thickness = -1, inplace=False): 490 | """ 491 | draws points over the top of an image given a list of (point,color) pairs 492 | point : (x,y) 493 | colors : (R,G,B) 494 | """ 495 | if inplace: 496 | outim = im 497 | else: 498 | outim = im.copy() 499 | 500 | for XY,color in XY_color_list[:-1]: 501 | xy = tuple([round(v) for v in XY]) 502 | outim = cv2.circle(outim, xy, radius=radius, color=color, thickness=round(thickness * 0.8)) 503 | 504 | # last point is larger 505 | XY,color = XY_color_list[-1] 506 | xy = tuple([round(v) for v in XY]) 507 | outim = cv2.circle(outim, xy, radius=radius, color=color, thickness=round(thickness)) 508 | 509 | return outim 510 | 511 | if __name__ == "__main__": 512 | pass -------------------------------------------------------------------------------- /detect/scripts/ObjectDetection/inpaintRemote.py: -------------------------------------------------------------------------------- 1 | # Utilities to run DeepFlow Inpaint from remote container 2 | from time import time 3 | from paramiko import SSHClient, AutoAddPolicy 4 | from threading import Thread 5 | 6 | # primarily utilize parent methods, where possible 7 | class InpaintRemote(SSHClient): 8 | def __init__(self, *args, **kwargs): 9 | super(InpaintRemote,self).__init__(*args, **kwargs) 10 | self.isConnected = False 11 | self.set_missing_host_key_policy(AutoAddPolicy()) 12 | self.c = { 13 | 'pythonPath': "/usr/bin/python3", 14 | 'workingDir': "/home/appuser/Deep-Flow", 15 | 'scriptPath': "/home/appuser/Deep-Flow/tools/video_inpaint.py", 16 | 'pretrainedModel': "/home/appuser/Deep-Flow/pretrained_models/FlowNet2_checkpoint.pth.tar", 17 | 'optionsString' : "--FlowNet2 --DFC --ResNet101 --Propagation" 18 | } 19 | 20 | def __del__(self): 21 | self.close() 22 | 23 | def connectInpaint(self,hostname='inpaint', username='appuser', password='appuser'): 24 | self.connect(hostname,username=username,password=password) 25 | self.isConnected = True 26 | 27 | def executeCommandsInpaint(self,commands): 28 | """ 29 | Executes specified commands in container, returns results 30 | """ 31 | assert self.isConnected, "Client was not connected!" 32 | 33 | start = time() 34 | results = { 'stdin': [], 'stdout': [], 'stderr': [] } 35 | for cmd in commands: 36 | stdin, stdout, stderr = self.exec_command(cmd) # non-blocking call 37 | exit_status = stdout.channel.recv_exit_status() # blocking call 38 | results['stdin'].append(stdin) 39 | results['stdout'].append(stdout) 40 | results['stderr'].append(stderr) 41 | 42 | finish = time() 43 | return results 44 | 45 | 46 | def testConnectionInpaint(self,testCommands=None,hardErrors=True): 47 | """ 48 | Tests simple connectivity to the container 49 | to see if host is accessible, and all command paths are accessible 50 | """ 51 | assert self.isConnected, "Client was not connected!" 52 | 53 | if testCommands is None: 54 | testCommands = [ f'cd {self.c["workingDir"]} ; pwd', 55 | f'ls {self.c["pythonPath"]}', 56 | f'ls {self.c["scriptPath"]}', 57 | f'ls {self.c["pretrainedModle"]}', 58 | ] 59 | 60 | start = time() 61 | errors = [] 62 | for cmd in testCommands: 63 | stdin, stdout, stderr = self.exec_command(cmd) # non-blocking call 64 | exit_status = stdout.channel.recv_exit_status() # blocking call 65 | if stderr: 66 | errors.append({'cmd': cmd, 'message': stderr}) 67 | finish = time() 68 | 69 | if any(errors): 70 | disconnectInpaint() 71 | if hardErrors: 72 | for err in errors: 73 | print(f"Error executing remote command:\n<<{err['cmd']}>>") 74 | print(f"\nResult output:\n") 75 | for l in err['message']: 76 | print(l.strip()) 77 | return False 78 | 79 | raise Exception("Errors encountered while testing remote execution") 80 | else: 81 | return errors 82 | 83 | return True 84 | 85 | 86 | def disconnectInpaint(self): 87 | self.close() 88 | self.isConnected = False 89 | 90 | def runInpaint(self, 91 | frameDirPath, maskDirPath, 92 | commandScript=None, # default pre-baked script will be used 93 | inputHeight=512, inputWidth=1024, # maximum size limited to 512x1024 94 | CUDA_VISIBLE_DEVICES='', # specify specific device if required, otherwise default 95 | optionsString='' # optional parameters string 96 | ): 97 | """ 98 | 'runInpaint' will execute a 'pre-baked' formula for inpainting based on the example from 99 | the https://github.com/nbei/Deep-Flow-Guided-Video-Inpainting definition. 100 | 101 | """ 102 | assert self.isConnected, "Client was not connected!" 103 | 104 | cudaString = f"CUDA_VISIBLE_DEVICES={CUDA_VISIBLE_DEVICES}; " if CUDA_VISIBLE_DEVICES else "" 105 | if not optionsString: 106 | optionsString = self.c['optionsString'] 107 | 108 | if commandScript is None: 109 | commandScript = cudaString + \ 110 | f"cd {self.c['workingDir']}; " + \ 111 | f"{self.c['pythonPath']} {self.c['scriptPath']} " + \ 112 | f"--frame_dir {frameDirPath} --MASK_ROOT {maskDirPath} " + \ 113 | f"--img_size {inputHeight} {inputWidth} " + \ 114 | optionsString 115 | 116 | start = time() 117 | stdin, stdout, stderr = self.exec_command(commandScript) # non-blocking call 118 | exit_status = stdout.channel.recv_exit_status() # blocking call 119 | finish = time() 120 | 121 | return (stdin, stdout, stderr) 122 | 123 | if __name__ == "__main__": 124 | pass 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | -------------------------------------------------------------------------------- /detect/scripts/demo.py: -------------------------------------------------------------------------------- 1 | # Demo script to process command line level 2 | # run full video object removal from the command line 3 | import os 4 | import sys 5 | import cv2 6 | import argparse 7 | import tempfile 8 | from glob import glob 9 | from time import sleep 10 | import numpy as np 11 | import ObjectDetection.imutils as imu 12 | from ObjectDetection.detect import GroupSequence 13 | from ObjectDetection.inpaintRemote import InpaintRemote 14 | from threading import Thread 15 | 16 | # ------------ 17 | # helper functions 18 | 19 | class ThreadWithReturnValue(Thread): 20 | def __init__(self, group=None, target=None, name=None, 21 | args=(), kwargs={}, Verbose=None): 22 | Thread.__init__(self, group, target, name, args, kwargs) 23 | self._return = None 24 | def run(self): 25 | if self._target is not None: 26 | self._return = self._target(*self._args, **self._kwargs) 27 | def join(self, *args): 28 | Thread.join(self, *args) 29 | return self._return 30 | 31 | 32 | # ------------ 33 | parser = argparse.ArgumentParser(description='Automatic Video Object Removal') 34 | 35 | parser.add_argument('--input',type=str,required=True, 36 | help="Video (.mp4,.avi,.mov) or directory of frames") 37 | 38 | parser.add_argument('--start', type=int, required=False, default=None, 39 | help="start at frame number") 40 | 41 | parser.add_argument('--finish', type=int, required=False, default=None, 42 | help="finish at frame number (negative numbers indicate from end)") 43 | 44 | parser.add_argument('--outfile',type=str,required=False, default="results.mp4", 45 | help="Output file (.mp4), default='results.mp4'") 46 | 47 | parser.add_argument('--objlist',type=str,nargs='+', default=None, 48 | help='object list, quote delimited per instance: "person:1,5,2" "car:0,2"') 49 | 50 | parser.add_argument('--confidence',type=float,default=0.5, 51 | help='prediction probablility threshold for object classification (default=0.5)') 52 | 53 | parser.add_argument('--minCount', type=int, default=None, 54 | help="minimum length of sequence for object class filtering") 55 | 56 | parser.add_argument('--dilationW',type=int, default=0, 57 | help="Use dilation, sets mask half width (default=0, no dilation)") 58 | 59 | parser.add_argument('--dilationK',type=str, default='el', choices=['re','cr','el'], 60 | help="Use kernel shape elipse(el), cross(cr), or rectangle(re) (default=el)") 61 | 62 | parser.add_argument('--useBBmasks', action='store_true', 63 | help="Utilize Bounding Box mask substituion") 64 | 65 | parser.add_argument('--annotateOnly', action='store_true', 66 | help="Only perform dection and annotates images, skip inpainting") 67 | 68 | parser.add_argument('--sequenceOnly', action='store_true', 69 | help="Perform detection, sequencing, skip inpainting") 70 | 71 | if __name__ == '__main__': 72 | 73 | #-------------- 74 | # preprocessing 75 | 76 | # get commands 77 | args = parser.parse_args() 78 | 79 | # check args 80 | assert sum([args.annotateOnly, args.sequenceOnly]) <= 1, \ 81 | "Ambiguous arguments for 'annotateOnly' and 'sequenceOnly' given" 82 | 83 | # make sure output file is mp4 84 | assert ".mp4" in args.outfile, \ 85 | f"Only MP4 files are supported for output, got:{args.outfile}" 86 | 87 | # determine number of frames 88 | vfile = args.input 89 | assert os.path.exists(vfile), f"Could not determine the input file or directory: {vfile}" 90 | 91 | n_frames = imu.get_nframes(vfile) 92 | width,height = imu.get_WidthHeight(vfile) 93 | fps = imu.get_fps(vfile) 94 | 95 | # determine number of frames to process 96 | startframe = 0 97 | if args.start: 98 | assert abs(args.start) < n_frames, \ 99 | f"Invalid 'start'={startframe} frame specified, exceeds number of frames ({n_frames})" 100 | 101 | startframe = args.start if args.start >= 0 else n_frames + args.start # negative indicates from end 102 | 103 | finishframe = n_frames 104 | if args.finish is not None: 105 | assert abs(args.finish) < n_frames, \ 106 | f"Invalid 'finish'={finishframe} frame specified, exceeds number of frames({n_frames})" 107 | 108 | finishframe = args.finish if args.finish >= 0 else n_frames + args.finish # negative indicates from end 109 | 110 | assert finishframe > startframe, f"Invalid definition of 'start'={startframe} and 'finish'={finishframe}, start > finish" 111 | 112 | # acquire all frames 113 | frame_gen = imu.get_frame(vfile,n_frames=n_frames,startframe=startframe, finishframe=finishframe) 114 | 115 | imglist = [] 116 | for img in frame_gen: 117 | imglist.append(img) 118 | 119 | #-------------- 120 | # perform detection, determine number of objects 121 | objlistDict = {} 122 | objlistNames = None 123 | if args.objlist: 124 | for objnl in args.objlist: 125 | if ":" in objnl: 126 | objn,objl = objnl.strip('"\'').replace(" ","").split(":") 127 | objinds = [ int(v) for v in objl.split(",")] 128 | else: 129 | objn = objnl.strip('"\' ') 130 | objinds = [] 131 | 132 | if objlistDict.get(objn): 133 | objlistDict[objn].extend(objinds) 134 | else: 135 | objlistDict[objn] = objinds 136 | 137 | objlistNames = list(objlistDict.keys()) 138 | 139 | # intiate engine 140 | groupseq = GroupSequence(selectObjectNames=objlistNames, score_threshold=args.confidence) 141 | groupseq.set_imagelist(imglist) 142 | 143 | # perform grouping 144 | groupseq.groupObjBBMaskSequence(useBBmasks=args.useBBmasks) 145 | 146 | if args.annotateOnly: 147 | res = groupseq.get_groupedResults() 148 | annoImages = groupseq.get_annotatedResults() 149 | imu.writeFramesToVideo(imageList=annoImages,filePath=args.outfile,fps=fps) 150 | for objn,objl in res.items(): 151 | print(f"ObjName = {objn}, has {len(objl)} instances:") 152 | for i,obji in enumerate(objl): 153 | print(f"\t{objn}[{i}] has {len(obji)} frame instances") 154 | 155 | sys.exit(0) 156 | 157 | # filtered by object class, instance, and length 158 | if args.minCount is not None: 159 | groupseq.filter_ObjBBMaskSeq(allowObjNameInstances=objlistDict, 160 | minCount=args.minCount) 161 | 162 | # fill sequence 163 | groupseq.fill_ObjBBMaskSequence(specificObjectNameInstances=objlistDict if objlistDict else None) 164 | 165 | # use dilation 166 | if args.dilationW > 0: 167 | groupseq.combine_MaskSequence() 168 | groupseq.dilateErode_MaskSequence(kernelShape=args.dilationK, 169 | maskHalfWidth=args.dilationW) 170 | 171 | # output sequence video only 172 | if args.sequenceOnly: 173 | groupseq.create_animationObject(MPEGfile=args.outfile, 174 | interval=fps, 175 | useFFMPEGdirect=True) 176 | sys.exit(0) 177 | 178 | # perform inpainting 179 | with tempfile.TemporaryDirectory(dir=os.path.dirname(args.outfile)) as tempdir: 180 | 181 | frameDirPath =os.path.join(tempdir,"frames") 182 | maskDirPath = os.path.join(tempdir,"masks") 183 | resultDirPath = os.path.join(os.path.join(tempdir,"Inpaint_Res"),"inpaint_res") 184 | 185 | groupseq.write_ImageMaskSequence( 186 | writeImagesToDirectory=frameDirPath, 187 | writeMasksToDirectory=maskDirPath) 188 | 189 | rinpaint = InpaintRemote() 190 | rinpaint.connectInpaint() 191 | 192 | trd1 = ThreadWithReturnValue(target=rinpaint.runInpaint, 193 | kwargs={'frameDirPath':frameDirPath,'maskDirPath':maskDirPath}) 194 | trd1.start() 195 | 196 | print("working:",end='',flush=True) 197 | while trd1.is_alive(): 198 | print('.',end='',flush=True) 199 | sleep(1) 200 | 201 | print("\nfinished") 202 | rinpaint.disconnectInpaint() 203 | 204 | stdin,stdout,stderr = trd1.join() 205 | ok = False 206 | for l in stdout: 207 | if "Propagation has been finished" in l: 208 | ok = True 209 | print(l.strip()) 210 | 211 | assert ok, "Could not determine if results were valid!" 212 | 213 | print(f"\n....Writing results to {args.outfile}") 214 | 215 | resultfiles = sorted(glob(os.path.join(resultDirPath,"*.png"))) 216 | imgres = [ cv2.imread(f) for f in resultfiles] 217 | imu.writeFramesToVideo(imgres, filePath=args.outfile, fps=fps) 218 | print(f"Finished writing {args.outfile} ") 219 | 220 | print("Done") 221 | -------------------------------------------------------------------------------- /detect/scripts/run_detection.py: -------------------------------------------------------------------------------- 1 | # Test image detection implementation 2 | import cv2 3 | from time import time, sleep 4 | from glob import glob 5 | import numpy as np 6 | import ObjectDetection.imutils as imu 7 | from ObjectDetection.detect import DetectSingle, TrackSequence, GroupSequence 8 | from ObjectDetection.inpaintRemote import InpaintRemote 9 | from threading import Thread 10 | 11 | class ThreadWithReturnValue(Thread): 12 | def __init__(self, group=None, target=None, name=None, 13 | args=(), kwargs={}, Verbose=None): 14 | Thread.__init__(self, group, target, name, args, kwargs) 15 | self._return = None 16 | def run(self): 17 | if self._target is not None: 18 | self._return = self._target(*self._args, **self._kwargs) 19 | def join(self, *args): 20 | Thread.join(self, *args) 21 | return self._return 22 | 23 | test_imutils = False 24 | test_single = False 25 | test_dilateErode = False 26 | test_sequence = False 27 | test_grouping = False 28 | test_bbmasks = False 29 | test_maskFill = False 30 | test_maskoutput = False 31 | test_remoteInpaint = True 32 | 33 | if test_imutils: 34 | bbtest = [0.111, 0.123, 0.211, 0.312] 35 | bbc = imu.bboxCenter(bbtest) 36 | print(bbc) 37 | 38 | if test_single: 39 | detect = DetectSingle(selectObjectNames=['person','car']) 40 | imgfile = "../data/input.jpg" 41 | detect.predict(imgfile) 42 | imout = detect.annotate() 43 | 44 | cv2.imshow('results',imout) 45 | cv2.waitKey(0) 46 | cv2.destroyAllWindows() 47 | 48 | imout = detect.visualize_all(scale=1.2) 49 | cv2.imshow('results',imout) 50 | cv2.waitKey(0) 51 | cv2.destroyAllWindows(); 52 | 53 | if test_dilateErode: 54 | detect = DetectSingle(selectObjectNames=['person','car']) 55 | imgfile = "../data/input.jpg" 56 | detect.predict(imgfile) 57 | masks = detect.masks 58 | mask = imu.combineMasks(masks) 59 | 60 | orig = "original" 61 | modf = "DilationErosion" 62 | cv2.namedWindow(orig) 63 | cv2.namedWindow(modf) 64 | 65 | # single dilation operation 66 | modmask = imu.dilateErodeMask(mask) 67 | cv2.imshow(orig,imu.maskToImg(mask)) 68 | cv2.imshow(modf,imu.maskToImg(modmask)) 69 | cv2.waitKey(0) 70 | 71 | modmask = imu.dilateErodeMask(mask,actionList=['dilate','erode','dilate']) 72 | cv2.imshow(orig,imu.maskToImg(mask)) 73 | cv2.imshow(modf,imu.maskToImg(modmask)) 74 | cv2.waitKey(0) 75 | 76 | cv2.destroyAllWindows() 77 | 78 | 79 | if test_sequence: 80 | fnames = sorted(glob("../data/Colomar/frames/*.png"))[200:400] 81 | trackseq = TrackSequence(selectObjectNames=['person','car']) 82 | trackseq.predict_sequence(filelist=fnames) 83 | res = trackseq.get_sequenceResults() 84 | 85 | if test_grouping: 86 | fnames = sorted(glob("../data/Colomar/frames/*.png"))[200:400] 87 | groupseq = GroupSequence(selectObjectNames=['person','car']) 88 | groupseq.load_images(filelist=fnames) 89 | groupseq.groupObjBBMaskSequence(useBBmasks=test_bbmasks) 90 | res = groupseq.get_groupedResults(getSpecificObjNames='person') 91 | 92 | if test_grouping and test_maskFill: 93 | groupseq.filter_ObjBBMaskSeq(allowObjNameInstances={'person':[2]},minCount=70) 94 | #groupseq.fill_ObjBBMaskSequence(specificObjectNameInstances={'person':[0,1,2]}) 95 | groupseq.fill_ObjBBMaskSequence() 96 | 97 | if test_grouping and test_maskoutput: 98 | groupseq.combine_MaskSequence() 99 | groupseq.dilateErode_MaskSequence(kernelShape='elipse',maskHalfWidth=10) 100 | groupseq.write_ImageMaskSequence(cleanDirectory=True, 101 | writeImagesToDirectory="../data/Colomar/fourInpaint/frames", 102 | writeMasksToDirectory="../data/Colomar/fourInpaint/masks") 103 | groupseq.create_animationObject(MPEGfile="../data/Colomar/result.mp4") 104 | 105 | if test_remoteInpaint: 106 | rinpaint = InpaintRemote() 107 | rinpaint.connectInpaint() 108 | 109 | frameDirPath="/home/appuser/data/Colomar/threeInpaint/frames" 110 | maskDirPath="/home/appuser/data/Colomar/threeInpaint/masks" 111 | 112 | trd1 = ThreadWithReturnValue(target=rinpaint.runInpaint, 113 | kwargs={'frameDirPath':frameDirPath,'maskDirPath':maskDirPath}) 114 | trd1.start() 115 | 116 | print("working:",end='',flush=True) 117 | while trd1.is_alive(): 118 | print('.',end='',flush=True) 119 | sleep(1) 120 | 121 | print("\nfinished") 122 | rinpaint.disconnectInpaint() 123 | 124 | stdin,stdout,stderr = trd1.join() 125 | ok = False 126 | for l in stdout: 127 | if "Propagation has been finished" in l: 128 | ok = True 129 | print(l.strip()) 130 | 131 | print("done") 132 | -------------------------------------------------------------------------------- /detect/scripts/test_detect.sh: -------------------------------------------------------------------------------- 1 | # run the detector for a sample image: ../data/input.jpg 2 | # result is stored in ../data/outputs 3 | 4 | DATA=/home/appuser/data 5 | BASE=/home/appuser/detectron2_repo 6 | 7 | wget http://images.cocodataset.org/val2017/000000439715.jpg -O ../../data/input.jpg 8 | 9 | docker exec -ti detect \ 10 | /usr/bin/python3 $BASE/demo/demo.py \ 11 | --config-file $BASE/configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml \ 12 | --input $DATA/input.jpg \ 13 | --output $DATA/outputs/ \ 14 | --opts MODEL.WEIGHTS detectron2://COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/model_final_f10217.pkl 15 | -------------------------------------------------------------------------------- /detect/scripts/test_inpaint_remote.sh: -------------------------------------------------------------------------------- 1 | cd /home/appuser/Deep-Flow 2 | 3 | /usr/bin/python3 /home/appuser/Deep-Flow/tools/video_inpaint.py \ 4 | --frame_dir /home/appuser/data/flamingo/origin \ 5 | --MASK_ROOT /home/appuser/data/flamingo/masks \ 6 | --img_size 512 832 \ 7 | --FlowNet2 --DFC --ResNet101 --Propagation 8 | -------------------------------------------------------------------------------- /detect/scripts/test_sshparamiko.py: -------------------------------------------------------------------------------- 1 | from threading import Thread 2 | from time import time, sleep 3 | import paramiko 4 | 5 | class ThreadWithReturnValue(Thread): 6 | def __init__(self, group=None, target=None, name=None, 7 | args=(), kwargs={}, Verbose=None): 8 | Thread.__init__(self, group, target, name, args, kwargs) 9 | self._return = None 10 | def run(self): 11 | #print(type(self._target)) 12 | if self._target is not None: 13 | self._return = self._target(*self._args, 14 | **self._kwargs) 15 | def join(self, *args): 16 | Thread.join(self, *args) 17 | return self._return 18 | 19 | def boring(waittime): 20 | stdin, stdout, stderr = client.exec_command(f"sleep {waittime}; echo I did something") 21 | exit_status = stdout.channel.recv_exit_status() 22 | return [l.strip() for l in stdout] 23 | 24 | 25 | client = paramiko.SSHClient() 26 | client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 27 | client.connect('inpaint', username='appuser', password='appuser') 28 | 29 | #stdin, stdout, stderr = client.exec_command('ls -l') 30 | start = time() 31 | t1 = ThreadWithReturnValue(target=boring,args=(10,)) 32 | t1.start() 33 | #t1.join() 34 | 35 | while t1.is_alive(): 36 | print("apparently not done") 37 | sleep(1) 38 | 39 | res = t1.join() 40 | finish = time() 41 | print("That took ",finish - start, " seconds") 42 | 43 | print("\n".join(res)) 44 | 45 | 46 | client.close() -------------------------------------------------------------------------------- /inpaint/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:8.0-cudnn7-devel-ubuntu16.04 2 | 3 | RUN apt update -y; apt install -y \ 4 | python3 \ 5 | python3-pip \ 6 | libglib2.0-0 \ 7 | libsm6 \ 8 | libxext6 \ 9 | libxrender-dev \ 10 | openssh-server sudo \ 11 | wget git vim ffmpeg 12 | 13 | # main setup 14 | RUN pip3 --disable-pip-version-check install \ 15 | addict==2.2.1 \ 16 | certifi==2019.6.16 \ 17 | cffi==1.12.3 \ 18 | chardet==3.0.4 \ 19 | cvbase==0.5.5 \ 20 | Cython==0.29.12 \ 21 | idna==2.8 \ 22 | mkl-fft \ 23 | mkl-random \ 24 | numpy==1.16.4 \ 25 | olefile==0.46 \ 26 | opencv-python==4.1.0.25 \ 27 | Pillow==6.2.0 \ 28 | protobuf==3.8.0 \ 29 | pycparser==2.19 \ 30 | PyYAML==5.1.1 \ 31 | requests==2.22.0 \ 32 | scipy==1.2.1 \ 33 | six==1.12.0 \ 34 | tensorboardX==1.8 \ 35 | terminaltables==3.1.0 \ 36 | torch==0.4.0 \ 37 | torchvision==0.2.1 \ 38 | tqdm==4.32.1 \ 39 | urllib3==1.25.3 \ 40 | pytest-runner 41 | 42 | RUN pip3 --no-dependencies --disable-pip-version-check install mmcv==0.2.10 43 | 44 | # create a non-root user 45 | ARG USER_ID=1000 46 | RUN useradd -m --no-log-init --system --uid ${USER_ID} appuser -s /bin/bash -g sudo -G root 47 | RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers 48 | RUN echo 'appuser:appuser' | chpasswd 49 | USER appuser 50 | WORKDIR /home/appuser 51 | 52 | # setup ssh 53 | RUN sudo service ssh start 54 | EXPOSE 22 55 | 56 | 57 | # install Deep-Flow 58 | RUN git clone https://github.com/RexBarker/Deep-Flow.git Deep-Flow 59 | 60 | # install scripts 61 | WORKDIR /home/appuser/Deep-Flow 62 | RUN chmod +x /home/appuser/Deep-Flow/install_scripts.sh && \ 63 | bash ./install_scripts.sh 64 | 65 | # useful for remotely debugging with VScode from remote workstation 66 | #CMD ["/usr/sbin/sshd","-D"] 67 | #CMD ["jupyter", "notebook", "--ip=0.0.0.0", "--no-browser"] 68 | -------------------------------------------------------------------------------- /inpaint/docker/build_inpaint.sh: -------------------------------------------------------------------------------- 1 | docker build --tag inpainting . 2 | -------------------------------------------------------------------------------- /inpaint/docker/run_inpainting.sh: -------------------------------------------------------------------------------- 1 | #docker run --ipc=host --gpus all -ti -v"${PWD}/..:/inpainting" inpainting:latest bash 2 | docker run --ipc=host --gpus all -ti \ 3 | -v"${PWD}/../../data:/home/appuser/data" \ 4 | -v"${PWD}/../../setup:/home/appuser/setup" \ 5 | -v"${PWD}/../scripts:/home/appuser/scripts" \ 6 | -v"${PWD}/../pretrained_models:/home/appuser/Deep-Flow/pretrained_models" \ 7 | -p 48172:22 \ 8 | --network detectinpaint \ 9 | --name=inpaint inpainting:latest bash 10 | 11 | -------------------------------------------------------------------------------- /inpaint/docker/set_X11.sh: -------------------------------------------------------------------------------- 1 | # Grant docker access to host X server to show images 2 | xhost +local:`docker inspect --format='{{ .Config.Hostname }}' detectron2` 3 | -------------------------------------------------------------------------------- /inpaint/pretrained_models/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RexBarker/VideoObjectRemoval/26b8648645044389e9b7311f609c04b41f92f0b7/inpaint/pretrained_models/.gitkeep -------------------------------------------------------------------------------- /inpaint/scripts/test_inpaint_ex_container.sh: -------------------------------------------------------------------------------- 1 | # to be run in host environment (where docker daemon is running) 2 | 3 | # run the demo flamingo model using docker 4 | docker exec -ti inpaint \ 5 | /usr/bin/python3 /home/appuser/Deep-Flow/tools/video_inpaint.py \ 6 | --frame_dir /home/appuser/data/flamingo/origin \ 7 | --MASK_ROOT /home/appuser/data/flamingo/masks \ 8 | --img_size 512 832 \ 9 | --FlowNet2 --DFC --ResNet101 --Propagation 10 | -------------------------------------------------------------------------------- /inpaint/scripts/test_inpaint_in_container.sh: -------------------------------------------------------------------------------- 1 | # to be run from within the inpaint container 2 | 3 | # run the demo flamingo model using docker 4 | /usr/bin/python3 /home/appuser/Deep-Flow/tools/video_inpaint.py \ 5 | --frame_dir /home/appuser/data/flamingo/origin \ 6 | --MASK_ROOT /home/appuser/data/flamingo/masks \ 7 | --img_size 512 832 \ 8 | --FlowNet2 --DFC --ResNet101 --Propagation 9 | -------------------------------------------------------------------------------- /setup/_download_googledrive.sh: -------------------------------------------------------------------------------- 1 | # from https://gist.github.com/iamtekeste/3cdfd0366ebfd2c0d805#gistcomment-2359248 2 | # use: 3 | # $1 = long_google_drive_file_id 4 | # $2 = local_file_name 5 | 6 | function gdrive_download () { 7 | CONFIRM=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate "https://docs.google.com/uc?export=download&id=$1" -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p') 8 | wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$CONFIRM&id=$1" -O $2 9 | rm -rf /tmp/cookies.txt 10 | } 11 | 12 | gdrive_download $1 $2 13 | -------------------------------------------------------------------------------- /setup/download_inpaint_models.sh: -------------------------------------------------------------------------------- 1 | # this script automatically downloads from a google drive location 2 | # as originally posted by https://drive.google.com/drive/folders/1Nh6eJsue2IkP_bsN02SRPvWzkIi6cNbE 3 | 4 | # you can use this script, or go to this link if you don't trust my script 5 | 6 | DownloadScript=./_download_googledrive.sh 7 | DownloadDir=../inpaint/pretrained_models 8 | 9 | $DownloadScript 1rKr1HtqjJ5gBdOA8fTgJ99brWDIJi46v $DownloadDir/FlowNet2_checkpoint.pth.tar 10 | $DownloadScript 1ZxyGeWk1d37QdZkx1d2aBseXSfYL4uuJ $DownloadDir/resnet101-5d3b4d8f.pth 11 | $DownloadScript 1i0fZ37se14p7-MW5fxi6O4rv3VjHCDFd $DownloadDir/resnet101_movie.pth 12 | $DownloadScript 1dZQjITK8bOWuS4yQeC_WVddqT_QkFXFO $DownloadDir/resnet50-19c8e357.pth 13 | $DownloadScript 16hmQgpp_cPBzw5Dug9EnjVnwy7a6KF9M $DownloadDir/resnet50_stage1.pth 14 | $DownloadScript 1jltdGzyZaJ1RGpeMf9Ns6ofyEJMsyboe $DownloadDir/imagenet_deepfill.pth 15 | -------------------------------------------------------------------------------- /setup/setup_network.sh: -------------------------------------------------------------------------------- 1 | # create docker network for interprocess communication 2 | if [ -z `docker network ls -f name=detectinpaint -q` ]; then 3 | docker network create -d bridge detectinpaint 4 | echo "created detectinpaint network" 5 | else 6 | echo "detectinpaint network already exists" 7 | fi 8 | 9 | -------------------------------------------------------------------------------- /setup/setup_venv.sh: -------------------------------------------------------------------------------- 1 | # to be run by docker build 2 | # activate virtualenv 3 | . /home/appuser/venv/bin/activate 4 | 5 | # install into virtualenv 6 | apt update -y; 7 | sudo apt install -y python3 python3-pip libglib2.0-0 libsm6 libxext6 libxrender-dev 8 | pip3 --disable-pip-version-check install addict==2.2.1 certifi==2019.6.16 cffi==1.12.3 chardet==3.0.4 cvbase==0.5.5 Cython==0.29.12 idna==2.8 mkl-fft mkl-random numpy==1.16.4 olefile==0.46 opencv-python==4.1.0.25 Pillow==6.2.0 protobuf==3.8.0 pycparser==2.19 PyYAML==5.1.1 requests==2.22.0 scipy==1.2.1 six==1.12.0 tensorboardX==1.8 terminaltables==3.1.0 torch==0.4.0 torchvision==0.2.1 tqdm==4.32.1 urllib3==1.25.3 9 | pip3 --disable-pip-version-check install mmcv==0.2.10 10 | 11 | # install kernel for jupyter notebook to this venv 12 | pip install ipykernel 13 | python -m ipykernel install --user --name venv --display-name "venv" 14 | 15 | 16 | -------------------------------------------------------------------------------- /setup/ssh/sshd_config: -------------------------------------------------------------------------------- 1 | # $OpenBSD: sshd_config,v 1.101 2017/03/14 07:19:07 djm Exp $ 2 | 3 | # This is the sshd server system-wide configuration file. See 4 | # sshd_config(5) for more information. 5 | 6 | # This sshd was compiled with PATH=/usr/bin:/bin:/usr/sbin:/sbin 7 | 8 | # The strategy used for options in the default sshd_config shipped with 9 | # OpenSSH is to specify options with their default value where 10 | # possible, but leave them commented. Uncommented options override the 11 | # default value. 12 | 13 | #Port 22 14 | #AddressFamily any 15 | #ListenAddress 0.0.0.0 16 | #ListenAddress :: 17 | 18 | #HostKey /etc/ssh/ssh_host_rsa_key 19 | #HostKey /etc/ssh/ssh_host_ecdsa_key 20 | #HostKey /etc/ssh/ssh_host_ed25519_key 21 | 22 | # Ciphers and keying 23 | #RekeyLimit default none 24 | 25 | # Logging 26 | #SyslogFacility AUTH 27 | #LogLevel INFO 28 | 29 | # Authentication: 30 | 31 | #LoginGraceTime 2m 32 | #PermitRootLogin prohibit-password 33 | #StrictModes yes 34 | #MaxAuthTries 6 35 | #MaxSessions 10 36 | 37 | #PubkeyAuthentication yes 38 | 39 | # Expect .ssh/authorized_keys2 to be disregarded by default in future. 40 | #AuthorizedKeysFile .ssh/authorized_keys .ssh/authorized_keys2 41 | 42 | #AuthorizedPrincipalsFile none 43 | 44 | #AuthorizedKeysCommand none 45 | #AuthorizedKeysCommandUser nobody 46 | 47 | # For this to work you will also need host keys in /etc/ssh/ssh_known_hosts 48 | #HostbasedAuthentication no 49 | # Change to yes if you don't trust ~/.ssh/known_hosts for 50 | # HostbasedAuthentication 51 | #IgnoreUserKnownHosts no 52 | # Don't read the user's ~/.rhosts and ~/.shosts files 53 | #IgnoreRhosts yes 54 | 55 | # To disable tunneled clear text passwords, change to no here! 56 | #PasswordAuthentication yes 57 | #PermitEmptyPasswords no 58 | 59 | # Change to yes to enable challenge-response passwords (beware issues with 60 | # some PAM modules and threads) 61 | ChallengeResponseAuthentication no 62 | 63 | # Kerberos options 64 | #KerberosAuthentication no 65 | #KerberosOrLocalPasswd yes 66 | #KerberosTicketCleanup yes 67 | #KerberosGetAFSToken no 68 | 69 | # GSSAPI options 70 | #GSSAPIAuthentication no 71 | #GSSAPICleanupCredentials yes 72 | #GSSAPIStrictAcceptorCheck yes 73 | #GSSAPIKeyExchange no 74 | 75 | # Set this to 'yes' to enable PAM authentication, account processing, 76 | # and session processing. If this is enabled, PAM authentication will 77 | # be allowed through the ChallengeResponseAuthentication and 78 | # PasswordAuthentication. Depending on your PAM configuration, 79 | # PAM authentication via ChallengeResponseAuthentication may bypass 80 | # the setting of "PermitRootLogin without-password". 81 | # If you just want the PAM account and session checks to run without 82 | # PAM authentication, then enable this but set PasswordAuthentication 83 | # and ChallengeResponseAuthentication to 'no'. 84 | UsePAM yes 85 | 86 | #AllowAgentForwarding yes 87 | #AllowTcpForwarding yes 88 | #GatewayPorts no 89 | X11Forwarding yes 90 | #X11DisplayOffset 10 91 | X11UseLocalhost no 92 | #PermitTTY yes 93 | PrintMotd no 94 | #PrintLastLog yes 95 | #TCPKeepAlive yes 96 | #UseLogin no 97 | #PermitUserEnvironment no 98 | #Compression delayed 99 | #ClientAliveInterval 0 100 | #ClientAliveCountMax 3 101 | #UseDNS no 102 | #PidFile /var/run/sshd.pid 103 | #MaxStartups 10:30:100 104 | #PermitTunnel no 105 | #ChrootDirectory none 106 | #VersionAddendum none 107 | 108 | # no default banner path 109 | #Banner none 110 | 111 | # Allow client to pass locale environment variables 112 | AcceptEnv LANG LC_* 113 | 114 | # override default of no subsystems 115 | Subsystem sftp /usr/lib/openssh/sftp-server 116 | 117 | # Example of overriding settings on a per-user basis 118 | #Match User anoncvs 119 | # X11Forwarding no 120 | # AllowTcpForwarding no 121 | # PermitTTY no 122 | # ForceCommand cvs server 123 | -------------------------------------------------------------------------------- /tools/convert_frames2video.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from glob import glob 3 | import cv2 4 | import os 5 | import numpy as np 6 | import subprocess as sp 7 | import ffmpeg 8 | 9 | def parse_args(): 10 | parser = argparse.ArgumentParser() 11 | parser.add_argument('--input_dir', type=str, required=True, default=None, 12 | help="input directory of frames (assuming numeric ordering)") 13 | 14 | parser.add_argument('--mask_dir', type=str, required=False, default=None, 15 | help="(optional) input directory of masks (assuming numeric ordering)") 16 | 17 | parser.add_argument('--rotate_right', action='store_true', help="Rotate image by 90 deg clockwise") 18 | parser.add_argument('--rotate_left', action='store_true', help="Rotate image by 90 deg anticlockwise") 19 | parser.add_argument('--fps', type=int, default=25, help="frames per second encoding speed (default=25 fps)") 20 | parser.add_argument('--output_file', type=str, default=None, 21 | help="name of output mp4 file (default = input directory name") 22 | 23 | args = parser.parse_args() 24 | 25 | return args 26 | 27 | 28 | def createVideoClip_Cmd(clip, ouputfile, fps, size=[256, 256]): 29 | 30 | vf = clip.shape[0] 31 | command = ['ffmpeg', 32 | '-y', # overwrite output file if it exists 33 | '-f', 'rawvideo', 34 | '-s', '%dx%d' % (size[1], size[0]), # '256x256', # size of one frame 35 | '-pix_fmt', 'rgb24', 36 | '-r', '25', # frames per second 37 | '-an', # Tells FFMPEG not to expect any audio 38 | '-i', '-', # The input comes from a pipe 39 | '-vcodec', 'libx264', 40 | '-b:v', '1500k', 41 | '-vframes', str(vf), # 5*25 42 | '-s', '%dx%d' % (size[1], size[0]), # '256x256', # size of one frame 43 | outputfile] 44 | # sfolder+'/'+name 45 | pipe = sp.Popen(command, stdin=sp.PIPE, stderr=sp.PIPE) 46 | out, err = pipe.communicate(clip.tostring()) 47 | pipe.wait() 48 | pipe.terminate() 49 | print(err) 50 | 51 | 52 | def createVideoClip(clip, outputfile, fps, size=[256, 256]): 53 | 54 | vf = clip.shape[0] 55 | 56 | args = [#'ffmpeg', 57 | '-y', # overwrite output file if it exists 58 | '-f', 'rawvideo', 59 | '-s', '%dx%d' % (size[1], size[0]), # '256x256', # size of one frame 60 | '-pix_fmt', 'rgb24', 61 | '-r', str(fps), # frames per second 62 | '-an', # Tells FFMPEG not to expect any audio 63 | '-i', '-', # The input comes from a pipe 64 | '-vcodec', 'libx264', 65 | '-b:v', '1500k', 66 | '-vframes', str(vf), # 5*25 67 | '-s', '%dx%d' % (size[1], size[0]), # '256x256', # size of one frame 68 | outputfile] 69 | 70 | process = ( 71 | ffmpeg 72 | .input('pipe:', format='rawvideo', pix_fmt='rgb24', s='{}x{}'.format(size[1], size[0])) 73 | .output(outputfile, pix_fmt='yuv420p', format='mp4', video_bitrate='1500k', r=str(fps), s='{}x{}'.format(size[1], size[0])) 74 | .overwrite_output() 75 | ) 76 | 77 | command = ffmpeg.compile(process, overwrite_output=True) 78 | 79 | #command = ffmpeg.get_args(args, overwrite_output=True) 80 | # sfolder+'/'+name 81 | pipe = sp.Popen(command, stdin=sp.PIPE, stderr=sp.PIPE) 82 | out, err = pipe.communicate(clip.tostring()) 83 | pipe.wait() 84 | pipe.terminate() 85 | print(err) 86 | 87 | 88 | 89 | if __name__ == '__main__': 90 | args = parse_args() 91 | 92 | out_frames = [] 93 | 94 | assert os.path.exists(args.input_dir), f"Could not find input directory = {args.input_dir}" 95 | inputdir = args.input_dir 96 | 97 | imgfiles = [] 98 | for ftype in ("*.jpg", "*.png"): 99 | imgfiles = sorted(glob(os.path.join(inputdir,ftype))) 100 | if imgfiles: break 101 | 102 | assert imgfiles, f"Could not find any suitable *.jpg or *.png files in {inputdir}" 103 | 104 | # DAN, you left off here! 105 | if arg.mask_dir is not None: 106 | assert os.path.exists(args.mask_dir), f"Mask directory specified, but could not be found = {args.mask_dir}" 107 | 108 | fps = args.fps 109 | currdir = os.path.abspath(os.curdir) 110 | 111 | if args.output_file is not None: 112 | video_name = args.output_file 113 | else: 114 | video_name = os.path.basename(inputdir) 115 | if not video_name.endswith(".mp4"): video_name = video_name + ".mp4" 116 | 117 | for imgfile in imgfiles: 118 | print(imgfile) 119 | out_frame = cv2.imread(imgfile) 120 | 121 | if args.rotate_left: 122 | out_frame = cv2.rotate(out_frame,cv2.ROTATE_90_COUNTERCLOCKWISE) 123 | elif args.rotate_right: 124 | out_frame = cv2.rotate(out_frame,cv2.ROTATE_90_CLOCKWISE) 125 | 126 | shape = out_frame.shape 127 | out_frames.append(out_frame[:, :, ::-1]) 128 | 129 | final_clip = np.stack(out_frames) 130 | 131 | outputfile = os.path.join(currdir,video_name) 132 | 133 | #createVideoClip(final_clip, outputfile, [shape[0], shape[1]]) 134 | createVideoClip_Cmd(final_clip, outputfile, fps, [shape[0], shape[1]]) 135 | print(f"\nVideo output file:{outputfile}") 136 | print("\nCompleted successfully") 137 | -------------------------------------------------------------------------------- /tools/convert_video2frames.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import cv2 3 | import os 4 | import numpy as np 5 | from math import log10, ceil 6 | 7 | def parse_args(): 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument('--input_file', type=str, required=True, default=None, 10 | help="input video file (.avi, .mp4, .mkv, mov)") 11 | parser.add_argument('--rotate_right', action='store_true', help="Rotate image by 90 deg clockwise") 12 | parser.add_argument('--rotate_left', action='store_true', help="Rotate image by 90 deg anticlockwise") 13 | parser.add_argument('--image_type', type=str, default='png', help="output frame file type (def=png)") 14 | parser.add_argument('--output_dir', type=str, default=None, 15 | help="name of output directory (default = base of input file name") 16 | 17 | args = parser.parse_args() 18 | 19 | return args 20 | 21 | def video_to_frames(inputfile,outputdir,imagetype='png'): 22 | 23 | if not os.path.exists(outputdir): 24 | dout = '.' 25 | for din in outputdir.split('/'): 26 | dout = dout + '/' + din 27 | if not os.path.exists(dout): 28 | os.mkdir(dout) 29 | 30 | cap = cv2.VideoCapture(inputfile) 31 | 32 | length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) 33 | 34 | padlength = ceil(log10(length)) 35 | 36 | n = 0 37 | while True: 38 | ret, frame = cap.read() 39 | 40 | if not ret: break 41 | 42 | if args.rotate_left: 43 | frame = cv2.rotate(frame,cv2.ROTATE_90_COUNTERCLOCKWISE) 44 | elif args.rotate_right: 45 | frame = cv2.rotate(frame,cv2.ROTATE_90_CLOCKWISE) 46 | 47 | fname = str(n).rjust(padlength,'0') + '.' + imagetype 48 | cv2.imwrite(os.path.join(outputdir,fname),frame) 49 | 50 | n += 1 51 | 52 | return n # number of frames processed 53 | 54 | 55 | if __name__ == '__main__': 56 | args = parse_args() 57 | 58 | assert os.path.exists(args.input_file), f"Could not find input file = {args.input_file}" 59 | inputfile = args.input_file 60 | 61 | currdir = os.path.abspath(os.curdir) 62 | 63 | if args.output_dir is not None: 64 | outputdir = args.output_dir 65 | else: 66 | outputdir = os.path.basename(inputdir).split('.')[0] 67 | outputdir = os.path.join(currdir,outputdir + "_frames") 68 | 69 | n = video_to_frames(inputfile,outputdir,imagetype=args.image_type) 70 | 71 | print(f"\nCompleted successfully, processed {n} frames") 72 | -------------------------------------------------------------------------------- /tools/play_video.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import argparse 4 | from glob import glob 5 | from time import time, sleep 6 | 7 | fontconfig = { 8 | "font" : cv2.FONT_HERSHEY_SIMPLEX, 9 | "rel_coords" : (0.8, 0.05), 10 | "cornercoords" : (10,500), 11 | "minY" : 30, 12 | "fontScale" : 1, 13 | "fontColor" : (0,255,0), 14 | "lineType" : 2 15 | } 16 | 17 | parser = argparse.ArgumentParser() 18 | 19 | parser.add_argument('--infile', type=str, required=None, 20 | help="input file in .mp4, .avi, .mov, or .mkv format") 21 | 22 | parser.add_argument('--maskdir', type=str, required=None, 23 | help="mask directory (*.jpg or *.png), total must be same as frame count") 24 | 25 | parser.add_argument('--fps', type=int, default=None, 26 | help="video replay frame rate, frames per second (default=60 fps)") 27 | 28 | parser.add_argument('--rotate_right', action='store_true', 29 | help="Rotate image by 90 deg clockwise") 30 | 31 | parser.add_argument('--rotate_left', action='store_true', 32 | help="Rotate image by 90 deg anticlockwise") 33 | 34 | parser.add_argument('--frame_num', action='store_true', 35 | help="display frame number") 36 | 37 | parser.add_argument('--start', type=int, default= 0, help="start from frame#") 38 | 39 | parser.add_argument('--finish', type=int, default= None, help="finish at frame#") 40 | 41 | parser.add_argument('--info', action='store_true', 42 | help="output video information") 43 | 44 | parser.add_argument('other', nargs=argparse.REMAINDER) # catch unnamed arguments 45 | 46 | 47 | ##### Helper functions ##### 48 | def get_fps(vfile): 49 | if not os.path.isdir(vfile): 50 | cap = cv2.VideoCapture(vfile) 51 | fps = cap.get(cv2.CAP_PROP_FPS) 52 | print(f"File spec FPS ={fps}") 53 | cap.release() 54 | return fps 55 | else: 56 | return None 57 | 58 | def get_nframes(vfile): 59 | if not os.path.isdir(vfile): 60 | cap = cv2.VideoCapture(vfile) 61 | n_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) 62 | print(f"File spec n_frames ={n_frames}") 63 | cap.release() 64 | else: 65 | images = glob(os.path.join(vfile, '*.jp*')) 66 | if not images: 67 | images = glob(os.path.join(vfile, '*.png')) 68 | assert images, f"No image file (*.jpg or *.png) found in {vfile}" 69 | n_frames = len(images) 70 | 71 | return n_frames 72 | 73 | 74 | def get_frame(vfile, n_frames, startframe=0, finishframe=None): 75 | if os.path.isdir(vfile): 76 | images = glob(os.path.join(vfile, '*.jp*')) 77 | if not images: 78 | images = glob(os.path.join(vfile, '*.png')) 79 | assert images, f"No image file (*.jpg or *.png) found in {vfile}" 80 | 81 | assert len(images) == n_frames, \ 82 | f"Mismatch in number of mask files versus number of frames\n" + \ 83 | f"n_frames={n_frames}, n_masks={len(images)}" 84 | 85 | images = sorted(images, 86 | key=lambda x: int(x.split('/')[-1].split('.')[0])) 87 | 88 | if finishframe is None: 89 | finishframe = n_frames 90 | 91 | images = images[startframe:finishframe] 92 | 93 | for img in images: 94 | frame = cv2.imread(img) 95 | yield frame 96 | 97 | else: 98 | cap = cv2.VideoCapture(vfile) 99 | 100 | # start frame is indexed 101 | # stop frame is set by controlling loop (caller) 102 | if startframe != 0: 103 | cap.set(cv2.CAP_PROP_POS_FRAMES, startframe) 104 | 105 | while True: 106 | ret, frame = cap.read() 107 | 108 | if ret: 109 | yield frame 110 | else: 111 | cap.release() 112 | break 113 | 114 | def get_mask(maskdir,n_frames, startframe=0, finishframe=None): 115 | assert os.path.isdir(maskdir), \ 116 | "Use masks specified, however supplied path was not a directory:\n{maskdir}" 117 | 118 | images = glob(os.path.join(maskdir, '*.jp*')) 119 | if not images: 120 | images = glob(os.path.join(maskdir, '*.png')) 121 | assert images, f"No mask files (*.jpg or *.png) found in {maskdir}" 122 | assert len(images) == n_frames, \ 123 | f"Mismatch in number of mask files versus number of frames\n" + \ 124 | f"n_frames={n_frames}, n_masks={len(images)}" 125 | 126 | images = sorted(images, 127 | key=lambda x: int(x.split('/')[-1].split('.')[0])) 128 | 129 | if finishframe is None: 130 | finishframe = n_frames 131 | 132 | images = images[startframe:finishframe] 133 | 134 | for img in images: 135 | mask = cv2.imread(img) 136 | yield mask 137 | 138 | 139 | if __name__ == '__main__': 140 | args = parser.parse_args() 141 | 142 | if args.infile: 143 | vfile = args.infile 144 | elif args.other: 145 | vfile = args.other[0] 146 | else: 147 | assert False,"No input file was specified" 148 | 149 | assert os.path.exists(vfile), f"Input file was not found: {vfile}" 150 | 151 | if args.fps is not None: 152 | fps = args.fps 153 | else: 154 | fps = get_fps(vfile) 155 | if fps is None: 156 | fps = 60 157 | 158 | spf = float(1.0/fps) 159 | 160 | n_frames = get_nframes(vfile) 161 | width,height = 0,0 162 | current = 0.0 163 | 164 | startframe = 0 165 | if args.start: 166 | assert abs(args.start) < n_frames, \ 167 | f"Invalid 'start'={startframe} frame specified, exceeds number of frames ({n_frames})" 168 | 169 | startframe = args.start if args.start >= 0 else n_frames + args.start # negative indicates from end 170 | 171 | finishframe = n_frames 172 | if args.finish is not None: 173 | assert abs(args.finish) < n_frames, \ 174 | f"Invalid 'finish'={finishframe} frame specified, exceeds number of frames({n_frames})" 175 | 176 | finishframe = args.finish if args.finish >= 0 else n_frames + args.finish # negative indicates from end 177 | 178 | assert finishframe > startframe, f"Invalid definition of 'start'={startframe} and 'finish'={finishframe}, start > finish" 179 | 180 | replay = 1 181 | 182 | while replay: 183 | start = time() 184 | 185 | frame_gen = get_frame(vfile, n_frames, startframe, finishframe) 186 | mask_gen = get_mask(args.maskdir,n_frames, startframe, finishframe) if args.maskdir else None 187 | 188 | i_frames = 0 189 | for i in range(startframe,finishframe): 190 | frame = next(frame_gen) 191 | mask = next(mask_gen) if mask_gen else None 192 | 193 | timediff = time() - current 194 | 195 | if timediff < spf: 196 | sleep(spf - timediff) 197 | 198 | current = time() 199 | 200 | height,width = frame.shape[:2] 201 | 202 | ### optional add mask 203 | # modify existing frame to include mask 204 | if mask is not None: 205 | if len(mask.shape) == 3: 206 | mask = mask[:,:,0] 207 | frame[:, :, 2] = (mask > 0) * 255 + (mask == 0) * frame[:, :, 2] 208 | 209 | ### optional rotations 210 | if args.rotate_left: 211 | frame = cv2.rotate(frame,cv2.ROTATE_90_COUNTERCLOCKWISE) 212 | elif args.rotate_right: 213 | frame = cv2.rotate(frame,cv2.ROTATE_90_CLOCKWISE) 214 | 215 | ### add frame number to image 216 | if args.frame_num: 217 | real_x = round(fontconfig["rel_coords"][0] * width) 218 | real_y = max(round(fontconfig["rel_coords"][1] * height), fontconfig['minY']) 219 | cv2.putText(frame, str(i), 220 | (real_x, real_y), 221 | fontconfig['font'], 222 | fontconfig['fontScale'], 223 | fontconfig['fontColor'], 224 | fontconfig['lineType'] ) 225 | 226 | ### show image 227 | cv2.imshow('frame',frame) 228 | keycode = cv2.waitKey(10) 229 | if keycode & 0xFF == ord('p'): # pause 230 | while True: 231 | keycode = cv2.waitKey(0) 232 | if keycode & 0xFF == ord('p'): 233 | break 234 | 235 | if keycode & 0xFF == ord('q'): # quit (immediately) 236 | replay = 0 237 | break 238 | elif keycode & 0xFF == ord('e'): # end (eventually) 239 | replay = 0 240 | elif keycode & 0xFF == ord('r'): # restart 241 | replay = 1 242 | break 243 | 244 | i_frames += 1 245 | 246 | #cap.release() 247 | cv2.destroyAllWindows() 248 | 249 | actual_fps = i_frames / (time() - start) 250 | 251 | if args.info: 252 | print(f"Number of frames: {n_frames}") 253 | print(f"Width x height = ({width},{height})") 254 | print(f"Actual replay speed = {actual_fps:.3f}/s") 255 | --------------------------------------------------------------------------------