├── .gitattributes ├── .gitignore ├── LICENSE ├── README ├── RUNME.m ├── RUNME_param_tune.m ├── VERSION ├── VISION ├── bb ├── bb_content.m ├── bb_content_safe.m ├── bb_feature_distance.m ├── bb_grid.m ├── bb_grid_adjustment.m ├── bb_load_boxes.m ├── bb_minus_log_likelihood.m ├── bb_normalize_likelihood.m ├── bb_normalize_minus_log_likelihood.m ├── bb_random.m ├── bb_range_check.m └── bb_resample_roulette_wheel.m ├── bkg ├── bkg_subtraction.m ├── bkg_subtraction_thresholding.m └── offline_bkg_detection.m ├── distance ├── dist_bhattacharyya.m ├── dist_chisquare.m ├── dist_correlation.m ├── dist_intersection.m ├── dist_l1.m └── dist_l2.m ├── eval ├── calculate_CPE.m ├── calculate_S.m ├── draw_cpe_plot.m ├── draw_success_plot.m ├── evaluate_center_position_error.m ├── evaluate_scale_adaptation.m └── evaluate_success_plot_OPE.m ├── feature ├── depth │ └── median_of_depth.m ├── hoc │ ├── color_clustering.m │ ├── histogram_of_colors.m │ └── hoc_vis.m └── initialize_features.m ├── mu └── model_update.m ├── tracker ├── loader │ ├── loader_init.m │ ├── loader_test.m │ └── loader_test.txt ├── lower_bounds │ ├── lb_center_bb_init.m │ ├── lb_center_bb_test.m │ ├── lb_crazy_init.m │ ├── lb_crazy_test.m │ ├── lb_first_bb_init.m │ ├── lb_first_bb_test.m │ ├── lb_invisible_init.m │ ├── lb_invisible_test.m │ ├── lb_rand_loc_init.m │ ├── lb_rand_loc_test.m │ ├── lb_rand_size_init.m │ ├── lb_rand_size_loc_init.m │ ├── lb_rand_size_loc_test.m │ └── lb_rand_size_test.m ├── particle_filter │ ├── bb_feature_distance_normalization.m │ ├── bb_feature_extraction.m │ ├── bb_likelihood.m │ ├── bb_occlusion_state_transition.m │ ├── bb_positioning_noise.m │ ├── bb_prob_color_indicator.m │ ├── bb_resample.m │ ├── box_confidence.m │ ├── expected_target.m │ ├── initialize_particles_bb.m │ ├── particle_filter_demo.m │ ├── particle_filter_init.m │ ├── particle_filter_output.m │ ├── particle_filter_test.m │ ├── particle_filter_train.m │ └── vis_particle.m └── upper_bounds │ ├── ub_gt_first_ratio_init.m │ ├── ub_gt_first_ratio_test.m │ ├── ub_gt_first_size_init.m │ ├── ub_gt_first_size_test.m │ ├── ub_gt_init.m │ ├── ub_gt_no_occ_last_init.m │ ├── ub_gt_no_occ_last_test.m │ ├── ub_gt_no_occ_rand_init.m │ ├── ub_gt_no_occ_rand_test.m │ └── ub_gt_test.m ├── tracker_history.m ├── trackers_evaluate.m ├── trackers_initialize.m ├── trackers_output.m ├── trackers_test.m ├── trackers_train.m ├── util ├── clip_range.m ├── console_messages.m ├── read_frame.m ├── rect_span.m ├── setprod.m └── video_info.m └── vis ├── vis_error_types.m ├── vis_final_tracker_cpe.m ├── vis_final_tracker_sae.m ├── vis_final_tracker_track.m ├── vis_final_tracker_trajectory.m ├── vis_particle_hoc_distances.m ├── vis_particle_medd_distances.m ├── vis_particle_position_occlusion.m ├── vis_particle_position_probability.m ├── vis_particle_probability.m ├── vis_particle_probability_comparison.m ├── vis_particle_template_hoc_comparison.m ├── vis_particle_template_medd_comparison.m └── vis_point_cloud.m /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | *.sln merge=union 7 | *.csproj merge=union 8 | *.vbproj merge=union 9 | *.fsproj merge=union 10 | *.dbproj merge=union 11 | 12 | # Standard to msysgit 13 | *.doc diff=astextplain 14 | *.DOC diff=astextplain 15 | *.docx diff=astextplain 16 | *.DOCX diff=astextplain 17 | *.dot diff=astextplain 18 | *.DOT diff=astextplain 19 | *.pdf diff=astextplain 20 | *.PDF diff=astextplain 21 | *.rtf diff=astextplain 22 | *.RTF diff=astextplain 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ################# 2 | ## Eclipse 3 | ################# 4 | 5 | *.pydevproject 6 | .project 7 | .metadata 8 | bin/ 9 | tmp/ 10 | *.tmp 11 | *.bak 12 | *.swp 13 | *~.nib 14 | local.properties 15 | .classpath 16 | .settings/ 17 | .loadpath 18 | 19 | # External tool builders 20 | .externalToolBuilders/ 21 | 22 | # Locally stored "Eclipse launch configurations" 23 | *.launch 24 | 25 | # CDT-specific 26 | .cproject 27 | 28 | # PDT-specific 29 | .buildpath 30 | 31 | 32 | ################# 33 | ## Visual Studio 34 | ################# 35 | 36 | ## Ignore Visual Studio temporary files, build results, and 37 | ## files generated by popular Visual Studio add-ons. 38 | 39 | # User-specific files 40 | *.suo 41 | *.user 42 | *.sln.docstates 43 | 44 | # Build results 45 | 46 | [Dd]ebug/ 47 | [Rr]elease/ 48 | x64/ 49 | build/ 50 | [Bb]in/ 51 | [Oo]bj/ 52 | 53 | # MSTest test Results 54 | [Tt]est[Rr]esult*/ 55 | [Bb]uild[Ll]og.* 56 | 57 | *_i.c 58 | *_p.c 59 | *.ilk 60 | *.meta 61 | *.obj 62 | *.pch 63 | *.pdb 64 | *.pgc 65 | *.pgd 66 | *.rsp 67 | *.sbr 68 | *.tlb 69 | *.tli 70 | *.tlh 71 | *.tmp 72 | *.tmp_proj 73 | *.log 74 | *.vspscc 75 | *.vssscc 76 | .builds 77 | *.pidb 78 | *.log 79 | *.scc 80 | 81 | # Visual C++ cache files 82 | ipch/ 83 | *.aps 84 | *.ncb 85 | *.opensdf 86 | *.sdf 87 | *.cachefile 88 | 89 | # Visual Studio profiler 90 | *.psess 91 | *.vsp 92 | *.vspx 93 | 94 | # Guidance Automation Toolkit 95 | *.gpState 96 | 97 | # ReSharper is a .NET coding add-in 98 | _ReSharper*/ 99 | *.[Rr]e[Ss]harper 100 | 101 | # TeamCity is a build add-in 102 | _TeamCity* 103 | 104 | # DotCover is a Code Coverage Tool 105 | *.dotCover 106 | 107 | # NCrunch 108 | *.ncrunch* 109 | .*crunch*.local.xml 110 | 111 | # Installshield output folder 112 | [Ee]xpress/ 113 | 114 | # DocProject is a documentation generator add-in 115 | DocProject/buildhelp/ 116 | DocProject/Help/*.HxT 117 | DocProject/Help/*.HxC 118 | DocProject/Help/*.hhc 119 | DocProject/Help/*.hhk 120 | DocProject/Help/*.hhp 121 | DocProject/Help/Html2 122 | DocProject/Help/html 123 | 124 | # Click-Once directory 125 | publish/ 126 | 127 | # Publish Web Output 128 | *.Publish.xml 129 | *.pubxml 130 | 131 | # NuGet Packages Directory 132 | ## TODO: If you have NuGet Package Restore enabled, uncomment the next line 133 | #packages/ 134 | 135 | # Windows Azure Build Output 136 | csx 137 | *.build.csdef 138 | 139 | # Windows Store app package directory 140 | AppPackages/ 141 | 142 | # Others 143 | sql/ 144 | *.Cache 145 | ClientBin/ 146 | [Ss]tyle[Cc]op.* 147 | ~$* 148 | *~ 149 | *.dbmdl 150 | *.[Pp]ublish.xml 151 | *.pfx 152 | *.publishsettings 153 | 154 | # RIA/Silverlight projects 155 | Generated_Code/ 156 | 157 | # Backup & report files from converting an old project file to a newer 158 | # Visual Studio version. Backup files are not needed, because we have git ;-) 159 | _UpgradeReport_Files/ 160 | Backup*/ 161 | UpgradeLog*.XML 162 | UpgradeLog*.htm 163 | 164 | # SQL Server files 165 | App_Data/*.mdf 166 | App_Data/*.ldf 167 | 168 | ############# 169 | ## Windows detritus 170 | ############# 171 | 172 | # Windows image file caches 173 | Thumbs.db 174 | ehthumbs.db 175 | 176 | # Folder config file 177 | Desktop.ini 178 | 179 | # Recycle Bin used on file shares 180 | $RECYCLE.BIN/ 181 | 182 | # Mac crap 183 | .DS_Store 184 | 185 | 186 | ############# 187 | ## Python 188 | ############# 189 | 190 | *.py[co] 191 | 192 | # Packages 193 | *.egg 194 | *.egg-info 195 | dist/ 196 | build/ 197 | eggs/ 198 | parts/ 199 | var/ 200 | sdist/ 201 | develop-eggs/ 202 | .installed.cfg 203 | 204 | # Installer logs 205 | pip-log.txt 206 | 207 | # Unit test / coverage reports 208 | .coverage 209 | .tox 210 | 211 | #Translations 212 | *.mo 213 | 214 | #Mr Developer 215 | .mr.developer.cfg 216 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 2013 lapwing 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | {signature of Ty Coon}, 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Enhancing Probabilistic Appearance-Based Object Tracking with Depth Information: Multiple Object Tracking under Occlusion 2 | 3 | Code by: 4 | Kourosh Meshgi 5 | PhD Student @ Kyoto University 6 | 7 | Initiated: 8 | July 2013 9 | 10 | Publication: 11 | 1. K. Meshgi, Y. Li, S. Oba, S. Maeda, S. Ishii, "Enhancing Probabilistic Appearance-Based Object Tracking with Depth Information: Object Tracking under Occlusion," in IEICE Tech. Rep., vol. 113, no. 197, IBISML2013-22, pp. 85-91, Sept. 2013 12 | 13 | 2. Language 14 | Matlab 15 | 16 | Contact: 17 | kouroshmeshgi@gmail.com 18 | 19 | More Information: 20 | http://hawaii.sys.i.kyoto-u.ac.jp/~meshgi-k/ -------------------------------------------------------------------------------- /RUNME.m: -------------------------------------------------------------------------------- 1 | addpath(genpath('.')); 2 | clear 3 | close all 4 | 5 | 6 | %% General Settings of Environment 7 | control.enable_learning = true; 8 | 9 | control.visualize_tracker_result = true; 10 | control.visualize_tracking_history = false; 11 | control.visualize_tracker_dynamics = false; 12 | 13 | control.video_tracker_result = false; 14 | control.video_tracking_history = false; 15 | control.video_tracker_dynamics = false; 16 | 17 | control.save_tracker_result = false; 18 | control.verbose = true; 19 | 20 | control.number_of_objects = 1; 21 | control.initialize_input_method = 'file'; %file,hand,detector 22 | control.start_frame = 1; 23 | control.success_plot_thresold_steps = 20; 24 | 25 | control.train_videos = {'new_ex_occ4'};%{ 'bear_front', 'face_occ5', 'zcup_move_1', 'child_no1'}; 26 | control.test_videos = {'new_ex_occ4'}; 27 | 28 | %% Tracker Initialization parameters 29 | % 'particle_filter' 30 | % 'ub_gt' 31 | % 'ub_gt_first_size' 32 | % 'ub_gt_first_ratio' 33 | % 'ub_gt_no_occ_rand' 34 | % 'ub_gt_no_occ_last' 35 | % 'lb_first_bb' 36 | % 'lb_center_bb' 37 | % 'lb_rand_size' 38 | % 'lb_rand_loc' 39 | % 'lb_rand_size_loc' 40 | % 'lb_crazy' 41 | % 'lb_invisible' 42 | 43 | console_messages ('clear'); 44 | console_messages ('add' , 'Initializing Trackers'); 45 | 46 | control.tracker_list = {'lb_rand_size', 'ub_gt_first_size', 'lb_rand_loc', 'particle_filter'}; 47 | params1.name = 'lb_rand_size'; 48 | % params1.name = 'loader'; 49 | % params1.filename = 'tracker/loader/loader_test.txt'; 50 | 51 | params2.name = 'ub_gt_first_size'; 52 | params3.name = 'lb_rand_loc'; 53 | 54 | params4.name = 'particle filter'; 55 | params4.number_of_particles = 100; 56 | 57 | 58 | params4.feature_name = {'HoC(RGB Clustering,Gridding with Confidence)', 'medD'}; %HoC (RGB Clustering,Grid2) , HoC (RGB Clustering,Grid3), HoC (HSV),... 59 | params4.similarity_measure = {'Bhattacharyya(Gridding with Confidence)' , 'L1' }; 60 | params4.feature_importance = [ 1 , 1]; 61 | params4.feature_normalizer = [ 1 , 1]; 62 | 63 | 64 | params4.occlusion_probability = 0.5; 65 | params4.occlusion_flag_threshold = 0.3; 66 | params4.state_transition_matrix = [0.1 0.9; 0.4 0.6]; % nocc->nocc = 0.99, nocc->occ = 0.01, occ->nocc = 0.25, occ->occ = 0.75 67 | params4.bkg_detection = 'temporal median'; 68 | params4.bkg_subtraction = 'thresholding'; 69 | params4.model_update = 'moving_average'; 70 | params4.enable_occ_flag = true; 71 | params4.grid_size = 2; 72 | 73 | tracker_parameters = {params1, params2, params3, params4}; 74 | 75 | %% Tracking Scenario 76 | % console_messages ('newline' , 'Tracking Scenario ... 1'); 77 | trackers = trackers_initialize ( control.tracker_list, tracker_parameters , control ); 78 | trackers = trackers_train ( control.train_videos , trackers , control ); 79 | [results,trackers] = trackers_test ( control.test_videos , trackers , control ); 80 | evaluation = trackers_evaluate ( control.test_videos , trackers , control , results ); 81 | trackers_output ( control.test_videos , trackers , control , results ); 82 | 83 | 84 | -------------------------------------------------------------------------------- /RUNME_param_tune.m: -------------------------------------------------------------------------------- 1 | addpath(genpath('.')); 2 | clear 3 | close all 4 | 5 | 6 | %% General Settings of Environment 7 | control.enable_learning = false; 8 | 9 | control.visualize_tracker_result = true; 10 | control.visualize_tracking_history = false; 11 | control.visualize_tracker_dynamics = false; 12 | 13 | control.video_tracker_result = false; 14 | control.video_tracking_history = false; 15 | control.video_tracker_dynamics = false; 16 | 17 | control.save_tracker_result = false; 18 | control.verbose = true; 19 | 20 | control.number_of_objects = 1; 21 | control.initialize_input_method = 'file'; %file,hand,detector 22 | control.start_frame = 1; 23 | control.success_plot_thresold_steps = 20; 24 | 25 | control.train_videos = { 'bear_front', 'face_occ5', 'zcup_move_1', 'child_no1'}; 26 | control.test_videos = {'new_ex_occ4'}; 27 | 28 | %% Tracker Initialization parameters 29 | % 'particle_filter' 30 | % 'ub_gt' 31 | % 'ub_gt_first_size' 32 | % 'ub_gt_first_ratio' 33 | % 'ub_gt_no_occ_rand' 34 | % 'ub_gt_no_occ_last' 35 | % 'lb_first_bb' 36 | % 'lb_center_bb' 37 | % 'lb_rand_size' 38 | % 'lb_rand_loc' 39 | % 'lb_rand_size_loc' 40 | % 'lb_crazy' 41 | % 'lb_invisible' 42 | 43 | console_messages ('clear'); 44 | console_messages ('add' , 'Initializing Trackers'); 45 | 46 | control.tracker_list = {'lb_rand_size', 'ub_gt_first_size', 'lb_rand_loc', 'particle_filter'}; 47 | params1.name = 'lb_rand_size'; 48 | % params1.name = 'loader'; 49 | % params1.filename = 'tracker/loader/loader_test.txt'; 50 | 51 | params2.name = 'ub_gt_first_size'; 52 | params3.name = 'lb_rand_loc'; 53 | 54 | params4.name = 'particle filter'; 55 | params4.number_of_particles = 100; 56 | 57 | 58 | params4.feature_name = {'HoC(RGB Clustering)', 'medD'}; %HoC (RGB Clustering,Grid2) , HoC (RGB Clustering,Grid3), HoC (HSV),... 59 | params4.similarity_measure = {'Bhattacharyya', 'L1' }; 60 | params4.feature_importance = [ 1, 1]; 61 | params4.feature_normalizer = [ 1, 1]; 62 | 63 | params4.bkg_detection = 'temporal median'; 64 | params4.bkg_subtraction = 'thresholding'; 65 | params4.model_update = 'moving_average'; 66 | params4.enable_occ_flag = true; 67 | 68 | params4.occlusion_probability = 0.3; 69 | params4.occlusion_flag_threshold = 0.5; 70 | params4.state_transition_matrix = [0.95 0.05; 0.25 0.75]; % nocc->nocc = 0.99, nocc->occ = 0.01, occ->nocc = 0.25, occ->occ = 0.75 71 | 72 | 73 | tracker_parameters = {params1, params2, params3, params4}; 74 | 75 | %% Tracking Scenario 76 | 77 | % three nested loops for occlusion_case_probability, and nocc->nocc and 78 | % occ-> occ 79 | 80 | for i = 1:10 81 | for j = 1:10 82 | for k = 1:10 83 | ocp = double(i)/10; 84 | z0 = double(j)/10; 85 | z1 = double(k)/10; 86 | 87 | params4.occlusion_probability = ocp; 88 | params4.state_transition_matrix = [z0 1-z0; z1 1-z1]; 89 | 90 | trackers = trackers_initialize ( control.tracker_list, tracker_parameters , control ); 91 | trackers = trackers_train ( control.train_videos , trackers , control ); 92 | tic 93 | [results,trackers] = trackers_test ( control.test_videos , trackers , control ); 94 | toc 95 | evaluation = trackers_evaluate ( control.test_videos , trackers , control , results ); 96 | % trackers_output ( control.test_videos , trackers , control , results ); 97 | 98 | auc_all(i,j,k) = evaluation(3,4); % row 3: auc_list, col 4: particle filter 99 | [i,j,k,auc_all(i,j,k)] 100 | end 101 | end 102 | end 103 | 104 | [ibest,jbest,kbest] = ind2sub(size(auc_all),find(max(auc_all(:)))); 105 | disp (['best occlusion case probability is: ' num2str(double(ibest)/10)]); 106 | 107 | best_stm = [double(jbest)/10 1-double(jbest)/10 ; double(kbest)/10 1-double(kbest)/10]; 108 | disp (['best state transition matrix is: ' mat2str(best_stm)]); 109 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | Version 0.4.0: 2 | ============== 3 | Architecture Design for Particle Filter: 4 | 5 | Input: 6 | 1- Number of Particles 7 | 2- Feature Name + Similarity Measure + Gaussian Variance 8 | 3- Occlusion Flag Threshold 9 | 10 | Particle Filter: 11 | 1- Initialize 12 | 1-1- Initialize First Bounding Box 13 | 1-2- Initialize Parameters 14 | 1-3- Initialize Particles 15 | 1-4- Initialize Target Model (Calculate Features Based on RGB + Depth + Init BB) 16 | 2- Particle Filter Train 17 | 3- Particle Filter Test Loop 18 | 3-1- Calculate Features 19 | 3-2- Measure Particle Features to Target 20 | 3-3- Calculate Particle Likelihood Based on Feature 21 | 3-4- Normalize Features Between All Particles 22 | 3-5- Calculate Particle Likelihood via Log-Sum-Exp 23 | 3-6- Normalize Particle Likelihood to Make Probability 24 | 3-7- Apply Occlusion Flag 25 | 3-8- Update Target Model 26 | 3-9- Resampling 27 | 28 | 29 | 30 | 31 | Version 0.3.0: 32 | ============== 33 | 1- Naive trackers (lower bound and upper bound trackers) 34 | 35 | VERSION 0.2.0: 36 | ============== 37 | 1- General Framework Planning: 38 | 39 | * General Settings 40 | * Training 41 | *** For Each Video Do 42 | ****** Read Video General Info 43 | ****** Initialize Trackers for Training Mode 44 | ****** For Each Frame of Video Do 45 | ********* For Each Tracker Do 46 | ************ Provide Tracker with Input Data and Ground Truth 47 | ****** Returning & Saving the Trained Parameters of Tracker / If No Video is Provided Load Default Values 48 | * Testing 49 | *** For Each Video Do 50 | ****** Read Video General Info 51 | ****** Initialize Trackers for Testing Mode: with Trained Parameters, Hand Tuned Parameters (Same Algorithm with Different Set of Parameters), 52 | ****** For Each Frame of Video Do 53 | ********* For Each Tracker Do 54 | ************ Provide Tracker with Input Data 55 | ************ Save the Result of Tracking 56 | *** Returning The Result for Each Video 57 | * Evaluations 58 | *** For Each Video Do 59 | ****** For Each Tracker Do 60 | ********* Calculate the Evaluation Metrics 61 | ****** Draw Result Table 62 | * Output Generation 63 | *** Save Overall Tracking Video 64 | *** Save Tracker Result Files 65 | 66 | VERSION 0.1.0: 67 | ============== 68 | 1- Read data from Princeton dataset 69 | 70 | -------------------------------------------------------------------------------- /VISION: -------------------------------------------------------------------------------- 1 | REQUIREMENTS: 2 | ============= 3 | 4 | * Various Detection Modes 5 | 1- Color Only 6 | 2- Depth Only 7 | 3- Color and Depth 8 | 9 | * Input Data from Public dataset 10 | 11 | * Occlusion Handling 12 | 1- Princeton 13 | 14 | * Occlusion Detection 15 | 1- Based on D 16 | 1-1- Bimodal Histogram of Depth 17 | 1-2- Variance of Gaussian Fit on HoD is Greater than a Threshold 18 | 2- Based on RGB 19 | 3- Based on Trackers Information Exchange 20 | 4- Track Occluder -> Create another Tracker with Shared Memory 21 | 5- Detect Partial Caption 22 | 23 | * Run Different Trackers Simultaneously & Assign Different Colors to Them 24 | 25 | * Target Initialization 26 | 1- By Hand 27 | 2- Text File 28 | 3- Detector Algorithm 29 | 30 | * Pre-processing 31 | 1- Background Subtraction 32 | 1-1- Using GMM for FG & BG 33 | 1-2- Using Temporal Median Background 34 | 2- Shadow Removal 35 | 3- Illumination Normalization 36 | 4- Blurring of Depth Map 37 | 38 | * 2D Features 39 | 1- Color Feature -> Histogram of Colors 40 | 1-1- Using K-Means & RGB Histogram 41 | 1-2- Using Fixed Grid & RGB Histogram 42 | 1-3- Using Fixed Grid & HSV Histogram 43 | 1-4- Using GMM for Target 44 | 2- Advanced Feature 45 | 2-1- HOG 46 | 2-2- SIFT 47 | 2-3- PCA-SIFT 48 | 2-4- Edge Orientation Histogram 49 | 2-5- Shape Contexts 50 | 2-6- Haar Wavelet 51 | 2-7- R-HOG 52 | 2-8- C-HOG 53 | 2-9- R2-HOG 54 | 2-10- E-ShapeC 55 | 2-11- G-ShapeC 56 | 2-12- SURF 57 | 2-13- Integral Histogram 58 | 2-14- Spatiogram 59 | 3- Textural Features 60 | 4- Segmentation 61 | 4-1- Color Clusters Adjacency Graph 62 | 63 | * 3D Features 64 | 1- Histogram of Colors in Color Namespace 65 | 2- Points Count in the Cell 66 | 3- 3D Shape Feature 67 | 3-1- Scatterness 68 | 3-2- Linearness 69 | 3-3- Surfaceness 70 | 71 | * Depth Features 72 | 1- Median of Depth 73 | 2- Histogram of Depth 74 | 3- Histogram of Oriented Depth (HOD) 75 | 76 | * Motion Features 77 | 1- Optical Flow 78 | 2- Mihaylon Paper 79 | 3- Inertial System -> Keep track of move pattern and predict next move -> Add Velocity to Particles (Vx,Vy,Vz) -> Kalman filter? 80 | 81 | * Distance Measures -> To be used in Gaussian 82 | 1- Point-to-Point Euclidean Distance 83 | 2- Mahalanobis Distance 84 | 3- Bhattacherya Distance 85 | 4- KL Divergance 86 | 87 | * Enable Train and Test Mode 88 | 1- Train: Learning Confidence, Tuning Parameters, Learning Weights of Different features on Current dataset, ... 89 | 2- Test: Seeing The Results 90 | 91 | * Enable Online Learning 92 | 1- Target Model 93 | 1-1- Supervised Learning -> N-P Learning 94 | 1-2- Statistically -> Weighted Mean Sliding Window 95 | 2- Occlusion Cases 96 | 3- State Transition Model 97 | 4- Confidence Map 98 | 5- Background 99 | 100 | * Switching Between RGB and D Channels 101 | 1- Different Confidence Measures: Feature Confidence, RGB Confidence, D Confidence 102 | 2- Linear Sum of RGB and D -> Adaptive Weights 103 | 3- Using Occlusion detection 104 | 105 | * Evaluation of Results 106 | 1- CLEAR MOT: MOTA, MOTP 107 | 2- PASCAL VOC: Area Under Curve of Success Plot, Error Type Analysis (localization error, missing object, false positive, identity loss/change) 108 | 3- Average Object Center Position Error(CPE) 109 | 4- Scale Error: Averagr Scale Adaptation(SA) 110 | 111 | * Evaluation Curves: 112 | 1- ROC 113 | 2- DET: error-tradeoff log-log sclae (miss rate vs. false positive) 114 | 3- Success Plot (Success vs. Overlapping Threshold) 115 | 4- Percision Plot (CPE vs Time) 116 | 5- Robustness Evaluation: One-Pass Eval(OPE), Temporal Robustness Eval(TRE), Spatial Robustness Eval (SRE) 117 | 6- SA vs. Time 118 | 119 | 120 | 121 | TECHNICAL FEATURES: 122 | =================== 123 | * Start Video from a Given Frame 124 | 125 | * Dataset Annotation Ability 126 | 127 | * Organized Source Code into Folders 128 | 129 | * Verbose Report 130 | 131 | * Saving Video of Results 132 | 133 | * Save Result Bounding Boxes in the File 134 | 135 | * Parameters of Tracker 136 | 1- Number of Particles, Features, Similarity Measures, Gaussian Variance, Occlusion Flag Threshold, ... 137 | 2- Run Each Tracker with its own set of parameters. 138 | 139 | * Stable Bounding Box Around The Target (Do not shake or jitter) 140 | 141 | * Visualization 142 | 1- Input Data: Color Channel, Depth Channel, Point Cloud, ... 143 | 2- Tracking History and Ground Truth: Type of Errors, Source of Errors, Target Bounding Box Center Point vs Ground Truth On Picture Precision Plot, The same for (w,h), Video Event Annotation, Video Occlusion State, ... 144 | 3- Particle Dynamics: Color Histogram, Local Features (e.g. iHOG), Depth Map, Confidence Map, Distance of Features to Template vs. Time, ... 145 | 4- Tracker Dynamics: All Particles, Expected New Target, Occlusion Suspects, Ground Truth, Background Update, Template Update, Probability Histogram of Particles (including probability of ground truth), Learnt Samples, Distance of Expected Target to Template vs. Time... 146 | 5- All Objects Silhouette Image 147 | 6- Stabilized Picture (only target bounding box), Stabilized Frame (fixed size centered on target center) 148 | 7- Error vs Time for All Trackers -> e.g. CPE vs time, ... 149 | 150 | * Data Exchange Between trackers -> Shared Memory pool 151 | 152 | -------------------------------------------------------------------------------- /bb/bb_content.m: -------------------------------------------------------------------------------- 1 | function out = bb_content (img, bb) 2 | bb = ceil(bb); 3 | out = img (bb(2):bb(2)+bb(4),bb(1):bb(1)+bb(3),:); 4 | end -------------------------------------------------------------------------------- /bb/bb_content_safe.m: -------------------------------------------------------------------------------- 1 | function out = bb_content_safe (img, bb) 2 | bb = ceil(bb); 3 | 4 | if ( bb(1) < 1 || bb(2) < 1 || bb(1)+bb(3)>size(img,2) || bb(2)+bb(4)>size(img,1)) 5 | 6 | x = 1; 7 | y = 1; 8 | 9 | if (bb(1) < 1) 10 | x = 1-bb(1); 11 | end 12 | if (bb(2) < 1) 13 | y = 1-bb(2); 14 | end 15 | 16 | img2 = repmat(uint8(255),[-y+size(img,1)+1,-x+size(img,2)+1,3]); 17 | img2(y:y+size(img,1)-1,x:x+size(img,2)-1,:) = img; 18 | 19 | img = img2; 20 | bb(1) = min(1,bb(1)); 21 | bb(2) = min(1,bb(2)); 22 | 23 | end 24 | 25 | out = bb_content(img,bb); 26 | 27 | end -------------------------------------------------------------------------------- /bb/bb_feature_distance.m: -------------------------------------------------------------------------------- 1 | function x = bb_feature_distance ( x ,y , g, features ) 2 | 3 | for f = 1:size(features,2) 4 | 5 | % Matlab is stupid! Array1 and Array2 can be uint8, so 6 | % array1-array2 can't be negative! 7 | array1 = double(x.cell(1,1).feature(f).val); 8 | array2 = double(y.cell(1,1).feature(f).val); 9 | 10 | % calculating the distance of each feature 11 | switch (features{f}.sim) 12 | 13 | case 'L1' 14 | d = dist_l1 ( array1 , array2 ); 15 | 16 | case 'L2' 17 | d = dist_l2 ( array1 , array2 ); 18 | 19 | case 'Bhattacharyya' 20 | d = dist_bhattacharyya ( array1 , array2 ); 21 | 22 | case 'Intersection' 23 | case 'KL-divergance' 24 | case 'GMM-dist' 25 | case 'Quadratic' 26 | case 'Cosine' 27 | case 'Chi-sqaure' 28 | case 'Diffusion' 29 | case 'Earth Mover' 30 | case 'Correlation' 31 | 32 | 33 | case 'Bhattacharyya(Gridding with Confidence)' 34 | x.dist(f) = 0; 35 | for i = 1:g 36 | for j = 1:g 37 | 38 | array1 = x.cell(i,j).feature(f).val; 39 | array2 = y.cell(i,j).feature(f).val; 40 | d = dist_bhattacharyya ( array1 , array2 ); 41 | c = x.cell(i,j).feature(f).coeff; 42 | x.dist(f) = x.dist(f) + c*d; 43 | end 44 | end 45 | 46 | case 'L2,Grid3' 47 | case 'L2,Grid2,Weighted' 48 | end 49 | 50 | if ( isnan(d) ) 51 | % e.g. HoC is empty 52 | d = Inf; 53 | end 54 | 55 | x.dist(f) = d; 56 | end 57 | 58 | % disp ([x.dist(1) x.dist(2)]); 59 | 60 | end 61 | 62 | 63 | -------------------------------------------------------------------------------- /bb/bb_grid.m: -------------------------------------------------------------------------------- 1 | function boy = bb_grid(box, g, i , j) 2 | 3 | boy = uint16([0 0 0 0]); 4 | boy(1) = box(1) + (j-1)*box(3)/g; 5 | boy(2) = box(2) + (i-1)*box(4)/g; 6 | boy(3) = box(3)/g; 7 | boy(4) = box(4)/g; -------------------------------------------------------------------------------- /bb/bb_grid_adjustment.m: -------------------------------------------------------------------------------- 1 | function boy = bb_grid_adjustment (box,g) 2 | 3 | boy = box; 4 | boy(3) = box(3) - mod(box(3),g); 5 | boy(4) = box(4) - mod(box(4),g); 6 | 7 | end 8 | 9 | -------------------------------------------------------------------------------- /bb/bb_load_boxes.m: -------------------------------------------------------------------------------- 1 | function bb_list = bb_load_boxes (filename) 2 | 3 | fid = fopen(filename); 4 | bb_list = fscanf(fid, '%g,%g,%g,%g', [4,inf]); 5 | fclose(fid); -------------------------------------------------------------------------------- /bb/bb_minus_log_likelihood.m: -------------------------------------------------------------------------------- 1 | function x = bb_minus_log_likelihood ( x , features ) 2 | 3 | % minus log probability 4 | l = 0; 5 | importance_sum = 0; 6 | for f = 1:size(features,2) 7 | if ( features{f}.imp == 0 ) % feature is set to be ignored 8 | continue; 9 | else 10 | importance_sum = importance_sum + 1.0 / features{f}.imp; 11 | feature_mlog_likelihood = x.dist(f) / features{f}.nrm / features{f}.imp; 12 | l = l + feature_mlog_likelihood; 13 | end 14 | end 15 | 16 | % disp (exp(-l)) 17 | % bringing the range from [0,1/imp1 + 1/ imp2 +...] to [0,1] 18 | x.minus_log_likelihood = l / importance_sum; 19 | -------------------------------------------------------------------------------- /bb/bb_normalize_likelihood.m: -------------------------------------------------------------------------------- 1 | function pr = bb_normalize_likelihood (li) 2 | 3 | normal_li = li / max(li); 4 | pr = li / sum (li); 5 | 6 | -------------------------------------------------------------------------------- /bb/bb_normalize_minus_log_likelihood.m: -------------------------------------------------------------------------------- 1 | function pr = bb_normalize_minus_log_likelihood (li) 2 | 3 | % K: a constant that transform range of sum of distances from [0,1] to [0,K] 4 | % using this constant the probability would be [exp(-K),1] 5 | % exp(-36) = 0 6 | K = 36; 7 | 8 | minli = min(li); 9 | maxli = max(li); 10 | 11 | % bring it to range [0,K] 12 | normal_li = (K / (maxli-minli)) * (-minli + li); 13 | 14 | % making probability 15 | pr = (1.0 / sum (exp(-normal_li))) * exp(-normal_li); 16 | 17 | 18 | % hist(pr,100); 19 | if any(isnan(pr)) 20 | disp ('HOLY CRAB!!!'); 21 | end 22 | 23 | -------------------------------------------------------------------------------- /bb/bb_random.m: -------------------------------------------------------------------------------- 1 | function bb = bb_random( img_size ) 2 | %BB_RANDOM Creates a random bounding box which fits completely in the image 3 | % dimensions given 4 | % This function serves as a utility function to create a random bounding 5 | % box on image, where tyhe size and position does not matter. By default 6 | % the size of box is larger than 30x30 pixels 7 | 8 | H = img_size(1); 9 | W = img_size(2); 10 | 11 | x = randi([1 W-30],1); %x in the image 12 | y = randi([1 H-30],1); %y in the image 13 | 14 | w = randi([30, W-x],1); 15 | h = randi([30, H-y],1); 16 | 17 | bb = [x,y,w,h]; 18 | end 19 | 20 | -------------------------------------------------------------------------------- /bb/bb_range_check.m: -------------------------------------------------------------------------------- 1 | function B = box_range_check (B, Wrange, Hrange, Isize, g) 2 | 3 | %decomposition of bounding box 4 | X = B(:,1); 5 | Y = B(:,2); 6 | W = B(:,3); 7 | H = B(:,4); 8 | N = size(B,1); 9 | 10 | % width of the box 11 | W(W < Wrange(1)) = Wrange(1); 12 | W(W > Wrange(2)) = Wrange(2); 13 | 14 | % height of the box 15 | H(H < Hrange(1)) = Hrange(1); 16 | H(H > Hrange(2)) = Hrange(2); 17 | 18 | % out of range: right 19 | Z = X+W - repmat(Isize(2)-10,N,1); 20 | Z (Z < 0 ) = 0; 21 | X = X -Z; 22 | 23 | % out of range: bottom 24 | Z = Y+H - repmat(Isize(1)-10,N,1); 25 | Z (Z < 0 ) = 0; 26 | Y = Y -Z; 27 | 28 | % out of range: left 29 | X (X <= 0) = 10; 30 | 31 | % out of range: top 32 | Y ( Y <= 0) = 10; 33 | 34 | % reconstructing bounding box and grid size adjustment 35 | B = bb_grid_adjustment(ceil([X Y W H]),g); 36 | -------------------------------------------------------------------------------- /bb/bb_resample_roulette_wheel.m: -------------------------------------------------------------------------------- 1 | function [X,ZZ] = bb_resample_roulette_wheel(X, Z, P, N) 2 | 3 | % calculating cumulative distribution 4 | R = cumsum(P, 2); 5 | 6 | % random numbers 7 | T = rand(1, N); 8 | 9 | % resampling 10 | [~, I] = histc(T, R); 11 | X = X(I+1,:); 12 | 13 | % occlusion flag of new particles 14 | ZZ = Z(I+1); 15 | 16 | % h = figure; subplot(1,2,1); plot(R); ylim([0,1]); subplot(1,2,2); hist(I,size(P,2)); % DEBUG MODE 17 | % close (h); % DEBUG MODE 18 | -------------------------------------------------------------------------------- /bkg/bkg_subtraction.m: -------------------------------------------------------------------------------- 1 | function [rgb_msk, dep_msk] = bkg_subtraction ( method , rgb_raw, dep_raw, rgb_bkg , dep_bkg) 2 | 3 | switch (method) 4 | case 'none' 5 | rgb_msk = ones(size(rgb_msk)); 6 | dep_msk = ones(size(dep_msk)); 7 | case 'thresholding' 8 | [rgb_msk, dep_msk] = bkg_subtraction_thresholding ( rgb_raw, dep_raw, rgb_bkg , dep_bkg); 9 | end -------------------------------------------------------------------------------- /bkg/bkg_subtraction_thresholding.m: -------------------------------------------------------------------------------- 1 | function [rgb_msk, dep_msk] = bkg_subtraction_thresholding ( rgb_raw, dep_raw, rgb_bkg , dep_bkg) 2 | 3 | % MATLAB is stupid! unit8 is not enough for calculations, it should be converted into double 4 | 5 | diff = double(rgb_raw) - double(rgb_bkg); 6 | s = sum(abs(diff),3); 7 | 8 | thr = 100; 9 | % 10 | % sc = histc(s(:),max(s(:))); 11 | % lnp = lognfit(sc); %log_normal_dist_params 12 | % energy = logncdf(thr,lnp(1),lnp(2)) 13 | 14 | 15 | mask = s>thr; 16 | 17 | rgb_msk = mask; dep_msk = mask; 18 | 19 | % % visualization 20 | % figure; 21 | % subplot (6,2,[1,3]); imshow(rgb_raw); 22 | % subplot (6,2,[5,7]); imshow(rgb_bkg); 23 | % subplot (6,2,[9,11]); imshow(mask); 24 | % subplot (6,2,[2,4,6]); image(s); colormap ('hot'); 25 | % subplot (6,2,[8,10,12]); hist(s(:),max(s(:))); 26 | 27 | 28 | -------------------------------------------------------------------------------- /bkg/offline_bkg_detection.m: -------------------------------------------------------------------------------- 1 | function [bkg_rgb,bkg_dep] = offline_bkg_detection( method, dataset_name , sample_count ) 2 | %UNTITLED Summary of this function goes here 3 | % Detailed explanation goes here 4 | 5 | [vid_param, directory, num_frames, ~, ~, ~] = video_info(dataset_name); 6 | 7 | 8 | %% method: temporal median 9 | tmp = randperm(num_frames); 10 | rand_frames = tmp(1:sample_count); % get m frames 11 | 12 | for fr = 1:sample_count 13 | [rgb, dep] = read_frame(vid_param, directory, rand_frames(fr)); 14 | bkg_sample_rgb(fr,:,:,:) = rgb; 15 | bkg_sample_dep(fr,:,:,:) = dep; 16 | % subplot(1,2,1); imshow(rgb); 17 | % subplot(1,2,2); imshow(dep); 18 | % drawnow; 19 | end 20 | 21 | bkg_rgb = bkg_median ( bkg_sample_rgb ); 22 | % subplot(1,2,1); imshow(bkg_rgb); 23 | % drawnow; 24 | bkg_dep = bkg_median ( bkg_sample_dep ); 25 | % subplot(1,2,2); imshow(bkg_dep); 26 | 27 | 28 | end 29 | 30 | function im = bkg_median ( samples ) 31 | 32 | % need to be vectorized 33 | for i = 1:size(samples,2) 34 | for j = 1:size(samples,3) 35 | for k = 1:size(samples,4) 36 | a = samples(:,i,j,k); 37 | m = median(a); 38 | im(i,j,k) = m; 39 | end 40 | end 41 | end 42 | end -------------------------------------------------------------------------------- /distance/dist_bhattacharyya.m: -------------------------------------------------------------------------------- 1 | function d = dist_bhattacharyya ( h1 , h2 ) 2 | 3 | % http://docs.opencv.org/doc/tutorials/imgproc/histograms/histogram_compari 4 | % son/histogram_comparison.html 5 | % note that OpenCV formulation includes histogram normalization, which is 6 | % done before in this implementation 7 | 8 | d1 = sum(sqrt(h1.*h2)); 9 | d = sqrt (1 - d1); 10 | -------------------------------------------------------------------------------- /distance/dist_chisquare.m: -------------------------------------------------------------------------------- 1 | function d = dist_chisquare ( h1 , h2 ) 2 | 3 | d_item = ( h1 - h2 ).^2 ./ h1; 4 | d_item(isnan(d_item) | isinf(d_item)) = 0; 5 | d = sum(d_item); -------------------------------------------------------------------------------- /distance/dist_correlation.m: -------------------------------------------------------------------------------- 1 | function d = dist_correlation ( h1 , h2 ) 2 | 3 | h1_m = mean (h1); 4 | h2_m = mean (h2); 5 | 6 | d_enu = (h1-h1_m)*(h2-h2_m)'; 7 | d_den = sqrt(((h1-h1_m)*(h1-h1_m)')*((h2-h2_m)*(h2-h2_m)')); 8 | 9 | if (d_den ~=0) 10 | d = d_enu / d_den; 11 | else 12 | d = inf; 13 | disp ('Singularity problem: one of the histograms is zero or uniform'); 14 | end -------------------------------------------------------------------------------- /distance/dist_intersection.m: -------------------------------------------------------------------------------- 1 | function d = dist_intersection ( h1 , h2 ) 2 | 3 | d = sum(min(h1,h2)); -------------------------------------------------------------------------------- /distance/dist_l1.m: -------------------------------------------------------------------------------- 1 | function d = dist_l1 ( h1 , h2 ) 2 | 3 | d = sum(abs(h1-h2)); 4 | 5 | end -------------------------------------------------------------------------------- /distance/dist_l2.m: -------------------------------------------------------------------------------- 1 | function d = dist_l2 ( h1 , h2 ) 2 | 3 | d = sum((h1-h2).^2); 4 | 5 | end -------------------------------------------------------------------------------- /eval/calculate_CPE.m: -------------------------------------------------------------------------------- 1 | function d = calculate_CPE (gt, alg) 2 | if (isnan(gt(1))) 3 | d = NaN; 4 | return; 5 | end 6 | 7 | cp_gt = gt(1:2) + gt(3:4)/2; 8 | cp_alg = alg(1:2) + alg(3:4)/2; 9 | 10 | diff2 = (cp_alg - cp_gt).^2; 11 | d = sqrt(diff2(1) + diff2(2)); 12 | end %====================================================================== -------------------------------------------------------------------------------- /eval/calculate_S.m: -------------------------------------------------------------------------------- 1 | function S = calculate_S ( gt, alg ) 2 | 3 | % Ground Truth says Occlusion 4 | if isnan(gt(1)) 5 | if isnan(alg(1)) 6 | S = 1; 7 | return; 8 | else 9 | S = -1; 10 | return; 11 | end 12 | end 13 | 14 | % Ground Truth says No Occlusion but Algorithm says Occlusion 15 | if isnan(alg(1)) 16 | S = -1; 17 | return; 18 | end 19 | 20 | % Compute the overlapping ratio 21 | box_intersect = rectint(gt,alg); 22 | box_union = gt(3)*gt(4) + alg(3)*alg(4) - box_intersect; 23 | S = double(box_intersect)/double(box_union); 24 | 25 | end %====================================================================== -------------------------------------------------------------------------------- /eval/draw_cpe_plot.m: -------------------------------------------------------------------------------- 1 | function cpe_plot = draw_cpe_plot (plot_title, mismatch, trackers, no_occlusion_marks) 2 | 3 | cpe_plot = figure; 4 | hold on; 5 | 6 | colors = {'r','g','b','y','m','c','k','r--','g--','b--','c--','m--','y--','k--'}; 7 | 8 | X = 1:length(no_occlusion_marks); 9 | maxY = 0; 10 | 11 | for tr = 1:size(trackers,2) 12 | Y = mismatch(tr,:); 13 | 14 | plot(X,Y, colors{1,tr}, 'LineWidth',2); 15 | 16 | maxY = max(maxY,max(Y(:))); 17 | end 18 | 19 | title(plot_title,'FontSize', 12); 20 | xlabel('Frames'); 21 | ylabel('CPE(pixel)'); 22 | 23 | % occlusion coloring 24 | for fr = 1:length(no_occlusion_marks) 25 | if no_occlusion_marks(fr) == 0 26 | line ([fr-1;fr-1],[0;maxY],'Color','k','LineStyle','-.'); 27 | line ([fr ;fr ],[0;maxY],'Color','k','LineStyle','-.'); 28 | line ([fr+1;fr+1],[0;maxY],'Color','k','LineStyle','-.'); 29 | end 30 | end 31 | 32 | ylim([0 maxY+5]); 33 | xlim([0 length(X)]); 34 | 35 | end %====================================================================== 36 | -------------------------------------------------------------------------------- /eval/draw_success_plot.m: -------------------------------------------------------------------------------- 1 | function [ figure_handle ] = draw_success_plot( plot_title, threshold_steps, alg_success, alg_names, alg_auc ) 2 | %UNTITLED Summary of this function goes here 3 | % Detailed explanation goes here 4 | 5 | figure_handle = figure; 6 | hold on; 7 | 8 | leg = cell(size(alg_names,2),1); 9 | for tr = 1:size(alg_names,2) 10 | leg{tr} = [alg_names{1,tr}.name, sprintf(' (%0.3f)', alg_auc(tr))]; 11 | end 12 | 13 | X = 0:1.0/threshold_steps:1; 14 | Y1 = alg_success(1,:); 15 | 16 | if size(alg_names,2) == 4 17 | Y2 = alg_success(2,:); 18 | Y3 = alg_success(3,:); 19 | Y4 = alg_success(4,:); 20 | 21 | pl = plot(X,Y1,X,Y2,X,Y3,X,Y4); 22 | legend(leg{1},leg{2},leg{3},leg{4},'Location','NorthEast'); 23 | else 24 | % unknown case, show only the first one 25 | pl = plot(X,Y1); 26 | legend(leg{1},'Location','NorthEast'); 27 | end 28 | 29 | set(pl, 'LineWidth', 3); 30 | xlim([0 1]); 31 | ylim([0 1.05]); 32 | title(plot_title,'FontSize', 12); 33 | xlabel('Overlap Threshold'); 34 | ylabel('Success Rate'); 35 | 36 | end 37 | 38 | -------------------------------------------------------------------------------- /eval/evaluate_center_position_error.m: -------------------------------------------------------------------------------- 1 | function cpe = evaluate_center_position_error (test_videos , trackers , control , results , ground_truth) 2 | %EVALUATE_CENTER_POSITION_ERROR Evaluate trackers accuracy in localization 3 | %of target using one point 4 | % This function calculates the center point of the bounding box as the 5 | % representative of the object and then measure ythe distance between 6 | % real and estimated central points 7 | % 8 | % code by: Kourosh Meshgi, Nov 2013 9 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 10 | 11 | for vid = 1:size(test_videos,2) 12 | for tr = 1:length(trackers) 13 | cpe(vid,tr) = 0; 14 | k = []; % marks non-occluded frames 15 | for fr = 1:size(results,4) 16 | gt = squeeze (ground_truth{1,vid}(1:4,fr)); 17 | alg = squeeze (results(vid,tr,:,fr)); 18 | mismatch(tr,fr) = calculate_CPE ( gt(:)' , alg(:)' ); 19 | if ( ~isnan(mismatch(tr,fr)) ) 20 | cpe(vid,tr) = cpe(vid,tr) + mismatch(tr,fr); 21 | end 22 | 23 | k = [k ~isnan(gt(1))]; 24 | 25 | end 26 | cpe(vid,tr) = cpe(vid,tr) / sum(k); % there is no division by zero since all sequences has at least one non occluded frame (first one) 27 | end 28 | % darw cpe vs time plots 29 | cpe_plots(vid) = draw_cpe_plot (['Sequence: ' test_videos{1,vid}], mismatch, trackers, k); 30 | end 31 | 32 | end %====================================================================== 33 | 34 | -------------------------------------------------------------------------------- /eval/evaluate_scale_adaptation.m: -------------------------------------------------------------------------------- 1 | function sa = evaluate_scale_adaptation (test_videos , trackers , control , results , ground_truth) 2 | %EVALUATE_SCALE_ADAPTATION Evaluate trackers scale adaptation ability 3 | % This function compares the dimensions of ground truth bounding box with 4 | % the one suggested by tracking algorithm. The results is calculated 5 | % error for this dimensions mismatches. 6 | % 7 | % code by: Kourosh Meshgi, Nov 2013 8 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 9 | 10 | for vid = 1:size(test_videos,2) 11 | for tr = 1:1:length(trackers) 12 | sa(vid,tr) = 0; 13 | k = 0; % counts non-occluded frames 14 | for fr = 1:size(results,4) 15 | gt = squeeze (ground_truth{1,vid}(1:4,fr)); 16 | alg = squeeze (results(vid,tr,:,fr)); 17 | mismatch = calculate_SA ( gt(:)' , alg(:)' ); 18 | if ( mismatch >= 0 ) 19 | sa(vid,tr) = sa(vid,tr) + mismatch; 20 | k = k+1; 21 | end 22 | end 23 | sa(vid,tr) = sa(vid,tr) / k; % there is no division by zero since all sequences has at least one non occluded frame (first one) 24 | end 25 | end 26 | 27 | end %====================================================================== 28 | 29 | 30 | 31 | function d = calculate_SA (gt, alg) 32 | if (isnan(gt(1))) 33 | d = -1; 34 | return; 35 | end 36 | 37 | diff2 = (alg-gt).^2; 38 | d = sqrt(diff2(3) + diff2(4)); 39 | end %====================================================================== 40 | -------------------------------------------------------------------------------- /eval/evaluate_success_plot_OPE.m: -------------------------------------------------------------------------------- 1 | function [ auc, success_plots ] = evaluate_success_plot_OPE (test_videos , trackers , control , results , ground_truth) 2 | %EVALUATE_SUCCESS_PLOT_OPE Evaluate trackers using Sucess plots with 3 | %One-Pass Evaluation 4 | % This function draw a success plot for all video seqeunces and the 5 | % average performance of trackers in separate plots. In each plot several 6 | % trackers are evaluated by their success to estimate the ground truth by 7 | % a certain degree. This degree, called overlapping threshold can be 8 | % between 0 to 100%. If the threshold is zero, it means that the 9 | % evaluation metric only cares about whether the tracker could 10 | % distinguish occlusions or not. In the case of 100% it should not only 11 | % detect occlusions correctly, but also outputs the exact bounding box of 12 | % ground thruth. The success plot, draws the success of each tracker 13 | % having different thresholds and calculates the area under curve (AUC) 14 | % of this plot as a measure for tracker computattion. 15 | % The conventional way to evaluate trackers is to run them throughout a 16 | % test sequence with initialization from the ground truth position in the 17 | % first frame and report the average precision or success rate. We refer 18 | % this as one-pass evaluation (OPE). 19 | % 20 | % code by: Kourosh Meshgi, Nov 2013 21 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 22 | 23 | for vid = 1:size(test_videos,2) 24 | for tr = 1:1:length(trackers) 25 | for th = 0:control.success_plot_thresold_steps 26 | k = 0; % success counter 27 | for fr = 1:size(results,4) 28 | gt = squeeze (ground_truth{1,vid}(1:4,fr)); 29 | alg = squeeze (results(vid,tr,:,fr)); 30 | S = calculate_S ( gt(:)' , alg(:)' ); 31 | if S >= (double(th)/control.success_plot_thresold_steps) 32 | k = k + 1; 33 | end; 34 | end 35 | r = k / size(results,4); % success ratio 36 | success(tr,th+1) = r; % success of each tracker for each overlap threshold 37 | end 38 | auc(vid,tr) = sum(success(tr,:)); % area under curve for each tracker 39 | end 40 | success_plots(vid) = draw_success_plot (['Sequence: ' test_videos{1,vid}], control.success_plot_thresold_steps, success, trackers, auc); 41 | % success_plots = []; % DEBUG MODE 42 | 43 | end 44 | % draw_success_plot ('Average Performance Over All Videos', control.success_plot_thresold_steps, AVERAGE(success_OVER_VIDEOS), trackers, AVERAGE_OVER_VIDEOS(auc)); 45 | 46 | 47 | end %====================================================================== 48 | 49 | -------------------------------------------------------------------------------- /feature/depth/median_of_depth.m: -------------------------------------------------------------------------------- 1 | function [nominal, variance, avg] = median_of_depth ( dmap, mask) 2 | 3 | pts = dmap(:); 4 | % pts = pts(find(mask(:) == 1)); % enables background sutraction 5 | 6 | if isempty(pts) 7 | nominal = 1; 8 | variance = -1; 9 | avg = -1; 10 | return; 11 | end 12 | 13 | nominal = double(median(pts))/255; 14 | variance = var(double(pts')/255); 15 | avg = mean(pts)/255; 16 | % 17 | % h = figure; subplot(2,2,1); imshow(dmap); subplot(2,2,2); imshow(mask); % DEBUG MODE 18 | % subplot(2,2,3); imhist(dmap(:)); subplot(2,2,4); imhist(pts); disp(nominal); % DEBUG MODE 19 | % close(h); % DEBUG MODE -------------------------------------------------------------------------------- /feature/hoc/color_clustering.m: -------------------------------------------------------------------------------- 1 | function rgb_ctr = color_clustering (img , sample_count , rgb_bins , init_bb) 2 | 3 | if isempty(img) 4 | rgb_ctr = []; 5 | return 6 | end 7 | 8 | % get all points of image, sample them choosing points in equivalent 9 | % distance 10 | all_pixels = reshape (img(:,:,1:3) , size(img,1) * size(img,2) , size(img,3)); 11 | sample_idx = floor (linspace(1,size(all_pixels,1),sample_count-100)); 12 | 13 | % adding subject appearance 14 | obj = bb_content(img,init_bb); 15 | obj_pixels = reshape (obj(:,:,1:3) , size(obj,1) * size(obj,2) , size(obj,3)); 16 | obj_sample_idx = floor (linspace(1,size(obj_pixels,1),100)); 17 | 18 | 19 | % feed these sample points (RGB) to K-means 20 | samples = double (all_pixels(sample_idx,:)); 21 | sample_obj = double (obj_pixels(obj_sample_idx,:)); 22 | samples = [samples ; sample_obj]; 23 | 24 | opts=statset('Display','final'); 25 | [~,rgb_ctr]=kmeans( samples ,rgb_bins,'Options',opts); 26 | 27 | -------------------------------------------------------------------------------- /feature/hoc/histogram_of_colors.m: -------------------------------------------------------------------------------- 1 | function freq = histogram_of_colors (img, msk, ctrs ) 2 | 3 | img_reshaped = reshape( img, size(img,1)*size(img,2), size(img,3) ); 4 | % img_valid = img_reshaped(find(msk==1),:); % enables background sutraction 5 | img_valid = img_reshaped; % disables background subtraction 6 | 7 | num_pixels = size( img_valid, 1 ); % fg pixel count 8 | num_bins = size(ctrs,1); % bin count 9 | freq = zeros( num_bins, 1 ); % bin counter initialization 10 | 11 | d = zeros( num_bins, num_pixels); % distance matrix 12 | for c = 1:size(img,3) 13 | col_cnt = repmat(ctrs(:,c),1,num_pixels); 14 | col_img = repmat(img_valid(:,c)',num_bins,1); 15 | d = d + (double(col_cnt) - double(col_img)).^2; 16 | end 17 | 18 | [~, idx] = min(d); % label of each pixel (=coresponding bin) 19 | freq = hist( idx, 1:num_bins ); % counting members of each bin 20 | freq = freq / num_pixels; 21 | 22 | % h = figure; subplot (2,2,1); imshow (img); subplot(2,2,2); imshow(msk); subplot(2,2,[3,4]); hist_vis (freq, ctrs); %DEBUG MODE 23 | % close(h); % DEBUG MODE 24 | 25 | if (isnan(freq(1))) 26 | disp ('Warning! Image has no foreground, thus num_pixels = 0'); 27 | end -------------------------------------------------------------------------------- /feature/hoc/hoc_vis.m: -------------------------------------------------------------------------------- 1 | function hoc_vis (h, ctrs) 2 | 3 | if (nargin == 0) 4 | % demo mode 5 | m = 4; % single color channel quantization 6 | q = linspace(1,255,m); 7 | ctrs = setprod (q,q,q); 8 | h = 0.05:0.05/(m^3):0.1; h = h(1:end-1); 9 | end 10 | 11 | ctrs = floor(ctrs); 12 | 13 | for i = 1:size(h,2) 14 | x = zeros(size(h)); 15 | x(i) = h(i); 16 | bar(x,'FaceColor',ctrs(i,:)/255,'EdgeColor','none'); 17 | % bar(x,'FaceColor',ctrs(i,:)/255); % show separating line 18 | hold on; 19 | end 20 | 21 | xlim([0 size(h,2)+1]); 22 | ylim([0 0.1]); 23 | 24 | hold off; -------------------------------------------------------------------------------- /feature/initialize_features.m: -------------------------------------------------------------------------------- 1 | function self = initialize_features (self, rgb_raw, video_name, init_bb) 2 | 3 | for f = 1:size(self.fn,2) 4 | % create space for each feature 5 | self.feature{f}.name = self.fn{1,f}; 6 | self.feature{f}.sim = self.sm{1,f}; 7 | self.feature{f}.imp = self.imp(f); 8 | self.feature{f}.nrm = self.nrm(f); 9 | 10 | % global initilization of feature who needs it 11 | switch (self.fn{1,f}) 12 | 13 | case 'HoC(RGB Clustering)' 14 | % color centers 15 | if ( self.rgb_bins_load ) 16 | load (['bkg/' video_name '/rgb_ctr.mat']); 17 | else 18 | rgb_ctr = color_clustering (rgb_raw , self.rgb_clustering_samples, self.rgb_bins, init_bb); 19 | fld = ['bkg/' video_name]; 20 | if (~exist(fld,'dir')) 21 | mkdir(fld); 22 | end 23 | save([fld '/rgb_ctr.mat'],'rgb_ctr'); 24 | end 25 | self.feature{f}.rgb_ctr = rgb_ctr; 26 | 27 | case 'HoC(Uniform)' 28 | m = self.rgb_bins; % single color channel quantization 29 | q = linspace(1,255,m); 30 | rgb_ctr = setprod (q,q,q); 31 | self.feature{f}.rgb_ctr = rgb_ctr; 32 | 33 | case 'HoC(RGB Clustering,Gridding with Confidence)' 34 | % color centers 35 | if ( self.rgb_bins_load ) 36 | load (['bkg/' video_name '/rgb_ctr.mat']); 37 | else 38 | rgb_ctr = color_clustering (rgb_raw , self.rgb_clustering_samples, self.rgb_bins, init_bb); 39 | fld = ['bkg/' video_name]; 40 | if (~exist(fld,'dir')) 41 | mkdir(fld); 42 | end 43 | save([fld '/rgb_ctr.mat'],'rgb_ctr'); 44 | end 45 | self.feature{f}.rgb_ctr = rgb_ctr; 46 | 47 | % confidence measure 48 | load (['beta_dist_cells_' num2str(self.g) 'x' num2str(self.g) '.mat']); 49 | self.feature{f}.rgb_cnf = pd; 50 | 51 | end 52 | end -------------------------------------------------------------------------------- /mu/model_update.m: -------------------------------------------------------------------------------- 1 | function model = model_update ( method , old_model , new_model , tracker ) 2 | 3 | switch (method) 4 | case 'none' 5 | model = old_model; 6 | case 'moving_average' 7 | a = tracker.target_update_forgetting_rate; 8 | % model = (1/1+a)*(new_model + a * old_model); % TO BE REVISED, 9 | % IT'S JUST THE CONCEPT 10 | 11 | for f = 1:size(tracker.feature,2) 12 | switch (tracker.feature{f}.name) 13 | 14 | case {'HoC(RGB Clustering)','HoC(Uniform)','medD'} 15 | x.cell(1,1).feature(f).val = (1/(1+a))*(new_model.cell(1,1).feature(f).val + a*old_model.cell(1,1).feature(f).val); 16 | 17 | case 'HoC (RGB Clustering,Grid2)' 18 | g = 2; 19 | for i = 1:g 20 | for j = 1:g 21 | x.cell(i,j).feature(f).val = (1/(1+a))*(new_model.cell(i,j).feature(f).val + a*old_model.cell(i,j).feature(f).val); 22 | end 23 | end 24 | 25 | case 'Grid Confidence (Beta)' 26 | for i = 1:g 27 | for j = 1:g 28 | x.cell(i,j).feature(f).val = 1; %pdf(prior_dist(i,j),f.cell(i,j).fg_ratio); 29 | end 30 | end 31 | 32 | end 33 | 34 | 35 | 36 | 37 | end 38 | model = x; 39 | 40 | case 'last_5' 41 | % enqueue new model 42 | % dequeue 5th model 43 | % average all of them 44 | case 'learning' 45 | case 'kalman' 46 | case 'grid_based' 47 | end 48 | 49 | end -------------------------------------------------------------------------------- /tracker/loader/loader_init.m: -------------------------------------------------------------------------------- 1 | function x = loader_init (params) 2 | 3 | x.name = params.name; 4 | x.type = 'loader'; 5 | 6 | x.bb_list = bb_load_boxes (params.filename); 7 | -------------------------------------------------------------------------------- /tracker/loader/loader_test.m: -------------------------------------------------------------------------------- 1 | function bb = loader_test (x, fr) 2 | %LOADER_TEST O... 3 | % This function outputs ... 4 | % 5 | % code by: Kourosh Meshgi, Dec 2013 6 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 7 | 8 | bb = x.bb_list (1:4,fr); 9 | end %====================================================================== -------------------------------------------------------------------------------- /tracker/loader/loader_test.txt: -------------------------------------------------------------------------------- 1 | 109,214,84,267 2 | 116,214,84,267 3 | 131,214,84,267 4 | 133,214,84,267 5 | 151,220,83,258 6 | 163,220,83,258 7 | 177,226,84,251 8 | 182,226,84,251 9 | 193,228,84,251 10 | NaN,NaN,NaN,NaN 11 | 225,228,76,249 12 | 233,227,76,249 13 | 248,227,76,249 14 | 257,228,76,249 15 | 264,234,76,249 16 | 275,231,76,249 17 | 289,231,76,249 18 | 294,231,76,249 19 | 302,230,76,245 20 | 309,230,76,245 21 | 331,231,69,244 22 | 339,228,69,244 23 | 352,230,69,244 24 | 133,214,84,267 25 | 370,237,69,241 26 | 378,236,54,238 27 | 384,238,43,236 28 | 386,309,36,149 29 | NaN,NaN,NaN,NaN 30 | NaN,NaN,NaN,NaN 31 | NaN,NaN,NaN,NaN 32 | 473,247,37,226 33 | 464,242,46,231 34 | 458,246,61,230 35 | 458,246,69,231 36 | 468,247,69,231 37 | 477,252,67,226 38 | 483,252,67,226 39 | 486,249,67,226 40 | 501,248,67,226 41 | 501,248,67,226 42 | 519,249,56,225 43 | 519,249,56,225 44 | 533,250,56,225 45 | 539,256,55,219 46 | 543,255,55,219 47 | 552,255,55,219 48 | 559,255,55,219 49 | 565,257,55,219 50 | 569,257,55,219 51 | 574,257,55,219 52 | -------------------------------------------------------------------------------- /tracker/lower_bounds/lb_center_bb_init.m: -------------------------------------------------------------------------------- 1 | function x = lb_center_bb_init (params) 2 | 3 | x.name = params.name; 4 | x.type = 'lb_center_bb'; 5 | 6 | x.bb = [0,0,0,0]; -------------------------------------------------------------------------------- /tracker/lower_bounds/lb_center_bb_test.m: -------------------------------------------------------------------------------- 1 | function [bb, self] = lb_center_bb_test (rgb, depth, init_bb , grt, self) 2 | %LB_CENTER_BB_TEST Always outputs the box locate at center of image, with first 3 | % frame box size 4 | % This function serves as a lower bound for tracker. 5 | % 6 | % code by: Kourosh Meshgi, Oct 2013 7 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 8 | 9 | if ( ~isempty(init_bb)) 10 | bb = init_bb; 11 | 12 | [m,n] = size(depth); 13 | w = init_bb(3); 14 | h = init_bb(4); 15 | 16 | self.bb = [(n-w)/2,(m-h)/2,w,h]; 17 | else 18 | bb = self.bb; 19 | end 20 | end %====================================================================== -------------------------------------------------------------------------------- /tracker/lower_bounds/lb_crazy_init.m: -------------------------------------------------------------------------------- 1 | function x = lb_crazy_init (params) 2 | 3 | x.name = params.name; 4 | x.type = 'lb_crazy'; 5 | 6 | -------------------------------------------------------------------------------- /tracker/lower_bounds/lb_crazy_test.m: -------------------------------------------------------------------------------- 1 | function bb = lb_crazy_test (rgb, depth, init_bb) 2 | %LB_CRAZY_TEST Outputs bounding boxes with random location and size without 3 | %any prior knowledge of dataset 4 | % This function serves as a lower bound for tracker, providing complete 5 | % random box position and size. Only at the first frame, it returns the 6 | % given bounding box. 7 | % 8 | % code by: Kourosh Meshgi, Oct 2013 9 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 10 | 11 | if ( ~isempty(init_bb)) 12 | bb = init_bb; 13 | else 14 | [m,n] = size(depth); 15 | 16 | x = randi(n-30,1); 17 | y = randi(m-30,1); 18 | w = randi([x,n],1) - x + 29; 19 | h = randi([y,m],1) - y + 29; 20 | 21 | bb = [x,y,w,h]; 22 | end 23 | end %====================================================================== -------------------------------------------------------------------------------- /tracker/lower_bounds/lb_first_bb_init.m: -------------------------------------------------------------------------------- 1 | function x = lb_first_bb_init (params) 2 | 3 | x.name = params.name; 4 | x.type = 'lb_first_bb'; 5 | 6 | x.bb = [0,0,0,0]; -------------------------------------------------------------------------------- /tracker/lower_bounds/lb_first_bb_test.m: -------------------------------------------------------------------------------- 1 | function [bb, self] = lb_first_bb_test (rgb, depth, init_bb , grt, self) 2 | %LB_FIRST_BB_TEST Always outputs the first frame bounding box for all 3 | %frames 4 | % This function serves as a lower bound for tracker. 5 | % 6 | % code by: Kourosh Meshgi, Oct 2013 7 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 8 | 9 | if ( ~isempty(init_bb)) 10 | bb = init_bb; 11 | self.bb = bb; 12 | else 13 | bb = self.bb; 14 | end 15 | end %====================================================================== -------------------------------------------------------------------------------- /tracker/lower_bounds/lb_invisible_init.m: -------------------------------------------------------------------------------- 1 | function x = lb_invisible_init (params) 2 | 3 | x.name = params.name; 4 | x.type = 'lb_invisible'; 5 | -------------------------------------------------------------------------------- /tracker/lower_bounds/lb_invisible_test.m: -------------------------------------------------------------------------------- 1 | function bb = lb_invisible_test (rgb, depth, init_bb) 2 | %LB_INVISIBLE_TEST Outputs bounding box to be occluded all the time 3 | % This function serves as a lower bound for tracker, outputing occlusion 4 | % at all frames of the sequence but gives the correct answer at the first 5 | % frame (since the target in the first frame is always visible) 6 | % 7 | % code by: Kourosh Meshgi, Nov 2013 8 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 9 | 10 | if ( ~isempty(init_bb)) 11 | bb = init_bb; 12 | else 13 | bb = NaN(1,4); 14 | end 15 | end %====================================================================== -------------------------------------------------------------------------------- /tracker/lower_bounds/lb_rand_loc_init.m: -------------------------------------------------------------------------------- 1 | function x = lb_rand_loc_init (params) 2 | 3 | x.name = params.name; 4 | x.type = 'lb_rand_loc'; 5 | 6 | x.gt_bb_mean = 0; 7 | x.gt_bb_std = NaN; -------------------------------------------------------------------------------- /tracker/lower_bounds/lb_rand_loc_test.m: -------------------------------------------------------------------------------- 1 | function [bb, self] = lb_rand_loc_test (rgb, depth, init_bb , grt, self) 2 | %LB_RAND_LOC_TEST Outputs bounding boxes with the first frame box size and 3 | %a random location based on dataset statistics 4 | % This function serves as a lower bound for tracker. It uses the mean and 5 | % variance for the bounding box location and sample new bounding box from 6 | % a Normal distribution based on them, the size is the same as the ground 7 | % truth 8 | % 9 | % code by: Kourosh Meshgi, Oct 2013 10 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 11 | 12 | if ( ~isempty(init_bb)) 13 | bb = init_bb; 14 | [~,gt_valid] = find(~isnan(grt(1,:))); 15 | gt_trimmed = grt(1:4,gt_valid); 16 | 17 | self.gt_bb_mean = mean(gt_trimmed'); 18 | self.gt_bb_std = std(gt_trimmed'); 19 | else 20 | [m,n] = size(depth); 21 | 22 | % size 23 | w = grt(3,1); 24 | h = grt(4,1); 25 | 26 | % location 27 | x = self.gt_bb_mean(1) + randn(1) * self.gt_bb_std(1); 28 | y = self.gt_bb_mean(2) + randn(1) * self.gt_bb_std(2); 29 | x = clip_range(x,[w/2+1, n-w/2-1]); 30 | y = clip_range(y,[h/2+1, m-h/2-1]); 31 | 32 | bb = floor([x,y,w,h]); 33 | end 34 | end %====================================================================== -------------------------------------------------------------------------------- /tracker/lower_bounds/lb_rand_size_init.m: -------------------------------------------------------------------------------- 1 | function x = lb_rand_size_init (params) 2 | 3 | x.name = params.name; 4 | x.type = 'lb_rand_size'; 5 | 6 | x.gt_bb_mean = 0; 7 | x.gt_bb_std = NaN; -------------------------------------------------------------------------------- /tracker/lower_bounds/lb_rand_size_loc_init.m: -------------------------------------------------------------------------------- 1 | function x = lb_rand_size_loc_init (params) 2 | 3 | x.name = params.name; 4 | x.type = 'lb_rand_size_loc'; 5 | 6 | x.gt_bb_mean = 0; 7 | x.gt_bb_std = NaN; -------------------------------------------------------------------------------- /tracker/lower_bounds/lb_rand_size_loc_test.m: -------------------------------------------------------------------------------- 1 | function [bb, self] = lb_rand_size_loc_test (rgb, depth, init_bb , grt, self) 2 | %LB_RAND_SIZE_LOC_TEST Outputs bounding boxes with random location and size 3 | %based on dataset statistics 4 | % This function serves as a lower bound for tracker, providing complete 5 | % box position and size, but with prior knowledge about data set 6 | % statistics. It uses the mean and variance for the bounding box location 7 | % and size and sample new bounding box from a Normal distribution based 8 | % on them 9 | % 10 | % code by: Kourosh Meshgi, Oct 2013 11 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 12 | 13 | if ( ~isempty(init_bb)) 14 | bb = init_bb; 15 | [~,gt_valid] = find(~isnan(grt(1,:))); 16 | gt_trimmed = grt(1:4,gt_valid); 17 | 18 | self.gt_bb_mean = mean(gt_trimmed'); 19 | self.gt_bb_std = std(gt_trimmed'); 20 | else 21 | [m,n] = size(depth); 22 | 23 | % location 24 | x = self.gt_bb_mean(1) + randn(1) * self.gt_bb_std(1); 25 | y = self.gt_bb_mean(2) + randn(1) * self.gt_bb_std(2); 26 | x = clip_range(x,[30, n-30]); 27 | y = clip_range(y,[30, m-30]); 28 | 29 | % size 30 | w = self.gt_bb_mean(3) + randn(1) * self.gt_bb_std(3); 31 | h = self.gt_bb_mean(4) + randn(1) * self.gt_bb_std(4); 32 | w = clip_range(w,[29,n-x]); 33 | h = clip_range(h,[29,m-y]); 34 | 35 | bb = floor([x,y,w,h]); 36 | end 37 | end %====================================================================== -------------------------------------------------------------------------------- /tracker/lower_bounds/lb_rand_size_test.m: -------------------------------------------------------------------------------- 1 | function [bb, self] = lb_rand_size_test (rgb, depth, init_bb , grt, self) 2 | %LB_RAND_SIZE_TEST OuOutputs bounding boxes with the first frame box location 3 | %and a random size based on dataset statistics 4 | % This function serves as a lower bound for tracker. It uses the mean and 5 | % variance for the bounding box size and sample new bounding box from a 6 | % Normal distribution based on them with the first location object was 7 | % found 8 | % 9 | % code by: Kourosh Meshgi, Oct 2013 10 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 11 | 12 | if ( ~isempty(init_bb)) 13 | bb = init_bb; 14 | [~,gt_valid] = find(~isnan(grt(1,:))); 15 | gt_trimmed = grt(1:4,gt_valid); 16 | 17 | self.gt_bb_mean = mean(gt_trimmed'); 18 | self.gt_bb_std = std(gt_trimmed'); 19 | else 20 | [m,n] = size(depth); 21 | 22 | % location 23 | x = grt(1,1); 24 | y = grt(2,1); 25 | 26 | % size 27 | w = self.gt_bb_mean(3) + randn(1) * self.gt_bb_std(3); 28 | h = self.gt_bb_mean(4) + randn(1) * self.gt_bb_std(4); 29 | w = clip_range(w,[30,min((n-x)*2,(x-1)*2)]); 30 | h = clip_range(h,[30,min((m-y)*2,(y-1)*2)]); 31 | 32 | bb = floor([x,y,w,h]); 33 | end 34 | end %====================================================================== -------------------------------------------------------------------------------- /tracker/particle_filter/bb_feature_distance_normalization.m: -------------------------------------------------------------------------------- 1 | function p = bb_feature_distance_normalization ( p ) 2 | 3 | N = length(p); 4 | for i = 1:N 5 | if (p{i}.z == 0) 6 | d(i,:) = p{i}.dist; 7 | else 8 | d(i,:) = NaN(1,2); 9 | end 10 | end 11 | 12 | mind = min(d); 13 | maxd = max(d); 14 | 15 | d = (d - repmat(mind,N,1)) ./ repmat(maxd-mind,N,1); % bring it to range [0,1] 16 | 17 | for i = 1:N 18 | p{i}.dist = d(i,:); 19 | end 20 | end -------------------------------------------------------------------------------- /tracker/particle_filter/bb_feature_extraction.m: -------------------------------------------------------------------------------- 1 | function x = bb_feature_extraction ( bb , g, rgb_raw, dep_raw, rgb_msk, dep_msk, features ) 2 | 3 | for f = 1:size(features,2) 4 | 5 | % global initilization of feature who needs it 6 | switch (features{f}.name) 7 | 8 | case 'HoC(RGB Clustering)' 9 | im = bb_content(rgb_raw,bb); 10 | ms = bb_content(rgb_msk,bb); 11 | x.cell(1,1).feature(f).val = histogram_of_colors (im, ms, features{f}.rgb_ctr ); 12 | 13 | case 'HoC(Uniform)' 14 | x.cell(1,1).feature(f).val = histogram_of_colors (bb_content(rgb_raw,bb), bb_content(rgb_msk,bb), features{f}.rgb_ctr ); 15 | 16 | case 'HoC(RGB Clustering,Gridding with Confidence)' 17 | for i = 1:g 18 | for j = 1:g 19 | cell_bb = bb_grid (bb,g,i,j); 20 | x.cell(i,j).feature(f).val = histogram_of_colors (bb_content(rgb_raw,cell_bb), bb_content(rgb_msk,cell_bb), features{f}.rgb_ctr ); 21 | x.cell(i,j).fg_ratio = box_confidence (bb_content(rgb_msk,cell_bb)); 22 | x.cell(i,j).feature(f).coeff = pdf(features{f}.rgb_cnf(i,j),x.cell(i,j).fg_ratio); 23 | end 24 | end 25 | 26 | case 'medD' 27 | x.cell(1,1).feature(f).val = median_of_depth (bb_content(dep_raw,bb),bb_content(dep_msk,bb)); 28 | end 29 | end 30 | 31 | 32 | 33 | % 34 | % for i = 1:g 35 | % for j = 1:g 36 | % cell_bb = bb_grid (bb,g,i,j); 37 | % 38 | % x.cell(i,j).rgb_hist = histogram_of_colors (bb_content(rgb_raw,cell_bb), bb_content(rgb_msk,cell_bb), rgb_ctr ); 39 | % x.cell(i,j).fg_ratio = box_confidence (bb_content(rgb_msk,cell_bb)); 40 | % x.cell(i,j).confidence = 1; %pdf(prior_dist(i,j),f.cell(i,j).fg_ratio); 41 | % x.cell(i,j).depth_med = median_of_depth (bb_content(dep_raw,cell_bb),bb_content(dep_msk,cell_bb)); 42 | % end 43 | % end 44 | -------------------------------------------------------------------------------- /tracker/particle_filter/bb_likelihood.m: -------------------------------------------------------------------------------- 1 | function x = bb_likelihood ( x , features ) 2 | 3 | likelihood = 1; 4 | for f = 1:size(features,2) 5 | feature_likelihood = exp( - x.dist(f) / features{f}.var ); 6 | likelihood = likelihood * feature_likelihood; 7 | end 8 | 9 | x.likelihood = likelihood; 10 | 11 | 12 | -------------------------------------------------------------------------------- /tracker/particle_filter/bb_occlusion_state_transition.m: -------------------------------------------------------------------------------- 1 | function Z = box_occlusion_state_transition (oldZ, T) 2 | 3 | R = rand(size(oldZ,1),1); 4 | Z = uint8(R > T(uint8(oldZ)+1,1)); 5 | 6 | -------------------------------------------------------------------------------- /tracker/particle_filter/bb_positioning_noise.m: -------------------------------------------------------------------------------- 1 | function B = box_positioning_noise (B, Wrange, Hrange, Xnoise, Ynoise) 2 | 3 | N = size(B,1); 4 | 5 | % random numbers between -1 to +1 6 | R1 = 2* (rand(N,1) - 0.5); 7 | R2 = 2* (rand(N,1) - 0.5); 8 | R3 = 2* (rand(N,1) - 0.5); 9 | R4 = 2* (rand(N,1) - 0.5); 10 | 11 | % noised bounding boxes 12 | X = B(:,1) + Xnoise * R1; 13 | Y = B(:,2) + Ynoise * R2; 14 | W = B(:,3) + ((Wrange(2)-Wrange(1))/10) * R3; %white noise 15 | H = B(:,4) + ((Hrange(2)-Hrange(1))/10) * R4; %white noise 16 | 17 | B = [X Y W H]; 18 | -------------------------------------------------------------------------------- /tracker/particle_filter/bb_prob_color_indicator.m: -------------------------------------------------------------------------------- 1 | function col = bb_prob_color_indicator (L , Z) 2 | N = length(L); 3 | col = zeros(N,3); 4 | 5 | if (any(L)) 6 | mxl = log(max(L)); 7 | mnl = log(min(L(find(L>0)))); 8 | 9 | col(find(L>0),1) = ceil(254 / (mxl-mnl) * bsxfun(@minus,log(L(find(L>0))),mnl)); 10 | 11 | col(find(col>255)) = 255; 12 | col(find(col<0)) = 0; 13 | col(isnan(col)) = 0; 14 | 15 | col(:,2) = col(:,1); 16 | col(:,3) = col(:,1); 17 | end 18 | 19 | % make occluded boxes red 20 | col(find(Z == 1),:) = repmat([255,0,0],sum(Z==1),1); 21 | 22 | col = double(col) / 255; 23 | -------------------------------------------------------------------------------- /tracker/particle_filter/bb_resample.m: -------------------------------------------------------------------------------- 1 | function [B, Z] = bb_resample ( B, Z, P, expT, expZ, I, Wrange, Hrange, Xnoise, Ynoise, g, T ) 2 | 3 | N = size(B,1); 4 | 5 | % resample N-1 particles from N particles proportional to their prob 6 | [B, Z] = bb_resample_roulette_wheel (B, Z, P, N-1); 7 | 8 | % add noise to position of resampled boxes 9 | B = bb_positioning_noise (B, Wrange, Hrange, Xnoise, Ynoise); 10 | 11 | % check whether the boxes are valid ones 12 | B = bb_range_check (B, Wrange, Hrange, size(I), g); 13 | 14 | % state transition of boxes based on STM 15 | Z = bb_occlusion_state_transition (Z, T); 16 | 17 | % archive the expected target, to avoid un-neccessary jiters in box 18 | B = [B; bb_grid_adjustment(ceil(expT),g)]; % the expected box adjusted to grid 19 | Z = [Z; uint8(expZ>0.5)]; %% TO BE CONSIDERED % if the occlusion prob is higher than 0.5 it is probably occluded 20 | -------------------------------------------------------------------------------- /tracker/particle_filter/box_confidence.m: -------------------------------------------------------------------------------- 1 | function c = box_confidence (mask) 2 | 3 | c = sum(sum(mask == 1)); % number of fg pixels 4 | 5 | if (c == 0) % handles zero fg pixel case 6 | c = 1; 7 | end 8 | 9 | c = c / (numel(mask)+eps); % divided by number of all pixels in roi -------------------------------------------------------------------------------- /tracker/particle_filter/expected_target.m: -------------------------------------------------------------------------------- 1 | function [bb_hat, z_hat] = expected_target ( bbs, z, prob ) 2 | 3 | bb_hat = prob * bbs; 4 | z_hat = prob * double(z); -------------------------------------------------------------------------------- /tracker/particle_filter/initialize_particles_bb.m: -------------------------------------------------------------------------------- 1 | function [box, z] = initialize_particles_bb (img, N, w_rng, h_rng, grid, w_noise, h_noise, mask, init_bb, nocc_init) 2 | % Create bounding boxes as particles in particle filter 3 | 4 | %N: number of boxes 5 | box=zeros(N,4); 6 | 7 | if (nargin < 5) 8 | grid = 1; 9 | end 10 | 11 | if (nargin < 6) 12 | mask = []; 13 | w_noise = 0; 14 | h_noise = 0; 15 | end 16 | 17 | if (nargin < 10) 18 | nocc_init = 1; % no occlusion handled 19 | end 20 | 21 | % limit mask to desired foreground object only 22 | trimmed_mask = zeros(size(mask)); % create new mask, initialized by nothing 23 | [a,b] = rect_span(init_bb); % define the rectangle around the init_bb 24 | trimmed_mask(a,b) = bb_content(mask,init_bb); % copy the defined rectangle to the trimmed mask 25 | 26 | % figure; subplot(1,2,1) ;imshow(mask); subplot(1,2,2) ;imshow(trimmed_mask); hold on %DEBUG MODE 27 | for i = 1:N 28 | box(i,:) = create_box(img, w_rng, h_rng, uint16(grid), w_noise, h_noise, trimmed_mask , init_bb); 29 | % rectangle('Position',box(i,:),'EdgeColor','y'); %DEBUG MODE 30 | end 31 | % hold off % DEBUG MODE 32 | 33 | z = rand(N,1) > nocc_init; 34 | end 35 | 36 | 37 | function box=create_box(im, w_range, h_range, g, w_noise, h_noise, mask, init_bb) 38 | % Create bouding box in 3 different cases: 39 | % 1- normal bb, 40 | % 2- grid bb, and 41 | % 3- grid bb centered on foreground 42 | 43 | % initial sizes should be like init_bb if present 44 | 45 | 46 | 47 | if (isempty(mask)) 48 | % uniformely distributed particles 49 | 50 | % box(3)= w_range(1)+rand*(w_range(2)-w_range(1)); %w[50 200] 51 | % box(4)= h_range(1)+rand*(h_range(2)-h_range(1)); %h [200 400] 52 | box(3) = min(init_bb(3),w_range(2)); %%%% NEW 53 | box(4) = min(init_bb(4),h_range(2)); %%%% NEW 54 | 55 | box(1)=1+rand*(size(im,2)-box(3)-10); %set restriction [1 640-w-2] 56 | box(2)=1+rand*(size(im,1)-box(4)-10); %set restriction [1 480-h-2] 57 | 58 | box=ceil(box); 59 | 60 | else 61 | % the particle centers are foreground points 62 | [fg_y,fg_x] = find(mask ==1); 63 | 64 | r = randi(size(fg_x,1)); 65 | box = can_make_box ( fg_x(r) + randi(w_noise), ... 66 | fg_y(r) + randi(h_noise), ... 67 | h_range(1), h_range(2), ... 68 | w_range(1), w_range(2), ... 69 | size(im,1), size(im,2), ... 70 | init_bb); 71 | while ( isempty(box) ) 72 | r = randi(size(fg_x,1)); 73 | box = can_make_box ( fg_x(r) + randi(w_noise), ... 74 | fg_y(r) + randi(h_noise), ... 75 | h_range(1), h_range(2), ... 76 | w_range(1), w_range(2), ... 77 | size(im,1), size(im,2), ... 78 | init_bb); 79 | end 80 | 81 | end 82 | 83 | % grid size adjustment 84 | box = bb_grid_adjustment(uint16(box),g); 85 | 86 | end 87 | 88 | 89 | 90 | function box = can_make_box(x,y,min_h,max_h,min_w,max_w,img_h,img_w,init_bb) 91 | % Checks whether it is possible to make bb with valid sizes, if it is 92 | % possible make a valid random one centered on given coordinates 93 | 94 | % center range check 95 | if (x < 1 || x > img_w || y < 1 || y > img_h) 96 | box = []; 97 | return 98 | end 99 | 100 | % calcuate distance of center to edges of screen 101 | lft_margin = x - 1; 102 | rgt_margin = img_w - x - 1; 103 | top_margin = y - 1; 104 | dwn_margin = img_h - y - 1; 105 | 106 | % check if the margins are enough to make at least smallest box 107 | if lft_margin < min_w/2 || rgt_margin < min_w/2 || top_margin < min_h/2 || dwn_margin < min_h/2 108 | box = []; 109 | return 110 | end 111 | 112 | % set the biggest possible box dimensions 113 | max_w = min3(max_w, 2*lft_margin, 2*rgt_margin); 114 | max_h = min3(max_h, 2*top_margin, 2*dwn_margin); 115 | 116 | % make a random box in valid size 117 | % w = min_w + randi(max_w-min_w + 1) - 1; 118 | % h = min_h + randi(max_h-min_h + 1) - 1; 119 | w = min(max_w,init_bb(3)); %%%% NEW 120 | h = min(max_h,init_bb(4)); %%%% NEW 121 | 122 | box = ceil([x-w/2,y-h/2,w,h]); 123 | end 124 | 125 | function m = min3(a,b,c) 126 | m = min(a,min(b,c)); 127 | end 128 | 129 | -------------------------------------------------------------------------------- /tracker/particle_filter/particle_filter_demo.m: -------------------------------------------------------------------------------- 1 | function particle_filter_demo (gt, self , vid_param, cam_param, directory, num_frames, start) 2 | fh = figure('toolbar','none','menubar','none','color','k','units','normalized','outerposition',[0 0 1 1],'name','Occlusion Aware Particle Framework'); 3 | colormap('jet'); 4 | 5 | 6 | 7 | % ===================================================================== 8 | writerObj = VideoWriter('output/v1.avi'); % VID 9 | writerObj.FrameRate = 10; % VID 10 | open(writerObj); % VID 11 | title('Final Tracking Results: Ground Truth (green) vs Tracker Output (Yellow)','Color','w'); 12 | 13 | for fr = start:num_frames 14 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 15 | 16 | bb = floor(self.history.target(fr,:)); 17 | bb_gt = (gt(1:4,fr))'; 18 | 19 | vis_final_tracker_track (rgb_raw, bb, bb_gt ); 20 | drawnow; 21 | 22 | frame = getframe(fh); % VID 23 | writeVideo(writerObj,frame); % VID 24 | 25 | 26 | end 27 | 28 | clf; 29 | close(writerObj); % VID 30 | % ===================================================================== 31 | writerObj = VideoWriter('output/v9.avi'); % VID 32 | writerObj.FrameRate = 10; % VID 33 | open(writerObj); % VID 34 | title('Particle Dynamics - HOC Distances: Particle Distances (red), Average Distances (blue), Weighted Distances (black)','Color','w'); 35 | set(gca, 'YColor', 'w'); 36 | for fr = start:num_frames 37 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 38 | 39 | bb = floor(self.history.target(fr,:)); 40 | bb_gt = (gt(1:4,fr))'; 41 | 42 | vis_particle_hoc_distances ( self, num_frames, fr); 43 | drawnow; 44 | 45 | 46 | frame = getframe(fh); % VID 47 | writeVideo(writerObj,frame); % VID 48 | 49 | end 50 | clf; 51 | close(writerObj); % VID 52 | % ===================================================================== 53 | writerObj = VideoWriter('output/v2.avi'); % VID 54 | writerObj.FrameRate = 10; % VID 55 | open(writerObj); % VID 56 | title('Final Tracking Results (Depth Map): Ground Truth (green) vs Tracker Output (Yellow)','Color','w'); 57 | for fr = start:num_frames 58 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 59 | 60 | bb = floor(self.history.target(fr,:)); 61 | bb_gt = (gt(1:4,fr))'; 62 | 63 | vis_final_tracker_track (dep_raw, bb, bb_gt ); 64 | drawnow; 65 | 66 | frame = getframe(fh); % VID 67 | writeVideo(writerObj,frame); % VID 68 | 69 | end 70 | clf; 71 | close(writerObj); % VID 72 | % ===================================================================== 73 | 74 | % title('Video Sequence 3D View (2.5 D)','Color','w'); 75 | % vis_point_cloud (rgb_raw, dep_raw, cam_param, [0,-90] ); 76 | 77 | % ===================================================================== 78 | writerObj = VideoWriter('output/v3.avi'); % VID 79 | writerObj.FrameRate = 10; % VID 80 | open(writerObj); % VID 81 | title('Tracker Error Types','Color','w'); 82 | set(gca, 'XColor', 'w', 'YColor', 'w'); 83 | error_type = zeros (1,4); 84 | for fr = start:num_frames 85 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 86 | 87 | bb = floor(self.history.target(fr,:)); 88 | bb_gt = (gt(1:4,fr))'; 89 | 90 | error_type = vis_error_types (bb, bb_gt, 0.5 , error_type , num_frames); 91 | set(gca, 'XColor', 'w', 'YColor', 'w'); 92 | 93 | frame = getframe(fh); % VID 94 | writeVideo(writerObj,frame); % VID 95 | 96 | end 97 | clf; 98 | close(writerObj); % VID 99 | % ===================================================================== 100 | writerObj = VideoWriter('output/v4.avi'); % VID 101 | writerObj.FrameRate = 10; % VID 102 | open(writerObj); % VID 103 | title('Final Tracking Trajectory: Ground Truth (green) vs Tracker Output (Yellow)','Color','w'); 104 | for fr = start:num_frames 105 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 106 | 107 | bb = floor(self.history.target(fr,:)); 108 | bb_gt = (gt(1:4,fr))'; 109 | 110 | vis_final_tracker_trajectory (rgb_raw, gt , self, fr ); 111 | drawnow; 112 | 113 | frame = getframe(fh); % VID 114 | writeVideo(writerObj,frame); % VID 115 | 116 | end 117 | clf; 118 | close(writerObj); % VID 119 | % ===================================================================== 120 | writerObj = VideoWriter('output/v5.avi'); % VID 121 | writerObj.FrameRate = 10; % VID 122 | open(writerObj); % VID 123 | title('Final Tracking Central Point Error','Color','w'); 124 | set(gca, 'YColor', 'w'); 125 | 126 | for fr = start:num_frames 127 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 128 | 129 | bb = floor(self.history.target(fr,:)); 130 | bb_gt = (gt(1:4,fr))'; 131 | 132 | vis_final_tracker_cpe (num_frames, self, gt, fr ); 133 | drawnow; 134 | 135 | frame = getframe(fh); % VID 136 | writeVideo(writerObj,frame); % VID 137 | 138 | end 139 | clf; 140 | close(writerObj); % VID 141 | % ===================================================================== 142 | writerObj = VideoWriter('output/v6.avi'); % VID 143 | writerObj.FrameRate = 10; % VID 144 | open(writerObj); % VID 145 | title('Final Tracking Size Adaptation: Width(blue), Height(red), Ground Truth(dashed), Szie Adaptation Error(black)','Color','w'); 146 | set(gca, 'YColor', 'w'); 147 | 148 | for fr = start:num_frames 149 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 150 | 151 | bb = floor(self.history.target(fr,:)); 152 | bb_gt = (gt(1:4,fr))'; 153 | 154 | vis_final_tracker_sae (rgb_raw, num_frames, self, gt, fr ); 155 | drawnow; 156 | 157 | frame = getframe(fh); % VID 158 | writeVideo(writerObj,frame); % VID 159 | 160 | end 161 | clf; 162 | close(writerObj); % VID 163 | 164 | % ===================================================================== 165 | writerObj = VideoWriter('output/v10.avi'); % VID 166 | writerObj.FrameRate = 10; % VID 167 | open(writerObj); % VID 168 | title('Particle Dynamics - Median of Depth Distances: Particle Distances (red), Average Distances (blue), Weighted Distances (black)','Color','w'); 169 | set(gca, 'YColor', 'w'); 170 | for fr = start:num_frames 171 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 172 | 173 | bb = floor(self.history.target(fr,:)); 174 | bb_gt = (gt(1:4,fr))'; 175 | 176 | vis_particle_medd_distances ( self, num_frames, fr); 177 | drawnow; 178 | 179 | 180 | frame = getframe(fh); % VID 181 | writeVideo(writerObj,frame); % VID 182 | 183 | end 184 | clf; 185 | close(writerObj); % VID 186 | % ===================================================================== 187 | writerObj = VideoWriter('output/v11.avi'); % VID 188 | writerObj.FrameRate = 10; % VID 189 | open(writerObj); % VID 190 | title('Particle Dynamics - Particle Probabilities (occluded ones are in red)','Color','w'); 191 | set(gca, 'XColor', 'w','YColor', 'w'); 192 | for fr = start:num_frames 193 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 194 | 195 | bb = floor(self.history.target(fr,:)); 196 | bb_gt = (gt(1:4,fr))'; 197 | 198 | vis_particle_probability ( self, fr ); 199 | xlabel(['frame: ' num2str(fr)],'Color','w'); 200 | drawnow; 201 | 202 | 203 | frame = getframe(fh); % VID 204 | writeVideo(writerObj,frame); % VID 205 | 206 | end 207 | clf; 208 | close(writerObj); % VID 209 | % ===================================================================== 210 | writerObj = VideoWriter('output/v13.avi'); % VID 211 | writerObj.FrameRate = 10; % VID 212 | open(writerObj); % VID 213 | title('Stabilized Image','Color','w'); 214 | for fr = start:num_frames 215 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 216 | 217 | bb = floor(self.history.target(fr,:)); 218 | bb_gt = (gt(1:4,fr))'; 219 | 220 | subplot (1,2,1); imshow (bb_content(rgb_raw,bb)); 221 | xlabel(['frame: ' num2str(fr)],'Color','w'); 222 | drawnow; 223 | 224 | if (~isnan(bb_gt(1))) 225 | subplot (1,2,2); imshow (bb_content(rgb_raw,bb_gt-[0 0 3 3])); 226 | xlabel('ground truth','Color','w'); 227 | else 228 | subplot (1,2,2); cla; 229 | xlabel('OCCLUSION!','Color','w'); 230 | end 231 | drawnow; 232 | 233 | frame = getframe(fh); % VID 234 | writeVideo(writerObj,frame); % VID 235 | 236 | end 237 | clf; 238 | close(writerObj); % VID 239 | % % fix it later 240 | % cpx = bb(1) + bb(3)/2; 241 | % cpy = bb(2) + bb(4)/2; 242 | % subplot (1,2,2); imshow (bb_content_safe(rgb_raw,[cpx-150,cpy-150,300,300])); 243 | 244 | % ===================================================================== 245 | writerObj = VideoWriter('output/v15.avi'); % VID 246 | writerObj.FrameRate = 10; % VID 247 | open(writerObj); % VID 248 | title('Particle Dynamics - Particle MedD vs. Template MedD','Color','w'); 249 | set(gca, 'XColor', 'w'); 250 | for fr = start:num_frames 251 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 252 | 253 | bb = floor(self.history.target(fr,:)); 254 | bb_gt = (gt(1:4,fr))'; 255 | 256 | vis_particle_template_medd_comparison ( self, dep_raw , fr , bb); 257 | 258 | 259 | frame = getframe(fh); % VID 260 | writeVideo(writerObj,frame); % VID 261 | 262 | clf; 263 | end 264 | clf; 265 | close(writerObj); % VID 266 | % ===================================================================== 267 | writerObj = VideoWriter('output/v7.avi'); % VID 268 | writerObj.FrameRate = 10; % VID 269 | open(writerObj); % VID 270 | for fr = start:num_frames 271 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 272 | 273 | bb = floor(self.history.target(fr,:)); 274 | bb_gt = (gt(1:4,fr))'; 275 | 276 | title('Particle Dynamics - Box Position and Probability Indicator: Red(occlusion chance), Brighter (more probable), Darker(less probable)','Color','w'); 277 | 278 | vis_particle_position_probability (rgb_raw, self, fr , 1); 279 | for i = 2:self.N 280 | vis_particle_position_probability ([], self, fr , i); 281 | hold on; 282 | drawnow; 283 | 284 | frame = getframe(fh); % VID 285 | writeVideo(writerObj,frame); % VID 286 | 287 | end 288 | end 289 | clf; 290 | close(writerObj); % VID 291 | % ===================================================================== 292 | writerObj = VideoWriter('output/v12.avi'); % VID 293 | writerObj.FrameRate = 10; % VID 294 | open(writerObj); % VID 295 | title('Particle Dynamics - Particle Probabilities Compared (occluded ones are exploded)','Color','w'); 296 | set(gca, 'YColor', 'w'); 297 | for fr = start:num_frames 298 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 299 | 300 | bb = floor(self.history.target(fr,:)); 301 | bb_gt = (gt(1:4,fr))'; 302 | 303 | vis_particle_probability_comparison (self , fr , 1); 304 | hold on; 305 | 306 | for i = 1:self.N 307 | vis_particle_probability_comparison (self , fr , i); 308 | drawnow; 309 | 310 | frame = getframe(fh); % VID 311 | writeVideo(writerObj,frame); % VID 312 | end 313 | hold off; 314 | end 315 | clf; 316 | close(writerObj); % VID 317 | 318 | % if ( fr == 1 ) 319 | % continue; 320 | % end 321 | % 322 | % probs = self.history.probs(fr,:); 323 | % explode = self.history.z(fr,:); 324 | % for i = 1:self.N 325 | % pie(probs(1:i),explode(1:i)); 326 | % drawnow; pause (0.1); 327 | % end 328 | % 329 | 330 | % ===================================================================== 331 | writerObj = VideoWriter('output/v14.avi'); % VID 332 | writerObj.FrameRate = 10; % VID 333 | open(writerObj); % VID 334 | title('Particle Dynamics - Particle HOC vs. Template HOC and the Difference','Color','w'); 335 | for fr = start:num_frames 336 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 337 | 338 | bb = floor(self.history.target(fr,:)); 339 | bb_gt = (gt(1:4,fr))'; 340 | 341 | vis_particle_template_hoc_comparison (rgb_raw, self , fr , 1); 342 | set(gca, 'YColor', 'w'); 343 | hold on; 344 | for i = 1:self.N 345 | vis_particle_template_hoc_comparison (rgb_raw, self , fr , i); 346 | set(gca, 'YColor', 'w'); 347 | xlabel(['particle: ' num2str(i)],'Color','w'); 348 | drawnow; 349 | 350 | frame = getframe(fh); % VID 351 | writeVideo(writerObj,frame); % VID 352 | end 353 | hold off; 354 | end 355 | clf; 356 | close(writerObj); % VID 357 | 358 | % ===================================================================== 359 | 360 | writerObj = VideoWriter('output/v14.avi'); % VID 361 | writerObj.FrameRate = 10; % VID 362 | open(writerObj); % VID 363 | 364 | ctrs = self.feature{1}.rgb_ctr; 365 | bbs = squeeze(floor(self.history.bbs(fr,:,:))); 366 | 367 | hoc1 = self.history.model_rgb_hist(fr,:); 368 | subplot (3,1,1); hoc_vis (hoc1,ctrs); 369 | 370 | for i = 1:self.N 371 | hoc2 = histogram_of_colors(bb_content(rgb_raw,bbs(i,:)),[],ctrs); 372 | subplot (3,1,2); hoc_vis (hoc2,ctrs); 373 | subplot (3,1,3); bar (abs(hoc1-hoc2)); xlim([0 length(hoc1)]); ylim([0 0.2]); drawnow; 374 | drawnow; 375 | 376 | frame = getframe(fh); % VID 377 | writeVideo(writerObj,frame); % VID 378 | end 379 | close(writerObj); % VID 380 | % ===================================================================== 381 | writerObj = VideoWriter('output/v8.avi'); % VID 382 | writerObj.FrameRate = 50; % VID 383 | open(writerObj); % VID 384 | for fr = start:num_frames 385 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 386 | 387 | bb = floor(self.history.target(fr,:)); 388 | bb_gt = (gt(1:4,fr))'; 389 | 390 | title('Particle Dynamics - Occluded Particles','Color','w'); 391 | 392 | vis_particle_position_occlusion (rgb_raw, self, fr , 1); 393 | for i = 2:self.N 394 | vis_particle_position_occlusion ([], self, fr , i); 395 | hold on; 396 | 397 | frame = getframe(fh); % VID 398 | writeVideo(writerObj,frame); % VID 399 | 400 | end 401 | end 402 | clf; 403 | close(writerObj); % VID 404 | 405 | 406 | 407 | % ===================================================================== 408 | % CONFIDENCE MAP 409 | % TEMPLATE EVOLUTION (OLD TEMPLATE, NEW ENTRY, NEW TEMPLATE) 410 | 411 | 412 | 413 | % frame = getframe(fh); % VID 414 | % writeVideo(writerObj,frame); % VID 415 | 416 | 417 | % close(writerObj); % VID 418 | 419 | end 420 | 421 | 422 | 423 | 424 | 425 | 426 | function insert_title (str) 427 | annotation('textbox',... 428 | [0 0 1 1],'String',str,'LineWidth',2,'BackgroundColor',[0 0 0],... 429 | 'FontSize',100,'FontName','Arial','Color',[1 1 1]); 430 | end 431 | 432 | 433 | 434 | 435 | -------------------------------------------------------------------------------- /tracker/particle_filter/particle_filter_init.m: -------------------------------------------------------------------------------- 1 | function x = particle_filter_init (params) 2 | 3 | x.name = params.name; 4 | x.type = 'particle_filter'; 5 | 6 | %% Manual Parameter Handling 7 | x.N = params.number_of_particles; 8 | x.fn = params.feature_name; 9 | x.sm = params.similarity_measure; 10 | x.imp = params.feature_importance; 11 | x.nrm = params.feature_normalizer; 12 | x.occ_pr = params.occlusion_probability; 13 | x.occ_thr = params.occlusion_flag_threshold; 14 | x.target = NaN(1,4); 15 | x.bkg_det = params.bkg_detection; 16 | x.bkg_sub = params.bkg_subtraction; 17 | x.update = params.model_update; 18 | 19 | % x.g = params.grid_size; 20 | if ( isfield(params,'grid_size') ) x.g = params.grid_size; else x.g = 1; end 21 | 22 | %% Default Parameters 23 | % background subtraction 24 | x.bkg_det_samples = 30; % frames sampled for background subtraction 25 | x.bkg_det_load = true; 26 | 27 | % HoC RGB Clustering 28 | x.rgb_clustering_samples = 3000; 29 | x.rgb_bins = 40; % number of bins in RGB 30 | x.rgb_bins_load = true; 31 | 32 | % Cell confidence 33 | x.cell_conf_load = true; 34 | 35 | % particle dynamics 36 | x.box_w_range = [50, 250]; 37 | x.box_h_range = [50, 400]; 38 | 39 | x.max_velocity_x = 20; 40 | x.max_velocity_y = 20; 41 | 42 | % occlusion flag 43 | if (params.enable_occ_flag) 44 | x.state_transition_matrix = params.state_transition_matrix; % nocc->nocc = 0.99, nocc->occ = 0.01, occ->nocc = 0.25, occ->occ = 0.75 45 | else 46 | x.state_transition_matrix = [1 0; 1 0]; % disables occlusion flag by letting no particle to go to occlusion state 47 | end 48 | 49 | % model update 50 | x.target_update_forgetting_rate = 0.1; % 1: completely explore (forget old ones), 0: completely exploit 51 | -------------------------------------------------------------------------------- /tracker/particle_filter/particle_filter_output.m: -------------------------------------------------------------------------------- 1 | function particle_filter_output (gt, self , vid_param, cam_param, directory, num_frames, start) 2 | 3 | fh = figure('toolbar','none','menubar','none','color','k','units','normalized','outerposition',[0 0 1 1],'name','Occlusion Aware Particle Framework'); 4 | colormap('jet'); 5 | % writerObj = VideoWriter('output/v1.avi'); % VID 6 | % writerObj.FrameRate = 10; % VID 7 | % open(writerObj); % VID 8 | pause; 9 | 10 | title('Final Tracking Results: Ground Truth (green) vs Tracker Output (Yellow)','Color','w'); 11 | for fr = start:num_frames 12 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 13 | 14 | bb = floor(self.history.target(fr,:)); 15 | bb_gt = (gt(1:4,fr))'; 16 | 17 | vis_final_tracker_track (rgb_raw, bb, bb_gt ); 18 | drawnow; 19 | 20 | end 21 | 22 | pause; clf; 23 | % ===================================================================== 24 | title('Final Tracking Results (Depth Map): Ground Truth (green) vs Tracker Output (Yellow)','Color','w'); 25 | for fr = start:num_frames 26 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 27 | 28 | bb = floor(self.history.target(fr,:)); 29 | bb_gt = (gt(1:4,fr))'; 30 | 31 | vis_final_tracker_track (dep_raw, bb, bb_gt ); 32 | drawnow; 33 | end 34 | pause; clf; 35 | % ===================================================================== 36 | 37 | % title('Video Sequence 3D View (2.5 D)','Color','w'); 38 | % vis_point_cloud (rgb_raw, dep_raw, cam_param, [0,-90] ); 39 | 40 | % ===================================================================== 41 | title('Tracker Error Types','Color','w'); 42 | set(gca, 'XColor', 'w', 'YColor', 'w'); 43 | error_type = zeros (1,4); 44 | for fr = start:num_frames 45 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 46 | 47 | bb = floor(self.history.target(fr,:)); 48 | bb_gt = (gt(1:4,fr))'; 49 | 50 | error_type = vis_error_types (bb, bb_gt, 0.5 , error_type , num_frames); 51 | set(gca, 'XColor', 'w', 'YColor', 'w'); 52 | end 53 | pause; clf; 54 | % ===================================================================== 55 | title('Final Tracking Trajectory: Ground Truth (green) vs Tracker Output (Yellow)','Color','w'); 56 | for fr = start:num_frames 57 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 58 | 59 | bb = floor(self.history.target(fr,:)); 60 | bb_gt = (gt(1:4,fr))'; 61 | 62 | vis_final_tracker_trajectory (rgb_raw, gt , self, fr ); 63 | drawnow; 64 | end 65 | pause; clf; 66 | % ===================================================================== 67 | title('Final Tracking Central Point Error','Color','w'); 68 | set(gca, 'YColor', 'w'); 69 | 70 | for fr = start:num_frames 71 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 72 | 73 | bb = floor(self.history.target(fr,:)); 74 | bb_gt = (gt(1:4,fr))'; 75 | 76 | vis_final_tracker_cpe (num_frames, self, gt, fr ); 77 | drawnow; 78 | end 79 | pause; clf; 80 | % ===================================================================== 81 | title('Final Tracking Size Adaptation: Width(blue), Height(red), Ground Truth(dashed), Szie Adaptation Error(black)','Color','w'); 82 | set(gca, 'YColor', 'w'); 83 | for fr = start:num_frames 84 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 85 | 86 | bb = floor(self.history.target(fr,:)); 87 | bb_gt = (gt(1:4,fr))'; 88 | 89 | vis_final_tracker_sae (rgb_raw, num_frames, self, gt, fr ); 90 | drawnow; 91 | end 92 | pause; clf; 93 | % ===================================================================== 94 | % for fr = start:num_frames 95 | % [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 96 | % 97 | % bb = floor(self.history.target(fr,:)); 98 | % bb_gt = (gt(1:4,fr))'; 99 | % 100 | % title('Particle Dynamics - Box Position and Probability Indicator: Red(occlusion chance), Brighter (more probable). Darker(less probable)','Color','w'); 101 | % 102 | % vis_particle_position_probability (rgb_raw, self, fr , 1); 103 | % for i = 2:self.N 104 | % vis_particle_position_probability ([], self, fr , i); 105 | % hold on; 106 | % drawnow; 107 | % end 108 | % end 109 | % pause; clf; 110 | % % ===================================================================== 111 | % for fr = start:num_frames 112 | % [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 113 | % 114 | % bb = floor(self.history.target(fr,:)); 115 | % bb_gt = (gt(1:4,fr))'; 116 | % 117 | % title('Particle Dynamics - Occluded Particles','Color','w'); 118 | % 119 | % vis_particle_position_occlusion (rgb_raw, self, fr , 1); 120 | % for i = 2:self.N 121 | % vis_particle_position_occlusion ([], self, fr , i); 122 | % hold on; 123 | % end 124 | % end 125 | % pause; clf; 126 | % ===================================================================== 127 | title('Particle Dynamics - HOC Distances: Particle Distances (red), Average Distances (blue), Weighted Distances (black)','Color','w'); 128 | set(gca, 'YColor', 'w'); 129 | for fr = start:num_frames 130 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 131 | 132 | bb = floor(self.history.target(fr,:)); 133 | bb_gt = (gt(1:4,fr))'; 134 | 135 | vis_particle_hoc_distances ( self, num_frames, fr); 136 | drawnow; 137 | end 138 | pause; clf; 139 | % ===================================================================== 140 | title('Particle Dynamics - Median of Depth Distances: Particle Distances (red), Average Distances (blue), Weighted Distances (black)','Color','w'); 141 | set(gca, 'YColor', 'w'); 142 | for fr = start:num_frames 143 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 144 | 145 | bb = floor(self.history.target(fr,:)); 146 | bb_gt = (gt(1:4,fr))'; 147 | 148 | vis_particle_medd_distances ( self, num_frames, fr); 149 | drawnow; 150 | end 151 | pause; clf; 152 | % ===================================================================== 153 | title('Particle Dynamics - Particle Probabilities (occluded ones are in red)','Color','w'); 154 | set(gca, 'XColor', 'w','YColor', 'w'); 155 | for fr = start:num_frames 156 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 157 | 158 | bb = floor(self.history.target(fr,:)); 159 | bb_gt = (gt(1:4,fr))'; 160 | 161 | vis_particle_probability ( self, fr ); 162 | xlabel(['frame: ' num2str(fr)],'Color','w'); 163 | drawnow; 164 | end 165 | pause; clf; 166 | % ===================================================================== 167 | % title('Particle Dynamics - Particle Probabilities Compared (occluded ones are exploded)','Color','w'); 168 | % set(gca, 'YColor', 'w'); 169 | % for fr = start:num_frames 170 | % [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 171 | % 172 | % bb = floor(self.history.target(fr,:)); 173 | % bb_gt = (gt(1:4,fr))'; 174 | % 175 | % vis_particle_probability_comparison (self , fr , 1); 176 | % hold on; 177 | % 178 | % for i = 1:self.N 179 | % vis_particle_probability_comparison (self , fr , i); 180 | % drawnow; 181 | % end 182 | % hold off; 183 | % end 184 | % pause; clf; 185 | % 186 | % % if ( fr == 1 ) 187 | % % continue; 188 | % % end 189 | % % 190 | % % probs = self.history.probs(fr,:); 191 | % % explode = self.history.z(fr,:); 192 | % % for i = 1:self.N 193 | % % pie(probs(1:i),explode(1:i)); 194 | % % drawnow; pause (0.1); 195 | % % end 196 | % % 197 | 198 | % ===================================================================== 199 | title('Stabilized Image','Color','w'); 200 | for fr = start:num_frames 201 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 202 | 203 | bb = floor(self.history.target(fr,:)); 204 | bb_gt = (gt(1:4,fr))'; 205 | 206 | subplot (1,2,1); imshow (bb_content(rgb_raw,bb)); 207 | xlabel(['frame: ' num2str(fr)],'Color','w'); 208 | drawnow; 209 | 210 | if (~isnan(bb_gt(1))) 211 | subplot (1,2,2); imshow (bb_content(rgb_raw,bb_gt-[0 0 3 3])); 212 | xlabel('ground truth','Color','w'); 213 | else 214 | subplot (1,2,2); cla; 215 | xlabel('OCCLUSION!','Color','w'); 216 | end 217 | drawnow; 218 | end 219 | pause; clf; 220 | % % fix it later 221 | % cpx = bb(1) + bb(3)/2; 222 | % cpy = bb(2) + bb(4)/2; 223 | % subplot (1,2,2); imshow (bb_content_safe(rgb_raw,[cpx-150,cpy-150,300,300])); 224 | 225 | % ===================================================================== 226 | % title('Particle Dynamics - Particle HOC vs. Template HOC and the Difference','Color','w'); 227 | % for fr = start:num_frames 228 | % [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 229 | % 230 | % bb = floor(self.history.target(fr,:)); 231 | % bb_gt = (gt(1:4,fr))'; 232 | % 233 | % vis_particle_template_hoc_comparison (rgb_raw, self , fr , 1); 234 | % set(gca, 'YColor', 'w'); 235 | % hold on; 236 | % for i = 1:self.N 237 | % vis_particle_template_hoc_comparison (rgb_raw, self , fr , i); 238 | % set(gca, 'YColor', 'w'); 239 | % xlabel(['particle: ' num2str(i)],'Color','w'); 240 | % drawnow; 241 | % end 242 | % hold off; 243 | % end 244 | % pause; clf; 245 | % 246 | % ctrs = self.feature{1}.rgb_ctr; 247 | % bbs = squeeze(floor(self.history.bbs(fr,:,:))); 248 | % 249 | % hoc1 = self.history.model_rgb_hist(fr,:); 250 | % subplot (3,1,1); hoc_vis (hoc1,ctrs); 251 | % 252 | % for i = 1:self.N 253 | % hoc2 = histogram_of_colors(bb_content(rgb_raw,bbs(i,:)),[],ctrs); 254 | % subplot (3,1,2); hoc_vis (hoc2,ctrs); 255 | % subplot (3,1,3); bar (abs(hoc1-hoc2)); xlim([0 length(hoc1)]); ylim([0 0.2]); drawnow; 256 | % drawnow; 257 | % pause (0.1); 258 | % end 259 | 260 | % ===================================================================== 261 | title('Particle Dynamics - Particle MedD vs. Template MedD','Color','w'); 262 | set(gca, 'XColor', 'w'); 263 | for fr = start:num_frames 264 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 265 | 266 | bb = floor(self.history.target(fr,:)); 267 | bb_gt = (gt(1:4,fr))'; 268 | 269 | vis_particle_template_medd_comparison ( self, dep_raw , fr , bb); 270 | clf; 271 | end 272 | pause; clf; 273 | 274 | 275 | % ===================================================================== 276 | % CONFIDENCE MAP 277 | % TEMPLATE EVOLUTION (OLD TEMPLATE, NEW ENTRY, NEW TEMPLATE) 278 | 279 | 280 | 281 | % frame = getframe; % VID 282 | % writeVideo(writerObj,frame); % VID 283 | 284 | 285 | % close(writerObj); % VID 286 | 287 | 288 | 289 | 290 | 291 | -------------------------------------------------------------------------------- /tracker/particle_filter/particle_filter_test.m: -------------------------------------------------------------------------------- 1 | function [bb, self] = particle_filter_test (rgb_raw, dep_raw, init_bb , self, video_name) 2 | %PARTICLE_FILTER_TEST Uses a particle filter to track the given objects 3 | %using selected features 4 | % ---------------description 5 | % 6 | % 7 | % 8 | % SELF = tracker 9 | % code by: Kourosh Meshgi, Dec 2013 10 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 11 | 12 | if ( ~isempty(init_bb)) 13 | self.frame = 1; 14 | 15 | % 1-1- Initialize Parameters 16 | % 1-2- Initialize Particles 17 | % 1-3- Initialize Target Model 18 | % 1-4- Save Tracker parameters 19 | 20 | 21 | % background detection 22 | if ( self.bkg_det_load ) 23 | load (['bkg/' video_name '/rgb_bkg.mat']); 24 | load (['bkg/' video_name '/dep_bkg.mat']); 25 | else 26 | [rgb_bkg,dep_bkg] = offline_bkg_detection ( self.bkg_det, video_name , self.bkg_det_samples ); 27 | fld = ['bkg/' video_name]; 28 | if (~exist(fld,'dir')) 29 | mkdir(fld); 30 | end 31 | save([fld '/rgb_bkg.mat'],'rgb_bkg'); 32 | save([fld '/dep_bkg.mat'],'dep_bkg'); 33 | end 34 | [rgb_msk, dep_msk] = bkg_subtraction ( self.bkg_sub , rgb_raw, dep_raw, rgb_bkg , dep_bkg); 35 | 36 | % create feature space 37 | self = initialize_features (self, rgb_raw, video_name , init_bb); 38 | 39 | % initialize bounding boxes around foreground points 40 | [boxes , z] = initialize_particles_bb (rgb_raw, self.N, ... 41 | self.box_w_range , self.box_h_range, ... 42 | self.g, ... 43 | self.max_velocity_x, self.max_velocity_x, ... 44 | rgb_msk, init_bb); 45 | 46 | % initial model 47 | model = bb_feature_extraction ( init_bb , self.g, ... 48 | rgb_raw, dep_raw, ... 49 | rgb_msk, dep_msk, ... 50 | self.feature ); 51 | 52 | % first bounding box (bb = target) 53 | bb = init_bb; 54 | 55 | % saving tracker parameters 56 | self.target = init_bb; 57 | self.target_z = 0; 58 | self.rgb_bkg = rgb_bkg; 59 | self.dep_bkg = dep_bkg; 60 | self.bbs = boxes; 61 | self.z = z; 62 | self.model = model; 63 | 64 | else 65 | self.frame = self.frame + 1; 66 | % console_messages ('newline',['frame: ' num2str(self.frame)]); 67 | 68 | 69 | % 3-1- Calculate Features for Bounding Boxes and Target 70 | % 3-2- Measure Particle Features to Target 71 | % 3-3- Calculate Particle Likelihood Based on Feature 72 | % 3-4- Normalize Features Between All Particles 73 | % 3-5- Calculate Particle Likelihood via Log-Sum-Exp 74 | % 3-6- Apply Occlusion Flag 75 | % 3-7- Normalize Particle Likelihood to Make Probability 76 | % 3-8- Update Target 77 | % 3-9- Resampling & Anti-Jitter policy 78 | % 3-10- Update Target Model 79 | 80 | % global data 81 | [rgb_msk, dep_msk] = bkg_subtraction ( self.bkg_sub , rgb_raw, dep_raw, self.rgb_bkg , self.dep_bkg); 82 | 83 | % particles characteristics 84 | particle = cell(self.N); 85 | 86 | parfor i = 1:self.N 87 | 88 | % extract features for no occlusion case 89 | if (self.z(i) == 0 ) 90 | % calculate features 91 | particle{i} = bb_feature_extraction ( self.bbs(i,:) , self.g, ... 92 | rgb_raw, dep_raw, ... 93 | rgb_msk, dep_msk, ... 94 | self.feature ); 95 | 96 | % features distance to template 97 | particle{i} = bb_feature_distance ( particle{i} , self.model ,self.g ,self.feature ); 98 | 99 | % h =figure('toolbar','none','menubar','none','color','k','units','normalized','outerposition',[0 0 1 1]); 100 | % subplot(2,1,1); hoc_vis (particle{i}.cell(1,1).feature(1).val, self.feature{1}.rgb_ctr); 101 | % subplot(2,1,2); hoc_vis (self.model.cell(1,1).feature(1).val, self.feature{1}.rgb_ctr); 102 | % close (h); 103 | end 104 | 105 | particle{i}.z = self.z(i); 106 | particle{i}.bb = self.bbs(i,:); 107 | 108 | % particles likelihood calculation considering occlusion vs. no occlusion case 109 | if (particle{i}.z == 0 ) % not occluded 110 | particle{i} = bb_minus_log_likelihood ( particle{i} , self.feature ); 111 | else % occluded 112 | particle{i}.minus_log_likelihood = self.occ_pr; 113 | end 114 | likelihood(i) = particle{i}.minus_log_likelihood; 115 | 116 | 117 | % h = vis_particle (i,rgb_raw,dep_raw,rgb_msk,particle{i}.bb,particle{i}.cell(1,1).feature(1).val,particle{i}.cell(1,1).feature(2).val,particle{i}.dist(1),particle{i}.dist(2),likelihood(i),self.feature{1}.rgb_ctr,self.model.cell(1,1).feature(1).val,self.target); 118 | % close (h); 119 | end 120 | 121 | % ATTENTION! particles feature channels normalization was a mistake, added 122 | % normalization factor as a input... 123 | 124 | % particle probability calculation 125 | bb_prob = bb_normalize_minus_log_likelihood (likelihood); 126 | 127 | % expected target and its state of visibility 128 | [proposed_bb, proposed_z] = expected_target ( self.bbs, self.z, bb_prob ); 129 | 130 | % particle resampling proportional to probability 131 | [resampled_bbs, resampled_z] = bb_resample ( self.bbs, self.z, bb_prob, ... 132 | proposed_bb, proposed_z, ... 133 | rgb_raw, ... 134 | self.box_w_range , self.box_h_range, ... 135 | self.max_velocity_x, self.max_velocity_y, ... 136 | self.g, self.state_transition_matrix ); 137 | 138 | 139 | 140 | % MESSY CODE, FIX IT!!!!!!!!!!!!!!!!!!! 141 | 142 | 143 | 144 | % % target model update, only when no occlusion is expected 145 | % if ( proposed_z < self.occ_thr ) 146 | % proposed_t = bb_feature_extraction ( self.target , self.g, ... 147 | % rgb_raw, dep_raw, ... 148 | % rgb_msk, dep_msk, ... 149 | % self.feature ); 150 | % model = model_update ( self.update , self.model , proposed_t , self); 151 | % else 152 | % model = self.model; 153 | % end 154 | 155 | 156 | % target model update, only when no occlusion is expected 157 | 158 | proposed_t = bb_feature_extraction ( self.target , self.g, ... 159 | rgb_raw, dep_raw, ... 160 | rgb_msk, dep_msk, ... 161 | self.feature ); 162 | model_nocc = model_update ( self.update , self.model , proposed_t , self); 163 | model_occ = self.model; 164 | 165 | model = self.model; 166 | model.cell(1,1).feature(2).val = proposed_z*model_occ.cell(1,1).feature(2).val + (1-proposed_z)*model_nocc.cell(1,1).feature(2).val; 167 | 168 | 169 | 170 | % MESSY CODE, FIX IT!!!!!!!!!!!!!!!!!!! 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | % col = bb_prob_color_indicator (bb_prob , self.z); % DEBUG MODE 185 | % h = figure; imshow(rgb_raw); pause(0.05); % DEBUG MODE 186 | % for i = 1:size(self.bbs,1) 187 | % rectangle('Position',self.bbs(i,:),'EdgeColor',col(i,:)); pause(0.01); drawnow; % DEBUG MODE 188 | % end % DEBUG MODE 189 | % close (h); % DEBUG MODE 190 | 191 | % save data to tracker 192 | self.target = proposed_bb; 193 | self.target_z = proposed_z; 194 | self.bbs = floor(resampled_bbs); 195 | self.z = resampled_z; 196 | self.model = model; 197 | 198 | if (self.z > self.occ_thr ) 199 | bb = NaN(1,4); 200 | else 201 | bb = floor(proposed_bb); %% iteration output 202 | end 203 | 204 | end 205 | 206 | % % history saving 207 | if self.frame > 1 208 | self.history = tracker_history ( self, particle, bb_prob ); 209 | else 210 | self.history = tracker_history ( self, [], [] ); 211 | end 212 | 213 | end %====================================================================== 214 | 215 | 216 | 217 | % TO DO: 218 | % model update 219 | % motion model adding (constant velocity x(t) = 2 x(t-1) - x(t-2) + noise) 220 | % add griding 221 | % add confidence 222 | % importance factor for depth 223 | % forgetting rate -------------------------------------------------------------------------------- /tracker/particle_filter/particle_filter_train.m: -------------------------------------------------------------------------------- 1 | function self = particle_filter_train ( self , bd ) 2 | 3 | % background detection 4 | if ( ~ self.cell_conf_load ) 5 | 6 | for k = 1:length(bd) 7 | for i = 1:self.g 8 | for j = 1:self.g 9 | bb = bb_grid ([1,1,size(bd.bb_msk{k},1),size(bd.bb_msk{k},2)],self.g,i,j); 10 | fg = sum(sum(bb_content(bd.bb_msk{k} , bb))); 11 | ratio(k,i,j) = fg / numel(bb_content(bd.bb_msk{k} , bb)); 12 | end 13 | end 14 | end 15 | 16 | 17 | for i = 1:self.g 18 | for j = 1:self.g 19 | x = ratio(:,i,j); 20 | pd((i-1)*self.g+j) = fitdist(x,'Beta'); 21 | end 22 | end 23 | pd = reshape(pd',self.g,self.g); 24 | 25 | filename = ['tracker/particle_filter/beta_dist_cells_' num2str(self.g) 'x' num2str(self.g) '1.mat']; 26 | 27 | save(filename,'pd'); 28 | end 29 | 30 | -------------------------------------------------------------------------------- /tracker/particle_filter/vis_particle.m: -------------------------------------------------------------------------------- 1 | function fh = vis_particle (no, rgb_raw, dep_raw, rgb_msk, bb , rgb_hist , medD , rgb_dist, dep_dist , likelihood, rgb_centers , target_hist,target_bb) 2 | 3 | g = 1; 4 | 5 | % big bad black figuure -> full screen: 'units','normalized','outerposition',[0 0 1 1] 6 | if isempty(get(0,'Children')) || strcmp(get(gcf,'name'),'Particle Visualization') == false 7 | fh = figure('toolbar','none','menubar','none','color','k','units','normalized','outerposition',[0 0 1 1],'name','Particle Visualization'); 8 | axis tight 9 | end 10 | fh = gcf; 11 | 12 | % particle histogram 13 | s1 = subplot(4,4,[1,2,5,6]); 14 | hoc_vis (rgb_hist, rgb_centers); 15 | 16 | % target histogram 17 | s2 = subplot(4,4,[9,10,13,14]); 18 | hoc_vis (target_hist, rgb_centers); 19 | 20 | % tracking image 21 | s3 = subplot(4,4,7); 22 | imshow(rgb_msk); 23 | hold on; 24 | rectangle('Position',bb,'EdgeColor','y'); 25 | 26 | % rgb image 27 | s4 = subplot(4,4,3); 28 | imshow(rgb_raw); 29 | hold on; 30 | rectangle('Position',bb,'EdgeColor','y'); 31 | rectangle('Position',target_bb,'EdgeColor','g','LineWidth',2); 32 | 33 | % depth image 34 | s5 = subplot(4,4,4); 35 | imshow(dep_raw); 36 | hold on; 37 | rectangle('Position',bb,'EdgeColor','y'); 38 | rectangle('Position',target_bb,'EdgeColor','g','LineWidth',2); 39 | 40 | % depth distribution 41 | s6 = subplot(4,4,[11,12,15,16]); 42 | 43 | imhist(bb_content(dep_raw,bb)); 44 | % hold on; 45 | % line([medD, medD],[0 max(max*histc(dep_raw,256))]); 46 | 47 | % confidence map 48 | s7 = subplot(4,4,8); 49 | % conf_map = visualization_confidence_map (dmap, box, g, box_f, conf_dists ); 50 | % visualization_particle_box(target.box,'g',conf_map, g); 51 | % hold on 52 | % visualization_particle_box(box,'y',[],g); 53 | % hold off 54 | 55 | %information 56 | str0 = sprintf('Particle # %i: (Press any key to load next particle)',no); 57 | str1 = sprintf('RGB Hist Distance: %10f',rgb_dist); 58 | str2 = sprintf('D Median Distance: %10f',dep_dist); 59 | str3 = sprintf('D Median: %10f',medD); 60 | str4 = sprintf('Sum Log-Likelihood: %10f',likelihood); 61 | 62 | 63 | annotation('textbox',... 64 | [0.13 0.01 0.775 0.095],... 65 | 'String',{str0, str1,str2,str3,str4},... 66 | 'LineWidth',2,... 67 | 'BackgroundColor',[1 1 1],... 68 | 'Color',[0.84 0.16 0]); 69 | % 'FontSize',14,... 70 | % 'FontName','Arial',... 71 | % 'LineStyle','--',... 72 | % 'EdgeColor',[1 1 0],... 73 | 74 | % terminating 75 | pause 76 | annotation('textbox',... 77 | [0.13 0.01 0.775 0.095],... 78 | 'String','Loading Next Particle Statistics, Please Wait A few Seconds...',... 79 | 'LineWidth',2,... 80 | 'BackgroundColor',[1 1 1],... 81 | 'Color',[0 0 0]); 82 | drawnow -------------------------------------------------------------------------------- /tracker/upper_bounds/ub_gt_first_ratio_init.m: -------------------------------------------------------------------------------- 1 | function x = ub_gt_first_ratio_init (params) 2 | 3 | x.name = params.name; 4 | x.type = 'ub_gt_first_ratio'; 5 | 6 | x.ratio = 1.0; -------------------------------------------------------------------------------- /tracker/upper_bounds/ub_gt_first_ratio_test.m: -------------------------------------------------------------------------------- 1 | function [bb,self] = ub_gt_first_ratio_test (depth,grt,frame_number,init_bb,self) 2 | %UB_GT_FIRST_RATIO_TEST Uses the GT location and first frame box ratio 3 | % This function outputs ground truth location in each frame, if the 4 | % object is occluded it returns NaN... Also the size of bounding box has 5 | % the same aspect ratio of the first frame 6 | % 7 | % code by: Kourosh Meshgi, Oct 2013 8 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 9 | 10 | if ( ~isempty(init_bb)) 11 | bb = init_bb; 12 | self.ratio = double(bb(4)) / double(bb(3)); 13 | return 14 | end 15 | 16 | if (isnan(grt(1,frame_number))) 17 | bb = NaN(1,4); 18 | else 19 | x = grt (1,frame_number); 20 | y = grt (2,frame_number); 21 | [H,W] = size(depth); 22 | 23 | w = randi([1,W-x]); 24 | h = floor(self.ratio * w); 25 | 26 | if (h > H ) 27 | h = H; 28 | w = floor(h/self.ratio); 29 | end 30 | 31 | bb = [x,y,w,h]; 32 | 33 | end 34 | 35 | end %====================================================================== -------------------------------------------------------------------------------- /tracker/upper_bounds/ub_gt_first_size_init.m: -------------------------------------------------------------------------------- 1 | function x = ub_gt_first_size_init (params) 2 | 3 | x.name = params.name; 4 | x.type = 'ub_gt_first_size'; 5 | 6 | x.bb = [0,0,0,0]; -------------------------------------------------------------------------------- /tracker/upper_bounds/ub_gt_first_size_test.m: -------------------------------------------------------------------------------- 1 | function [bb,self] = ub_gt_first_size_test (grt,frame_number,init_bb,self) 2 | %UB_GT_FIRST_SIZE_TEST Uses the GT location and first frame box size 3 | % This function outputs ground truth location in each frame, if the 4 | % object is occluded it returns NaN... Also the size of bounding box is 5 | % fixed to the first frame bounding box. 6 | % 7 | % code by: Kourosh Meshgi, Oct 2013 8 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 9 | 10 | if ( ~isempty(init_bb)) 11 | bb = init_bb; 12 | self.bb(3:4) = bb(3:4); 13 | return 14 | end 15 | 16 | if (isnan(grt(1,frame_number))) 17 | bb = [NaN,NaN,NaN,NaN]; 18 | else 19 | bb(3:4) = self.bb(3:4); 20 | bb(1:2) = grt (1:2,frame_number); 21 | end 22 | 23 | end %====================================================================== -------------------------------------------------------------------------------- /tracker/upper_bounds/ub_gt_init.m: -------------------------------------------------------------------------------- 1 | function x = ub_gt_init (params) 2 | 3 | x.name = params.name; 4 | x.type = 'ub_gt'; 5 | -------------------------------------------------------------------------------- /tracker/upper_bounds/ub_gt_no_occ_last_init.m: -------------------------------------------------------------------------------- 1 | function x = ub_gt_no_occ_last_init (params) 2 | 3 | x.name = params.name; 4 | x.type = 'ub_gt_no_occ_last'; 5 | -------------------------------------------------------------------------------- /tracker/upper_bounds/ub_gt_no_occ_last_test.m: -------------------------------------------------------------------------------- 1 | function bb = ub_gt_no_occ_last_test (grt,frame_number,init_bb) 2 | %UB_GT_NO_OCC_LAST_TEST Outputs ground truth as an upper bound, if it's not 3 | %available inputs the last known bounding box for that 4 | % This function outputs ground truth in each frame, if the object is 5 | % occluded (GT = NaN) it shows the last known position of the object... 6 | % 7 | % code by: Kourosh Meshgi, Oct 2013 8 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 9 | 10 | if ( ~isempty(init_bb)) 11 | bb = init_bb; 12 | return 13 | end 14 | 15 | while (isnan(grt(1,frame_number))) 16 | frame_number = frame_number - 1; 17 | end 18 | 19 | bb = grt (1:4,frame_number); 20 | end %====================================================================== -------------------------------------------------------------------------------- /tracker/upper_bounds/ub_gt_no_occ_rand_init.m: -------------------------------------------------------------------------------- 1 | function x = ub_gt_no_occ_rand_init (params) 2 | 3 | x.name = params.name; 4 | x.type = 'ub_gt_no_occ_rand'; 5 | -------------------------------------------------------------------------------- /tracker/upper_bounds/ub_gt_no_occ_rand_test.m: -------------------------------------------------------------------------------- 1 | function bb = ub_gt_no_occ_rand_test (depth,grt,frame_number,init_bb) 2 | %UB_GT_NO_OCC_RAND_TEST Outputs the GT box, if exists, and a random box 3 | %otherwise 4 | % This function outputs ground truth in each frame, if the object is 5 | % occluded (GT = NaN) it shows a random bounding box... 6 | % 7 | % code by: Kourosh Meshgi, Oct 2013 8 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 9 | 10 | if ( ~isempty(init_bb)) 11 | bb = init_bb; 12 | return 13 | end 14 | 15 | if (isnan(grt(1,frame_number))) 16 | bb = bb_random (size(depth)); 17 | else 18 | bb = grt (1:4,frame_number); 19 | end 20 | end %====================================================================== -------------------------------------------------------------------------------- /tracker/upper_bounds/ub_gt_test.m: -------------------------------------------------------------------------------- 1 | function bb = ub_gt_test (grt,frame_number,init_bb) 2 | %UB_GT_TEST Outputs ground truth as an upper bound 3 | % This function outputs ground truth in each frame, if the object is 4 | % occluded (GT = NaN) it outputs NaN 5 | % 6 | % code by: Kourosh Meshgi, Oct 2013 7 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 8 | 9 | if ( ~isempty(init_bb)) 10 | bb = init_bb; 11 | return 12 | end 13 | 14 | bb = grt (1:4,frame_number); 15 | end %====================================================================== -------------------------------------------------------------------------------- /tracker_history.m: -------------------------------------------------------------------------------- 1 | function x = tracker_history ( self, particle, bb_prob ) 2 | 3 | fr = self.frame; 4 | if (fr > 1) 5 | x = self.history; 6 | 7 | for i = 1:self.N 8 | if (particle{i}.z == 0 ) 9 | dist_hoc(1,i) = particle{i}.dist(1) / self.nrm(1); 10 | dist_medd(1,i) = particle{i}.dist(2) / self.nrm(2); 11 | else 12 | % if the particle is occluded no feature is calculated 13 | dist_hoc(1,i) = NaN; 14 | dist_medd(1,i) = NaN; 15 | end 16 | likelihood(1,i) = particle{i}.minus_log_likelihood; 17 | end 18 | x.dist_hoc(fr,:) = dist_hoc; 19 | x.dist_medd(fr,:) = dist_medd; 20 | x.likelihoods(fr,:) = likelihood; 21 | x.probs(fr,:) = bb_prob; 22 | else 23 | % first frame we have the bounding box, and everything is initialized, 24 | % so there is no calculated features for the particles 25 | x.dist_hoc(1,:) = Inf(1,self.N); 26 | x.dist_medd(fr,:) = Inf(1,self.N); 27 | x.likelihoods(fr,:) = zeros(1,self.N); 28 | x.probs(fr,:) = zeros(1,self.N); 29 | 30 | end 31 | 32 | x.target(fr,:) = self.target; 33 | x.target_z(fr,:) = self.target_z; 34 | x.model_rgb_hist(fr,:) = self.model.cell(1,1).feature(1).val; 35 | x.model_med_depth(fr,:) = self.model.cell(1,1).feature(2).val; 36 | x.bbs(fr,:,:) = self.bbs; 37 | x.z(fr,:) = self.z; 38 | -------------------------------------------------------------------------------- /trackers_evaluate.m: -------------------------------------------------------------------------------- 1 | function eval = trackers_evaluate ( test_videos , trackers , control , results ) 2 | %TRACKERS_EVALUATE Evaluate tracker to extract metrics to compare with 3 | %other algorithms 4 | % This function 5 | % 6 | % code by: Kourosh Meshgi, Oct 2013 7 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 8 | 9 | ground_truth = {}; 10 | initial_bb = {}; 11 | 12 | for vid = 1:size(test_videos,2) 13 | [~, ~, num_frames, ~, gt, init_bb] = video_info(test_videos{1,vid}); 14 | number_of_frames (vid) = num_frames; 15 | ground_truth{1,vid} = gt; 16 | initial_bb{1,vid} = init_bb; 17 | ground_truth{1,vid}(1:4,1) = init_bb'; % crazy error in data set that ground_truth of frame one is different from initial boundinx box passed to the trackers, here we adjust it 18 | end 19 | 20 | 21 | 22 | % the tracking result then returned in a T-4-by-M array, where M is the 23 | % number of frames of each video and T is the number of trackers. In the 24 | % case of multiple videos, the result will be N-by-T-by-4-by-M array, 25 | % where N is the number of videos. 26 | 27 | cpe = evaluate_center_position_error (test_videos , trackers , control , results , ground_truth); 28 | 29 | [ auc_list, success_plots ] = evaluate_success_plot_OPE (test_videos , trackers , control , results , ground_truth); 30 | 31 | sa = evaluate_scale_adaptation (test_videos , trackers , control , results , ground_truth); 32 | 33 | % cpe = [0,0,0,0]; % DEBUG 34 | % sa = [0,0,0,0]; % DEBUG 35 | eval = [cpe; sa; auc_list]; 36 | 37 | % disp(auc_list); 38 | % disp(sa); 39 | 40 | end 41 | -------------------------------------------------------------------------------- /trackers_initialize.m: -------------------------------------------------------------------------------- 1 | function [trackers] = trackers_initialize ( tracker_list, tracker_parameters , control ) 2 | %TRACKERS_INITIALIZE Initialize Trackers of given types with given 3 | %parameters 4 | % This function initialize multiple trackers to work on the same training 5 | % and test set. Types of trackers are included in tracker_list which could 6 | % contain multiple instances of the same tracker. The parameters of each 7 | % tracker are passed via dedicated parameters. Control argument contains 8 | % general information about tracking environment. 9 | % types of use: 10 | % 1- a tracker with its lower bound and upper bound heuristics 11 | % 2- multiple instances of the same tracker with different parameter 12 | % setting 13 | % 3- multiple instances of the same base tracker with features added to 14 | % different versions of them (e.g. comparison of SIFT with HOG feature, 15 | % wether to have Occlusion Flag or not, etc) 16 | % 4- different tracking algorithms for the sake of comparison 17 | % 5- multiple trackers track different objects in the same video 18 | % use your imagination! 19 | % 20 | % code by: Kourosh Meshgi, Oct 2013 21 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 22 | 23 | trackers = {}; 24 | for tr = 1:size(tracker_list,2) 25 | x = []; 26 | x.status = 'init'; 27 | switch tracker_list{1,tr} 28 | case 'particle_filter' 29 | x = particle_filter_init (tracker_parameters{1,tr}); 30 | case 'rgbd_pf_grid_occ' 31 | disp '4' 32 | % e.g. the object to track in the multiple object cases 33 | case 'loader' 34 | x = loader_init (tracker_parameters{1,tr}); 35 | case 'ub_gt' 36 | x = ub_gt_init (tracker_parameters{1,tr}); 37 | case 'ub_gt_first_size' 38 | x = ub_gt_first_size_init (tracker_parameters{1,tr}); 39 | case 'ub_gt_best_size' 40 | disp '6' 41 | case 'ub_gt_first_ratio' 42 | x = ub_gt_first_ratio_init (tracker_parameters{1,tr}); 43 | case 'ub_gt_best_ratio' 44 | disp '8' 45 | case 'ub_gt_no_occ_rand' 46 | x = ub_gt_no_occ_rand_init (tracker_parameters{1,tr}); 47 | case 'ub_gt_no_occ_last' 48 | x = ub_gt_no_occ_last_init (tracker_parameters{1,tr}); 49 | case 'lb_first_bb' 50 | x = lb_first_bb_init (tracker_parameters{1,tr}); 51 | case 'lb_center_bb' 52 | x = lb_center_bb_init (tracker_parameters{1,tr}); 53 | case 'lb_rand_size' 54 | x = lb_rand_size_init (tracker_parameters{1,tr}); 55 | case 'lb_rand_loc' 56 | x = lb_rand_loc_init (tracker_parameters{1,tr}); 57 | case 'lb_rand_size_loc' 58 | x = lb_rand_size_loc_init (tracker_parameters{1,tr}); 59 | case 'lb_crazy' 60 | x = lb_crazy_init (tracker_parameters{1,tr}); 61 | case 'lb_invisible' 62 | x = lb_invisible_init (tracker_parameters{1,tr}); 63 | end 64 | trackers{tr} = x; 65 | disp (x.name); 66 | end 67 | 68 | end %====================================================================== 69 | 70 | -------------------------------------------------------------------------------- /trackers_output.m: -------------------------------------------------------------------------------- 1 | function trackers_output ( test_videos , trackers , control , results ) 2 | %TRACKERS_OUTPUT Outputs the dynamics of each tracker and resulting track 3 | % This function 4 | % 5 | % code by: Kourosh Meshgi, Dec 2013 6 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 7 | 8 | ground_truth = {}; 9 | 10 | for vid = 1:size(test_videos,2) 11 | [vid_param, directory, num_frames, cam_param, grt, init_bb] = video_info(test_videos{vid}); 12 | number_of_frames (vid) = num_frames; 13 | ground_truth{vid} = grt; 14 | ground_truth{vid}(1:4,1) = init_bb'; % crazy error in data set that ground_truth of frame one is different from initial boundinx box passed to the trackers, here we adjust it 15 | 16 | end 17 | 18 | % particle_filter_output (ground_truth{1}, trackers{4} , vid_param, cam_param , directory, num_frames, control.start_frame); 19 | 20 | particle_filter_demo (ground_truth{1}, trackers{4} , vid_param, cam_param , directory, num_frames, control.start_frame); -------------------------------------------------------------------------------- /trackers_test.m: -------------------------------------------------------------------------------- 1 | function [results, trackers] = trackers_test ( test_videos , trackers , control ) 2 | %TRACKERS_TEST Test trackers wether they can track the given object 3 | % This function tests the trackers using given video. It provides them 4 | % the bounding box in first frame either given on file, via an 5 | % interactive tool, or using object detector as an input. 6 | % If the tagging method is 'file', the method automatically read the file 7 | % 'init.txt' in the dataset and extract bounding box from it. In the case 8 | % of multiple object tracking, the file contains multiple lines. 9 | % If the tagging method is 'hand' an interactive image will be opened to 10 | % input bounding boxes of objects one-by-one. 11 | % In the case of 'detector', the algorithm select the object candidates 12 | % from top of the list provided. 13 | % the tracking result then returned in a T-4-by-M array, where M is the 14 | % number of frames of each video and T is the number of trackers. In the 15 | % case of multiple videos, the result will be N-by-T-by-4-by-M array, 16 | % where N is the number of videos. 17 | % 18 | % code by: Kourosh Meshgi, Oct 2013 19 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 20 | 21 | results = []; 22 | 23 | for vid = 1:size(test_videos,2) 24 | [vid_param, directory, num_frames, cam_param, grt, init_bb] = video_info(test_videos{1,vid}); 25 | 26 | % if initialization method is not 'file' then initialize it!!!! 27 | 28 | for fr = control.start_frame:num_frames 29 | [rgb, depth] = read_frame(vid_param, directory, fr); 30 | 31 | for tr = 1:length(trackers) 32 | % give tracker input data, without ground truth and save the 33 | % results 34 | initial_target_bb = []; 35 | if (fr == 1) 36 | initial_target_bb = init_bb; 37 | end 38 | 39 | trackers{1,tr}.status = 'test'; 40 | switch trackers{1,tr}.type 41 | case 'particle_filter' 42 | [res,trackers{1,tr}] = particle_filter_test ( rgb, depth, initial_target_bb, trackers{1,tr}, test_videos{1,vid}); 43 | case 'rgbd_pf_grid_occ' 44 | disp '4' 45 | % e.g. the object to track in the multiple object cases 46 | case 'loader' 47 | res = loader_test (trackers{1,tr}, fr); 48 | case 'ub_gt' 49 | res = ub_gt_test (grt,fr,initial_target_bb); 50 | case 'ub_gt_first_size' 51 | [res,trackers{1,tr}] = ub_gt_first_size_test (grt,fr,initial_target_bb, trackers{1,tr}); 52 | case 'ub_gt_best_size' 53 | disp '6' 54 | case 'ub_gt_first_ratio' 55 | [res,trackers{1,tr}] = ub_gt_first_ratio_test (depth,grt,fr,initial_target_bb, trackers{1,tr}); 56 | case 'ub_gt_best_ratio' 57 | disp '8' 58 | case 'ub_gt_no_occ_rand' 59 | res = ub_gt_no_occ_rand_test (depth,grt,fr,initial_target_bb); 60 | case 'ub_gt_no_occ_last' 61 | res = ub_gt_no_occ_last_test (grt,fr,initial_target_bb); 62 | case 'lb_first_bb' 63 | [res,trackers{1,tr}] = lb_first_bb_test ( rgb, depth, initial_target_bb, grt, trackers{1,tr} ); 64 | case 'lb_center_bb' 65 | [res,trackers{1,tr}] = lb_center_bb_test ( rgb, depth, initial_target_bb, grt, trackers{1,tr} ); 66 | case 'lb_rand_size' 67 | [res,trackers{1,tr}] = lb_rand_size_test ( rgb, depth, initial_target_bb, grt, trackers{1,tr} ); 68 | case 'lb_rand_loc' 69 | [res,trackers{1,tr}] = lb_rand_loc_test ( rgb, depth, initial_target_bb, grt, trackers{1,tr} ); 70 | case 'lb_rand_size_loc' 71 | [res,trackers{1,tr}] = lb_rand_size_loc_test (rgb, depth, initial_target_bb, grt, trackers{1,tr}); 72 | case 'lb_crazy' 73 | res = lb_crazy_test (rgb, depth, initial_target_bb); 74 | case 'lb_invisible' 75 | res = lb_invisible_test (rgb, depth, initial_target_bb); 76 | end 77 | tracker_output(tr,1:4,fr) = res(:); 78 | end 79 | 80 | 81 | % TEMPORARY VISUALIZATION 82 | tracker_output(1:4,1:4,fr); %%%%%%%%%% PRINT OUT 83 | 84 | imshow(rgb); 85 | if ~isnan(tracker_output(1,1:4,fr)), rectangle('Position',tracker_output(1,1:4,fr),'EdgeColor','r'); end 86 | if ~isnan(tracker_output(2,1:4,fr)), rectangle('Position',tracker_output(2,1:4,fr),'EdgeColor','g'); end; 87 | if ~isnan(tracker_output(3,1:4,fr)), rectangle('Position',tracker_output(3,1:4,fr),'EdgeColor','b'); end; 88 | if ~isnan(tracker_output(4,1:4,fr)), rectangle('Position',tracker_output(4,1:4,fr),'EdgeColor','y'); end; 89 | drawnow; 90 | 91 | end 92 | video_tracking_footage(vid,:,:,:) = tracker_output; 93 | 94 | 95 | end 96 | 97 | results = video_tracking_footage; 98 | end %====================================================================== 99 | 100 | 101 | % * Testing 102 | % *** For Each Video Do 103 | % ****** Read Video General Info 104 | % ****** Initialize Trackers for Testing Mode 105 | % ****** For Each Frame of Video Do 106 | % ********* For Each Tracker Do 107 | % ************ Provide Tracker with Input Data 108 | % ************ Save the Result of Tracking 109 | % *** Returning The Result for Each Video -------------------------------------------------------------------------------- /trackers_train.m: -------------------------------------------------------------------------------- 1 | function trackers = trackers_train ( train_videos , trackers , control ) 2 | %TRACKER_TRAIN Train tracker parameters with annotated dataset 3 | % This function let all of the trackers to tune up their parameters with 4 | % the given videos and annotated tracking box. 5 | % 6 | % code by: Kourosh Meshgi, Oct 2013 7 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 8 | 9 | if (~ control.enable_learning) 10 | return; 11 | end 12 | 13 | self = trackers{4}; 14 | k = 0; 15 | for vid = 1:size(train_videos,2) 16 | [vid_param, directory, num_frames, cam_param, grt, init_bb] = video_info(train_videos{vid}); 17 | 18 | % background detection 19 | if ( self.bkg_det_load ) 20 | load (['bkg/' train_videos{vid} '/rgb_bkg.mat']); 21 | load (['bkg/' train_videos{vid} '/dep_bkg.mat']); 22 | else 23 | [rgb_bkg,dep_bkg] = offline_bkg_detection ( self.bkg_det, train_videos{vid} , self.bkg_det_samples ); 24 | fld = ['bkg/' train_videos{vid}]; 25 | if (~exist(fld,'dir')) 26 | mkdir(fld); 27 | end 28 | save([fld '/rgb_bkg.mat'],'rgb_bkg'); 29 | save([fld '/dep_bkg.mat'],'dep_bkg'); 30 | end 31 | 32 | % if initialization method is not 'file' then initialize it!!!! 33 | for fr = control.start_frame:num_frames 34 | [rgb_raw, dep_raw] = read_frame(vid_param, directory, fr); 35 | [rgb_msk, dep_msk] = bkg_subtraction ( self.bkg_sub , rgb_raw, dep_raw, rgb_bkg , dep_bkg); 36 | bb = grt(1:4,fr); 37 | 38 | if (isnan(bb(1))) 39 | continue; 40 | end 41 | 42 | bb(3) = min(bb(3)+bb(1),size(rgb_raw,2))-bb(1); 43 | bb(4) = min(bb(4)+bb(2),size(rgb_raw,1))-bb(2); 44 | 45 | [k , bb(4)+bb(2), bb(3)+bb(1)] 46 | 47 | k = k + 1; 48 | batch_data{k}.bb_rgb = bb_content(rgb_raw,bb); 49 | batch_data{k}.bb_dep = bb_content(dep_raw,bb); 50 | batch_data{k}.bb_msk = bb_content(rgb_msk,bb); 51 | 52 | end 53 | end 54 | 55 | 56 | 57 | for tr = 1:length(trackers) 58 | % give tracker input data, without ground truth and save the 59 | % results 60 | 61 | 62 | trackers{tr}.status = 'train'; 63 | switch trackers{tr}.type 64 | case 'particle_filter' 65 | trackers{tr} = particle_filter_train ( trackers{tr}, batch_data ); 66 | end 67 | end 68 | 69 | 70 | end %====================================================================== -------------------------------------------------------------------------------- /util/clip_range.m: -------------------------------------------------------------------------------- 1 | function x = clip_range (x, R) 2 | 3 | x = min(max(x,R(1)),R(2)); -------------------------------------------------------------------------------- /util/console_messages.m: -------------------------------------------------------------------------------- 1 | function console_messages( comm , str ) 2 | %UNTITLED Summary of this function goes here 3 | % Detailed explanation goes here 4 | 5 | global s; 6 | 7 | switch (comm) 8 | case 'clear' 9 | clc; 10 | s = ''; 11 | case 'add' 12 | clc; 13 | s = [s str]; 14 | disp(s); 15 | case 'newline' 16 | s = sprintf('%s\n%s',s,str); 17 | disp(str); 18 | end 19 | 20 | 21 | end 22 | 23 | -------------------------------------------------------------------------------- /util/read_frame.m: -------------------------------------------------------------------------------- 1 | function [ rgb, depth, XYZpoints, RGBpoints ] = read_frame( frames, directory, frame_no, cam_param) 2 | %READ_FRAME reads a frame of dataset given video info and frame number 3 | % readframe( frames, directory, frame_no, cam_param) reads data from 4 | % the corresponding frame in the dataset, and returns the color image, the 5 | % depth map, and the constructed 3D world in the point cloud format, two 6 | % arrays of RGB and XYZ value with the column number works as their 7 | % index. 8 | % readframe( frames, directory, frame_no) works similarly, but does 9 | % not construct the 3D world saving the computation power and returns 10 | % empty matrices for XYZ- and RGBpoints. 11 | % Modified from the code of Shuran Song, code was obtained Sept 2013 from 12 | % http://vision.princeton.edu/projects/2013/tracking/dataset.html 13 | % 14 | % code by: Kourosh Meshgi, Oct 2013 15 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 16 | 17 | 18 | % imageNames = cell(1,numOfFrames*2); 19 | % XYZcam = zeros(480,640,4,numOfFrames); 20 | 21 | %% Input Check 22 | if nargin < 3, frame_no = 1; end 23 | 24 | %% Read Frame Information 25 | imageName = fullfile(directory,sprintf('rgb/r-%d-%d.png', frames.imageTimestamp(frame_no), frames.imageFrameID(frame_no))); 26 | rgb = imread(imageName); 27 | depthName = fullfile(directory,sprintf('depth/d-%d-%d.png', frames.depthTimestamp(frame_no), frames.depthFrameID(frame_no))); 28 | depth = imread(depthName); 29 | depth = bitor(bitshift(depth,-3), bitshift(depth,3-16)); 30 | depth = double(depth); 31 | 32 | %% Create 3D World 33 | 34 | % if nargin > 3 35 | % 36 | % % Camera Intrinsic parameters K is [fx 0 cx; 0 fy cy; 0 0 1]; 37 | % K = cam_param; 38 | % cx = K(1,3);cy = K(2,3); 39 | % fx = K(1,1);fy = K(2,2); 40 | % 41 | % % 3D point for the frame 42 | % depthInpaint = depth/1000; 43 | % [x,y] = meshgrid(1:640, 1:480); 44 | % Xworld = (x-cx).*depthInpaint*1/fx; 45 | % Yworld = (y-cy).*depthInpaint*1/fy; 46 | % Zworld = depthInpaint; 47 | % validM = depth~=0; 48 | % XYZworldframe = [Xworld(:)'; Yworld(:)'; Zworld(:)']; 49 | % valid = validM(:)'; 50 | % 51 | % % XYZworldframe 3xn and RGB 3xn 52 | % RGB = [reshape(rgb(:,:,1),1,[]);reshape(rgb(:,:,2),1,[]);reshape(rgb(:,:,3),1,[])]; 53 | % XYZpoints = XYZworldframe(:,valid); 54 | % RGBpoints = RGB(:,valid); 55 | % 56 | % else 57 | % 58 | % XYZpoints = []; 59 | % RGBpoints = []; 60 | % 61 | % end 62 | 63 | %% Rescale depth 64 | depth(depth==0) = 10000; 65 | depth = (depth-500)/8500; %only use the data from 0.5-8m 66 | depth(depth<0) = 0; 67 | depth(depth>1) = 1; 68 | depth = uint8(floor(255*(1-depth))); 69 | 70 | 71 | end %====================================================================== -------------------------------------------------------------------------------- /util/rect_span.m: -------------------------------------------------------------------------------- 1 | function [x,y] = rect_span (box) 2 | x = box(2):box(2)+box(4); 3 | y = box(1):box(1)+box(3); -------------------------------------------------------------------------------- /util/setprod.m: -------------------------------------------------------------------------------- 1 | function C = setprod(varargin) 2 | % SETPROD product of multiple sets. 3 | % 4 | % X = SETPROD(A,B,C,...) returns the cartesian product of the sets 5 | % A,B,C, etc, where A,B,C, are numeric or character arrays. 6 | % 7 | % Example: A = [-1 -3 -5]; B = [10 11]; C = [0 1]; 8 | % 9 | % X = SETPROD(A,B,C) 10 | % X = 11 | % 12 | % -5 10 0 13 | % -3 10 0 14 | % -1 10 0 15 | % -5 11 0 16 | % -3 11 0 17 | % -1 11 0 18 | % -5 10 1 19 | % -3 10 1 20 | % -1 10 1 21 | % -5 11 1 22 | % -3 11 1 23 | % -1 11 1 24 | 25 | % Mukhtar Ullah 26 | % mukhtar.ullah@informatic.uni-rostock.de 27 | % September 20, 2004 28 | 29 | args = varargin; 30 | 31 | if any([cellfun('isclass',args,'cell') cellfun('isclass',args,'struct')]) 32 | error(' SETPROD only supports numeric/character arrays ') 33 | end 34 | 35 | n = nargin; 36 | 37 | [F{1:n}] = ndgrid(args{:}); 38 | 39 | for i=n:-1:1 40 | G(:,i) = F{i}(:); 41 | end 42 | 43 | C = unique(G , 'rows'); -------------------------------------------------------------------------------- /util/video_info.m: -------------------------------------------------------------------------------- 1 | function [vid_param, directory, num_frames, cam_param, ground_truth, init_bb] = video_info (dataset_name) 2 | %VIDEO_INFO Grabs essential information about video dataset 3 | % This function reads the folder corrensponds to the dataset, and extract 4 | % essential informations from it such as camera parameters, The path to 5 | % dataset, the length of the video and returns it. 6 | % 7 | % code by: Kourosh Meshgi, Oct 2013 8 | % https://github.com/meshgi/RGBD_Particle_Filter_Tracker 9 | 10 | directory = [ './data/' dataset_name '/']; 11 | load([directory 'frames']); 12 | 13 | vid_param = frames; 14 | cam_param = frames.K; 15 | num_frames = frames.length; 16 | 17 | if exist([directory dataset_name '.txt'],'file') == 2 18 | fid = fopen([directory dataset_name '.txt']); 19 | ground_truth = fscanf(fid, '%g,%g,%g,%g,%g', [5,inf]); 20 | fclose(fid); 21 | num_frames = min (num_frames,size(ground_truth,2)); 22 | else 23 | ground_truth = []; 24 | end 25 | 26 | if exist([directory 'init.txt'],'file') == 2 27 | fid = fopen([directory 'init.txt']); 28 | init_bb = fscanf(fid, '%g,%g,%g,%g', [4,inf]); 29 | fclose(fid); 30 | else 31 | init_bb = []; 32 | end 33 | 34 | end %====================================================================== -------------------------------------------------------------------------------- /vis/vis_error_types.m: -------------------------------------------------------------------------------- 1 | function error_type = vis_error_types (bb, bb_gt, to, error_type , num_frames) 2 | 3 | bbz = isnan(bb(1)); 4 | s = calculate_S (bb_gt,bb); 5 | 6 | if (s == -1) 7 | if (bbz == 1) 8 | % error TYPE II: missed the target 9 | k = 2; 10 | else 11 | % error TYPE III: false positive, identity loss 12 | k = 3; 13 | end 14 | else 15 | if (s < to ) 16 | % error TYPE I: overlap not significant 17 | k = 1; 18 | else 19 | % tracker did a good job 20 | k = 4; 21 | end 22 | end 23 | error_type(k) = error_type(k) + 1; 24 | 25 | bar(error_type,'FaceColor','b'); 26 | hold on 27 | overlay = zeros(1,4); overlay(k) = error_type(k); 28 | bar(overlay,'FaceColor','r'); 29 | ylim([0 num_frames]) 30 | set(gca,'XTickLabel',{'low overlap','miss','false positive','good detection'}); 31 | xlabel (['Success rate (t_o = ' num2str(to) ') - frame: ' num2str(sum(error_type(:)))]); 32 | drawnow; 33 | 34 | end -------------------------------------------------------------------------------- /vis/vis_final_tracker_cpe.m: -------------------------------------------------------------------------------- 1 | function vis_final_tracker_cpe (num_frames, self, gt, fr ) 2 | 3 | for i = 1:num_frames 4 | bbi = floor(self.history.target(i,:)); 5 | bbi_gt = (gt(1:4,i))'; 6 | 7 | cpe(i) = calculate_CPE (bbi_gt,bbi); 8 | end 9 | 10 | plot(1:fr,cpe(1:fr),'LineWidth',2); 11 | hold on; 12 | plot(fr,cpe(fr),'ro'); 13 | xlabel('Frames'); 14 | ylabel('CPE(pixel)'); 15 | 16 | mc = max(cpe); 17 | xlim([1 num_frames]); 18 | ylim([0 mc]); 19 | hold on 20 | 21 | % http://stackoverflow.com/questions/6245626/matlab-filling-in-the-area-between-two-sets-of-data-lines-in-one-figure 22 | occ = isnan(cpe); 23 | % line ([find(occ)' find(occ)'+1],repmat(mc,sum(occ),2),'LineWidth',2); 24 | % line ([find(occ)' find(occ)'+1],repmat(0,sum(occ),2),'LineWidth',2); 25 | % fill ([find(occ)' find(occ)'+1],repmat(mc,sum(occ),2) 26 | % -------------------------------------------------------------------------------- /vis/vis_final_tracker_sae.m: -------------------------------------------------------------------------------- 1 | function vis_final_tracker_sae (rgb_raw, num_frames, self, gt, fr ) 2 | 3 | for i = 1:fr 4 | bbi(i,:) = floor(self.history.target(i,:)); 5 | bbi_gt(i,:) = (gt(1:4,i))'; 6 | end 7 | 8 | hold on 9 | plot(1:fr, bbi(:,3), 'b-','LineWidth',2); 10 | plot(1:fr, bbi_gt(:,3), 'b--','LineWidth',2); 11 | 12 | plot(1:fr, bbi(:,4), 'r-','LineWidth',2); 13 | plot(1:fr, bbi_gt(:,4), 'r--','LineWidth',2); 14 | 15 | sa = sqrt((bbi(:,3)-bbi_gt(:,3)).^2 + (bbi(:,4)-bbi_gt(:,4)).^2); 16 | plot(1:fr, sa, 'k-','LineWidth',3); 17 | 18 | xlim([1 num_frames]); 19 | ylim([0 sqrt(sum(size(rgb_raw).^2))]); 20 | hold off; 21 | 22 | 23 | end -------------------------------------------------------------------------------- /vis/vis_final_tracker_track.m: -------------------------------------------------------------------------------- 1 | function vis_final_tracker_track (img_raw, bb, bb_gt ) 2 | 3 | imshow(img_raw); 4 | hold on; 5 | if (~isnan(bb)) 6 | rectangle('Position',bb,'EdgeColor','y'); 7 | end 8 | hold on; 9 | if (~isnan(bb_gt)) 10 | rectangle('Position',bb_gt,'EdgeColor','g'); 11 | end 12 | 13 | end -------------------------------------------------------------------------------- /vis/vis_final_tracker_trajectory.m: -------------------------------------------------------------------------------- 1 | function vis_final_tracker_trajectory (rgb_raw, gt , self, fr ) 2 | 3 | imshow(rgb_raw); 4 | for i = 1:fr 5 | bbi = floor(self.history.target(i,:)); 6 | bbi_gt = (gt(1:4,i))'; 7 | 8 | bb_traj_x(i) = bbi(1) + bbi(3)/2; 9 | bb_traj_y(i) = bbi(2) + bbi(4)/2; 10 | bbg_traj_x(i) = bbi_gt(1) + bbi_gt(3)/2; 11 | bbg_traj_y(i) = bbi_gt(2) + bbi_gt(4)/2; 12 | end 13 | hold on; 14 | plot(bb_traj_x,bb_traj_y,'-yo','LineWidth',2,'MarkerEdgeColor','y','MarkerFaceColor','y','MarkerSize',8); 15 | hold on; 16 | plot(bbg_traj_x,bbg_traj_y,'-go','LineWidth',2,'MarkerEdgeColor','g','MarkerFaceColor','g','MarkerSize',8); 17 | hold off; 18 | drawnow; 19 | end -------------------------------------------------------------------------------- /vis/vis_particle_hoc_distances.m: -------------------------------------------------------------------------------- 1 | function vis_particle_hoc_distances ( self, num_frames, fr) 2 | 3 | if ( fr == 1 ) 4 | return; 5 | end 6 | 7 | hoc_dist = self.history.dist_hoc(2:fr,:); 8 | probs = self.history.probs(2:fr,:); 9 | avg_hoc_dist = nanmean(hoc_dist'); 10 | 11 | hold on; 12 | for i = 2:fr 13 | semilogy (i*ones(1,self.N),hoc_dist(i-1,:),'ro'); 14 | hoc_dist(isnan(hoc_dist)) = 0; % to ignore NaN 15 | w_avg_hoc_dist(i-1) = hoc_dist(i-1,:)*probs(i-1,:)'; 16 | end 17 | semilogy (2:fr,avg_hoc_dist,'b--','LineWidth',2); 18 | semilogy (2:fr,w_avg_hoc_dist,'k-','LineWidth',4); 19 | 20 | xlim([1 num_frames+1]); 21 | ylim([0 1]); 22 | drawnow; 23 | 24 | end -------------------------------------------------------------------------------- /vis/vis_particle_medd_distances.m: -------------------------------------------------------------------------------- 1 | function vis_particle_medd_distances ( self, num_frames, fr) 2 | 3 | if ( fr == 1 ) 4 | return; 5 | end 6 | 7 | medd_dist = self.history.dist_medd(2:fr,:); 8 | probs = self.history.probs(2:fr,:); 9 | avg_medd_dist = nanmean(medd_dist'); 10 | 11 | hold on; 12 | for i = 2:fr 13 | plot (i*ones(1,self.N),medd_dist(i-1,:),'ro'); 14 | medd_dist(isnan(medd_dist)) = 0; % to ignore NaN 15 | w_avg_medd_dist(i-1) = medd_dist(i-1,:)*probs(i-1,:)'; 16 | end 17 | plot (2:fr,avg_medd_dist,'b--','LineWidth',2); 18 | plot (2:fr,w_avg_medd_dist,'k-','LineWidth',4); 19 | 20 | xlim([1 num_frames+1]); 21 | ylim([0 0.5]); 22 | drawnow 23 | 24 | end -------------------------------------------------------------------------------- /vis/vis_particle_position_occlusion.m: -------------------------------------------------------------------------------- 1 | function vis_particle_position_occlusion (rgb_raw, self, frame_no , particle_no) 2 | 3 | if ( self.history.z(frame_no,particle_no) == 1 ) 4 | bbs = squeeze(floor(self.history.bbs(frame_no,:,:))); 5 | rectangle('Position',bbs(particle_no,:),'EdgeColor','r'); 6 | else 7 | if (~isempty(rgb_raw)) 8 | imshow(rgb_raw); 9 | end 10 | end 11 | 12 | 13 | end 14 | 15 | -------------------------------------------------------------------------------- /vis/vis_particle_position_probability.m: -------------------------------------------------------------------------------- 1 | function vis_particle_position_probability (rgb_raw, self, frame_no , particle_no) 2 | 3 | col = bb_prob_color_indicator (self.history.probs(frame_no,:) , self.history.z(frame_no,:)); 4 | bbs = squeeze(floor(self.history.bbs(frame_no,:,:))); 5 | 6 | if (~isempty(rgb_raw)) 7 | imshow(rgb_raw); 8 | hold on 9 | end 10 | 11 | rectangle('Position',bbs(particle_no,:),'EdgeColor',col(particle_no,:)); 12 | drawnow; 13 | 14 | end -------------------------------------------------------------------------------- /vis/vis_particle_probability.m: -------------------------------------------------------------------------------- 1 | function vis_particle_probability ( self, fr ) 2 | 3 | if ( fr == 1 ) 4 | return; 5 | end 6 | 7 | maxp = max(max(self.history.probs)); 8 | probs = self.history.probs(fr,:); 9 | z = self.history.z(fr,:); 10 | 11 | bins = 0: 0.001 : maxp+0.001; 12 | h1 = histc(probs,bins); 13 | bar (bins,h1,'FaceColor','c'); 14 | hold on; 15 | 16 | p2 = probs; 17 | p2(z==0) = 0; 18 | h2 = histc(p2,bins); 19 | h2(1) = 0; 20 | bar (bins,h2,'FaceColor','r'); 21 | hold off; 22 | 23 | xlim([0,maxp+0.001]); 24 | ylim([0,10]); 25 | set(gca, 'XColor', 'w', 'YColor', 'w'); %% No Good- Interface dependant 26 | 27 | end -------------------------------------------------------------------------------- /vis/vis_particle_probability_comparison.m: -------------------------------------------------------------------------------- 1 | function vis_particle_probability_comparison (self , frame_no , particle_no ) 2 | 3 | if ( frame_no == 1 ) 4 | return; 5 | end 6 | 7 | probs = self.history.probs(frame_no,:); 8 | explode = self.history.z(frame_no,:); 9 | 10 | pie(probs(1:particle_no),explode(1:particle_no)); 11 | drawnow; 12 | 13 | end -------------------------------------------------------------------------------- /vis/vis_particle_template_hoc_comparison.m: -------------------------------------------------------------------------------- 1 | function vis_particle_template_hoc_comparison (rgb_raw, self , fr , particle_no) 2 | 3 | ctrs = self.feature{1}.rgb_ctr; 4 | bbs = squeeze(floor(self.history.bbs(fr,:,:))); 5 | 6 | hoc1 = self.history.model_rgb_hist(fr,:); 7 | ylabel('template HOC','Color','w'); 8 | subplot (3,1,1); hoc_vis (hoc1,ctrs); 9 | 10 | hoc2 = histogram_of_colors(bb_content(rgb_raw,bbs(particle_no,:)),[],ctrs); 11 | ylabel('particle HOC','Color','w'); 12 | 13 | subplot (3,1,2); hoc_vis (hoc2,ctrs); 14 | subplot (3,1,3); hoc_vis (abs(hoc1-hoc2),ctrs);%bar (abs(hoc1-hoc2)); xlim([0 length(hoc1)]); ylim([0 0.2]); drawnow; 15 | 16 | end -------------------------------------------------------------------------------- /vis/vis_particle_template_medd_comparison.m: -------------------------------------------------------------------------------- 1 | function vis_particle_template_medd_comparison ( self, dep_raw , fr , bb) 2 | 3 | bbs = squeeze(floor(self.history.bbs(fr,:,:))); 4 | 5 | medd_template = 255* self.history.model_med_depth(fr); 6 | subplot(2,1,1); imhist(dep_raw); 7 | hold on; 8 | plot (medd_template,5,'r^','MarkerEdgeColor','r','MarkerFaceColor','r','MarkerSize',15); 9 | hold off 10 | 11 | subplot(2,1,2); imhist(bb_content(dep_raw,bb)); 12 | hold on; 13 | line ([medd_template medd_template], ylim() , 'Color', 'r', 'LineStyle','--', 'LineWidth',6); 14 | ylabel('median of depth of the template', 'Color' , 'w'); 15 | 16 | last = -1; %FLASH EFFECT 17 | for i = 1:self.N 18 | medd = 255 * median_of_depth (bb_content(dep_raw,bbs(i,:)),[]); 19 | 20 | if (last ~= -1) %FLASH EFFECT 21 | plot (last,5,'b^','MarkerEdgeColor','b','MarkerFaceColor','b','MarkerSize',10); 22 | end 23 | last = medd; %FLASH EFFECT 24 | 25 | plot (medd,5,'y^','MarkerEdgeColor','y','MarkerFaceColor','y','MarkerSize',10); 26 | ylabel(['frame :' num2str(fr) ' - particle :' num2str(i)], 'Color' , 'w'); 27 | 28 | drawnow; 29 | end 30 | 31 | plot (last,5,'b^','MarkerEdgeColor','b','MarkerFaceColor','b','MarkerSize',10); %FLASH EFFECT 32 | 33 | hold off; 34 | 35 | end -------------------------------------------------------------------------------- /vis/vis_point_cloud.m: -------------------------------------------------------------------------------- 1 | function vis_point_cloud (rgb_raw, dep_raw, cam_param, view_dir ) 2 | 3 | if ( nargin < 4 || isempty(view_dir)) 4 | view_dir = [0,-90]; 5 | end 6 | 7 | depthInpaint = double(dep_raw)/1000; % convert from mm to m 8 | [x,y] = meshgrid(1:640, 1:480); 9 | 10 | cx = cam_param(1,3); cy = cam_param(2,3); 11 | fx = cam_param(1,1); fy = cam_param(2,2); 12 | 13 | Xworld = (x-cx).*depthInpaint*1/fx; 14 | Yworld = (y-cy).*depthInpaint*1/fy; 15 | Zworld = depthInpaint; 16 | 17 | validM = dep_raw~=0; 18 | XYZworldframe = [Xworld(:)'; Yworld(:)'; Zworld(:)']; 19 | valid = validM(:)'; 20 | 21 | % XYZworldframe 3xn and RGB 3xn 22 | RGB = [reshape(rgb_raw(:,:,1),1,[]);reshape(rgb_raw(:,:,2),1,[]);reshape(rgb_raw(:,:,3),1,[])]; 23 | XYZpoints = XYZworldframe(:,valid); 24 | RGBpoints = RGB(:,valid); 25 | 26 | % display in 3D: subsample to avoid too much to display. 27 | XYZpoints = XYZpoints(:,1:1:end); 28 | RGBpoints = RGBpoints(:,1:1:end); 29 | scatter3(XYZpoints(1,:),XYZpoints(2,:),XYZpoints(3,:),5*ones(1,size(XYZpoints,2)),double(RGBpoints)'/255,'filled'); 30 | axis equal; 31 | 32 | view(view_dir); 33 | end 34 | 35 | --------------------------------------------------------------------------------