├── .gitignore ├── LICENSE ├── README.md ├── algorithms ├── baseline │ └── src │ │ └── main │ │ └── r │ │ └── segmentation-baseline.R ├── cormier │ ├── cormier.py │ └── requirements.txt ├── heps │ ├── .gitignore │ ├── compile.sh │ ├── heps.sh │ ├── scripts │ │ └── HEPSScript-1.0.0 │ │ │ ├── HEPSScript.jar │ │ │ ├── heps.js │ │ │ └── script.conf │ └── src │ │ └── main │ │ └── java │ │ └── HEPSScript.java ├── meier │ ├── README.md │ ├── docker │ │ └── Dockerfile │ ├── model │ │ ├── callback_epoch_end.py │ │ ├── clean.sh │ │ ├── copy.sh │ │ ├── custom_image_generator.py │ │ ├── custom_image_generator_test.py │ │ ├── custom_image_generator_validation.py │ │ ├── extract_rectangles.py │ │ ├── extract_rectangles.sh │ │ ├── helper.py │ │ ├── id_list.py │ │ ├── main.py │ │ ├── main_crossval.py │ │ ├── main_restart.py │ │ ├── main_validate.py │ │ ├── meier2017.py │ │ ├── my_metrics.py │ │ ├── plot_all_kernels.py │ │ ├── plot_ranking.py │ │ ├── plot_ranking_new.py │ │ ├── resize.sh │ │ ├── setup_folders.py │ │ ├── test.py │ │ ├── test.sh │ │ ├── to_test.txt │ │ ├── train.sh │ │ └── validate.sh │ └── setup-directories.sh ├── mmdetection │ ├── .gitignore │ ├── Dockerfile │ ├── Dockerfile-with-model │ ├── infer.py │ ├── infer_single.py │ └── requirements.txt └── vips │ ├── .gitignore │ ├── compile.sh │ ├── scripts │ └── VIPSScript-1.0.0 │ │ ├── VIPSScript.jar │ │ ├── script.conf │ │ ├── vips.conf │ │ └── vips.js │ ├── src │ └── main │ │ └── java │ │ └── VIPSScript.java │ └── vips.sh └── src └── main └── bash ├── average-evaluations.sh └── combine-evaluation-files.sh /.gitignore: -------------------------------------------------------------------------------- 1 | /cikm20-web-page-segmentation-revisited-evaluation-framework-and-dataset.zip 2 | /cikm20-web-page-segmentation-revisited-evaluation-framework-and-dataset-master.zip 3 | /cikm20-web-page-segmentation-revisited-evaluation-framework-and-dataset/ 4 | /cikm20-web-page-segmentation-revisited-evaluation-framework-and-dataset-master/ 5 | /master.zip 6 | /cikm20/ 7 | /cikm20 8 | /webis-webseg-20-000000.zip 9 | /webis-webseg-20-000000/ 10 | /webis-webseg-20/ 11 | /webis-webseg-20-algorithm-segmentations-final/ 12 | webis-web-archive-17-000000.zip 13 | webis-web-archive-17-000000/ 14 | webis-web-archive-17/ 15 | /segmentations/ 16 | /results/ 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 2.1, February 1999 3 | 4 | Copyright (C) 1991, 1999 Free Software Foundation, Inc. 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | [This is the first released version of the Lesser GPL. It also counts 10 | as the successor of the GNU Library Public License, version 2, hence 11 | the version number 2.1.] 12 | 13 | Preamble 14 | 15 | The licenses for most software are designed to take away your 16 | freedom to share and change it. By contrast, the GNU General Public 17 | Licenses are intended to guarantee your freedom to share and change 18 | free software--to make sure the software is free for all its users. 19 | 20 | This license, the Lesser General Public License, applies to some 21 | specially designated software packages--typically libraries--of the 22 | Free Software Foundation and other authors who decide to use it. You 23 | can use it too, but we suggest you first think carefully about whether 24 | this license or the ordinary General Public License is the better 25 | strategy to use in any particular case, based on the explanations below. 26 | 27 | When we speak of free software, we are referring to freedom of use, 28 | not price. Our General Public Licenses are designed to make sure that 29 | you have the freedom to distribute copies of free software (and charge 30 | for this service if you wish); that you receive source code or can get 31 | it if you want it; that you can change the software and use pieces of 32 | it in new free programs; and that you are informed that you can do 33 | these things. 34 | 35 | To protect your rights, we need to make restrictions that forbid 36 | distributors to deny you these rights or to ask you to surrender these 37 | rights. These restrictions translate to certain responsibilities for 38 | you if you distribute copies of the library or if you modify it. 39 | 40 | For example, if you distribute copies of the library, whether gratis 41 | or for a fee, you must give the recipients all the rights that we gave 42 | you. You must make sure that they, too, receive or can get the source 43 | code. If you link other code with the library, you must provide 44 | complete object files to the recipients, so that they can relink them 45 | with the library after making changes to the library and recompiling 46 | it. And you must show them these terms so they know their rights. 47 | 48 | We protect your rights with a two-step method: (1) we copyright the 49 | library, and (2) we offer you this license, which gives you legal 50 | permission to copy, distribute and/or modify the library. 51 | 52 | To protect each distributor, we want to make it very clear that 53 | there is no warranty for the free library. Also, if the library is 54 | modified by someone else and passed on, the recipients should know 55 | that what they have is not the original version, so that the original 56 | author's reputation will not be affected by problems that might be 57 | introduced by others. 58 | 59 | Finally, software patents pose a constant threat to the existence of 60 | any free program. We wish to make sure that a company cannot 61 | effectively restrict the users of a free program by obtaining a 62 | restrictive license from a patent holder. Therefore, we insist that 63 | any patent license obtained for a version of the library must be 64 | consistent with the full freedom of use specified in this license. 65 | 66 | Most GNU software, including some libraries, is covered by the 67 | ordinary GNU General Public License. This license, the GNU Lesser 68 | General Public License, applies to certain designated libraries, and 69 | is quite different from the ordinary General Public License. We use 70 | this license for certain libraries in order to permit linking those 71 | libraries into non-free programs. 72 | 73 | When a program is linked with a library, whether statically or using 74 | a shared library, the combination of the two is legally speaking a 75 | combined work, a derivative of the original library. The ordinary 76 | General Public License therefore permits such linking only if the 77 | entire combination fits its criteria of freedom. The Lesser General 78 | Public License permits more lax criteria for linking other code with 79 | the library. 80 | 81 | We call this license the "Lesser" General Public License because it 82 | does Less to protect the user's freedom than the ordinary General 83 | Public License. It also provides other free software developers Less 84 | of an advantage over competing non-free programs. These disadvantages 85 | are the reason we use the ordinary General Public License for many 86 | libraries. However, the Lesser license provides advantages in certain 87 | special circumstances. 88 | 89 | For example, on rare occasions, there may be a special need to 90 | encourage the widest possible use of a certain library, so that it becomes 91 | a de-facto standard. To achieve this, non-free programs must be 92 | allowed to use the library. A more frequent case is that a free 93 | library does the same job as widely used non-free libraries. In this 94 | case, there is little to gain by limiting the free library to free 95 | software only, so we use the Lesser General Public License. 96 | 97 | In other cases, permission to use a particular library in non-free 98 | programs enables a greater number of people to use a large body of 99 | free software. For example, permission to use the GNU C Library in 100 | non-free programs enables many more people to use the whole GNU 101 | operating system, as well as its variant, the GNU/Linux operating 102 | system. 103 | 104 | Although the Lesser General Public License is Less protective of the 105 | users' freedom, it does ensure that the user of a program that is 106 | linked with the Library has the freedom and the wherewithal to run 107 | that program using a modified version of the Library. 108 | 109 | The precise terms and conditions for copying, distribution and 110 | modification follow. Pay close attention to the difference between a 111 | "work based on the library" and a "work that uses the library". The 112 | former contains code derived from the library, whereas the latter must 113 | be combined with the library in order to run. 114 | 115 | GNU LESSER GENERAL PUBLIC LICENSE 116 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 117 | 118 | 0. This License Agreement applies to any software library or other 119 | program which contains a notice placed by the copyright holder or 120 | other authorized party saying it may be distributed under the terms of 121 | this Lesser General Public License (also called "this License"). 122 | Each licensee is addressed as "you". 123 | 124 | A "library" means a collection of software functions and/or data 125 | prepared so as to be conveniently linked with application programs 126 | (which use some of those functions and data) to form executables. 127 | 128 | The "Library", below, refers to any such software library or work 129 | which has been distributed under these terms. A "work based on the 130 | Library" means either the Library or any derivative work under 131 | copyright law: that is to say, a work containing the Library or a 132 | portion of it, either verbatim or with modifications and/or translated 133 | straightforwardly into another language. (Hereinafter, translation is 134 | included without limitation in the term "modification".) 135 | 136 | "Source code" for a work means the preferred form of the work for 137 | making modifications to it. For a library, complete source code means 138 | all the source code for all modules it contains, plus any associated 139 | interface definition files, plus the scripts used to control compilation 140 | and installation of the library. 141 | 142 | Activities other than copying, distribution and modification are not 143 | covered by this License; they are outside its scope. The act of 144 | running a program using the Library is not restricted, and output from 145 | such a program is covered only if its contents constitute a work based 146 | on the Library (independent of the use of the Library in a tool for 147 | writing it). Whether that is true depends on what the Library does 148 | and what the program that uses the Library does. 149 | 150 | 1. You may copy and distribute verbatim copies of the Library's 151 | complete source code as you receive it, in any medium, provided that 152 | you conspicuously and appropriately publish on each copy an 153 | appropriate copyright notice and disclaimer of warranty; keep intact 154 | all the notices that refer to this License and to the absence of any 155 | warranty; and distribute a copy of this License along with the 156 | Library. 157 | 158 | You may charge a fee for the physical act of transferring a copy, 159 | and you may at your option offer warranty protection in exchange for a 160 | fee. 161 | 162 | 2. You may modify your copy or copies of the Library or any portion 163 | of it, thus forming a work based on the Library, and copy and 164 | distribute such modifications or work under the terms of Section 1 165 | above, provided that you also meet all of these conditions: 166 | 167 | a) The modified work must itself be a software library. 168 | 169 | b) You must cause the files modified to carry prominent notices 170 | stating that you changed the files and the date of any change. 171 | 172 | c) You must cause the whole of the work to be licensed at no 173 | charge to all third parties under the terms of this License. 174 | 175 | d) If a facility in the modified Library refers to a function or a 176 | table of data to be supplied by an application program that uses 177 | the facility, other than as an argument passed when the facility 178 | is invoked, then you must make a good faith effort to ensure that, 179 | in the event an application does not supply such function or 180 | table, the facility still operates, and performs whatever part of 181 | its purpose remains meaningful. 182 | 183 | (For example, a function in a library to compute square roots has 184 | a purpose that is entirely well-defined independent of the 185 | application. Therefore, Subsection 2d requires that any 186 | application-supplied function or table used by this function must 187 | be optional: if the application does not supply it, the square 188 | root function must still compute square roots.) 189 | 190 | These requirements apply to the modified work as a whole. If 191 | identifiable sections of that work are not derived from the Library, 192 | and can be reasonably considered independent and separate works in 193 | themselves, then this License, and its terms, do not apply to those 194 | sections when you distribute them as separate works. But when you 195 | distribute the same sections as part of a whole which is a work based 196 | on the Library, the distribution of the whole must be on the terms of 197 | this License, whose permissions for other licensees extend to the 198 | entire whole, and thus to each and every part regardless of who wrote 199 | it. 200 | 201 | Thus, it is not the intent of this section to claim rights or contest 202 | your rights to work written entirely by you; rather, the intent is to 203 | exercise the right to control the distribution of derivative or 204 | collective works based on the Library. 205 | 206 | In addition, mere aggregation of another work not based on the Library 207 | with the Library (or with a work based on the Library) on a volume of 208 | a storage or distribution medium does not bring the other work under 209 | the scope of this License. 210 | 211 | 3. You may opt to apply the terms of the ordinary GNU General Public 212 | License instead of this License to a given copy of the Library. To do 213 | this, you must alter all the notices that refer to this License, so 214 | that they refer to the ordinary GNU General Public License, version 2, 215 | instead of to this License. (If a newer version than version 2 of the 216 | ordinary GNU General Public License has appeared, then you can specify 217 | that version instead if you wish.) Do not make any other change in 218 | these notices. 219 | 220 | Once this change is made in a given copy, it is irreversible for 221 | that copy, so the ordinary GNU General Public License applies to all 222 | subsequent copies and derivative works made from that copy. 223 | 224 | This option is useful when you wish to copy part of the code of 225 | the Library into a program that is not a library. 226 | 227 | 4. You may copy and distribute the Library (or a portion or 228 | derivative of it, under Section 2) in object code or executable form 229 | under the terms of Sections 1 and 2 above provided that you accompany 230 | it with the complete corresponding machine-readable source code, which 231 | must be distributed under the terms of Sections 1 and 2 above on a 232 | medium customarily used for software interchange. 233 | 234 | If distribution of object code is made by offering access to copy 235 | from a designated place, then offering equivalent access to copy the 236 | source code from the same place satisfies the requirement to 237 | distribute the source code, even though third parties are not 238 | compelled to copy the source along with the object code. 239 | 240 | 5. A program that contains no derivative of any portion of the 241 | Library, but is designed to work with the Library by being compiled or 242 | linked with it, is called a "work that uses the Library". Such a 243 | work, in isolation, is not a derivative work of the Library, and 244 | therefore falls outside the scope of this License. 245 | 246 | However, linking a "work that uses the Library" with the Library 247 | creates an executable that is a derivative of the Library (because it 248 | contains portions of the Library), rather than a "work that uses the 249 | library". The executable is therefore covered by this License. 250 | Section 6 states terms for distribution of such executables. 251 | 252 | When a "work that uses the Library" uses material from a header file 253 | that is part of the Library, the object code for the work may be a 254 | derivative work of the Library even though the source code is not. 255 | Whether this is true is especially significant if the work can be 256 | linked without the Library, or if the work is itself a library. The 257 | threshold for this to be true is not precisely defined by law. 258 | 259 | If such an object file uses only numerical parameters, data 260 | structure layouts and accessors, and small macros and small inline 261 | functions (ten lines or less in length), then the use of the object 262 | file is unrestricted, regardless of whether it is legally a derivative 263 | work. (Executables containing this object code plus portions of the 264 | Library will still fall under Section 6.) 265 | 266 | Otherwise, if the work is a derivative of the Library, you may 267 | distribute the object code for the work under the terms of Section 6. 268 | Any executables containing that work also fall under Section 6, 269 | whether or not they are linked directly with the Library itself. 270 | 271 | 6. As an exception to the Sections above, you may also combine or 272 | link a "work that uses the Library" with the Library to produce a 273 | work containing portions of the Library, and distribute that work 274 | under terms of your choice, provided that the terms permit 275 | modification of the work for the customer's own use and reverse 276 | engineering for debugging such modifications. 277 | 278 | You must give prominent notice with each copy of the work that the 279 | Library is used in it and that the Library and its use are covered by 280 | this License. You must supply a copy of this License. If the work 281 | during execution displays copyright notices, you must include the 282 | copyright notice for the Library among them, as well as a reference 283 | directing the user to the copy of this License. Also, you must do one 284 | of these things: 285 | 286 | a) Accompany the work with the complete corresponding 287 | machine-readable source code for the Library including whatever 288 | changes were used in the work (which must be distributed under 289 | Sections 1 and 2 above); and, if the work is an executable linked 290 | with the Library, with the complete machine-readable "work that 291 | uses the Library", as object code and/or source code, so that the 292 | user can modify the Library and then relink to produce a modified 293 | executable containing the modified Library. (It is understood 294 | that the user who changes the contents of definitions files in the 295 | Library will not necessarily be able to recompile the application 296 | to use the modified definitions.) 297 | 298 | b) Use a suitable shared library mechanism for linking with the 299 | Library. A suitable mechanism is one that (1) uses at run time a 300 | copy of the library already present on the user's computer system, 301 | rather than copying library functions into the executable, and (2) 302 | will operate properly with a modified version of the library, if 303 | the user installs one, as long as the modified version is 304 | interface-compatible with the version that the work was made with. 305 | 306 | c) Accompany the work with a written offer, valid for at 307 | least three years, to give the same user the materials 308 | specified in Subsection 6a, above, for a charge no more 309 | than the cost of performing this distribution. 310 | 311 | d) If distribution of the work is made by offering access to copy 312 | from a designated place, offer equivalent access to copy the above 313 | specified materials from the same place. 314 | 315 | e) Verify that the user has already received a copy of these 316 | materials or that you have already sent this user a copy. 317 | 318 | For an executable, the required form of the "work that uses the 319 | Library" must include any data and utility programs needed for 320 | reproducing the executable from it. However, as a special exception, 321 | the materials to be distributed need not include anything that is 322 | normally distributed (in either source or binary form) with the major 323 | components (compiler, kernel, and so on) of the operating system on 324 | which the executable runs, unless that component itself accompanies 325 | the executable. 326 | 327 | It may happen that this requirement contradicts the license 328 | restrictions of other proprietary libraries that do not normally 329 | accompany the operating system. Such a contradiction means you cannot 330 | use both them and the Library together in an executable that you 331 | distribute. 332 | 333 | 7. You may place library facilities that are a work based on the 334 | Library side-by-side in a single library together with other library 335 | facilities not covered by this License, and distribute such a combined 336 | library, provided that the separate distribution of the work based on 337 | the Library and of the other library facilities is otherwise 338 | permitted, and provided that you do these two things: 339 | 340 | a) Accompany the combined library with a copy of the same work 341 | based on the Library, uncombined with any other library 342 | facilities. This must be distributed under the terms of the 343 | Sections above. 344 | 345 | b) Give prominent notice with the combined library of the fact 346 | that part of it is a work based on the Library, and explaining 347 | where to find the accompanying uncombined form of the same work. 348 | 349 | 8. You may not copy, modify, sublicense, link with, or distribute 350 | the Library except as expressly provided under this License. Any 351 | attempt otherwise to copy, modify, sublicense, link with, or 352 | distribute the Library is void, and will automatically terminate your 353 | rights under this License. However, parties who have received copies, 354 | or rights, from you under this License will not have their licenses 355 | terminated so long as such parties remain in full compliance. 356 | 357 | 9. You are not required to accept this License, since you have not 358 | signed it. However, nothing else grants you permission to modify or 359 | distribute the Library or its derivative works. These actions are 360 | prohibited by law if you do not accept this License. Therefore, by 361 | modifying or distributing the Library (or any work based on the 362 | Library), you indicate your acceptance of this License to do so, and 363 | all its terms and conditions for copying, distributing or modifying 364 | the Library or works based on it. 365 | 366 | 10. Each time you redistribute the Library (or any work based on the 367 | Library), the recipient automatically receives a license from the 368 | original licensor to copy, distribute, link with or modify the Library 369 | subject to these terms and conditions. You may not impose any further 370 | restrictions on the recipients' exercise of the rights granted herein. 371 | You are not responsible for enforcing compliance by third parties with 372 | this License. 373 | 374 | 11. If, as a consequence of a court judgment or allegation of patent 375 | infringement or for any other reason (not limited to patent issues), 376 | conditions are imposed on you (whether by court order, agreement or 377 | otherwise) that contradict the conditions of this License, they do not 378 | excuse you from the conditions of this License. If you cannot 379 | distribute so as to satisfy simultaneously your obligations under this 380 | License and any other pertinent obligations, then as a consequence you 381 | may not distribute the Library at all. For example, if a patent 382 | license would not permit royalty-free redistribution of the Library by 383 | all those who receive copies directly or indirectly through you, then 384 | the only way you could satisfy both it and this License would be to 385 | refrain entirely from distribution of the Library. 386 | 387 | If any portion of this section is held invalid or unenforceable under any 388 | particular circumstance, the balance of the section is intended to apply, 389 | and the section as a whole is intended to apply in other circumstances. 390 | 391 | It is not the purpose of this section to induce you to infringe any 392 | patents or other property right claims or to contest validity of any 393 | such claims; this section has the sole purpose of protecting the 394 | integrity of the free software distribution system which is 395 | implemented by public license practices. Many people have made 396 | generous contributions to the wide range of software distributed 397 | through that system in reliance on consistent application of that 398 | system; it is up to the author/donor to decide if he or she is willing 399 | to distribute software through any other system and a licensee cannot 400 | impose that choice. 401 | 402 | This section is intended to make thoroughly clear what is believed to 403 | be a consequence of the rest of this License. 404 | 405 | 12. If the distribution and/or use of the Library is restricted in 406 | certain countries either by patents or by copyrighted interfaces, the 407 | original copyright holder who places the Library under this License may add 408 | an explicit geographical distribution limitation excluding those countries, 409 | so that distribution is permitted only in or among countries not thus 410 | excluded. In such case, this License incorporates the limitation as if 411 | written in the body of this License. 412 | 413 | 13. The Free Software Foundation may publish revised and/or new 414 | versions of the Lesser General Public License from time to time. 415 | Such new versions will be similar in spirit to the present version, 416 | but may differ in detail to address new problems or concerns. 417 | 418 | Each version is given a distinguishing version number. If the Library 419 | specifies a version number of this License which applies to it and 420 | "any later version", you have the option of following the terms and 421 | conditions either of that version or of any later version published by 422 | the Free Software Foundation. If the Library does not specify a 423 | license version number, you may choose any version ever published by 424 | the Free Software Foundation. 425 | 426 | 14. If you wish to incorporate parts of the Library into other free 427 | programs whose distribution conditions are incompatible with these, 428 | write to the author to ask for permission. For software which is 429 | copyrighted by the Free Software Foundation, write to the Free 430 | Software Foundation; we sometimes make exceptions for this. Our 431 | decision will be guided by the two goals of preserving the free status 432 | of all derivatives of our free software and of promoting the sharing 433 | and reuse of software generally. 434 | 435 | NO WARRANTY 436 | 437 | 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO 438 | WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. 439 | EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR 440 | OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY 441 | KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE 442 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 443 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE 444 | LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME 445 | THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 446 | 447 | 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN 448 | WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY 449 | AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU 450 | FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR 451 | CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE 452 | LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING 453 | RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A 454 | FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF 455 | SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH 456 | DAMAGES. 457 | 458 | END OF TERMS AND CONDITIONS 459 | 460 | How to Apply These Terms to Your New Libraries 461 | 462 | If you develop a new library, and you want it to be of the greatest 463 | possible use to the public, we recommend making it free software that 464 | everyone can redistribute and change. You can do so by permitting 465 | redistribution under these terms (or, alternatively, under the terms of the 466 | ordinary General Public License). 467 | 468 | To apply these terms, attach the following notices to the library. It is 469 | safest to attach them to the start of each source file to most effectively 470 | convey the exclusion of warranty; and each file should have at least the 471 | "copyright" line and a pointer to where the full notice is found. 472 | 473 | 474 | Copyright (C) 475 | 476 | This library is free software; you can redistribute it and/or 477 | modify it under the terms of the GNU Lesser General Public 478 | License as published by the Free Software Foundation; either 479 | version 2.1 of the License, or (at your option) any later version. 480 | 481 | This library is distributed in the hope that it will be useful, 482 | but WITHOUT ANY WARRANTY; without even the implied warranty of 483 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 484 | Lesser General Public License for more details. 485 | 486 | You should have received a copy of the GNU Lesser General Public 487 | License along with this library; if not, write to the Free Software 488 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 489 | USA 490 | 491 | Also add information on how to contact you by electronic and paper mail. 492 | 493 | You should also get your employer (if you work as a programmer) or your 494 | school, if any, to sign a "copyright disclaimer" for the library, if 495 | necessary. Here is a sample; alter the names: 496 | 497 | Yoyodyne, Inc., hereby disclaims all copyright interest in the 498 | library `Frob' (a library for tweaking knobs) written by James Random 499 | Hacker. 500 | 501 | , 1 April 1990 502 | Ty Coon, President of Vice 503 | 504 | That's all there is to it! 505 | 506 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Repository for the ECIR21 Reproducibility Paper "An Empirical Comparison of Web Page Segmentation Algorithms" 2 | Paper by Johannes Kiesel, Lars Meyer, Florian Kneist, Benno Stein, and Martin Potthast. 3 | 4 | This repository enables the reproduction of the experiments from the paper, but also to run the algorithms on new data. 5 | 6 | Outline: 7 | - [Common Preparations](#common-preparations): necessary setup steps 8 | - [Algorithms](#algorithms): running each segmentation algorithm on the Webis-WebSeg-20: [Baseline](#baseline), [VIPS](#vips), [HEPS](#heps), [Cormier et al.](#cormier-et-al), [MMDetection](#mmdetection), [Meier et al.](#meier-et-al), [Ensemble](#ensemble) 9 | - [Evaluation](#evaluation): evaluating the segmentations 10 | - [Plotting Segmentations](#plotting-segmentations): visually checking on segmentations 11 | 12 | ## Common Preparations 13 | - Check out this repository. 14 | - If not done already, get the [source code](https://github.com/webis-de/cikm20-web-page-segmentation-revisited-evaluation-framework-and-dataset/archive/master.zip) of the evaluation framework paper, extract it next to this README, and rename the extracted directory (`cikm20-web-page-...`) to `cikm20`. 15 | - Make sure your system fulfills all the [requirements of the evaluation framework](https://github.com/webis-de/cikm20-web-page-segmentation-revisited-evaluation-framework-and-dataset/tree/235bb0b1b673da351e267b3966da811021c20e63#requirements). 16 | - If it does not exist yet, create the directory `segmentations` next to this README. 17 | 18 | 19 | ## Algorithms 20 | We here describe how to get the code and how to run each algorithm for one page, so that it produces a segmentation in the common format (a JSON file in the `segmentations` directory) which can then be used in the [evaluation](#evaluation). 21 | 22 | The instructions here use the page with ID 000000 so that they work with the sample ZIP archives, `webis-webseg-20-000000.zip` and `webis-web-archive-17-000000.zip`, as well as with the full datasets of [segmentations](https://doi.org/10.5281/zenodo.3354902) and [archives](https://doi.org/10.5281/zenodo.1002203). For the sample ZIP archives, download them from the respective full dataset pages, extract them next to this README, and rename them by removing the `-000000` suffix. If you download and extract the full datasets, they already have the correct name. Then follow the instructions below. 23 | 24 | 25 | ### Baseline 26 | The baseline creates a single segment that contains the entire web page. 27 | 28 | - In a shell, go to the directory that contains this README. 29 | 30 | ``` 31 | Rscript algorithms/baseline/src/main/r/segmentation-baseline.R \ 32 | --input webis-webseg-20/000000/screenshot.png \ 33 | --output segmentations/baseline.json 34 | ``` 35 | 36 | 37 | ### VIPS 38 | We use a TypeScript port of Tomáš Popela's [vips_java](https://github.com/tpopela/vips_java), transpiled to JavaScript. We thank the original author for providing his implementation. 39 | 40 | The implementation is in the [vips.js](algorithms/vips/scripts/VIPSScript-1.0.0/vips.js). This file is loaded into the webis-web-archiver to run on web pages that are reproduced from web archives. If needed, you can use the [compile.sh](algorithms/vips/compile.sh) to re-compile the Java part that controls the browser and executes the VIPS JavaScript (re-compilation requires a Java 8 JDK or above installed). 41 | 42 | - Install [Docker](https://www.docker.com/). 43 | - In a shell, go to the directory that contains this README. 44 | 45 | You can find the corresponding URL for an archive of the webis-web-archive-17 in the [sites-and-pages.txt](https://zenodo.org/record/4064019/files/sites-and-pages.txt). Note that the docker image may take quite some time to download when you run it the first time. 46 | ``` 47 | # Execute VIPS while reproducing the web page from the archive 48 | ./algorithms/vips/vips.sh \ 49 | --archive webis-web-archive-17/pages/000000/ \ 50 | --pdoc 5 \ 51 | --url "http://008345152.blog.fc2.com/blog-date-201305.html" \ 52 | --id 000000 \ 53 | --output segmentations 54 | 55 | # Convert hierarchical segmentation to a flat one 56 | Rscript cikm20/src/main/r/flatten-segmentations.R \ 57 | --input segmentations/vips.json \ 58 | --output segmentations/vips-flattened.json 59 | ``` 60 | 61 | 62 | ### HEPS 63 | We use a slightly modified version of Manabe et al.'s [HEPS implementation](https://github.com/tmanabe/HEPS) that outputs bounding box coordinates instead of text segments. We thank the original authors for providing their implementation. 64 | 65 | The implementation is in the [heps.js](algorithms/heps/scripts/HEPSScript-1.0.0/heps.js). This file is loaded into the webis-web-archiver to run on web pages that are reproduced from web archives. If needed, you can use the [compile.sh](algorithms/heps/compile.sh) to re-compile the Java part that controls the browser and executes the HEPS JavaScript (re-compilation requires a Java 8 JDK or above installed). 66 | 67 | - Install [Docker](https://www.docker.com/). 68 | - In a shell, go to the directory that contains this README. 69 | 70 | You can find the corresponding URL for an archive of the webis-web-archive-17 in the [sites-and-pages.txt](https://zenodo.org/record/4064019/files/sites-and-pages.txt). Note that the docker image may take quite some time to download when you run it the first time. 71 | ``` 72 | # Execute HEPS while reproducing the web page from the archive 73 | ./algorithms/heps/heps.sh \ 74 | --archive webis-web-archive-17/pages/000000/ \ 75 | --url "http://008345152.blog.fc2.com/blog-date-201305.html" \ 76 | --id 000000 \ 77 | --output segmentations 78 | ``` 79 | 80 | 81 | ### Cormier et al. 82 | We use a Python implementation graciously provided by [Michael Cormier](https://cs.uwaterloo.ca/~m4cormie/) and Zhuofu Tao, to whom we express our gratitude. 83 | 84 | You may adjust the `min_l` and `line_length` parameters in [`cormier.py`](algorithms/cormier/cormier.py). 85 | 86 | - Install [Python 3](https://www.python.org/downloads/) (e.g., for Debian/Ubuntu: `sudo apt install python3`). 87 | - Install `pip3` (e.g., for Debian/Ubuntu: `sudo apt install python3-pip`). 88 | - In a shell, go to the directory that contains this README. 89 | - Install the required Python packages: `pip3 install -r algorithms/cormier/requirements.txt`. 90 | 91 | ``` 92 | python3 algorithms/cormier/cormier.py \ 93 | --image webis-webseg-20/000000/screenshot.png \ 94 | --id 000000 \ 95 | --output segmentations 96 | ``` 97 | 98 | 99 | ### MMDetection 100 | We use the [original implementation](https://github.com/open-mmlab/mmdetection) provided by the authors. The provided inference scripts are suitable for use with Nvidia GPUs. By default, the container uses the first available GPU. 101 | 102 | - Install [Docker](https://www.docker.com/). 103 | - Install the [Nvidia Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker). 104 | 105 | ``` 106 | # Infer segments for page with ID 000000 (use 'infer.py' to segment all) 107 | nvidia-docker run -it \ 108 | -v ${PWD}/webis-webseg-20/:/pages \ 109 | -v ${PWD}/segmentations/mmdetection:/out \ 110 | ghcr.io/webis-de/mmdetection19-web-page-segmentation:1.0.0 \ 111 | python infer_single.py 000000 112 | 113 | # Fit segments 114 | Rscript cikm20/src/main/r/fit-segmentations-to-dom-nodes.R \ 115 | --input segmentations/mmdetection/000000.json \ 116 | --segmentations mmdetection_segms \ 117 | --nodes webis-webseg-20/000000/nodes.csv \ 118 | --output segmentations/mmdetection.json 119 | 120 | # Rename segmentation 121 | sed -i 's/mmdetection_segms.fitted/mmdetection/' segmentations/mmdetection.json 122 | 123 | # Convert hierarchical segmentation to a flat one 124 | Rscript cikm20/src/main/r/flatten-segmentations.R \ 125 | --input segmentations/mmdetection.json \ 126 | --output segmentations/mmdetection-flattened.json 127 | ``` 128 | 129 | 130 | ### Meier et al. 131 | The neural network is implemented in [Keras](https://keras.io) using the [TensorFlow](https://www.tensorflow.org) backend. We provide a [Docker container](https://hub.docker.com/layers/webis/meier17-web-page-segmentation/1.0.4/images/sha256-24ee082e1bee6f20b5e41140d081a52e5048298ff56e59a634a00239960d68db?context=explore) that can be used to train the model and perform inference with Nvidia GPUs. By default, the container uses the first available GPU. 132 | 133 | - Install [Docker](https://www.docker.com/). 134 | - Install the [Nvidia Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker). 135 | - In a shell, go to the directory that contains this README. 136 | 137 | The algorithm expects a specific directory structure for training and testing: 138 | - Download the `webis-webseg-20-folds.txt` and `webis-webseg-20-4096px.zip` from [Zenodo](https://doi.org/10.5281/zenodo.4146889) and extract the ZIP file. 139 | - Use `./algorithms/meier/setup-directories.sh `. The directory structure will be created in `./webis-webseg-20-meier`. 140 | - Download the `webis-webseg-20-meier-models.zip` from [Zenodo](https://doi.org/10.5281/zenodo.4146889) and extract it into the created `./webis-webseg-20-meier` directory. 141 | 142 | Instructions to create the input files and to train the models are provided in our [README for the algorithm](algorithms/meier/README.md). 143 | 144 | ``` 145 | # Run the algorithm on all screenshots of a fold, resizes the output to the original size, and extracts the segments from the masks. 146 | gpu=0 # The ID of the GPU to use 147 | fold=0 # Do this for each integer from 0 to 9 148 | sudo nvidia-docker run \ 149 | -it --rm -u $(id -u):$(id -g) \ 150 | --env NVIDIA_VISIBLE_DEVICES=$gpu \ 151 | --env KERAS_BACKEND=tensorflow \ 152 | -v ${PWD}/webis-webseg-20-meier/:/src/workspace/data \ 153 | ghcr.io/webis-de/meier17-web-page-segmentation:1.0.4 \ 154 | ./test.sh \ 155 | ../data/input/test/ \ 156 | $fold \ 157 | ../data/webis-webseg-20-meier-models/model-fold$fold-weights.h5 \ 158 | ../data/segmentations-fold$fold 159 | ``` 160 | 161 | 162 | ### Ensemble 163 | The Ensemble uses the segmentation fusion algorithm. 164 | 165 | - If one of these files is missing in `segmentations`, run the corresponding algorithm as described above: `vips.json`, `heps.json`, `cormier.json`, and `mmdetection.json`. 166 | - In a shell, go to the directory that contains this README. 167 | 168 | ``` 169 | # Create one segmentation file that contains VIPS, HEPS, Cormier et al., and MMDetection segmentations 170 | Rscript cikm20/src/main/r/combine-segmentation-files.R \ 171 | segmentations/vips.json \ 172 | segmentations/heps.json \ 173 | segmentations/cormier.json \ 174 | segmentations/mmdetection.json \ 175 | segmentations/all.json 176 | 177 | # Create the ensemble segmentation for Min-vote@2 178 | # Note that the implementation uses a *dis*agreement-threshold, thus use 1 - \theta_s of the paper! 179 | Rscript cikm20/src/main/r/fuse-segmentations.R \ 180 | --input segmentations/all.json \ 181 | --segments-min-annotators 2 \ 182 | --size-function pixels \ 183 | --disagreement-threshold 0.625 \ 184 | --output segmentations/ensemble.json 185 | ``` 186 | 187 | 188 | ## Evaluation 189 | The evaluation is here exemplified for the `baseline` algorithm and for `pixels` as atomic elements (the other options are `edges-fine`, `edges-coarse`, `nodes`, and `chars`). 190 | 191 | - The segmentation of the algorithm should be contained in a JSON file `segmentations/baseline.json`. If not, run the algorithm as described above. 192 | - If it does not exist yet, create the directory `results` next to this README. 193 | - In a shell, go to the directory that contains this README. 194 | 195 | ``` 196 | # Get BCubed precision, recall, and F-measure 197 | Rscript cikm20/src/main/r/evaluate-segmentation.R \ 198 | --algorithm segmentations/baseline.json \ 199 | --ground-truth webis-webseg-20/000000/ground-truth.json \ 200 | --size-function pixels \ 201 | --output results/baseline-pixels.csv 202 | ``` 203 | 204 | The agreement of two algorithms is calculated the same way, but with the segmentation of the second algorithm as the "ground-truth". 205 | 206 | 207 | ## Plotting Segmentations 208 | ``` 209 | Rscript cikm20/src/main/r/plot-segmentations.R \ 210 | --input .json \ 211 | --color-per-segment \ 212 | --output .png 213 | ``` 214 | 215 | -------------------------------------------------------------------------------- /algorithms/baseline/src/main/r/segmentation-baseline.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env Rscript 2 | 3 | ################################################################################ 4 | ## LOADING SEGMENTATION LIBRARY 5 | ################################################################################ 6 | 7 | rscript.options <- commandArgs(trailingOnly = FALSE) 8 | source.dir <- dirname(sub(".*=", "", rscript.options[grep("--file=", rscript.options)])) 9 | source(paste(source.dir, "..", "..", "..", "..", "..", "cikm20", "src", "main", "r", "segmentations", "lib.R", sep="/")) 10 | 11 | 12 | 13 | ################################################################################ 14 | ## OPTIONS 15 | ################################################################################ 16 | 17 | library("optparse") 18 | 19 | option_list <- list( 20 | make_option("--input", type="character", help="Screenshot to segment"), 21 | make_option("--id", type="character", default=NULL, help="ID of the task (instead of using the screenshot's directory name)"), 22 | make_option("--output", type="character", help="Output JSON file with the segmentation") 23 | ) 24 | 25 | options.parser <- OptionParser(option_list=option_list) 26 | options <- parse_args(options.parser) 27 | if (is.null(options$input)) { 28 | print_help(options.parser) 29 | stop("Missing input file", call.=FALSE) 30 | } 31 | if (is.null(options$output)) { 32 | print_help(options.parser) 33 | stop("Missing output file", call.=FALSE) 34 | } 35 | 36 | 37 | 38 | ################################################################################ 39 | ## EXECUTION 40 | ################################################################################ 41 | 42 | task <- Task(id = options$id, screenshotFile = options$input) 43 | segment <- Segment(c(0,0,task$width,task$width), c(0,task$height,task$height,0)) 44 | task$segmentations[["baseline"]] <- list(segment) 45 | 46 | WriteTask(task, options$output) 47 | 48 | -------------------------------------------------------------------------------- /algorithms/cormier/cormier.py: -------------------------------------------------------------------------------- 1 | import random 2 | import math 3 | import numpy as np 4 | from scipy.stats import norm 5 | from scipy.ndimage import convolve 6 | 7 | import cv2 8 | 9 | import sys 10 | import json 11 | 12 | import argparse 13 | 14 | # Label line directions 15 | horiz = 0 16 | vert = 1 17 | dir_h = 0 18 | dir_v = 1 19 | # Define Sobel filters 20 | v_filter = np.multiply(np.array([[1], [2], [1]]), np.array([[-1, 0, 1]])) 21 | h_filter = np.transpose(v_filter) 22 | # Set parameters 23 | min_l = 45 # Minimum size of a region; also half-window width 24 | threshold = 0.5 # Threshold probability to accept a line as semantically significant 25 | prior = 0.01 # Prior probability that a given pixel is an edge 26 | line_length = 256 # Maximum line segment length for recursion 27 | mcs_rp = 100 # Trials for Monte Carlo simulation (lower than original papers, but seems to work fine) 28 | mcs_prob = 0.3 # Minimum proportion of locally significant edges in Monte Carlo trials 29 | 30 | # Class representing an edge (used for data storage) 31 | class Edge: 32 | # Edge constructor 33 | def __init__(self, start_row, start_col, end_row, end_col): 34 | if start_row == end_row and start_col == end_col: 35 | raise ValueError("found pixel edge at", start_row, start_col) 36 | if start_row != end_row and start_col != end_col: 37 | raise ValueError("found diagonal edge from", start_row, start_col, "to", end_row, end_col) 38 | if start_row == end_row: 39 | self.direction = horiz 40 | self.start_row = start_row 41 | self.start_col = min(start_col, end_col) 42 | self.length = max(start_col, end_col) - self.start_col + 1 43 | else: 44 | self.direction = vert 45 | self.start_col = start_col 46 | self.start_row = min(start_row, end_row) 47 | self.length = max(start_row, end_row) - self.start_row + 1 48 | # Utility function to produce a list of pixel coordinates 49 | def to_pixels(self): 50 | pixels = [] 51 | if self.direction == horiz: 52 | for cur_col in range(self.start_col, self.start_col + self.length): 53 | pixels.append([self.start_row, cur_col]) 54 | else: 55 | for cur_row in range(self.start_row, self.start_row + self.length): 56 | pixels.append([cur_row, self.start_col]) 57 | return pixels 58 | 59 | # Class representing a node in a segmentation tree 60 | class SegmentNode: 61 | def __init__(self, t, b, l, r): 62 | self.t = t 63 | self.b = b 64 | self.l = l 65 | self.r = r 66 | self.children = [] 67 | 68 | def to_list(self): 69 | result = [[self.t, self.b, self.l, self.r]] 70 | for child in self.children: 71 | result.extend(child.to_list()) 72 | return result 73 | 74 | 75 | # Function for converting to greyscale by summing all channels 76 | def to_grayscale(img): 77 | return np.sum(img[:, :, :3], axis=2, dtype='float') 78 | 79 | # Perform convolution with Sobel filters and determine the probability that 80 | # each edge is locally significant 81 | def sobel(img): 82 | bw_img = to_grayscale(img) 83 | h = convolve(bw_img, h_filter) 84 | v = convolve(bw_img, v_filter) 85 | h, v = cdf(h, v) 86 | print(np.histogram(h, bins=20)) 87 | return h, v 88 | 89 | # Calculate CDF of gradient magnitudes 90 | def cdf(h, v): 91 | h, v = np.abs(np.array(h, dtype='float')), np.abs(np.array(v, dtype='float')) 92 | res_h, res_v = np.ones_like(h, dtype='float'), np.ones_like(v, dtype='float') 93 | nrows, ncols = len(h), len(v[0]) 94 | for i in range(0, nrows): 95 | print("cdf row", i) 96 | for j in range(0, ncols): 97 | bound_t = max(0, i - min_l) 98 | bound_b = min(ncols, i + min_l + 1) 99 | bound_l = max(0, j - min_l) 100 | bound_r = min(nrows, j + min_l + 1) 101 | local_t = h[bound_t:i, bound_l:bound_r] 102 | local_b = h[i+1:bound_b, bound_l:bound_r] 103 | local_l = v[bound_t:bound_b, bound_l:j] 104 | local_r = v[bound_t:bound_b, j+1:bound_r] 105 | flat_t, flat_b = local_t.flatten(), local_b.flatten() 106 | flat_l, flat_r = local_l.flatten(), local_r.flatten() 107 | count_t, count_b = float(len(flat_t) + 2), float(len(flat_b) + 2) 108 | count_l, count_r = float(len(flat_l) + 2), float(len(flat_r) + 2) 109 | target_h, target_v = h[i][j], v[i][j] 110 | 111 | lower_t = np.sum(norm.cdf(target_h, flat_t, 0.1)) 112 | lower_b = np.sum(norm.cdf(target_h, flat_b, 0.1)) 113 | lower_l = np.sum(norm.cdf(target_v, flat_l, 0.1)) 114 | lower_r = np.sum(norm.cdf(target_v, flat_r, 0.1)) 115 | #lower_t = np.sum(np.where(flat_t < target_h, 1.0, 0.0)) 116 | #lower_b = np.sum(np.where(flat_b < target_h, 1.0, 0.0)) 117 | #lower_l = np.sum(np.where(flat_l < target_v, 1.0, 0.0)) 118 | #lower_r = np.sum(np.where(flat_r < target_v, 1.0, 0.0)) 119 | sig_t = (lower_t + 0.5) / count_t # 1 - Pr(S_{x,y,s} | !E_{x,y}, P) 120 | sig_b = (lower_b + 0.5) / count_b 121 | sig_l = (lower_l + 0.5) / count_l 122 | sig_r = (lower_r + 0.5) / count_r 123 | sig_t = 1 - sig_t 124 | sig_b = 1 - sig_b 125 | sig_l = 1 - sig_l 126 | sig_r = 1 - sig_r 127 | res_t = (prior * (1 + sig_t - prior * sig_t)) / (prior + sig_t - prior * sig_t) 128 | res_b = (prior * (1 + sig_b - prior * sig_b)) / (prior + sig_b - prior * sig_b) 129 | res_l = (prior * (1 + sig_l - prior * sig_l)) / (prior + sig_l - prior * sig_l) 130 | res_r = (prior * (1 + sig_r - prior * sig_r)) / (prior + sig_r - prior * sig_r) 131 | res_h[i][j] = res_t + res_b - res_t * res_b # getting OR of two independent events 132 | res_v[i][j] = res_l + res_r - res_l * res_r 133 | # res_h[i][j], res_v[i][j] = max(sig_t, sig_b), max(sig_l, sig_r) 134 | return res_h, res_v 135 | 136 | # Function to segment a page (image stored in the specified file) 137 | def segment(filename): 138 | img = cv2.imread(filename) 139 | horiz, vert = sobel(img) 140 | global edge_list 141 | edge_list = [] 142 | segment_list = segment_rec(horiz, vert, 0, 0, len(horiz), 0, len(horiz[0])).to_list() 143 | # print(segment_list) 144 | return segment_list, img 145 | 146 | # Function to recursively segment a page 147 | def segment_rec(h, v, level, t, b, l, r, d=None, empty=False): 148 | self = SegmentNode(t, b, l, r) 149 | if level > 20 or b - t < (min_l * 2) or r - l < (min_l * 2): 150 | return self 151 | print(level, t, b, l, r, d) 152 | h_curr, v_curr = h[t:b, l:r], v[t:b, l:r].transpose() 153 | h_prob, v_prob = np.ones(len(h_curr) - min_l * 2, dtype='float'), np.ones(len(v_curr) - min_l * 2, dtype='float') 154 | if d is None or d == dir_h: 155 | for i, row in enumerate(h_curr): 156 | if i < min_l or i >= len(h_curr) - min_l: 157 | continue 158 | h_prob[i - min_l] = evaluate_line(row) 159 | #print(min(h_prob), max(h_prob)) 160 | if d is None or d == dir_v: 161 | for i, col in enumerate(v_curr): 162 | if i < min_l or i >= len(v_curr) - min_l: 163 | continue 164 | v_prob[i - min_l] = evaluate_line(col) 165 | #print(min(v_prob), max(v_prob)) 166 | h_pass, v_pass = [], [] 167 | h_max, v_max = 0, 0 168 | prev = 0 169 | if d is None or d == dir_h: 170 | for i in range(len(h_prob)): 171 | if i < prev: 172 | continue 173 | if h_prob[i] > threshold and h_prob[i] == np.max(h_prob[max(0, i - min_l):min(i + min_l, len(h_prob))]): 174 | h_pass.append(i + min_l) 175 | h_max = max(h_max, h_prob[i]) 176 | prev = i + min_l 177 | prev = 0 178 | if d is None or d == dir_v: 179 | for i in range(len(v_prob)): 180 | if i < prev: 181 | continue 182 | if v_prob[i] > threshold and v_prob[i] == np.max(v_prob[max(0, i - min_l):min(i + min_l, len(v_prob))]): 183 | v_pass.append(i + min_l) 184 | v_max = max(v_max, v_prob[i]) 185 | prev = i + min_l 186 | if d is None: 187 | d = dir_h if h_max > v_max else dir_v 188 | prev = 0 189 | if d == dir_h: 190 | if len(h_pass) == 0 and empty: 191 | return self 192 | for h_p in h_pass: 193 | start, end = prev + t, h_p + t 194 | self.children.append(segment_rec(h, v, level + 1, start, end, l, r, dir_v)) 195 | edge_list.append(Edge(end, l, end, r - 1)) 196 | prev = h_p 197 | self.children.append(segment_rec(h, v, level + 1, prev + t, b, l, r, dir_v, len(h_pass) == 0)) 198 | else: 199 | if len(v_pass) == 0 and empty: 200 | return self 201 | for v_p in v_pass: 202 | start, end = prev + l, v_p + l 203 | self.children.append(segment_rec(h, v, level + 1, t, b, start, end, dir_h)) 204 | edge_list.append(Edge(t, end, b - 1, end)) 205 | prev = v_p 206 | self.children.append(segment_rec(h, v, level + 1, t, b, prev + l, r, dir_h, len(v_pass) == 0)) 207 | return self 208 | 209 | # Function to estimate the probability that a line is semantically significant 210 | def evaluate_line(line): 211 | if len(line) < line_length: 212 | return monte_carlo_simulation(line) 213 | return evaluate_line(line[:math.ceil(len(line) / 2)]) * evaluate_line(line[math.ceil(len(line) / 2):]) 214 | 215 | # Function to use a Monte Carlo simulation to estimate the probability that a 216 | # minimum proportion of the pixels in a line are locally significant 217 | def monte_carlo_simulation(line): 218 | over = 0 219 | line_len = float(len(line)) 220 | for _ in range(mcs_rp): 221 | sim = np.array([1 if random.random() < p else 0 for p in line], dtype='float') 222 | total = np.sum(sim) / line_len 223 | #if total > mcs_prob: 224 | # print(total) 225 | over += 1.0 if (np.sum(sim) / line_len) > mcs_prob else 0.0 226 | #print(over) 227 | return over / float(mcs_rp) 228 | 229 | # Function to show edges in an image 230 | def _mark_edges(data, edges): 231 | for e in edges: 232 | for pixel in e.to_pixels(): 233 | if pixel[0] >= len(data) or pixel[1] >= len(data[pixel[0]]): 234 | continue 235 | data[pixel[0]][pixel[1]] = [255, 0, 0] 236 | return data 237 | 238 | edge_list = [] 239 | 240 | 241 | parser = argparse.ArgumentParser(description='Segmentation algorithm of Cormier et al.') 242 | parser.add_argument('--image', help="The screenshot of the web page", required=True) 243 | parser.add_argument('--id', help="The ID of the web page", required=True) 244 | parser.add_argument('--output', dest="output_directory", help="The output directory for the segmentation", required=True) 245 | parser.add_argument('--min-l', dest="min_l", type=int, default=min_l, help="The minimum size of a region (default: " + str(min_l) + ")") 246 | parser.add_argument('--line-length', dest="line_length", type=int, default=line_length, help="The maximum line segment length for recursion (default: " + str(line_length) + ")") 247 | args = parser.parse_args() 248 | 249 | min_l = args.min_l 250 | line_length = args.line_length 251 | 252 | try: 253 | outfile = open(args.output_directory + "/cormier.json", 'w') 254 | s_list, pic = segment(args.image) 255 | 256 | polygon_list = [] 257 | for segment in s_list: 258 | top = segment[0] 259 | bottom = segment[1] 260 | left = segment[2] 261 | right = segment[3] 262 | 263 | polygon_list.append([[[[left, top], [left, bottom], [right, bottom], [right, top], [left, top]]]]) 264 | 265 | out_obj = dict(height=pic.shape[0], width=pic.shape[1], id=args.id, segmentations=dict(cormier=polygon_list)) 266 | json.dump(out_obj, outfile) 267 | except FileNotFoundError: 268 | print("Unable to create file " + args.output_directory + "/cormier.json") 269 | -------------------------------------------------------------------------------- /algorithms/cormier/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | scipy 3 | opencv-python 4 | -------------------------------------------------------------------------------- /algorithms/heps/.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | webis-web-archiver.jar 3 | -------------------------------------------------------------------------------- /algorithms/heps/compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Make sure we are in the same directory as this file 4 | pushd $(dirname "${BASH_SOURCE[0]}") >/dev/null 2>&1 5 | 6 | if [ -e webis-web-archiver.jar ];then 7 | echo "Using existing webis-web-archiver.jar" 8 | else 9 | echo "Downloading JAR of the Webis Web Archiver for compilation" 10 | wget https://github.com/webis-de/webis-web-archiver/releases/download/0.1.0/webis-web-archiver.jar 11 | fi 12 | 13 | echo "Compiling script class src/main/java/HEPSScript.class" 14 | javac -cp "webis-web-archiver.jar:." \ 15 | --release 8 \ 16 | src/main/java/HEPSScript.java 17 | 18 | echo "Creating script JAR" 19 | jar cfM scripts/HEPSScript-1.0.0/HEPSScript.jar \ 20 | -C src/main/java HEPSScript.class 21 | 22 | -------------------------------------------------------------------------------- /algorithms/heps/heps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | script="HEPSScript" 4 | script_version=1.0.0 5 | scripts_directory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/scripts/ 6 | id=UNKNOWN 7 | keep=0 8 | 9 | OPTS=$(getopt --name $(basename $0) --options a:i:kr:u:o: --longoptions archive:,id:,keep,reproductionmode:,url:,output: -- $@) 10 | if [[ $? -ne 0 ]]; then 11 | exit 2 12 | fi 13 | eval set -- "$OPTS" 14 | 15 | while true;do 16 | case "$1" in 17 | -a|--archive) 18 | archive=$(readlink -f -- "$2") 19 | shift 2 20 | ;; 21 | -i|--id) 22 | id=$2 23 | shift 2 24 | ;; 25 | -k|--keep) 26 | keep=1 27 | shift 1 28 | ;; 29 | -r|--reproductionmode) 30 | mode="-$2" 31 | shift 2 32 | ;; 33 | -u|--url) 34 | url="$2" 35 | shift 2 36 | ;; 37 | -o|--output) 38 | output="$2" 39 | shift 2 40 | ;; 41 | --) 42 | break 43 | ;; 44 | esac 45 | done 46 | 47 | if [ \( -z "$archive" \) -o \( -z "$url" \) -o \( -z "$output" \) ];then 48 | echo "USAGE" 49 | echo " $0 [OPTIONS] --archive --url --output " 50 | echo "WHERE" 51 | echo " --archive " 52 | echo " Specifies the archive directory created by archive.sh" 53 | echo " --id " 54 | echo " Specifies the page ID (used in the output JSON)" 55 | echo " --keep" 56 | echo " Specifies to keep log files" 57 | echo " --url " 58 | echo " Specifies the start URL for the script" 59 | echo " --output " 60 | echo " Specifies the directory to which logs and the script output are written" 61 | echo " --reproductionmode [warcprox|pywb]" 62 | echo " Changes the program used for web page reproduction from a custom" 63 | echo " implementation to warcprox or pywb" 64 | exit 1 65 | fi 66 | 67 | mkdir -p $output # Creating directory is required to give web-archiver user permission to write 68 | output=$(readlink -f -- $output) 69 | 70 | maindir=$(readlink -f -- $(dirname $0)/..) 71 | 72 | is_in_docker_group=$(groups | sed 's/ /\n/g' | grep '^docker$' | wc -l) 73 | 74 | # Mounting /dev/shm is required for taking big screenshot in chrome 75 | # /warcs/ can not be mounted read-only for warcprox mode (which does, however, not change anything, but tests that it could write on startup) 76 | command="docker run --rm --user $(id -u) --env URL=\"$url\" --env DBUS_SESSION_BUS_ADDRESS=/dev/null --env MODE="reproduce$mode" --env SCRIPT=$script --env SCRIPT_VERSION=$script_version --volume $scripts_directory:/resources/scripts/:ro --volume $archive:/warcs/ --volume $output:/output/ --volume /dev/shm/:/dev/shm -a stdout -a stderr webis/web-archive-environment:1.2.1" 77 | if [ $is_in_docker_group -eq 0 ];then 78 | sudo $command 79 | else 80 | $command 81 | fi 82 | 83 | # Getting output segmentation (replacing ID) 84 | cat $output/script/heps.json \ 85 | | sed "s/TBFWID/$id/" \ 86 | | sed 's/"\[/[/' \ 87 | | sed 's/\]"/]/' \ 88 | > $output/heps.json 89 | rm -rf $output/script 90 | 91 | if [ $keep -eq 0 ];then 92 | # Removing logs 93 | rm -rf $output/logs 94 | fi 95 | 96 | -------------------------------------------------------------------------------- /algorithms/heps/scripts/HEPSScript-1.0.0/HEPSScript.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webis-de/ecir21-an-empirical-comparison-of-web-page-segmentation-algorithms/e9df2cc8baa757913c31a9281dc053eae4efeff0/algorithms/heps/scripts/HEPSScript-1.0.0/HEPSScript.jar -------------------------------------------------------------------------------- /algorithms/heps/scripts/HEPSScript-1.0.0/script.conf: -------------------------------------------------------------------------------- 1 | script = HEPSScript 2 | environment.name = de.webis.java 3 | environment.version = 1.0.0 4 | -------------------------------------------------------------------------------- /algorithms/heps/src/main/java/HEPSScript.java: -------------------------------------------------------------------------------- 1 | import de.webis.webarchive.common.Version; 2 | import de.webis.webarchive.environment.browsers.Browser; 3 | import de.webis.webarchive.environment.browsers.Windows; 4 | import de.webis.webarchive.environment.scripts.InteractionScript; 5 | import org.openqa.selenium.JavascriptExecutor; 6 | import org.openqa.selenium.WebDriver; 7 | 8 | import java.io.FileOutputStream; 9 | import java.io.IOException; 10 | import java.io.OutputStreamWriter; 11 | import java.io.Writer; 12 | import java.nio.file.Path; 13 | import java.util.Scanner; 14 | import java.util.logging.Logger; 15 | 16 | public class HEPSScript extends InteractionScript { 17 | ////////////////////////////////////////////////////////////////////////////// 18 | // LOGGING 19 | ////////////////////////////////////////////////////////////////////////////// 20 | 21 | private static final Logger LOG = 22 | Logger.getLogger(HEPSScript.class.getName()); 23 | 24 | ////////////////////////////////////////////////////////////////////////////// 25 | // MEMBERS 26 | ////////////////////////////////////////////////////////////////////////////// 27 | 28 | private final String hepsJs; 29 | 30 | ////////////////////////////////////////////////////////////////////////////// 31 | // CONSTRUCTORS 32 | ////////////////////////////////////////////////////////////////////////////// 33 | 34 | public HEPSScript(final Path scriptDirectory) 35 | throws IOException { 36 | super(scriptDirectory); 37 | 38 | LOG.info("Loading HEPS script"); 39 | this.hepsJs = new Scanner(scriptDirectory.resolve("heps.js")).useDelimiter("\\A").next() 40 | + "\nreturn window.HEPS.json;"; 41 | } 42 | 43 | ////////////////////////////////////////////////////////////////////////////// 44 | // FUNCTIONALITY 45 | ////////////////////////////////////////////////////////////////////////////// 46 | 47 | @Override 48 | protected void executeInteraction( 49 | final Browser browser, final String startUrl, final Path outputDirectory) 50 | throws Throwable { 51 | final WebDriver window = browser.openWindow(startUrl); 52 | this.scrollDown(browser, window); 53 | this.executeHeps(browser, window, outputDirectory); 54 | } 55 | 56 | protected void scrollDown(final Browser browser, final WebDriver window) { 57 | final long quietPeriodInSeconds = 3; 58 | final long waitTimeoutInSeconds = 10; 59 | browser.waitForQuiescence(quietPeriodInSeconds, waitTimeoutInSeconds); 60 | 61 | // Enough to reach "click for more"-button of google image search 62 | final int maxScrollings = 25; 63 | for (int scrollings = 0; scrollings < maxScrollings; ++scrollings) { 64 | final int scrollPosition = Windows.getScrollYPosition(window); 65 | final int scrollHeight = Windows.getScrollHeight(window); 66 | if (scrollPosition >= scrollHeight) { break; } 67 | 68 | LOG.info("Scrolling down " + (scrollings + 1) 69 | + " from " + scrollPosition + "/" + scrollHeight); 70 | Windows.scrollDownOneWindow(window); 71 | browser.waitForQuiescence(quietPeriodInSeconds, waitTimeoutInSeconds); 72 | } 73 | 74 | final int scrollPosition = Windows.getScrollYPosition(window); 75 | final int scrollHeight = Windows.getScrollHeight(window); 76 | LOG.info("Scrolled down to " + scrollPosition + "/" + scrollHeight); 77 | 78 | Windows.scrollToTop(window); 79 | LOG.info("Resize viewport height to " + scrollHeight); 80 | Windows.resizeViewportHeight(window, scrollHeight); 81 | browser.waitForQuiescence(quietPeriodInSeconds, waitTimeoutInSeconds); 82 | } 83 | 84 | protected void executeHeps( 85 | final Browser browser, final WebDriver window, final Path outputDirectory) 86 | throws Throwable { 87 | LOG.info("Executing HEPS"); 88 | JavascriptExecutor jsExecutor = (JavascriptExecutor) window; 89 | String json = (String) jsExecutor.executeScript(this.hepsJs); 90 | LOG.info("Writing result to " + outputDirectory.toString() + "/heps.json"); 91 | try (final Writer writer = new OutputStreamWriter(new FileOutputStream( 92 | outputDirectory.resolve("heps.json").toFile()), "UTF-8")) { 93 | writer.write(json); 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /algorithms/meier/README.md: -------------------------------------------------------------------------------- 1 | Meier et al. 2 | ============ 3 | 4 | Detailed instructions for the approach of Meier et al. 5 | 6 | Make sure your system fulfills the requirements as listed [in the main README](../../README.md#meier-et-al). 7 | 8 | Build the docker image 9 | ---------------------- 10 | You probably don't need to do this, as the image is hosted on [dockerhub](https://hub.docker.com/r/webis/meier17-web-page-segmentation) and Docker will automatically fetch it from there. 11 | ``` 12 | sudo docker build -f docker/Dockerfile . -t webis/meier17-web-page-segmentation:1.0.4 13 | ``` 14 | 15 | ### Re-creation 16 | If you want to re-create the data instead of downloading it, use these commands (the use of `optipng` is optional, but greatly reduces the size of the PNG files): 17 | 18 | Cropping/extending a `screenshot.png` to 4096px (`screenshot-4096px.png`): 19 | ``` 20 | convert -extent 1366x4096+0x0 -background white path/to/screenshot.png path/to/screenshot-4096px.png 21 | optipng path/to/screenshot-4096px.png 22 | ``` 23 | 24 | Creating the ground truth masks (`ground-truth-mask-4096px.png`): 25 | ``` 26 | Rscript path/to/cikm20-web-page-segmentation-revisited-evaluation-framework-and-dataset/src/main/r/plot-segmentation-mask.R --input path/to/ground-truth.json --height 4096 --output path/to/ground-truth-mask-4096px.png 27 | optipng path/to/ground-truth-mask-4096px.png 28 | ``` 29 | 30 | Creating the text masks (`text-mask-4096px.png`): 31 | ``` 32 | Rscript path/to/cikm20-web-page-segmentation-revisited-evaluation-framework-and-dataset/src/main/r/plot-text-mask.R --input path/to/nodes.csv --height 4096 --output path/to/text-mask-4096px.png 33 | optipng path/to//text-mask-4096px.png 34 | ``` 35 | 36 | Cropping/extending the ground truth (ground-truth-4096px.json): 37 | ``` 38 | Rscript path/to/cikm20-web-page-segmentation-revisited-evaluation-framework-and-dataset/src/main/r/crop-segmentations.R --input path/to/ground-truth.json --height 4096 --output path/to/ground-truth-4096px.json 39 | ``` 40 | 41 | 42 | Train the model 43 | --------------- 44 | You first need to [setup the directory structure](../../README.md#meier-et-al). 45 | 46 | For each fold number (0 to 9), you can train the corresponding model like this: 47 | ``` 48 | gpu=0 # The ID of the GPU to use 49 | fold=0 # The fold left for testing 50 | batch_size=16 # decrease for smaller GPUs, increase for larger ones 51 | epochs=100 # maximum number of epochs for training 52 | nvidia-docker run -it --rm -u $(id -u):$(id -g) --env NVIDIA_VISIBLE_DEVICES=$gpu -v ${PWD}/webis-webseg-20-meier/:/src/workspace/data -e KERAS_BACKEND=tensorflow webis/meier17-web-page-segmentation:1.0.4 python main.py --batch-size=$batch_size --epochs=$epochs --cross-val-fold=$fold 1> log$fold.txt 2>&1 53 | ``` 54 | 55 | -------------------------------------------------------------------------------- /algorithms/meier/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | #ARG cuda_version=9.0 2 | #ARG cudnn_version=7 3 | FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04 4 | 5 | # Install system packages 6 | RUN apt-get update && apt-get install -y --no-install-recommends \ 7 | bzip2 \ 8 | g++ \ 9 | git \ 10 | graphviz \ 11 | libgl1-mesa-glx \ 12 | libhdf5-dev \ 13 | openmpi-bin \ 14 | libpng12-0 \ 15 | libpng-dev \ 16 | wget \ 17 | software-properties-common && \ 18 | add-apt-repository -y ppa:cheah/imagemagick6-xenial && \ 19 | apt-get update && \ 20 | apt-get -y install imagemagick && \ 21 | rm -rf /var/lib/apt/lists/* 22 | 23 | # Install conda 24 | ENV CONDA_DIR /opt/conda 25 | ENV PATH $CONDA_DIR/bin:$PATH 26 | 27 | RUN wget --quiet --no-check-certificate https://repo.continuum.io/miniconda/Miniconda3-4.2.12-Linux-x86_64.sh && \ 28 | echo "c59b3dd3cad550ac7596e0d599b91e75d88826db132e4146030ef471bb434e9a *Miniconda3-4.2.12-Linux-x86_64.sh" | sha256sum -c - && \ 29 | /bin/bash /Miniconda3-4.2.12-Linux-x86_64.sh -f -b -p $CONDA_DIR && \ 30 | rm Miniconda3-4.2.12-Linux-x86_64.sh && \ 31 | echo export PATH=$CONDA_DIR/bin:'$PATH' > /etc/profile.d/conda.sh 32 | 33 | # Install Python packages and keras 34 | ENV NB_USER keras 35 | ENV NB_UID 1009 36 | 37 | RUN useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \ 38 | chown $NB_USER $CONDA_DIR -R && \ 39 | mkdir -p /src && \ 40 | chown $NB_USER /src 41 | 42 | USER $NB_USER 43 | 44 | ARG python_version=3.6 45 | 46 | RUN conda install -y python=${python_version} 47 | RUN pip install \ 48 | imageio==2.5.0 \ 49 | plotnine==0.6.0 \ 50 | cython==0.29.13 \ 51 | numpy==1.16.5 \ 52 | tensorflow-gpu==1.13.1 \ 53 | h5py==2.9.0 \ 54 | matplotlib==3.1.1 \ 55 | mkl==2019.0 \ 56 | nose==1.3.7 \ 57 | notebook==6.0.1 \ 58 | Pillow==6.1.0 \ 59 | pandas==0.25.1 \ 60 | pydot==1.4.1 \ 61 | pyyaml==5.1.2 \ 62 | scikit-learn==0.21.2 \ 63 | six==1.12.0 \ 64 | keras==2.2.5 \ 65 | Keras-Preprocessing==1.1.0 \ 66 | sklearn_pandas==1.8.0 && \ 67 | pip install bcolz && \ 68 | conda clean -yt && \ 69 | rm -r /home/$NB_USER/.cache 70 | 71 | ENV PYTHONPATH='/src/:$PYTHONPATH' 72 | 73 | WORKDIR /src/workspace/model 74 | 75 | COPY ./model /src/workspace/model 76 | -------------------------------------------------------------------------------- /algorithms/meier/model/callback_epoch_end.py: -------------------------------------------------------------------------------- 1 | ###################################################################################### 2 | # Custom Callback to save a sample mask after each epoch 3 | ###################################################################################### 4 | 5 | import numpy as np 6 | import keras 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | from PIL import Image 10 | 11 | import time 12 | 13 | import helper 14 | import plot_ranking 15 | import plot_all_kernels 16 | 17 | class PlotStuff(keras.callbacks.Callback): 18 | 19 | def on_train_begin(self, logs={}): 20 | self.i = 0 21 | self.x = [] 22 | self.losses = [] 23 | self.val_losses = [] 24 | self.acc = [] 25 | self.val_acc = [] 26 | self.fig = plt.figure() 27 | self.logs = [] 28 | 29 | def __init__(self, model, folder, test_X, test_y, ids): 30 | self.model_for_saving = model 31 | self.folder = folder 32 | self.test_X = test_X 33 | self.test_y = test_y 34 | self.ids = ids 35 | 36 | def on_epoch_end(self, epoch, logs={}): 37 | # print(self.model_for_saving.layers) 38 | global plt 39 | 40 | # Single Plot 41 | # helper.plotSingle(np, plt, Image, self, epoch, self.folder, logs) 42 | 43 | # Combined Plot 44 | G = gridspec.GridSpec(12, 3) 45 | inner = gridspec.GridSpecFromSubplotSpec(4,8, subplot_spec=G[5:8, :], wspace=0.01, hspace=0.05) 46 | # plt = helper.plotCombined(Image, plt, G, self, epoch, inner, np, self.folder, self.test_X) 47 | 48 | # Plot all Kernels 49 | plot_all_kernels.plot_kernels(plt, self, epoch) 50 | 51 | 52 | # Find best and worst accuracy 53 | # opti = keras.optimizers.Adam() 54 | # self.model_for_saving.compile(opti, loss='mean_squared_error', metrics=['acc']) # adam, adamax 55 | 56 | # accs = [] 57 | # for i in range(len(self.test_X)): 58 | # acc = self.model_for_saving.test_on_batch( 59 | # np.expand_dims(self.test_X[i,...],axis=0), # loss 60 | # np.expand_dims(self.test_y[i,...],axis=0) # accuracy 61 | # ) 62 | # accs.append((self.ids[i]['id_url'], self.ids[i]['id_assignment'], i, acc[1])) 63 | # accs.sort(key=lambda tup: tup[3]) 64 | 65 | # # Save List of accuracies 66 | # with open(self.folder + 'accuracy/ranking-epoch-'+str(epoch).zfill(10) +'.txt', 'w') as fp: 67 | # fp.write('\n'.join('%s %s %s %s' % x for x in accs)) 68 | 69 | # grid_h = gridspec.GridSpec(10, 4, wspace=0.01, hspace=0.05) 70 | 71 | # plot_ranking.plot_rows(epoch, "highest", gridspec, plt, self.folder, self.model, Image, self.test_X, self.test_y, accs, np) 72 | # plot_ranking.plot_rows(epoch, "lowest", gridspec, plt, self.folder, self.model, Image, self.test_X, self.test_y, accs, np) 73 | 74 | 75 | -------------------------------------------------------------------------------- /algorithms/meier/model/clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | folders="training validation test" 4 | inputs="annotation-boxes screenshots text-boxes" 5 | 6 | cd /src/workspace/data/input 7 | 8 | for folder in $folders; 9 | do 10 | for input in $inputs; 11 | do 12 | rm $folder/$input/cross-val-0/*; 13 | done; 14 | done; 15 | -------------------------------------------------------------------------------- /algorithms/meier/model/copy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | FOLD_FOLDER=${1%/} 4 | RESULT_FOLDER=${2%/} 5 | OUTPUT_FOLDER=${3%/} 6 | 7 | if [ $# -ne 3 ] 8 | then 9 | echo "Invalid number of parameters." 10 | echo "Usage:" "bash" "$0" "" "" "" 11 | exit 1 12 | fi 13 | 14 | if [ ! -d $OUTPUT_FOLDER ] 15 | then 16 | mkdir -p $OUTPUT_FOLDER 17 | fi 18 | 19 | echo "Copying folder" "$RESULT_FOLDER" "to" "$OUTPUT_FOLDER" 20 | k=0 21 | while IFS= read -r line; 22 | do 23 | cp ${RESULT_FOLDER}/`printf %06d $k`.png ${OUTPUT_FOLDER}/${line} 24 | let "k=k+1" 25 | done < <(ls -l ${FOLD_FOLDER}/*.png | sed 's/.*\///') 26 | -------------------------------------------------------------------------------- /algorithms/meier/model/custom_image_generator.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import random 3 | from keras.preprocessing.image import ImageDataGenerator 4 | 5 | def invert_colors(input_image): 6 | test = random.random() 7 | if test < 0.5: 8 | return input_image 9 | else: 10 | return 255 - input_image 11 | 12 | def generator_train(input_folder, output_folder, hyper): 13 | img_cols = 256 14 | img_rows = 768 15 | seed = 1 16 | 17 | args_flow_common = { 18 | 'target_size':(img_rows, img_cols), 19 | 'classes':['cross-val-' + str(hyper['cross_val_fold'])], 20 | 'class_mode':None, 21 | 'seed':seed, 22 | 'batch_size': hyper['batch_size'], 23 | 'color_mode':'grayscale' 24 | } 25 | 26 | args_gen_common = { 27 | 'horizontal_flip': True, 28 | 'zoom_range': 0.3 29 | } 30 | 31 | args_gen_input = { 32 | 'brightness_range': (0.5,1.0), 33 | 'channel_shift_range': 0.3 34 | } 35 | 36 | if hyper['no_augmentation']: 37 | args_gen_common = {} 38 | args_gen_input = {} 39 | 40 | if hyper['single_augmentation'] == "zoom": 41 | args_gen_common = { 'zoom_range': 0.3 } 42 | args_gen_input = {} 43 | 44 | if hyper['single_augmentation'] == "horizontal_flip": 45 | args_gen_common = { 'horizontal_flip': True } 46 | args_gen_input = {} 47 | 48 | if hyper['single_augmentation'] == "brightness_range": 49 | args_gen_common = {} 50 | args_gen_input = {'brightness_range': (0.5,1.0)} 51 | 52 | if hyper['single_augmentation'] == "channel_shift_range": 53 | args_gen_common = {} 54 | args_gen_input = {'channel_shift_range': 0.3} 55 | 56 | # Screenshots 57 | genX1 = ImageDataGenerator(**args_gen_common, **args_gen_input).flow_from_directory( 58 | input_folder + '/training/screenshots', 59 | **args_flow_common, 60 | #save_to_dir=output_folder + 'train-generator', 61 | #save_prefix='input-screenshot', 62 | #save_format='png' 63 | ) 64 | 65 | # Text Bounding Boxes 66 | genX2 = ImageDataGenerator(**args_gen_common, **args_gen_input).flow_from_directory( 67 | input_folder + '/training/text-boxes', 68 | **args_flow_common, 69 | #save_to_dir=output_folder + 'train-generator', 70 | #save_prefix='input-textbox', 71 | #save_format='png' 72 | ) 73 | 74 | # Annotations 75 | genY = ImageDataGenerator(**args_gen_common).flow_from_directory( 76 | input_folder + '/training/annotation-boxes', 77 | **args_flow_common 78 | ) 79 | 80 | while True: 81 | 82 | if hyper['no_text_boxes']: 83 | input_x = np.divide(genX1.next(),255) 84 | else: 85 | input_x = np.concatenate((np.divide(genX1.next(),255), np.divide(genX2.next(),255)), axis=3) 86 | 87 | output_y = np.divide(genY.next(), 255) # normalize! 88 | 89 | yield input_x, output_y 90 | -------------------------------------------------------------------------------- /algorithms/meier/model/custom_image_generator_test.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import random 3 | from keras.preprocessing.image import ImageDataGenerator 4 | 5 | def generator_test(input_folder, no_text_boxes, cross_val_fold): 6 | img_cols = 256 7 | img_rows = 768 8 | batchsize = 1 9 | seed = 1 10 | fold = 'cross-val-' + str(cross_val_fold) 11 | 12 | # Screenshots 13 | genX1 = ImageDataGenerator().flow_from_directory( 14 | input_folder + '/screenshots', 15 | target_size=(img_rows,img_cols), 16 | classes=[fold], 17 | class_mode=None, 18 | shuffle=False, 19 | batch_size=batchsize, 20 | color_mode='grayscale', 21 | #save_to_dir=output_path + 'validation-generator', 22 | #save_prefix='input-screenshot', 23 | #save_format='png' 24 | ) 25 | 26 | # Text Bounding Boxes 27 | genX2 = ImageDataGenerator().flow_from_directory( 28 | input_folder + '/text-boxes', 29 | target_size=(img_rows,img_cols), 30 | classes=[fold], 31 | class_mode=None, 32 | shuffle=False, 33 | batch_size=batchsize, 34 | color_mode='grayscale', 35 | #save_to_dir=output_path + 'validation-generator', 36 | #save_prefix='input-textbox', 37 | #save_format='png' 38 | ) 39 | 40 | count = 0 41 | 42 | while True: 43 | if(no_text_boxes): 44 | input_x = np.divide(genX1.next(),255) 45 | else: 46 | input_x = np.concatenate((np.divide(genX1.next(),255), np.divide(genX2.next(),255)), axis=3) 47 | 48 | yield input_x 49 | -------------------------------------------------------------------------------- /algorithms/meier/model/custom_image_generator_validation.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import random 3 | from keras.preprocessing.image import ImageDataGenerator 4 | 5 | def generator_val(input_folder, output_path, input_size, no_text_boxes, cross_val_fold): 6 | img_cols = 256 7 | img_rows = 768 8 | batchsize = 1 9 | seed = 1 10 | fold = 'cross-val-' + str(cross_val_fold) 11 | 12 | # Screenshots 13 | genX1 = ImageDataGenerator().flow_from_directory( 14 | input_folder + '/validation/screenshots', 15 | target_size=(img_rows,img_cols), 16 | classes=[fold], 17 | class_mode=None, 18 | seed=seed, 19 | batch_size=batchsize, 20 | color_mode='grayscale', 21 | #save_to_dir=output_path + 'validation-generator', 22 | #save_prefix='input-screenshot', 23 | #save_format='png' 24 | ) 25 | 26 | # Text Bounding Boxes 27 | genX2 = ImageDataGenerator().flow_from_directory( 28 | input_folder + '/validation/text-boxes', 29 | target_size=(img_rows, img_cols), 30 | classes=[fold], 31 | class_mode=None, 32 | seed=seed, 33 | batch_size=batchsize, 34 | color_mode='grayscale', 35 | #save_to_dir=output_path + 'validation-generator', 36 | #save_prefix='input-textbox', 37 | #save_format='png' 38 | ) 39 | 40 | # Annotations 41 | genY = ImageDataGenerator().flow_from_directory( 42 | input_folder + '/validation/annotation-boxes', 43 | target_size=(img_rows, img_cols), 44 | classes=[fold], 45 | class_mode=None, 46 | seed=seed, 47 | batch_size=batchsize, 48 | color_mode='grayscale') 49 | 50 | count = 0 51 | 52 | while True: 53 | if(no_text_boxes): 54 | input_x = np.divide(genX1.next(),255) 55 | else: 56 | input_x = np.concatenate((np.divide(genX1.next(),255), np.divide(genX2.next(),255)), axis=3) 57 | 58 | output_y = np.divide(genY.next(), 255) # normalize! 59 | yield input_x, output_y 60 | -------------------------------------------------------------------------------- /algorithms/meier/model/extract_rectangles.py: -------------------------------------------------------------------------------- 1 | import re 2 | import subprocess 3 | import sys 4 | import json 5 | 6 | if len(sys.argv) != 3: 7 | print("Invalid number of arguments.") 8 | print("Usage: python3 " + sys.argv[0] + " ") 9 | sys.exit(1) 10 | 11 | infile = sys.argv[1] 12 | filename = infile.rsplit('/', 1)[1].rsplit('.', 1)[0] 13 | outfilename = sys.argv[2] 14 | 15 | cmd = ('convert ' + infile + ' ' + 16 | '-threshold 35% ' 17 | '-define connected-components:verbose=true ' 18 | '-define connected-components:area-threshold=50 ' 19 | '-connected-components 1 ' 20 | + '/tmp/tmp.png') 21 | 22 | # Leave out first and last line, because they don't contain rectangle coordinates 23 | out = subprocess.check_output(cmd, shell=True).decode("utf-8").split('\n')[1:-1] 24 | 25 | re_coords = re.compile('\s*?\d+:\s*?(\d+)x(\d+)\+(\d+)\+(\d+).*srgb\((\d+),(\d+),(\d+)\)') 26 | 27 | rects = list() 28 | rect_str = '' 29 | for line in out: 30 | m = re_coords.match(line) 31 | 32 | r = int(m.group(5)) 33 | g = int(m.group(6)) 34 | b = int(m.group(7)) 35 | 36 | if r == 0 and g == 0 and b == 0: 37 | w = int(m.group(1)) 38 | h = int(m.group(2)) 39 | 40 | # Top left corner 41 | x1 = int(m.group(3)) 42 | y1 = int(m.group(4)) 43 | 44 | # Bottom right corner 45 | x2 = x1 + w 46 | y2 = y1 + h 47 | 48 | rects.append({'x1': x1, 'x2': x2, 'y1':y1, 'y2': y2}) 49 | 50 | try: 51 | outfile = open(outfilename, 'w') 52 | 53 | polygon_list = [] 54 | for rect in rects: 55 | polygon_list.append([[[[rect['x1'],rect['y1']], [rect['x1'],rect['y2']], [rect['x2'],rect['y2']], [rect['x2'],rect['y1']], [rect['x1'],rect['y1']]]]]) 56 | 57 | out_obj = dict(height=4096, width=1366, id=filename, segmentations=dict(meier=polygon_list)) 58 | json.dump(out_obj, outfile) 59 | except FileNotFoundError: 60 | print("Unable to create file " + filename + ".json") 61 | -------------------------------------------------------------------------------- /algorithms/meier/model/extract_rectangles.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | INPUT_FOLDER=${1%/} 4 | OUTPUT_FOLDER=${2%/} 5 | SRC_FOLDER=$(dirname $0) 6 | 7 | if [ $# -ne 2 ] 8 | then 9 | echo "Invalid number of arguments." 10 | echo "Usage:" "bash" "$0" "" "" 11 | exit 1 12 | fi 13 | 14 | if [ ! -d $OUTPUT_FOLDER ] 15 | then 16 | mkdir -p $OUTPUT_FOLDER 17 | fi 18 | 19 | for i in ${INPUT_FOLDER}/*.png 20 | do 21 | FILENAME=$(basename ${i}) 22 | OUTNAME=${FILENAME%.*} 23 | echo "Extracting bounding boxes for" "$i" "to" "${OUTPUT_FOLDER}/${OUTNAME}.json" 24 | python $SRC_FOLDER/extract_rectangles.py "$i" "${OUTPUT_FOLDER}/${OUTNAME}.json" 25 | done 26 | -------------------------------------------------------------------------------- /algorithms/meier/model/helper.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from PIL import Image 3 | from plotnine import * 4 | import pandas as pd 5 | 6 | ###################################################################################### 7 | # Filters 8 | ###################################################################################### 9 | 10 | def filter_no_rect(screens, rects, ids): 11 | 12 | to_remove = [] 13 | 14 | for i in range(rects.shape[0]): 15 | if not 1 in rects[i]: 16 | to_remove.append(i) 17 | 18 | keep_screens = np.delete(screens, to_remove, 0) 19 | keep_rects = np.delete(rects ,to_remove, 0) 20 | keep_ids = np.delete(ids, to_remove, 0) 21 | 22 | return keep_screens, keep_rects, keep_ids 23 | 24 | 25 | def filter_complete_rect(screens, rects, ids): 26 | 27 | to_remove = [] 28 | for i in range(rects.shape[0]): 29 | not_zero = np.count_nonzero(rects[i] == 1) 30 | if not_zero == 256 * 256: 31 | to_remove.append(i) 32 | 33 | keep_screens = np.delete(screens, to_remove, 0) 34 | keep_rects = np.delete(rects ,to_remove, 0) 35 | keep_ids = np.delete(ids, to_remove, 0) 36 | 37 | return keep_screens, keep_rects, keep_ids 38 | 39 | def coverage_ranking(screens, rects, ids): 40 | 41 | print(screens.shape) 42 | print(rects.shape) 43 | print(len(ids)) 44 | 45 | coverages = [] 46 | coverage_sum = 0 47 | bigger_95 = [] 48 | 49 | for i in range(rects.shape[0]): 50 | not_zero = np.count_nonzero(rects[i] == 1) / (256 * 256) * 100 51 | if not_zero >= 95: 52 | image = Image.fromarray(np.squeeze(rects[i] * 255).astype('uint8') ) 53 | image.save('test-bigger-95/' + str(i).zfill(10) + '-mask.png') 54 | 55 | if not_zero >= 99: 56 | np.savetxt('test-rects-txt/'+str(i).zfill(10) + '.txt', np.squeeze(rects[i])) 57 | coverage_sum += not_zero 58 | coverages.append(not_zero) 59 | 60 | coverages.sort() 61 | coverage_sum /= len(rects) 62 | 63 | return coverages, coverage_sum 64 | 65 | ###################################################################################### 66 | # Single Plot 67 | ###################################################################################### 68 | 69 | def plotSingle(np, plt, Image, self, epoch, folder, logs): 70 | 71 | # Filters 72 | layer_num = 2 73 | 74 | rows = len(self.model_for_saving.layers[layer_num].get_weights()[0]) # 5 75 | cols = len(self.model_for_saving.layers[layer_num].get_weights()[0][0]) # 5 76 | channels = len(self.model_for_saving.layers[layer_num].get_weights()[0][0][0]) # 1 77 | filters = len(self.model_for_saving.layers[layer_num].get_weights()[0][0][0][0]) # 32 78 | 79 | fig = plt.figure() 80 | fig.suptitle('Filters of 1st Conv2D Layer', fontsize=20) 81 | 82 | for filter in range(filters): 83 | exampleFilter = np.zeros((rows,cols)) 84 | for row in range(rows): 85 | for col in range(cols): 86 | exampleFilter[row, col] = self.model_for_saving.layers[layer_num].get_weights()[0][row][col][0][filter] 87 | 88 | exampleFilter = exampleFilter - exampleFilter.min() 89 | exampleFilter *= 255 / exampleFilter.max() 90 | 91 | #if filter == 0: 92 | # np.savetxt(folder + 'filters-txt/filter-epoch-' + str(epoch).zfill(8) +'-1.txt', exampleFilter.astype(int)) 93 | #if filter == 2: 94 | # np.savetxt(folder + 'filters-txt/filter-epoch-' + str(epoch).zfill(8) +'-3.txt', exampleFilter.astype(int)) 95 | #if filter == 4: 96 | # np.savetxt(folder + 'filters-txt/filter-epoch-' + str(epoch).zfill(8) +'-5.txt', exampleFilter.astype(int)) 97 | plt.subplot(4,8,filter+1) 98 | plt.rcParams["image.cmap"] = "Greys" 99 | plt.imshow(exampleFilter, vmin = 0, vmax = 255) 100 | plt.grid(False) 101 | frame1 = plt.gca() 102 | frame1.axes.get_xaxis().set_visible(False) 103 | frame1.axes.get_yaxis().set_visible(False) 104 | 105 | plt.savefig(folder + 'filters/filters-epoch' + str(epoch).zfill(8) +'.png') 106 | plt.close(fig) 107 | plt.close() 108 | 109 | # History 110 | plt.clf() 111 | self.logs.append(logs) 112 | self.x.append(self.i) 113 | self.losses.append(logs.get('loss')) 114 | self.val_losses.append(logs.get('val_loss')) 115 | self.acc.append(logs.get('acc')) 116 | self.val_acc.append(logs.get('val_acc')) 117 | 118 | self.precision.append(logs.get('precision')) 119 | self.val_precision.append(logs.get('val_precision')) 120 | self.recall.append(logs.get('recall')) 121 | self.val_recall.append(logs.get('val_recall')) 122 | 123 | self.bal_acc.append(logs.get('bal_acc')) 124 | self.val_bal_acc.append(logs.get('val_bal_acc')) 125 | self.mean_iou_gt.append(logs.get('mean_iou_gt')) 126 | self.mean_iou_pr.append(logs.get('mean_iou_pr')) 127 | 128 | self.iou.append(logs.get('iou')) 129 | self.val_iou.append(logs.get('val_iou')) 130 | 131 | 132 | self.i += 1 133 | 134 | print(logs) 135 | 136 | f, (ax1, ax2) = plt.subplots(1, 2, sharex=True, figsize=(15, 5)) 137 | 138 | ax1.set_yscale('log') 139 | ax1.plot(self.x, self.losses, label="loss") 140 | ax1.plot(self.x, self.val_losses, label="val_loss") 141 | ax1.legend() 142 | 143 | ax2.plot(self.x, self.acc, label="accuracy") 144 | ax2.plot(self.x, self.val_acc, label="validation accuracy") 145 | ax2.legend() 146 | 147 | plt.savefig(folder + 'history/history-epoch-' + str(epoch).zfill(8) +'.png') 148 | plt.close(fig) 149 | plt.close() 150 | 151 | df = { 152 | 'x': self.x, 153 | 'precision': self.precision, 154 | 'val_precision': self.val_precision, 155 | 'recall': self.recall, 156 | 'val_recall': self.val_recall, 157 | 'acc': self.acc, 158 | 'val_acc': self.val_acc, 159 | 'bal_acc': self.bal_acc, 160 | 'val_bal_acc': self.val_bal_acc, 161 | 'mean_iou_gt': self.mean_iou_gt, 162 | 'mean_iou_pr': self.mean_iou_pr, 163 | 'iou': self.iou, 164 | 'val_iou': self.val_iou 165 | #'loss': self.losses 166 | } 167 | 168 | d1 = pd.DataFrame(data=df) 169 | d = d1.melt(id_vars=['x'], value_vars=['acc', 'val_acc', 'bal_acc', 'val_bal_acc'], var_name='epoch', value_name='metric') 170 | 171 | p = (ggplot(d) 172 | + aes(x='x', y='metric',group='epoch',color='epoch') 173 | + scale_x_continuous(breaks=range(0,len(d1.loc[:,'x']))) 174 | + labs(color = "Metric Name") 175 | + xlab("Epoch") 176 | + ylab("Value") 177 | + geom_line(size=1) 178 | + theme_bw() 179 | + theme( 180 | #panel_border = element_blank(), 181 | panel_grid_major = element_blank(), 182 | panel_grid_minor = element_blank() 183 | #axis_line = element_line(colour = "black") 184 | ) 185 | 186 | ) 187 | p.save(filename=folder + 'history/history-accs-epoch-' + str(epoch).zfill(8) + '.png', verbose=False) 188 | 189 | # PRECISION AND RECALL 190 | 191 | d = pd.DataFrame(data=df) 192 | d = d.melt(id_vars=['x'], value_vars=['precision', 'val_precision', 'recall', 'val_recall'], var_name='epoch', value_name='metric') 193 | 194 | p = (ggplot(d) 195 | + aes(x='x', y='metric',group='epoch',color='epoch') 196 | + geom_line() 197 | #+ geom_hline(yintercept=0.4, linetype="dashed", color = "red") 198 | #+ geom_hline(yintercept=0.25, linetype="dashed", color = "cyan") 199 | + theme_bw() 200 | + theme( 201 | panel_grid_major = element_blank(), 202 | panel_grid_minor = element_blank() 203 | ) 204 | ) 205 | p.save(filename=folder + 'history/history-prre-epoch-' + str(epoch).zfill(8) + '.png', verbose=False) 206 | 207 | 208 | # IOU 209 | 210 | d = pd.DataFrame(data=df) 211 | d = d.melt(id_vars=['x'], value_vars=['mean_iou_gt', 'mean_iou_pr'], var_name='epoch', value_name='metric') 212 | 213 | p = (ggplot(d) 214 | + aes(x='x', y='metric',group='epoch',color='epoch') 215 | + geom_line() 216 | #+ geom_hline(yintercept=0.4, linetype="dashed", color = "red") 217 | #+ geom_hline(yintercept=0.25, linetype="dashed", color = "cyan") 218 | + theme_bw() 219 | + theme( 220 | panel_grid_major = element_blank(), 221 | panel_grid_minor = element_blank() 222 | ) 223 | ) 224 | p.save(filename=folder + 'history-iou/history-epoch-' + str(epoch).zfill(8) + '.png', verbose=False) 225 | 226 | 227 | # Write to csv 228 | d = pd.DataFrame(data=df) 229 | d.to_csv(folder + 'metrics-per-epoch/epoch-'+ str(epoch).zfill(8) + '.csv', index=False) 230 | 231 | 232 | 233 | ###################################################################################### 234 | # Combined Plot 235 | ###################################################################################### 236 | 237 | def plotCombined(Image, plt, G, self, epoch, inner, np, folder, test_X): 238 | # Mask 239 | num = 1109 240 | image = test_X[num] 241 | image = np.expand_dims(image, 0) 242 | image_plot = Image.fromarray(np.squeeze(test_X[num] * 255).astype('uint8') ) 243 | 244 | pred_test = self.model.predict(image) 245 | 246 | mask = pred_test[0] -pred_test[0].min() 247 | mask *= 255 / mask.max() 248 | mask = np.squeeze(mask.astype('uint8')) 249 | 250 | # End to End Output 251 | np.savetxt(folder + 'output-txt/output-epoch-' + str(epoch).zfill(8) +'.txt', np.squeeze(pred_test[0])) 252 | im_mask = Image.fromarray(mask) 253 | im_mask.save(folder + 'masks/mask-epoch-' + str(epoch).zfill(8) + '.png') 254 | 255 | multi_fig = plt.figure(figsize=(6, 10)) # TODO: move down (?) 256 | axes_1 = plt.subplot(G[:2, 0]) 257 | plt.imshow(image_plot, cmap="Greys", interpolation='nearest') 258 | axes_2 = plt.subplot(G[:2, 1]) 259 | plt.imshow(im_mask) 260 | axes_3 = plt.subplot(G[:2, 2]) 261 | blended = Image.blend(image_plot, im_mask, alpha=0.7) 262 | plt.imshow(blended) 263 | 264 | axes_4 = plt.subplot(G[2:5, :]) 265 | axes_4.set_title("Accuracy History") 266 | axes_4.plot(self.x, self.val_acc, label="validation accuracy") 267 | axes_4.plot(self.x, self.acc, label="training accuracy") 268 | axes_5 = plt.subplot(G[5:8, :]) 269 | axes_5.set_title("Filter visualization of Conv2D Layer 1_1") 270 | axes_5.axis('off') 271 | axes_5.text(0.0,-0.1, "Epoch: " + str(epoch), size=12, transform=axes_5.transAxes) 272 | 273 | axes_6 = plt.subplot(G[8:10, :]) 274 | 275 | # Filters of Conv2D Layer 6_2 276 | # TODO: 277 | weights_12_5 = self.model_for_saving.layers[60].get_weights() 278 | means = np.zeros(16) 279 | stds = np.zeros(16) 280 | np_weights_12_5 = np.array(weights_12_5[0]) 281 | print(np_weights_12_5.shape) 282 | for filter in range(len(weights_12_5[0][0][0][0])): 283 | #print(np_weights_12_5[...,filter]) 284 | means[filter] = np.mean(np_weights_12_5[...,filter].flatten()) 285 | stds[filter] = np.std(np_weights_12_5[...,filter].flatten()) 286 | #print(means) 287 | #print(stds) 288 | y = means 289 | x = range(16) 290 | e = stds 291 | 292 | axes_6.errorbar(x, y, e, linestyle='None', marker='^') 293 | axes_6.set_title("Filters of Conv2D Layer 6_2") 294 | 295 | # Filters 1st Conv2D Layer 296 | layer_num = 2 297 | 298 | rows = len(self.model_for_saving.layers[layer_num].get_weights()[0]) # 5 299 | cols = len(self.model_for_saving.layers[layer_num].get_weights()[0][0]) # 5 300 | channels = len(self.model_for_saving.layers[layer_num].get_weights()[0][0][0]) # 1 301 | filters = len(self.model_for_saving.layers[layer_num].get_weights()[0][0][0][0]) # 32 302 | 303 | for filter in range(filters): 304 | exampleFilter = np.zeros((rows,cols)) 305 | for row in range(rows): 306 | for col in range(cols): 307 | exampleFilter[row, col] = self.model_for_saving.layers[layer_num].get_weights()[0][row][col][0][filter] 308 | # TODO: Normalize filter because it has no bounds and we want an gray-scale image 309 | exampleFilter = (exampleFilter + 1) * 0.5 * 255 310 | 311 | # Distribute Subplots on rows of length 8 312 | ax = plt.Subplot(multi_fig, inner[filter // 8, filter % 8]) 313 | multi_fig.add_subplot(ax) 314 | plt.xticks(()) 315 | plt.yticks(()) 316 | plt.imshow(exampleFilter, vmin = 0, vmax = 255) 317 | 318 | plt.tight_layout() 319 | plt.savefig(folder + 'combined-plot/combined-plot-' + str(epoch).zfill(8) +'.png') 320 | plt.close(multi_fig) 321 | plt.close() 322 | return plt 323 | 324 | 325 | def plot_history(model, dest_folder): 326 | 327 | # summarize history for accuracy 328 | plt.subplot(121) 329 | plt.plot(history['acc']) 330 | plt.plot(history['val_acc']) 331 | plt.title('model accuracy') 332 | plt.ylabel('accuracy') 333 | plt.xlabel('epoch') 334 | plt.legend(['train', 'test'], loc='best') 335 | 336 | # summarize history for loss 337 | plt.subplot(122) 338 | plt.plot(history['loss']) 339 | plt.plot(history['val_loss']) 340 | plt.title('model loss') 341 | plt.ylabel('loss') 342 | plt.xlabel('epoch') 343 | plt.legend(['train', 'test'], loc='best') 344 | plt.show() 345 | plt.savefig(dest_folder + 'history-epoch-' + str(epoch).zfill(8) +'.png') 346 | plt.close(multi_fig) 347 | plt.close() 348 | 349 | 350 | -------------------------------------------------------------------------------- /algorithms/meier/model/id_list.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | def getIds(): 4 | file_path = '../../rectangles-fixed.json' 5 | 6 | ids = [] 7 | 8 | with open(file_path, 'r') as f: 9 | array = json.load(f) 10 | 11 | #for i in range(len(array)): 12 | # ids.append((array[i]['id_url'], array[i]['id_assignment'])) 13 | return array 14 | 15 | -------------------------------------------------------------------------------- /algorithms/meier/model/main.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import keras 4 | from keras import backend as K 5 | from keras.models import Model 6 | from keras.callbacks import ModelCheckpoint, EarlyStopping 7 | from keras.preprocessing import image 8 | from keras.utils import multi_gpu_model 9 | import tensorflow as tf 10 | from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img 11 | import random 12 | from PIL import Image 13 | import json 14 | import matplotlib.gridspec as gridspec 15 | import time 16 | from plotnine import * 17 | import pandas as pd 18 | import os 19 | import glob 20 | import re 21 | import json 22 | import random 23 | import argparse 24 | import sys # TODO: remove 25 | 26 | import meier2017 27 | import setup_folders 28 | import callback_epoch_end 29 | import id_list 30 | import helper 31 | import custom_image_generator 32 | import custom_image_generator_validation 33 | import plot_ranking_new 34 | import my_metrics 35 | 36 | 37 | # CLI Arguments 38 | parser = argparse.ArgumentParser() 39 | parser.add_argument("--random", action="store_true", help="random hyper parameters") 40 | parser.add_argument("--optimizer", type=str, help="set optimizer", choices=["SGD","Adam"]) 41 | parser.add_argument("--lr", type=float, help="set learning rate") 42 | parser.add_argument("--momentum", type=float, help="set momentum") 43 | parser.add_argument("--l2-lambda", type=float, help="set L2 lambda") 44 | parser.add_argument("--steps-train", type=int, help="set steps train") 45 | parser.add_argument("--steps-val", type=int, help="set steps validation") 46 | parser.add_argument("--batch-size", type=int, help="set batch size") 47 | parser.add_argument("--epochs", type=int, help="set number of epochs") 48 | parser.add_argument("--input-size", type=int, help="set input width and height in pixels") 49 | parser.add_argument("--no-text-boxes", action="store_true", help="deactivate multimodal input") 50 | parser.add_argument("--no-augmentation", action="store_true", help="deactivate input augmentation") 51 | parser.add_argument("--single-augmentation", type=str, help="select a single augmentation method", choices=["zoom","horizontal_flip", "brightness_range", "channel_shift_range"]) 52 | parser.add_argument("--cross-val-fold", type=int, help="which fold; k-fold cross validation (0,1, ..., 9)") 53 | args = parser.parse_args() 54 | 55 | output_folder = '../data/output' + str(getattr(args, 'cross_val_fold')) 56 | input_folder = '../data/input' 57 | 58 | first_results_folder = output_folder + '/results-' + '1'.zfill(7) + '/' 59 | folder = '' 60 | 61 | if not os.path.exists(first_results_folder): 62 | os.makedirs(first_results_folder) 63 | folder = first_results_folder 64 | else: 65 | output_folders = glob.glob(output_folder + '/results-*') 66 | output_folders.sort() 67 | output_folders_num = str(int(re.search('\d+', output_folders[-1])[0]) + 1) 68 | folder = output_folder + '/results-' + output_folders_num.zfill(7) + '/' 69 | 70 | setup_folders.create(folder) 71 | 72 | #sys.exit() 73 | 74 | 75 | # Standard Hyperparameters 76 | 77 | hyper = { 78 | 'optimizer': 'SGD', # SGD and Adam 79 | 'lr': 0.01, # meier2017 paper: 0.01 80 | 'momentum': 0.0, 81 | 'l2_lambda': 0.0001, # meier2017 paper: 0.0001 82 | 'steps_train': 50000, # 50000 83 | 'steps_val': 1581, # 1581 84 | 'batch_size': 16, 85 | 'epochs': 5, # 5 86 | 'input_size': 256, 87 | 'no_text_boxes': False, 88 | 'no_augmentation': False, 89 | 'cross_val_fold': 0, 90 | 'single_augmentation':"" 91 | } 92 | 93 | # Random Hyperparameters 94 | 95 | if args.random: 96 | hyper['optimizer'] = random.choice(['Adam', 'SGD']) 97 | hyper['lr'] = 10 ** random.uniform(-5,-1) 98 | hyper['l2_lambda'] = 10 ** random.uniform(-5,0) 99 | hyper['batch_size'] = random.choice([8, 12, 16, 24, 32]) 100 | 101 | if hyper['optimizer'] == 'SGD': 102 | hyper['momentum'] = 10 ** random.uniform(-0.3,0) # between ~0.5 and 1 103 | 104 | # Custom Hyperparameters 105 | 106 | for par in ['optimizer', 'lr', 'momentum', 'l2_lambda', 'steps_train', 'steps_val', 'batch_size', 'epochs', 'input_size', 'no_text_boxes', 'cross_val_fold', 'no_augmentation', 'single_augmentation']: 107 | val = getattr(args, par) 108 | if val is not None: 109 | hyper[par] = val 110 | 111 | if getattr(args, 'cross_val_fold') is not None: 112 | 113 | train_files_base = input_folder + '/training/annotation-boxes/' 114 | train_files = glob.glob(train_files_base + "cross-val-" + str(hyper["cross_val_fold"]) + "/*.png") 115 | train_files_num = len(train_files) 116 | 117 | val_files_base = input_folder + '/validation/annotation-boxes/' 118 | val_files = glob.glob(val_files_base + "cross-val-" + str(hyper["cross_val_fold"]) + "/*.png") 119 | val_files_num = len(val_files) 120 | 121 | if getattr(args, 'steps_train') is None: 122 | hyper['steps_train'] = train_files_num 123 | if getattr(args, 'steps_val') is None: 124 | hyper['steps_val'] = val_files_num 125 | 126 | with open(folder + 'hyperparameters.json', 'w') as outfile: 127 | json.dump(hyper, outfile, indent=2) 128 | 129 | 130 | # Model 131 | with tf.device("/gpu:0"): 132 | model_in, model_out = meier2017.model(hyper['l2_lambda'], hyper['input_size'], hyper['no_text_boxes']) 133 | 134 | model = Model(inputs=model_in, outputs=model_out) 135 | 136 | # model = multi_gpu_model(model, gpus=8) 137 | 138 | class printbatch(keras.callbacks.Callback): 139 | 140 | def __init__(self, model, folder, input_size): 141 | self.model_for_saving = model 142 | self.folder = folder 143 | self.input_size = input_size 144 | 145 | def on_train_begin(self, logs={}): 146 | self.i = 0 # counts the epochs 147 | self.x = [] # accumulates the epoch numbers 148 | self.losses = [] 149 | self.val_losses = [] 150 | self.acc = [] 151 | self.val_acc = [] 152 | self.precision = [] 153 | self.val_precision= [] 154 | self.recall = [] 155 | self.val_recall = [] 156 | self.bal_acc = [] 157 | self.val_bal_acc = [] 158 | self.fig = plt.figure() 159 | self.logs = [] 160 | self.mean_iou_gt = [] 161 | self.mean_iou_pr = [] 162 | self.iou = [] 163 | self.val_iou = [] 164 | 165 | def on_epoch_end(self, epoch, logs): 166 | 167 | if epoch > -1: 168 | 169 | # plot_ ranking before plotSingle because iou values get calculated in plot_ranking 170 | # plot_ranking_new.plot_rows(input_folder, folder, epoch, self, logs, hyper['steps_val'], hyper['no_text_boxes'], hyper['cross_val_fold']) 171 | 172 | # Accuracy Ranking 173 | # helper.plotSingle(np, plt, Image, self, epoch, self.folder, logs) 174 | 175 | # Save Model Weights 176 | self.model_for_saving.save_weights(self.folder + 'weights/epoch-' + str(epoch).zfill(8) + '-weights.h5') 177 | 178 | if hyper['optimizer'] == 'SGD': 179 | opti = keras.optimizers.SGD(lr=hyper['lr'], momentum=hyper['momentum'], nesterov=True) 180 | 181 | if hyper['optimizer'] == 'Adam': 182 | opti = keras.optimizers.Adam(lr=hyper['lr']) 183 | 184 | model.compile(opti, loss='binary_crossentropy', metrics=['acc', my_metrics.precision, my_metrics.recall, my_metrics.bal_acc, my_metrics.iou]) 185 | 186 | start = time.time() 187 | 188 | model.fit_generator(custom_image_generator.generator_train(input_folder, folder, hyper), 189 | steps_per_epoch= hyper['steps_train'] / hyper['batch_size'], 190 | callbacks=[printbatch(model, folder, hyper['input_size']), EarlyStopping(patience=10, verbose=1)], 191 | epochs=hyper['epochs'], 192 | verbose=1, 193 | validation_data=custom_image_generator_validation.generator_val(input_folder, folder, hyper['input_size'], hyper['no_text_boxes'], hyper['cross_val_fold']), 194 | validation_steps=hyper['steps_val'] 195 | ) 196 | 197 | end = time.time() 198 | training_duration = str(int((end - start) / 60)) 199 | 200 | # How long did the training take? 201 | #file = open(folder + "training-duration-minutes.txt","w") 202 | #file.write(training_duration) 203 | #file.close() 204 | 205 | exit() 206 | -------------------------------------------------------------------------------- /algorithms/meier/model/main_crossval.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import keras 4 | from keras import backend as K 5 | from keras.models import Model 6 | from keras.callbacks import ModelCheckpoint 7 | from keras.preprocessing import image 8 | from keras.utils import multi_gpu_model 9 | from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img 10 | import tensorflow as tf 11 | import random 12 | from PIL import Image 13 | import json 14 | import matplotlib.gridspec as gridspec 15 | import time 16 | from plotnine import * 17 | import pandas as pd 18 | import os 19 | import glob 20 | import re 21 | import json 22 | import random 23 | import argparse 24 | import sys # TODO: remove 25 | 26 | import meier2017 27 | import setup_folders 28 | import callback_epoch_end 29 | import id_list 30 | import helper 31 | import custom_image_generator 32 | import custom_image_generator_validation 33 | import plot_ranking_new 34 | import my_metrics 35 | 36 | 37 | # CLI Arguments 38 | parser = argparse.ArgumentParser() 39 | parser.add_argument("--random", action="store_true", help="random hyper parameters") 40 | parser.add_argument("--optimizer", type=str, help="set optimizer", choices=["SGD","Adam"]) 41 | parser.add_argument("--lr", type=float, help="set learning rate") 42 | parser.add_argument("--momentum", type=float, help="set momentum") 43 | parser.add_argument("--l2-lambda", type=float, help="set L2 lambda") 44 | parser.add_argument("--steps-train", type=int, help="set steps train") 45 | parser.add_argument("--steps-val", type=int, help="set steps validation") 46 | parser.add_argument("--batch-size", type=int, help="set batch size") 47 | parser.add_argument("--epochs", type=int, help="set number of epochs") 48 | parser.add_argument("--input-size", type=int, help="set input width and height in pixels") 49 | parser.add_argument("--no-text-boxes", action="store_true", help="deactivate multimodal input") 50 | parser.add_argument("--no-augmentation", action="store_true", help="deactivate input augmentation") 51 | parser.add_argument("--single-augmentation", type=str, help="select a single augmentation method", choices=["zoom","horizontal_flip", "brightness_range", "channel_shift_range"]) 52 | parser.add_argument("--cross-val-fold", type=int, help="how many folds for k-fold cross validation (e.g. 10: cross-val-{0,1, ..., 9}") 53 | args = parser.parse_args() 54 | 55 | output_folder = '../data/output' 56 | input_folder = '../data/input' 57 | 58 | first_results_folder = output_folder + '/results-' + '1'.zfill(7) + '/' 59 | folder = '' 60 | 61 | if not os.path.exists(first_results_folder): 62 | os.makedirs(first_results_folder) 63 | folder = first_results_folder 64 | else: 65 | output_folders = glob.glob(output_folder + '/results-*') 66 | output_folders.sort() 67 | output_folders_num = str(int(re.search('\d+', output_folders[-1])[0]) + 1) 68 | folder = output_folder + '/results-' + output_folders_num.zfill(7) + '/' 69 | 70 | setup_folders.create(folder) 71 | 72 | #sys.exit() 73 | 74 | 75 | # Standard Hyperparameters 76 | 77 | hyper = { 78 | 'optimizer': 'SGD', # SGD and Adam 79 | 'lr': 0.01, # meier2017 paper: 0.01 80 | 'momentum': 0.0, 81 | 'l2_lambda': 0.0001, # meier2017 paper: 0.0001 82 | 'steps_train': 50000, # 50000 83 | 'steps_val': 1581, # 1581 84 | 'batch_size': 16, 85 | 'epochs': 5, # 5 86 | 'input_size': 256, 87 | 'no_text_boxes': False, 88 | 'no_augmentation': False, 89 | 'cross_val_fold': 0, 90 | 'single_augmentation':"" 91 | } 92 | 93 | # Random Hyperparameters 94 | 95 | if args.random: 96 | hyper['optimizer'] = random.choice(['Adam', 'SGD']) 97 | hyper['lr'] = 10 ** random.uniform(-5,-1) 98 | hyper['l2_lambda'] = 10 ** random.uniform(-5,0) 99 | hyper['batch_size'] = random.choice([8, 12, 16, 24, 32]) 100 | 101 | if hyper['optimizer'] == 'SGD': 102 | hyper['momentum'] = 10 ** random.uniform(-0.3,0) # between ~0.5 and 1 103 | 104 | # Custom Hyperparameters 105 | 106 | for par in ['optimizer', 'lr', 'momentum', 'l2_lambda', 'steps_train', 'steps_val', 'batch_size', 'epochs', 'input_size', 'no_text_boxes', 'cross_val_fold', 'no_augmentation', 'single_augmentation']: 107 | val = getattr(args, par) 108 | if val is not None: 109 | hyper[par] = val 110 | 111 | # if getattr(args, 'cross_val_fold') is not None: 112 | # 113 | # train_files_base = input_folder + '/training/annotation-boxes/' 114 | # train_files = glob.glob(train_files_base + "cross-val-" + str(hyper["cross_val_fold"]) + "/*.png") 115 | # train_files_num = len(train_files) 116 | # 117 | # val_files_base = input_folder + '/validation/annotation-boxes/' 118 | # val_files = glob.glob(val_files_base + "cross-val-" + str(hyper["cross_val_fold"]) + "/*.png") 119 | # val_files_num = len(val_files) 120 | # 121 | # if getattr(args, 'steps_train') is None: 122 | # hyper['steps_train'] = train_files_num 123 | # if getattr(args, 'steps_val') is None: 124 | # hyper['steps_val'] = val_files_num 125 | 126 | with open(folder + 'hyperparameters.json', 'w') as outfile: 127 | json.dump(hyper, outfile, indent=2) 128 | 129 | 130 | # Model 131 | with tf.device("/gpu:0"): 132 | model_in, model_out = meier2017.model(hyper['l2_lambda'], hyper['input_size'], hyper['no_text_boxes']) 133 | model = Model(inputs=model_in, outputs=model_out) 134 | 135 | # model = multi_gpu_model(model, gpus=8) 136 | 137 | class printbatch(keras.callbacks.Callback): 138 | 139 | def __init__(self, model, folder, input_size): 140 | self.model_for_saving = model 141 | self.folder = folder 142 | self.input_size = input_size 143 | 144 | def on_train_begin(self, logs={}): 145 | self.i = 0 # counts the epochs 146 | self.x = [] # accumulates the epoch numbers 147 | self.losses = [] 148 | self.val_losses = [] 149 | self.acc = [] 150 | self.val_acc = [] 151 | self.precision = [] 152 | self.val_precision= [] 153 | self.recall = [] 154 | self.val_recall = [] 155 | self.bal_acc = [] 156 | self.val_bal_acc = [] 157 | self.fig = plt.figure() 158 | self.logs = [] 159 | self.mean_iou_gt = [] 160 | self.mean_iou_pr = [] 161 | self.iou = [] 162 | self.val_iou = [] 163 | 164 | def on_epoch_end(self, epoch, logs): 165 | 166 | if epoch > -1: 167 | 168 | # plot_ ranking before plotSingle because iou values get calculated in plot_ranking 169 | plot_ranking_new.plot_rows(input_folder, folder, epoch, self, logs, hyper['steps_val'], hyper['no_text_boxes'], hyper['cross_val_fold']) 170 | 171 | # Accuracy Ranking 172 | helper.plotSingle(np, plt, Image, self, epoch, self.folder, logs) 173 | 174 | # Save Model Weights 175 | self.model_for_saving.save_weights(self.folder + 'weights/epoch-' + str(epoch).zfill(8) + '-weights.h5') 176 | 177 | if hyper['optimizer'] == 'SGD': 178 | opti = keras.optimizers.SGD(lr=hyper['lr'], momentum=hyper['momentum'], nesterov=True) 179 | 180 | if hyper['optimizer'] == 'Adam': 181 | opti = keras.optimizers.Adam(lr=hyper['lr']) 182 | 183 | model.compile(opti, loss='binary_crossentropy', metrics=['acc', my_metrics.precision, my_metrics.recall, my_metrics.bal_acc, my_metrics.iou]) 184 | 185 | start = time.time() 186 | 187 | if getattr(args, 'cross_val_fold') is not None: 188 | folds = hyper['cross_val_fold'] 189 | print("Training and validating over " + str(folds) + " folds") 190 | for i in range(folds): 191 | print("Fold " + str(i)) 192 | hyper['cross_val_fold'] = i 193 | model.fit_generator(custom_image_generator.generator_train(input_folder, folder, hyper), 194 | steps_per_epoch= hyper['steps_train'] / hyper['batch_size'], 195 | #callbacks=[printbatch(model, folder, hyper['input_size'])], 196 | epochs=hyper['epochs'] / folds, 197 | verbose=1, 198 | validation_data=custom_image_generator_validation.generator_val(input_folder, folder, hyper['input_size'], hyper['no_text_boxes'], hyper['cross_val_fold']), 199 | validation_steps=hyper['steps_val'], 200 | initial_epoch=i * (hyper['epochs'] / folds) 201 | ) 202 | else: 203 | model.fit_generator(custom_image_generator.generator_train(input_folder, folder, hyper), 204 | steps_per_epoch= hyper['steps_train'] / hyper['batch_size'], 205 | #callbacks=[printbatch(model, folder, hyper['input_size'])], 206 | epochs=hyper['epochs'], 207 | verbose=1, 208 | validation_data=custom_image_generator_validation.generator_val(input_folder, folder, hyper['input_size'], hyper['no_text_boxes'], hyper['cross_val_fold']), 209 | validation_steps=hyper['steps_val'] 210 | ) 211 | 212 | end = time.time() 213 | training_duration = str(int((end - start) / 60)) 214 | 215 | # How long did the training take? 216 | #file = open(folder + "training-duration-minutes.txt","w") 217 | #file.write(training_duration) 218 | #file.close() 219 | 220 | exit() 221 | -------------------------------------------------------------------------------- /algorithms/meier/model/main_restart.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import keras 4 | from keras import backend as K 5 | from keras.models import Model 6 | from keras.callbacks import ModelCheckpoint, EarlyStopping 7 | from keras.preprocessing import image 8 | from keras.utils import multi_gpu_model 9 | import tensorflow as tf 10 | from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img 11 | import random 12 | from PIL import Image 13 | import json 14 | import matplotlib.gridspec as gridspec 15 | import time 16 | from plotnine import * 17 | import pandas as pd 18 | import os 19 | import glob 20 | import re 21 | import json 22 | import random 23 | import argparse 24 | import sys # TODO: remove 25 | 26 | import meier2017 27 | import setup_folders 28 | import callback_epoch_end 29 | import id_list 30 | import helper 31 | import custom_image_generator 32 | import custom_image_generator_validation 33 | import plot_ranking_new 34 | import my_metrics 35 | 36 | 37 | # CLI Arguments 38 | parser = argparse.ArgumentParser() 39 | parser.add_argument("--random", action="store_true", help="random hyper parameters") 40 | parser.add_argument("--optimizer", type=str, help="set optimizer", choices=["SGD","Adam"]) 41 | parser.add_argument("--lr", type=float, help="set learning rate") 42 | parser.add_argument("--momentum", type=float, help="set momentum") 43 | parser.add_argument("--l2-lambda", type=float, help="set L2 lambda") 44 | parser.add_argument("--steps-train", type=int, help="set steps train") 45 | parser.add_argument("--steps-val", type=int, help="set steps validation") 46 | parser.add_argument("--batch-size", type=int, help="set batch size") 47 | parser.add_argument("--epochs", type=int, help="set number of epochs") 48 | parser.add_argument("--input-size", type=int, help="set input width and height in pixels") 49 | parser.add_argument("--no-text-boxes", action="store_true", help="deactivate multimodal input") 50 | parser.add_argument("--no-augmentation", action="store_true", help="deactivate input augmentation") 51 | parser.add_argument("--single-augmentation", type=str, help="select a single augmentation method", choices=["zoom","horizontal_flip", "brightness_range", "channel_shift_range"]) 52 | parser.add_argument("--cross-val-fold", type=int, help="which fold; k-fold cross validation (0,1, ..., 9)") 53 | args = parser.parse_args() 54 | 55 | output_folder = '../data/output' 56 | input_folder = '../data/input' 57 | 58 | first_results_folder = output_folder + '/results-' + '1'.zfill(7) + '/' 59 | folder = '' 60 | 61 | if not os.path.exists(first_results_folder): 62 | os.makedirs(first_results_folder) 63 | folder = first_results_folder 64 | else: 65 | output_folders = glob.glob(output_folder + '/results-*') 66 | output_folders.sort() 67 | output_folders_num = str(int(re.search('\d+', output_folders[-1])[0]) + 1) 68 | folder = output_folder + '/results-' + output_folders_num.zfill(7) + '/' 69 | 70 | setup_folders.create(folder) 71 | 72 | #sys.exit() 73 | 74 | 75 | # Standard Hyperparameters 76 | 77 | hyper = { 78 | 'optimizer': 'SGD', # SGD and Adam 79 | 'lr': 0.01, # meier2017 paper: 0.01 80 | 'momentum': 0.0, 81 | 'l2_lambda': 0.0001, # meier2017 paper: 0.0001 82 | 'steps_train': 50000, # 50000 83 | 'steps_val': 1581, # 1581 84 | 'batch_size': 16, 85 | 'epochs': 5, # 5 86 | 'input_size': 256, 87 | 'no_text_boxes': False, 88 | 'no_augmentation': False, 89 | 'cross_val_fold': 0, 90 | 'single_augmentation':"" 91 | } 92 | 93 | # Random Hyperparameters 94 | 95 | if args.random: 96 | hyper['optimizer'] = random.choice(['Adam', 'SGD']) 97 | hyper['lr'] = 10 ** random.uniform(-5,-1) 98 | hyper['l2_lambda'] = 10 ** random.uniform(-5,0) 99 | hyper['batch_size'] = random.choice([8, 12, 16, 24, 32]) 100 | 101 | if hyper['optimizer'] == 'SGD': 102 | hyper['momentum'] = 10 ** random.uniform(-0.3,0) # between ~0.5 and 1 103 | 104 | # Custom Hyperparameters 105 | 106 | for par in ['optimizer', 'lr', 'momentum', 'l2_lambda', 'steps_train', 'steps_val', 'batch_size', 'epochs', 'input_size', 'no_text_boxes', 'cross_val_fold', 'no_augmentation', 'single_augmentation']: 107 | val = getattr(args, par) 108 | if val is not None: 109 | hyper[par] = val 110 | 111 | if getattr(args, 'cross_val_fold') is not None: 112 | 113 | train_files_base = input_folder + '/training/annotation-boxes/' 114 | train_files = glob.glob(train_files_base + "cross-val-" + str(hyper["cross_val_fold"]) + "/*.png") 115 | train_files_num = len(train_files) 116 | 117 | val_files_base = input_folder + '/validation/annotation-boxes/' 118 | val_files = glob.glob(val_files_base + "cross-val-" + str(hyper["cross_val_fold"]) + "/*.png") 119 | val_files_num = len(val_files) 120 | 121 | if getattr(args, 'steps_train') is None: 122 | hyper['steps_train'] = train_files_num 123 | if getattr(args, 'steps_val') is None: 124 | hyper['steps_val'] = val_files_num 125 | 126 | with open(folder + 'hyperparameters.json', 'w') as outfile: 127 | json.dump(hyper, outfile, indent=2) 128 | 129 | 130 | # Model 131 | with tf.device("/gpu:0"): 132 | model_in, model_out = meier2017.model(hyper['l2_lambda'], hyper['input_size'], hyper['no_text_boxes']) 133 | 134 | model = Model(inputs=model_in, outputs=model_out) 135 | 136 | # model = multi_gpu_model(model, gpus=8) 137 | 138 | class printbatch(keras.callbacks.Callback): 139 | 140 | def __init__(self, model, folder, input_size): 141 | self.model_for_saving = model 142 | self.folder = folder 143 | self.input_size = input_size 144 | 145 | def on_train_begin(self, logs={}): 146 | self.i = 0 # counts the epochs 147 | self.x = [] # accumulates the epoch numbers 148 | self.losses = [] 149 | self.val_losses = [] 150 | self.acc = [] 151 | self.val_acc = [] 152 | self.precision = [] 153 | self.val_precision= [] 154 | self.recall = [] 155 | self.val_recall = [] 156 | self.bal_acc = [] 157 | self.val_bal_acc = [] 158 | self.fig = plt.figure() 159 | self.logs = [] 160 | self.mean_iou_gt = [] 161 | self.mean_iou_pr = [] 162 | self.iou = [] 163 | self.val_iou = [] 164 | 165 | def on_epoch_end(self, epoch, logs): 166 | 167 | if epoch > -1: 168 | 169 | # plot_ ranking before plotSingle because iou values get calculated in plot_ranking 170 | # plot_ranking_new.plot_rows(input_folder, folder, epoch, self, logs, hyper['steps_val'], hyper['no_text_boxes'], hyper['cross_val_fold']) 171 | 172 | # Accuracy Ranking 173 | # helper.plotSingle(np, plt, Image, self, epoch, self.folder, logs) 174 | 175 | # Save Model Weights 176 | self.model_for_saving.save_weights(self.folder + 'weights/epoch-' + str(epoch).zfill(8) + '-weights.h5') 177 | 178 | if hyper['optimizer'] == 'SGD': 179 | opti = keras.optimizers.SGD(lr=hyper['lr'], momentum=hyper['momentum'], nesterov=True) 180 | 181 | if hyper['optimizer'] == 'Adam': 182 | opti = keras.optimizers.Adam(lr=hyper['lr']) 183 | 184 | model.compile(opti, loss='binary_crossentropy', metrics=['acc', my_metrics.precision, my_metrics.recall, my_metrics.bal_acc, my_metrics.iou]) 185 | 186 | model.load_weights("../data/output/results-0000130/weights/epoch-00000033-weights.h5") 187 | 188 | start = time.time() 189 | 190 | model.fit_generator(custom_image_generator.generator_train(input_folder, folder, hyper), 191 | steps_per_epoch= hyper['steps_train'] / hyper['batch_size'], 192 | callbacks=[printbatch(model, folder, hyper['input_size']), EarlyStopping(patience=10, verbose=1)], 193 | epochs=hyper['epochs'], 194 | verbose=1, 195 | validation_data=custom_image_generator_validation.generator_val(input_folder, folder, hyper['input_size'], hyper['no_text_boxes'], hyper['cross_val_fold']), 196 | validation_steps=hyper['steps_val'], 197 | initial_epoch=34 198 | ) 199 | 200 | end = time.time() 201 | training_duration = str(int((end - start) / 60)) 202 | 203 | # How long did the training take? 204 | #file = open(folder + "training-duration-minutes.txt","w") 205 | #file.write(training_duration) 206 | #file.close() 207 | 208 | exit() 209 | -------------------------------------------------------------------------------- /algorithms/meier/model/main_validate.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import keras 4 | from keras import backend as K 5 | from keras.models import Model 6 | from keras.callbacks import ModelCheckpoint, EarlyStopping 7 | from keras.preprocessing import image 8 | from keras.utils import multi_gpu_model 9 | import tensorflow as tf 10 | from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img 11 | import random 12 | from PIL import Image 13 | import json 14 | import matplotlib.gridspec as gridspec 15 | import time 16 | from plotnine import * 17 | import pandas as pd 18 | import os 19 | import glob 20 | import re 21 | import json 22 | import random 23 | import argparse 24 | import sys # TODO: remove 25 | 26 | import meier2017 27 | import setup_folders 28 | import callback_epoch_end 29 | import id_list 30 | import helper 31 | import custom_image_generator 32 | import custom_image_generator_validation 33 | import custom_image_generator_test 34 | import plot_ranking_new 35 | import my_metrics 36 | 37 | 38 | # CLI Arguments 39 | parser = argparse.ArgumentParser() 40 | parser.add_argument("--random", action="store_true", help="random hyper parameters") 41 | parser.add_argument("--optimizer", type=str, help="set optimizer", choices=["SGD","Adam"]) 42 | parser.add_argument("--lr", type=float, help="set learning rate") 43 | parser.add_argument("--momentum", type=float, help="set momentum") 44 | parser.add_argument("--l2-lambda", type=float, help="set L2 lambda") 45 | parser.add_argument("--steps-train", type=int, help="set steps train") 46 | parser.add_argument("--steps-val", type=int, help="set steps validation") 47 | parser.add_argument("--batch-size", type=int, help="set batch size") 48 | parser.add_argument("--epochs", type=int, help="set number of epochs") 49 | parser.add_argument("--input-size", type=int, help="set input width and height in pixels") 50 | parser.add_argument("--no-text-boxes", action="store_true", help="deactivate multimodal input") 51 | parser.add_argument("--no-augmentation", action="store_true", help="deactivate input augmentation") 52 | parser.add_argument("--single-augmentation", type=str, help="select a single augmentation method", choices=["zoom","horizontal_flip", "brightness_range", "channel_shift_range"]) 53 | parser.add_argument("--cross-val-fold", type=int, help="which fold; k-fold cross validation (0,1, ..., 9)") 54 | parser.add_argument("--weights-file", type=str, help="reload weights from previous training") 55 | args = parser.parse_args() 56 | 57 | output_folder = '../data/output' 58 | input_folder = '../data/input' 59 | 60 | first_results_folder = output_folder + '/results-' + '1'.zfill(7) + '/' 61 | folder = '' 62 | 63 | if not os.path.exists(first_results_folder): 64 | os.makedirs(first_results_folder) 65 | folder = first_results_folder 66 | else: 67 | output_folders = glob.glob(output_folder + '/results-*') 68 | output_folders.sort() 69 | output_folders_num = str(int(re.search('\d+', output_folders[-1])[0]) + 1) 70 | folder = output_folder + '/results-' + output_folders_num.zfill(7) + '/' 71 | 72 | setup_folders.create(folder) 73 | 74 | #sys.exit() 75 | 76 | 77 | # Standard Hyperparameters 78 | 79 | hyper = { 80 | 'optimizer': 'SGD', # SGD and Adam 81 | 'lr': 0.01, # meier2017 paper: 0.01 82 | 'momentum': 0.0, 83 | 'l2_lambda': 0.0001, # meier2017 paper: 0.0001 84 | 'steps_train': 50000, # 50000 85 | 'steps_val': 1581, # 1581 86 | 'batch_size': 16, 87 | 'epochs': 5, # 5 88 | 'input_size': 256, 89 | 'no_text_boxes': False, 90 | 'no_augmentation': False, 91 | 'cross_val_fold': 0, 92 | 'single_augmentation':"" 93 | } 94 | 95 | # Random Hyperparameters 96 | 97 | if args.random: 98 | hyper['optimizer'] = random.choice(['Adam', 'SGD']) 99 | hyper['lr'] = 10 ** random.uniform(-5,-1) 100 | hyper['l2_lambda'] = 10 ** random.uniform(-5,0) 101 | hyper['batch_size'] = random.choice([8, 12, 16, 24, 32]) 102 | 103 | if hyper['optimizer'] == 'SGD': 104 | hyper['momentum'] = 10 ** random.uniform(-0.3,0) # between ~0.5 and 1 105 | 106 | # Custom Hyperparameters 107 | 108 | for par in ['optimizer', 'lr', 'momentum', 'l2_lambda', 'steps_train', 'steps_val', 'batch_size', 'epochs', 'input_size', 'no_text_boxes', 'cross_val_fold', 'no_augmentation', 'single_augmentation']: 109 | val = getattr(args, par) 110 | if val is not None: 111 | hyper[par] = val 112 | 113 | if getattr(args, 'cross_val_fold') is not None: 114 | 115 | train_files_base = input_folder + '/training/annotation-boxes/' 116 | train_files = glob.glob(train_files_base + "cross-val-" + str(hyper["cross_val_fold"]) + "/*.png") 117 | train_files_num = len(train_files) 118 | 119 | val_files_base = input_folder + '/validation/annotation-boxes/' 120 | val_files = glob.glob(val_files_base + "cross-val-" + str(hyper["cross_val_fold"]) + "/*.png") 121 | val_files_num = len(val_files) 122 | 123 | if getattr(args, 'steps_train') is None: 124 | hyper['steps_train'] = train_files_num 125 | if getattr(args, 'steps_val') is None: 126 | hyper['steps_val'] = val_files_num 127 | 128 | with open(folder + 'hyperparameters.json', 'w') as outfile: 129 | json.dump(hyper, outfile, indent=2) 130 | 131 | 132 | # Model 133 | with tf.device("/gpu:0"): 134 | model_in, model_out = meier2017.model(hyper['l2_lambda'], hyper['input_size'], hyper['no_text_boxes']) 135 | 136 | model = Model(inputs=model_in, outputs=model_out) 137 | 138 | # model = multi_gpu_model(model, gpus=8) 139 | 140 | class printbatch(keras.callbacks.Callback): 141 | 142 | def __init__(self, model, folder, input_size): 143 | self.model_for_saving = model 144 | self.folder = folder 145 | self.input_size = input_size 146 | 147 | def on_train_begin(self, logs={}): 148 | self.i = 0 # counts the epochs 149 | self.x = [] # accumulates the epoch numbers 150 | self.losses = [] 151 | self.val_losses = [] 152 | self.acc = [] 153 | self.val_acc = [] 154 | self.precision = [] 155 | self.val_precision= [] 156 | self.recall = [] 157 | self.val_recall = [] 158 | self.bal_acc = [] 159 | self.val_bal_acc = [] 160 | self.fig = plt.figure() 161 | self.logs = [] 162 | self.mean_iou_gt = [] 163 | self.mean_iou_pr = [] 164 | self.iou = [] 165 | self.val_iou = [] 166 | 167 | def on_epoch_end(self, epoch, logs): 168 | 169 | if epoch > -1: 170 | 171 | # plot_ ranking before plotSingle because iou values get calculated in plot_ranking 172 | # plot_ranking_new.plot_rows(input_folder, folder, epoch, self, logs, hyper['steps_val'], hyper['no_text_boxes'], hyper['cross_val_fold']) 173 | 174 | # Accuracy Ranking 175 | # helper.plotSingle(np, plt, Image, self, epoch, self.folder, logs) 176 | 177 | # Save Model Weights 178 | self.model_for_saving.save_weights(self.folder + 'weights/epoch-' + str(epoch).zfill(8) + '-weights.h5') 179 | 180 | if hyper['optimizer'] == 'SGD': 181 | opti = keras.optimizers.SGD(lr=hyper['lr'], momentum=hyper['momentum'], nesterov=True) 182 | 183 | if hyper['optimizer'] == 'Adam': 184 | opti = keras.optimizers.Adam(lr=hyper['lr']) 185 | 186 | model.compile(opti, loss='binary_crossentropy', metrics=['acc', my_metrics.precision, my_metrics.recall, my_metrics.bal_acc, my_metrics.iou]) 187 | 188 | model.load_weights(getattr(args, 'weights_file')) 189 | 190 | start = time.time() 191 | 192 | # model.fit_generator(custom_image_generator.generator_train(input_folder, folder, hyper), 193 | # steps_per_epoch= hyper['steps_train'] / hyper['batch_size'], 194 | # callbacks=[printbatch(model, folder, hyper['input_size']), EarlyStopping(patience=10, verbose=1)], 195 | # epochs=hyper['epochs'], 196 | # verbose=1, 197 | # validation_data=custom_image_generator_validation.generator_val(input_folder, folder, hyper['input_size'], hyper['no_text_boxes'], hyper['cross_val_fold']), 198 | # validation_steps=hyper['steps_val'] 199 | # ) 200 | 201 | result = model.evaluate_generator(custom_image_generator_validation.generator_val(input_folder, folder, hyper['input_size'], hyper['no_text_boxes'], hyper['cross_val_fold']), callbacks=[printbatch(model, folder, hyper['input_size'])], steps=hyper['steps_val'], verbose=1) 202 | 203 | for i in range(len(result)): 204 | print(model.metrics_names[i] + " " + str(result[i])) 205 | 206 | end = time.time() 207 | training_duration = str(int((end - start) / 60)) 208 | 209 | # How long did the training take? 210 | #file = open(folder + "training-duration-minutes.txt","w") 211 | #file.write(training_duration) 212 | #file.close() 213 | 214 | exit() 215 | -------------------------------------------------------------------------------- /algorithms/meier/model/meier2017.py: -------------------------------------------------------------------------------- 1 | ###################################################################################### 2 | # Model meier2017 3 | ###################################################################################### 4 | 5 | #import keras 6 | #from keras.models import Sequential, Model 7 | from keras.layers import Add, Activation, Conv2D, Convolution2D, Conv2DTranspose, Dense, Dropout, Flatten, Input, MaxPooling2D, BatchNormalization, UpSampling2D 8 | #from keras.callbacks import ModelCheckpoint 9 | from keras.optimizers import SGD 10 | from keras.preprocessing import image 11 | from keras import regularizers 12 | 13 | def model(l2_lambda, input_size, no_text_boxes): 14 | 15 | 16 | # Multimodal input (screenshots and text boxes)? 17 | if no_text_boxes: 18 | input_num = 1 19 | else: 20 | input_num = 2 21 | 22 | # input = Input(shape=(256, 768, input_num)) # die 1 am ende weg!? 23 | input = Input(shape=(768, 256, input_num)) # rows, cols, input_num? 24 | 25 | # =========================================================================================== 26 | dropout1 = Dropout(0.3, input_shape=(256, 768, input_num))(input) # Step 1 27 | conv1_1 = Conv2D(filters=32, kernel_size=(5,5), strides=(1,1), padding="same", activation="relu", 28 | kernel_regularizer=regularizers.l2(l2_lambda))(dropout1) 29 | norm1_1 = BatchNormalization()(conv1_1) 30 | conv1_2 = Conv2D(filters=16, kernel_size=(3,3), strides=(1,1), padding="same", activation="relu", 31 | kernel_regularizer=regularizers.l2(l2_lambda))(norm1_1) 32 | 33 | norm1_2 = BatchNormalization()(conv1_2) 34 | # =========================================================================================== 35 | pool1 = MaxPooling2D(pool_size=(2,2), strides=(2,2), padding="valid")(norm1_2) # Step 2 36 | 37 | dropout2 = Dropout(0.3)(pool1) 38 | conv2_1 = Conv2D(filters=16, kernel_size=(5,5), strides=(1,1), padding="same", activation="relu", 39 | kernel_regularizer=regularizers.l2(l2_lambda))(dropout2) 40 | norm2_1 = BatchNormalization()(conv2_1) 41 | conv2_2 = Conv2D(filters=16, kernel_size=(3,3), strides=(1,1), padding="same", activation="relu", 42 | kernel_regularizer=regularizers.l2(l2_lambda))(norm2_1) 43 | norm2_2 = BatchNormalization()(conv2_2) 44 | # =========================================================================================== 45 | pool2 = MaxPooling2D(pool_size=(2,2), strides=(2,2), padding="valid")(norm2_2) # Step 3 46 | 47 | dropout3 = Dropout(0.5)(pool2) 48 | conv3_1 = Conv2D(filters=16, kernel_size=(3,3), strides=(1,1), padding="same", activation="relu", 49 | kernel_regularizer=regularizers.l2(l2_lambda))(dropout3) 50 | norm3_1 = BatchNormalization()(conv3_1) 51 | conv3_2 = Conv2D(filters=16, kernel_size=(3,3), strides=(1,1), padding="same", activation="relu", 52 | kernel_regularizer=regularizers.l2(l2_lambda))(norm3_1) 53 | norm3_2 = BatchNormalization()(conv3_2) 54 | # =========================================================================================== 55 | pool3 = MaxPooling2D(pool_size=(2,2), strides=(2,2), padding="valid")(norm3_2) # Step 4 56 | 57 | dropout4 = Dropout(0.5)(pool3) 58 | conv4_1 = Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), padding="same", activation="relu", 59 | kernel_regularizer=regularizers.l2(l2_lambda))(dropout4) 60 | norm4_1 = BatchNormalization()(conv4_1) 61 | conv4_2 = Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), padding="same", activation="relu", 62 | kernel_regularizer=regularizers.l2(l2_lambda))(norm4_1) 63 | norm4_2 = BatchNormalization()(conv4_2) 64 | # =========================================================================================== 65 | pool4 = MaxPooling2D(pool_size=(2,2), strides=(2,2), padding="valid")(norm4_2) # Step 5 66 | 67 | dropout5 = Dropout(0.5)(pool4) 68 | conv5_1 = Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), padding="same", activation="relu", 69 | kernel_regularizer=regularizers.l2(l2_lambda))(dropout5) 70 | norm5_1 = BatchNormalization()(conv5_1) 71 | conv5_2 = Conv2D(filters=128, kernel_size=(3,3), strides=(1,1), padding="same", activation="relu", 72 | kernel_regularizer=regularizers.l2(l2_lambda))(norm5_1) 73 | norm5_2 = BatchNormalization()(conv5_2) 74 | # =========================================================================================== 75 | pool5 = MaxPooling2D(pool_size=(2,2), strides=(2,2), padding="valid")(norm5_2) # Step 6 76 | 77 | dropout6 = Dropout(0.3)(pool5) 78 | conv6_1 = Conv2D(filters=128, kernel_size=(3,3), strides=(1,1), padding="same", activation="relu", 79 | kernel_regularizer=regularizers.l2(l2_lambda))(dropout6) 80 | norm6_1 = BatchNormalization()(conv6_1) 81 | conv6_2 = Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), padding="same", activation="relu", 82 | kernel_regularizer=regularizers.l2(l2_lambda))(norm6_1) 83 | norm6_2 = BatchNormalization()(conv6_2) 84 | # =========================================================================================== 85 | pool6 = MaxPooling2D(pool_size=(2,2), strides=(2,2), padding="valid")(norm6_2) # Step 7 86 | 87 | dropout7 = Dropout(0.3)(pool6) 88 | conv7_1 = Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), padding="same", activation="relu", 89 | kernel_regularizer=regularizers.l2(l2_lambda))(dropout7) 90 | norm7_1 = BatchNormalization()(conv7_1) 91 | # =========================================================================================== 92 | t_conv8_1 = Conv2DTranspose(filters=128, kernel_size=(2,2),strides=(2,2), activation="relu", 93 | kernel_regularizer=regularizers.l2(l2_lambda))(norm7_1) # Step 8 94 | #norm8_1 = BatchNormalization()(t_conv8_1) # WRONG! 95 | # add layers perform element-wise addition of their input and the given layer `p` 96 | add8 = Add()([pool5, t_conv8_1]) 97 | # =========================================================================================== 98 | t_conv9_1 = Conv2DTranspose(filters=64, kernel_size=(2,2),strides=(2,2), activation="relu", 99 | kernel_regularizer=regularizers.l2(l2_lambda))(add8) # Step 9 100 | # norm9_1 = BatchNormalization()(t_conv9_1) 101 | # =========================================================================================== 102 | add9 = Add()([pool4,t_conv9_1]) 103 | 104 | t_conv10_1 = Conv2DTranspose(filters=16, kernel_size=(2,2),strides=(2,2), activation="relu", 105 | kernel_regularizer=regularizers.l2(l2_lambda))(add9) # Step 10 106 | # norm10_1 = BatchNormalization()(t_conv10_1) 107 | # =========================================================================================== 108 | add10 = Add()([pool3,t_conv10_1]) 109 | # Step 11 110 | t_conv11_1 = Conv2DTranspose(filters=16, kernel_size=(4,4), strides=(4,4), padding="same", 111 | kernel_regularizer=regularizers.l2(l2_lambda))(add10) 112 | # norm11_1 = BatchNormalization()(t_conv11_1) 113 | # Step 12 114 | conv12_1 = Conv2D(filters=32, kernel_size=(5,5), strides=(1,1), padding="same", activation="relu", 115 | kernel_regularizer=regularizers.l2(l2_lambda))(t_conv11_1) 116 | 117 | norm12_1 = BatchNormalization()(conv12_1) 118 | conv12_2 = Conv2D(filters=32, kernel_size=(5,5), strides=(1,1), padding="same", activation="relu", 119 | kernel_regularizer=regularizers.l2(l2_lambda))(norm12_1) 120 | norm12_2 = BatchNormalization()(conv12_2) 121 | conv12_3_s = Conv2D(filters=8, kernel_size=(1,1), strides=(1,1), padding="same", activation="sigmoid", 122 | kernel_regularizer=regularizers.l2(l2_lambda))(norm12_2) 123 | # =========================================================================================== 124 | # norm12_3 = BatchNormalization()(conv12_3_s) # WRONG! 125 | dropout12 = Dropout(0.3)(conv12_3_s) 126 | conv12_4 = Conv2D(filters=32, kernel_size=(5,5), strides=(1,1), padding="same", activation="relu", 127 | kernel_regularizer=regularizers.l2(l2_lambda))(dropout12) 128 | norm12_4 = BatchNormalization()(conv12_4) 129 | conv12_5 = Conv2D(filters=16, kernel_size=(3,3), strides=(1,1), padding="same", activation="relu", 130 | kernel_regularizer=regularizers.l2(l2_lambda))(norm12_4) 131 | norm12_5 = BatchNormalization()(conv12_5) 132 | 133 | # Step 13 134 | conv13_1_s = Conv2D(filters=1, kernel_size=(1,1), strides=(1,1), padding="same", activation="sigmoid")(norm12_5) 135 | #norm13_1 = BatchNormalization()(conv13_1_s) # WRONG! 136 | # =========================================================================================== 137 | 138 | upscale13 = UpSampling2D(size=(2,2), interpolation='bilinear')(conv13_1_s) 139 | 140 | return input, upscale13 141 | -------------------------------------------------------------------------------- /algorithms/meier/model/my_metrics.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | 3 | def precision(y_true, y_pred): 4 | 5 | true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) 6 | predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1))) 7 | precision = true_positives / (predicted_positives + K.epsilon()) 8 | return precision 9 | 10 | def recall(y_true, y_pred): 11 | 12 | true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) 13 | possible_positives = K.sum(K.round(K.clip(y_true, 0, 1))) 14 | recall = true_positives / (possible_positives + K.epsilon()) 15 | return recall 16 | 17 | def bal_acc(y_true, y_pred): 18 | 19 | y_pred_pos = K.round(K.clip(y_pred, 0, 1)) 20 | y_pred_neg = 1 - y_pred_pos 21 | 22 | y_pos = K.round(K.clip(y_true, 0, 1)) 23 | y_neg = 1 - y_pos 24 | 25 | tp = K.sum(y_pos * y_pred_pos) 26 | tn = K.sum(y_neg * y_pred_neg) 27 | 28 | fp = K.sum(y_neg * y_pred_pos) 29 | fn = K.sum(y_pos * y_pred_neg) 30 | 31 | recall = tp / (tp + fn + K.epsilon()) 32 | specificity = tn / (tn + fp + K.epsilon()) 33 | return (recall + specificity) / 2 34 | 35 | def iou(y_true, y_pred): 36 | y_pred_pos = K.round(K.clip(y_pred, 0, 1)) 37 | y_pred_neg = 1 - y_pred_pos 38 | 39 | y_pos = K.round(K.clip(y_true, 0, 1)) 40 | y_neg = 1 - y_pos 41 | 42 | tp = K.sum(y_pos * y_pred_pos) 43 | tn = K.sum(y_neg * y_pred_neg) 44 | 45 | fp = K.sum(y_neg * y_pred_pos) 46 | fn = K.sum(y_pos * y_pred_neg) 47 | 48 | return tp / (tp + fp + fn) 49 | 50 | 51 | def debug_tp(y_true, y_pred): 52 | y_pred_pos = K.round(K.clip(y_pred, 0, 1)) 53 | y_pred_neg = 1 - y_pred_pos 54 | 55 | y_pos = K.round(K.clip(y_true, 0, 1)) 56 | y_neg = 1 - y_pos 57 | 58 | tp = K.sum(y_pos * y_pred_pos) 59 | tn = K.sum(y_neg * y_pred_neg) 60 | 61 | fp = K.sum(y_neg * y_pred_pos) 62 | fn = K.sum(y_pos * y_pred_neg) 63 | return tp 64 | 65 | def debug_tn(y_true, y_pred): 66 | y_pred_pos = K.round(K.clip(y_pred, 0, 1)) 67 | y_pred_neg = 1 - y_pred_pos 68 | 69 | y_pos = K.round(K.clip(y_true, 0, 1)) 70 | y_neg = 1 - y_pos 71 | 72 | tp = K.sum(y_pos * y_pred_pos) 73 | tn = K.sum(y_neg * y_pred_neg) 74 | 75 | fp = K.sum(y_neg * y_pred_pos) 76 | fn = K.sum(y_pos * y_pred_neg) 77 | return tn 78 | 79 | def debug_fp(y_true, y_pred): 80 | y_pred_pos = K.round(K.clip(y_pred, 0, 1)) 81 | y_pred_neg = 1 - y_pred_pos 82 | 83 | y_pos = K.round(K.clip(y_true, 0, 1)) 84 | y_neg = 1 - y_pos 85 | 86 | tp = K.sum(y_pos * y_pred_pos) 87 | tn = K.sum(y_neg * y_pred_neg) 88 | 89 | fp = K.sum(y_neg * y_pred_pos) 90 | fn = K.sum(y_pos * y_pred_neg) 91 | return fp 92 | 93 | def debug_fn(y_true, y_pred): 94 | y_pred_pos = K.round(K.clip(y_pred, 0, 1)) 95 | y_pred_neg = 1 - y_pred_pos 96 | 97 | y_pos = K.round(K.clip(y_true, 0, 1)) 98 | y_neg = 1 - y_pos 99 | 100 | tp = K.sum(y_pos * y_pred_pos) 101 | tn = K.sum(y_neg * y_pred_neg) 102 | 103 | fp = K.sum(y_neg * y_pred_pos) 104 | fn = K.sum(y_pos * y_pred_neg) 105 | return fn 106 | 107 | def debug_nd(y_true, y_pred): 108 | return K.ndim(y_true) 109 | -------------------------------------------------------------------------------- /algorithms/meier/model/plot_all_kernels.py: -------------------------------------------------------------------------------- 1 | # import plotnine 2 | import numpy as np 3 | import matplotlib.gridspec as gridspec 4 | 5 | def plot_layer(plt, f, G, weights, pos, pos_to): 6 | 7 | rows = len(weights[0]) # 5 8 | cols = len(weights[0][0]) # 5 9 | channels = len(weights[0][0][0]) # 1 10 | filters = len(weights[0][0][0][0]) # 32 11 | 12 | if filters == 16: 13 | row_length = 8 14 | inner = gridspec.GridSpecFromSubplotSpec(int(filters/row_length),row_length, subplot_spec=G[pos, 0], wspace=0.01, hspace=0.05) 15 | elif filters == 32: 16 | row_length = 16 17 | inner = gridspec.GridSpecFromSubplotSpec(int(filters/row_length),row_length, subplot_spec=G[pos, 0], wspace=0.01, hspace=0.05) 18 | elif filters == 64: 19 | row_length = 16 20 | inner = gridspec.GridSpecFromSubplotSpec(int(filters/row_length),row_length, subplot_spec=G[pos:pos_to, 0], wspace=0.01, hspace=0.05) 21 | 22 | 23 | # sub = plt.figure(figsize =(int(filters/8),8)) 24 | 25 | for filter in range(filters): 26 | exampleFilter = np.zeros((rows,cols)) 27 | for row in range(rows): 28 | for col in range(cols): 29 | exampleFilter[row, col] = weights[0][row][col][0][filter] 30 | # TODO: Normalize filter because it has no bounds and we want an gray-scale image 31 | exampleFilter = (exampleFilter + 1) * 0.5 * 255 32 | 33 | sub_ax = plt.Subplot(f, inner[filter // row_length, filter % row_length]) 34 | # sub_ax.set_title('test') 35 | f.add_subplot(sub_ax) 36 | plt.xticks(()) 37 | plt.yticks(()) 38 | plt.imshow(exampleFilter, vmin = 0, vmax = 255) 39 | 40 | def plot_kernels(plt, self, epoch): 41 | 42 | # (plotnine.ggplot(self.model.model_for_saving) 43 | # + aes(layers[2].get_weights() 44 | # ) 45 | plt.gray() 46 | f = plt.figure() 47 | G = gridspec.GridSpec(10,1, figure=f) 48 | 49 | # First Kernels 50 | axis_1 = plt.subplot(G[0,0]) 51 | axis_1.set_title('test', fontdict={'fontsize': 8}) 52 | axis_1.axis('off') 53 | plot_layer(plt, f, G, self.model_for_saving.layers[2].get_weights(), 0, -1) 54 | plot_layer(plt, f, G, self.model_for_saving.layers[4].get_weights(), 1, -1) 55 | 56 | plot_layer(plt, f, G, self.model_for_saving.layers[8].get_weights(), 2, -1) 57 | plot_layer(plt, f, G, self.model_for_saving.layers[10].get_weights(), 3, -1) 58 | 59 | plot_layer(plt, f, G, self.model_for_saving.layers[14].get_weights(), 4, -1) 60 | plot_layer(plt, f, G, self.model_for_saving.layers[16].get_weights(), 5, -1) 61 | 62 | plot_layer(plt, f, G, self.model_for_saving.layers[20].get_weights(), 6, 8) 63 | plot_layer(plt, f, G, self.model_for_saving.layers[22].get_weights(), 8, 10) 64 | 65 | plt.tight_layout() 66 | plt.savefig(self.folder + 'all-kernels/all-kernels-' + str(epoch).zfill(8) +'.png') 67 | plt.close(f) 68 | plt.close() 69 | -------------------------------------------------------------------------------- /algorithms/meier/model/plot_ranking.py: -------------------------------------------------------------------------------- 1 | ###################################################################################### 2 | # Plot the n best or worst outputs 3 | ###################################################################################### 4 | 5 | def plot_rows (epoch, pos, gridspec, plt, folder, model, Image, test_X, test_y, accs, np): 6 | grid_h = gridspec.GridSpec(10, 4, wspace=0.01, hspace=0.05) 7 | 8 | for i in range(1,11): 9 | 10 | if pos == "highest": 11 | j = -i 12 | elif pos == "lowest": 13 | j = i 14 | 15 | # Worker Annotation Rectangles 16 | axes_1 = plt.subplot(grid_h[i-1,0]) 17 | axes_1.axis('off') 18 | mask_worker = np.squeeze(test_y[accs[j][2]] * 255).astype('uint8') 19 | mask_worker = Image.fromarray(mask_worker) 20 | axes_1.imshow(mask_worker) 21 | 22 | # Original Screenshot 23 | axes_2 = plt.subplot(grid_h[i-1,1]) 24 | axes_2.axis('off') 25 | original_image = np.squeeze(test_X[accs[j][2]] * 255).astype('uint8') 26 | original_image = Image.fromarray(original_image) 27 | plt.imshow(original_image) 28 | 29 | # Network Output 30 | axes_3 = plt.subplot(grid_h[i-1,2]) 31 | axes_3.axis('off') 32 | # TODO: normalize over all masks instead of per mask 33 | # TODO: color bar 34 | prediction = model.predict(np.expand_dims(test_X[accs[j][2]], 0)) 35 | prediction[0] = prediction[0] - prediction[0].min() 36 | prediction[0] *= 255 / prediction[0].max() 37 | prediction_image = np.squeeze(prediction[0].astype('uint8')) 38 | prediction_image = Image.fromarray(prediction_image) 39 | axes_3.imshow(prediction_image) 40 | 41 | # Network Output blended on Original Screenshot 42 | axes_4 = plt.subplot(grid_h[i-1,3]) 43 | axes_4.axis('off') 44 | blended_image = Image.blend(original_image, prediction_image, alpha=0.7) 45 | axes_4.imshow(blended_image) 46 | 47 | #plt.tight_layout() 48 | plt.savefig(folder + "accuracy/" + pos + "-10-epoch-" + str(epoch).zfill(10) + ".png") 49 | plt.close() -------------------------------------------------------------------------------- /algorithms/meier/model/plot_ranking_new.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import matplotlib.image as mpimg 4 | import keras 5 | from keras.models import Model 6 | from keras.preprocessing import image 7 | from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img 8 | import random 9 | from PIL import Image 10 | import json 11 | import matplotlib.gridspec as gridspec 12 | import time 13 | import pandas as pd 14 | import os 15 | import subprocess 16 | import re 17 | import json 18 | 19 | def iou_single(rect_gt, rect_pr): 20 | # determine the (x, y)-coordinates of the intersection rectangle 21 | x1 = max(rect_gt['x1'], rect_pr['x1']) 22 | y1 = max(rect_gt['y1'], rect_pr['y1']) 23 | x2 = min(rect_gt['x2'], rect_pr['x2']) 24 | y2 = min(rect_gt['y2'], rect_pr['y2']) 25 | 26 | interArea = max(0, x2 - x1) * max(0, y2 - y1) 27 | 28 | if interArea > 0: 29 | 30 | boxAArea = (rect_gt['x2'] - rect_gt['x1']) * (rect_gt['y2'] - rect_gt['y1']) 31 | boxBArea = (rect_pr['x2'] - rect_pr['x1']) * (rect_pr['y2'] - rect_pr['y1']) 32 | 33 | iou = interArea / (boxAArea + boxBArea - interArea) 34 | 35 | if interArea == boxBArea and boxAArea == boxBArea: 36 | iou = 1 37 | 38 | elif interArea == 0: 39 | iou = 0 40 | 41 | return iou 42 | 43 | def iou_slice(rects_pr, filename): 44 | 45 | with open('rectangles-per-slice.json') as json_file: 46 | data = json.load(json_file) 47 | 48 | 49 | filename_correct = re.sub('-epoch-\d+', '', filename) 50 | 51 | rects_gt = data[filename_correct]['rectangles'] 52 | 53 | max_per_gt_sum = 0 54 | 55 | iou_per_gt = 0 56 | iou_per_pr = 0 57 | 58 | if len(rects_gt) > 0 and len(rects_pr) > 0: 59 | # Iterate ground truth 60 | for rect_gt in rects_gt: 61 | 62 | ious_per_gt = list() 63 | 64 | # Iterate predictions 65 | for rect_pr in rects_pr: 66 | 67 | iou = iou_single(rect_gt[0], rect_pr) 68 | ious_per_gt.append(iou) 69 | 70 | max_per_gt_sum += max(ious_per_gt) 71 | 72 | iou_per_gt = max_per_gt_sum / len(rects_gt) 73 | iou_per_pr = max_per_gt_sum / len(rects_pr) 74 | 75 | return {'iou_per_gt': iou_per_gt, 'iou_per_pr': iou_per_pr} 76 | 77 | 78 | def extract_rectangles(folder, output_path, output_name): 79 | 80 | mask_rect_temp_path = folder + 'masks-rectangles-temp/' + output_name 81 | mask_rect_path = folder + 'masks-rectangles/' + output_name 82 | 83 | cmd = ('convert ' + output_path + ' ' 84 | '-threshold 50% ' 85 | '-channel RGB -negate ' 86 | '-channel rgba ' 87 | '-alpha set ' 88 | '-define connected-components:verbose=true ' 89 | '-define connected-components:mean-color=true ' 90 | '-define connected-components:area-threshold=50 ' 91 | '-connected-components 4 ' 92 | '-auto-level ' 93 | + mask_rect_temp_path) 94 | 95 | # Leave out first and last line, because they don't contain rectangle coordinates 96 | out = subprocess.check_output(cmd, shell=True).decode("utf-8").split('\n')[1:-1] 97 | 98 | re_coords = re.compile('.*\d+: (\d+)x(\d+)\+(\d+)\+(\d+).*\s+(\d+)\s+srgba\((\d+)') 99 | 100 | rects = list() 101 | rect_str = '' 102 | for line in out: 103 | m = re_coords.match(line) 104 | 105 | # Alpha value 106 | a = int(m.group(5)) 107 | 108 | if a != 0 and m.group(6) == u'0': 109 | w = int(m.group(1)) 110 | h = int(m.group(2)) 111 | 112 | # Top left corner 113 | x1 = int(m.group(3)) 114 | y1 = int(m.group(4)) 115 | 116 | # Bottom right corner 117 | x2 = x1 + w 118 | y2 = y1 + h 119 | 120 | rect_str += ('-draw "rectangle ' + str(x1) + ',' + str(y1) + ' ' + str(x2-1) + ',' + str(y2-1) + '" ') 121 | rects.append({'x1': x1, 'x2': x2, 'y1':y1, 'y2': y2}) 122 | 123 | # Binarized output mask with bounding boxes 124 | 125 | draw = 'convert ' + mask_rect_temp_path + ' -fill none -strokewidth 2 -stroke red ' + rect_str + ' ' + mask_rect_path 126 | subprocess.check_output(draw, shell=True) 127 | return rects, mask_rect_path 128 | 129 | def iou_all(folder, epoch, accs_df): 130 | 131 | for i in range(accs_df.shape[0]): 132 | base = os.path.basename(accs_df.iloc[i]['filename']) 133 | output_name = os.path.splitext(base)[0] + '-epoch-' + str(epoch) + '.png' 134 | output_path = folder + 'masks/' + output_name 135 | rects_pr, mask_rect_path = extract_rectangles(folder, output_path, output_name) 136 | # Workaroun, not using this iou anymore 137 | #iou = iou_slice(rects_pr, output_name) 138 | iou = {'iou_per_gt': 0, 'iou_per_pr': 0} 139 | 140 | accs_df.at[i, 'iou_per_gt'] = iou['iou_per_gt'] 141 | accs_df.at[i, 'iou_per_pr'] = iou['iou_per_pr'] 142 | accs_df.at[i, 'mask_rect_path'] = mask_rect_path 143 | 144 | return accs_df 145 | 146 | 147 | ###################################################################################### 148 | # Plot the n best or worst outputs 149 | ###################################################################################### 150 | 151 | def plot_row(folder, epoch, accs_df, pos): 152 | 153 | fig = plt.figure(figsize=(20, 32)) 154 | fig.patch.set_facecolor('lightgray') 155 | 156 | grid_h = gridspec.GridSpec( 157 | 20, 6, 158 | wspace=0.06, hspace=0.07, 159 | height_ratios=[ 160 | 0.2, 1, 161 | 0.2, 1, 162 | 0.2, 1, 163 | 0.2 ,1, 164 | 0.2, 1, 165 | 0.2, 1, 166 | 0.2, 1, 167 | 0.2, 1, 168 | 0.2, 1, 169 | 0.2, 1 170 | ]) 171 | 172 | for i in range(1,21,2): 173 | df_row = int((i+1) / 2 - 1) 174 | 175 | # Row Headline 176 | axes_text = plt.subplot(grid_h[i-1,0:6]) 177 | axes_text.text( 178 | 0,0.5, 179 | "file: " + accs_df.iloc[df_row]["filename"] + ", accuracy: " + str(np.round_(accs_df.iloc[df_row]["acc"],3)), 180 | horizontalalignment='left', 181 | verticalalignment='center', 182 | fontsize=16) 183 | axes_text.axis('off') 184 | fig.add_subplot(axes_text) 185 | 186 | # Original Screenshot 187 | axes_1 = plt.subplot(grid_h[i,0]) 188 | axes_1.axis('off') 189 | original_image = np.squeeze(accs_df.iloc[df_row]['x1'] * 255).astype('uint8') 190 | 191 | args_grayscale = { 192 | 'cmap': 'gray', 193 | 'vmin': 0, 194 | 'vmax': 256 195 | } 196 | 197 | original_image = Image.fromarray(original_image, mode='L') # mode `L` stands for 8-bit pixels, black and white 198 | axes_1.imshow(original_image, **args_grayscale) 199 | fig.add_subplot(axes_1) 200 | 201 | # Worker Annotation Rectangles 202 | axes_2 = plt.subplot(grid_h[i,1]) 203 | axes_2.axis('off') 204 | mask_worker = np.squeeze(accs_df.iloc[df_row]['y'] * 255).astype('uint8') 205 | mask_worker = Image.fromarray(mask_worker) 206 | axes_2.imshow(mask_worker, **args_grayscale) 207 | fig.add_subplot(axes_2) 208 | 209 | # Text Boxes 210 | axes_3 = plt.subplot(grid_h[i,2]) 211 | axes_3.axis('off') 212 | text_boxes = np.squeeze(accs_df.iloc[df_row]['x2'] * 255).astype('uint8') 213 | text_boxes = Image.fromarray(text_boxes) 214 | axes_3.imshow(text_boxes, **args_grayscale) 215 | fig.add_subplot(axes_3) 216 | 217 | # Network Output 218 | axes_3 = plt.subplot(grid_h[i,3]) 219 | axes_3.axis('off') 220 | prediction_im = accs_df.iloc[df_row]['prediction'] 221 | prediction_im = prediction_im - prediction_im.min() 222 | prediction_im *= 255 / prediction_im.max() 223 | prediction_image = np.squeeze(prediction_im).astype('uint8') 224 | prediction_image_inv = np.squeeze(255 - prediction_im).astype('uint8') 225 | prediction_image_inv = Image.fromarray(prediction_image_inv) 226 | prediction_image = Image.fromarray(prediction_image) 227 | axes_3.imshow(prediction_image, **args_grayscale) 228 | fig.add_subplot(axes_3) 229 | 230 | # Network Output blended on Original Screenshot 231 | axes_4 = plt.subplot(grid_h[i,4]) 232 | axes_4.axis('off') 233 | blended_image = Image.blend(original_image, prediction_image_inv, alpha=0.7) 234 | axes_4.imshow(blended_image, **args_grayscale) 235 | fig.add_subplot(axes_4) 236 | 237 | # Binarized Mask with bounding boxes 238 | axes_5 = plt.subplot(grid_h[i,5]) 239 | axes_5.axis('off') 240 | img_bb = mpimg.imread(accs_df.iloc[df_row]['mask_rect_path']) 241 | axes_5.imshow(img_bb) 242 | fig.add_subplot(axes_5) 243 | 244 | plt.savefig(fname=folder + "accuracy/" + pos + "-10-epoch-" + str(epoch).zfill(10), format='png', facecolor=fig.get_facecolor()) 245 | plt.close(fig) 246 | plt.close() 247 | 248 | def plot_rows (input_folder, folder, epoch, self, logs, steps_val, no_text_boxes, cross_val_fold): 249 | start = time.time() 250 | 251 | # Note: shuffle must be `False` in order for the filenames to be correct later 252 | shared_args = { 253 | "target_size": (self.input_size, self.input_size), 254 | "classes": ["cross-val-" + str(cross_val_fold)], 255 | "class_mode": None, 256 | "batch_size": 1, 257 | "color_mode": "grayscale", 258 | "shuffle": False 259 | } 260 | 261 | # Original Screenshots 262 | image_gen = ImageDataGenerator().flow_from_directory( 263 | input_folder + '/validation/screenshots', 264 | **shared_args 265 | ) 266 | 267 | # Text Boxes 268 | text_gen = ImageDataGenerator().flow_from_directory( 269 | input_folder + '/validation/text-boxes', 270 | **shared_args 271 | ) 272 | 273 | # Annotations 274 | genY = ImageDataGenerator().flow_from_directory( 275 | input_folder + '/validation/annotation-boxes', 276 | **shared_args 277 | ) 278 | 279 | accs = list() 280 | 281 | # for i in range(0, steps_val): 282 | for i in range(0, 11): 283 | x1 = np.divide(image_gen.next(),255) 284 | x2 = np.divide(text_gen.next(),255) 285 | 286 | if no_text_boxes: 287 | x = x1 288 | else: 289 | x = np.concatenate((x1,x2), axis=3) 290 | 291 | y = np.divide(genY.next(),255) 292 | 293 | # Metrics order: loss, acc, precision, recall, bal_acc 294 | 295 | metrics = self.model_for_saving.test_on_batch(x, y) 296 | pred = self.model_for_saving.predict(x) 297 | 298 | prediction_image = pred 299 | prediction_image = prediction_image - prediction_image.min() 300 | prediction_image *= 255 / prediction_image.max() 301 | prediction_image = np.squeeze(255 - prediction_image).astype('uint8') 302 | prediction_image = Image.fromarray(prediction_image) 303 | 304 | base = os.path.basename(image_gen.filenames[i]) 305 | output_name = os.path.splitext(base)[0] + '-epoch-' + str(epoch) + '.png' 306 | output_path = folder + 'masks/' + output_name 307 | prediction_image.save(output_path) 308 | 309 | accs.append({ 310 | "loss": metrics[0], 311 | "acc": metrics[1], 312 | "precision": metrics[2], 313 | "recall": metrics[3], 314 | "bal_acc": metrics[4], 315 | "x1":x1,"x2":x2,"y":y, 316 | "prediction": pred, 317 | "iou_per_gt": None, "iou_per_pr": None, 318 | "filename": image_gen.filenames[i], 319 | "mask_rect_path": None 320 | }) 321 | 322 | accs_df = pd.DataFrame(accs) 323 | 324 | # IOU calculation 325 | accs_df = iou_all(folder, epoch, accs_df) 326 | 327 | logs['mean_iou_gt'] = accs_df.loc[:,'iou_per_gt'].mean() 328 | logs['mean_iou_pr'] = accs_df.loc[:,'iou_per_pr'].mean() 329 | 330 | accs_export = accs_df.drop(['prediction','x1','x2','y', 'mask_rect_path'], axis=1) 331 | accs_export.to_csv (folder + 'history-csv/history-epoch-' + str(epoch) + '.csv', index = None, header=True) 332 | 333 | # Highest acc: 334 | plot_row(folder, epoch, accs_df.sort_values('acc', ascending=False), 'highest') 335 | # Lowest acc: 336 | plot_row(folder, epoch, accs_df.sort_values('acc'), 'lowest') 337 | # Mid range acc: 338 | num_row_middle = int(accs_df.shape[0] / 2) 339 | plot_row(folder, epoch, accs_df.sort_values('acc').iloc[num_row_middle-5 : num_row_middle+5, ],'middle') 340 | # Fixed sample": 341 | np.random.seed(42) 342 | #plot_row(folder, epoch, accs_df.iloc[np.random.choice(range(0,steps_val),10).tolist()],'fixed') 343 | plot_row(folder, epoch, accs_df.iloc[np.random.choice(range(0,11),10).tolist()],'fixed') 344 | 345 | end = time.time() 346 | print(str(steps_val), " predictions took", str(end - start)) -------------------------------------------------------------------------------- /algorithms/meier/model/resize.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | INPUT_FOLDER=${1%/} 4 | OUTPUT_FOLDER=${2%/} 5 | 6 | if [ $# -ne 2 ] 7 | then 8 | echo "Invalid number of arguments." 9 | echo "Usage:" "bash" "$0" "" "" 10 | exit 1 11 | fi 12 | 13 | if [ ! -d $OUTPUT_FOLDER ] 14 | then 15 | mkdir -p $OUTPUT_FOLDER 16 | fi 17 | 18 | for i in ${INPUT_FOLDER}/*.png 19 | do 20 | FILENAME=$(basename ${i}) 21 | echo "Resizing" "${i}" "to" "${OUTPUT_FOLDER}/${FILENAME}" 22 | convert "$i" -interpolate Integer -filter point -resize 1366x4096\! ${OUTPUT_FOLDER}/${FILENAME} || break 23 | done 24 | -------------------------------------------------------------------------------- /algorithms/meier/model/setup_folders.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | def create(folder): 4 | 5 | if not os.path.exists(folder): 6 | os.makedirs(folder) 7 | 8 | if not os.path.exists(folder + 'masks'): 9 | os.makedirs(folder + 'masks') 10 | 11 | if not os.path.exists(folder + 'masks-rectangles'): 12 | os.makedirs(folder + 'masks-rectangles') 13 | 14 | if not os.path.exists(folder + 'masks-rectangles-temp'): 15 | os.makedirs(folder + 'masks-rectangles-temp') 16 | 17 | if not os.path.exists(folder + 'metrics-per-epoch'): 18 | os.makedirs(folder + 'metrics-per-epoch') 19 | 20 | if not os.path.exists(folder + 'filters'): 21 | os.makedirs(folder + 'filters') 22 | 23 | if not os.path.exists(folder + 'history'): 24 | os.makedirs(folder + 'history') 25 | 26 | if not os.path.exists(folder + 'history-iou'): 27 | os.makedirs(folder + 'history-iou') 28 | 29 | if not os.path.exists(folder + 'history-csv'): 30 | os.makedirs(folder + 'history-csv') 31 | 32 | if not os.path.exists(folder + 'combined-plot'): 33 | os.makedirs(folder + 'combined-plot') 34 | 35 | if not os.path.exists(folder + 'accuracy'): 36 | os.makedirs(folder + 'accuracy') 37 | 38 | if not os.path.exists(folder + 'weights'): 39 | os.makedirs(folder + 'weights') 40 | 41 | 42 | -------------------------------------------------------------------------------- /algorithms/meier/model/test.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import imageio 3 | import meier2017 4 | from keras.models import Model 5 | from keras.utils import multi_gpu_model 6 | from keras.optimizers import SGD, Adam 7 | from tensorflow import device 8 | import custom_image_generator_test 9 | import my_metrics 10 | import setup_folders 11 | import sys 12 | import os 13 | from glob import glob 14 | from re import search 15 | 16 | if len(sys.argv) != 5: 17 | print("Usage: python " + sys.argv[0] + " ") 18 | sys.exit(1) 19 | 20 | input_folder = sys.argv[1] 21 | fold = sys.argv[2] 22 | input_files = glob(input_folder + '/screenshots/cross-val-' + fold + "/*.png") 23 | input_files_num = len(input_files) 24 | 25 | folder = sys.argv[4] 26 | 27 | setup_folders.create(folder) 28 | 29 | # Standard Hyperparameters 30 | 31 | hyper = { 32 | 'optimizer': 'SGD', # SGD and Adam 33 | 'lr': 0.01, # meier2017 paper: 0.01 34 | 'momentum': 0.0, 35 | 'l2_lambda': 0.0001, # meier2017 paper: 0.0001 36 | 'steps_train': 50000, # 50000 37 | 'steps_val': 1581, # 1581 38 | 'batch_size': 16, 39 | 'epochs': 5, # 5 40 | 'input_size': 256, 41 | 'no_text_boxes': False, 42 | 'no_augmentation': False, 43 | 'cross_val_fold': 0, 44 | 'single_augmentation':"" 45 | } 46 | 47 | # Model 48 | with(device("/gpu:0")): 49 | model_in, model_out = meier2017.model(hyper['l2_lambda'], hyper['input_size'], hyper['no_text_boxes']) 50 | 51 | model = Model(inputs=model_in, outputs=model_out) 52 | 53 | # model = multi_gpu_model(model, gpus=8) 54 | 55 | weights_file = sys.argv[3] 56 | 57 | if hyper['optimizer'] == 'SGD': 58 | opti = SGD(lr=hyper['lr'], momentum=hyper['momentum'], nesterov=True) 59 | 60 | if hyper['optimizer'] == 'Adam': 61 | opti = Adam(lr=hyper['lr']) 62 | 63 | model.compile(opti, loss='binary_crossentropy', metrics=['acc', my_metrics.precision, my_metrics.recall, my_metrics.bal_acc, my_metrics.iou]) 64 | 65 | model.load_weights(weights_file) 66 | 67 | out = model.predict_generator(custom_image_generator_test.generator_test(input_folder, hyper['no_text_boxes'], fold), steps=input_files_num, verbose=1) 68 | 69 | for i in range(len(out)): 70 | mask = out[i] -out[i].min() 71 | mask *= 255 / mask.max() 72 | mask = np.squeeze(mask.astype('uint8')) 73 | imageio.imwrite(folder + str(i).zfill(6) + '.png', mask) 74 | -------------------------------------------------------------------------------- /algorithms/meier/model/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | input=$1 3 | fold=$2 4 | weights=$3 5 | output=$4 6 | tmp_output=$output.tmp/ 7 | tmp_output_renamed=$output.renamed.tmp/ 8 | tmp_output_resized=$output.resized.tmp/ 9 | 10 | python test.py $input $fold $weights $tmp_output 11 | ./copy.sh $input/screenshots/cross-val-$fold $tmp_output $tmp_output_renamed 12 | ./resize.sh $tmp_output_renamed $tmp_output_resized 13 | ./extract_rectangles.sh $tmp_output_resized $output/ 14 | 15 | -------------------------------------------------------------------------------- /algorithms/meier/model/to_test.txt: -------------------------------------------------------------------------------- 1 | ../data/output/results-0000132/weights/epoch-00000026-weights.h5 2 | ../data/output/results-0000118/weights/epoch-00000040-weights.h5 3 | ../data/output/results-0000120/weights/epoch-00000033-weights.h5 4 | ../data/output/results-0000122/weights/epoch-00000029-weights.h5 5 | ../data/output/results-0000123/weights/epoch-00000090-weights.h5 6 | ../data/output/results-0000125/weights/epoch-00000027-weights.h5 7 | ../data/output/results-0000127/weights/epoch-00000064-weights.h5 8 | ../data/output/results-0000128/weights/epoch-00000084-weights.h5 9 | ../data/output/results-0000130/weights/epoch-00000023-weights.h5 10 | ../data/output/results-0000131/weights/epoch-00000030-weights.h5 11 | -------------------------------------------------------------------------------- /algorithms/meier/model/train.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | folders="training validation test" 4 | inputs="annotation-boxes screenshots text-boxes" 5 | 6 | start=$1 7 | end=$2 8 | epochs=$3 9 | 10 | cd /src/workspace/data/input 11 | 12 | for i in $(seq $start $end); 13 | do 14 | val=$i 15 | test=$((($i + 1) % 10)) 16 | for j in $(seq 0 9); 17 | do 18 | if [ $j -ne $val ] && [ $j -ne $test ]; 19 | then 20 | for input in $inputs; 21 | do 22 | mkdir -p training/$input/cross-val-0; 23 | cd training/$input/cross-val-0; 24 | ln -s ../../../folds/$input/cross-val-${j}/* .; 25 | cd ../../../; 26 | done; 27 | fi; 28 | done; 29 | for input in $inputs; 30 | do 31 | mkdir -p validation/$input/cross-val-0; 32 | cd validation/$input/cross-val-0; 33 | ln -s ../../../folds/$input/cross-val-${val}/* .; 34 | cd ../../../; 35 | mkdir -p test/$input/cross-val-0; 36 | cd test/$input/cross-val-0; 37 | ln -s ../../../folds/$input/cross-val-${test}/* .; 38 | cd ../../../; 39 | done; 40 | cd /src/workspace/model; 41 | python main.py --batch-size=128 --epochs=$epochs --cross-val-fold=0; 42 | cd /src/workspace/data/input; 43 | for folder in $folders; 44 | do 45 | rm -r $folder/$input; 46 | done; 47 | done 48 | -------------------------------------------------------------------------------- /algorithms/meier/model/validate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | folders="training validation test" 4 | inputs="annotation-boxes screenshots text-boxes" 5 | 6 | start=$1 7 | end=$2 8 | weights=$3 9 | 10 | cd /src/workspace/data/input 11 | 12 | for i in $(seq $start $end); 13 | do 14 | val=$i 15 | test=$((($i + 1) % 10)) 16 | for j in $(seq 0 9); 17 | do 18 | if [ $j -ne $val ] && [ $j -ne $test ]; 19 | then 20 | for input in $inputs; 21 | do 22 | cd training/$input/cross-val-0; 23 | ln -s ../../../folds/$input/cross-val-${j}/* .; 24 | cd ../../../; 25 | done; 26 | fi; 27 | done; 28 | for input in $inputs; 29 | do 30 | cd validation/$input/cross-val-0; 31 | ln -s ../../../folds/$input/cross-val-${val}/* .; 32 | cd ../../../; 33 | cd test/$input/cross-val-0; 34 | ln -s ../../../folds/$input/cross-val-${test}/* .; 35 | cd ../../../; 36 | done; 37 | cd /src/workspace/model; 38 | python main_validate.py --batch-size=128 --weights-file=$weights --cross-val-fold=0; 39 | cd /src/workspace/data/input; 40 | for folder in $folders; 41 | do 42 | for input in $inputs; 43 | do 44 | rm $folder/$input/cross-val-0/*; 45 | done; 46 | done; 47 | done 48 | -------------------------------------------------------------------------------- /algorithms/meier/setup-directories.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -ne 2 ];then 4 | echo "Usage: $0 " 5 | exit 1 6 | fi 7 | 8 | webseg=$(readlink -f $1) 9 | folds_file=$(readlink -f $2) 10 | output=$(readlink -f webis-webseg-20-meier) 11 | 12 | 13 | folds_directory=$output/folds 14 | echo "Setup folds in $folds_directory" 15 | 16 | function cp_files() { 17 | local fold=$1 18 | local original_filename=$2 19 | local links_directory=$3 20 | 21 | mkdir -p $links_directory 22 | pushd $links_directory 23 | for task in $(cat $folds_file | grep "^$fold " | cut -d" " -f2);do 24 | cp $webseg/$task/$original_filename $task.png 25 | done 26 | popd > /dev/zero 27 | } 28 | 29 | for fold in $(seq 0 9);do 30 | fold_directory=$folds_directory/fold$fold 31 | cp_files $fold screenshot-4096px.png $fold_directory/screenshots 32 | cp_files $fold text-mask-4096px.png $fold_directory/text-boxes 33 | cp_files $fold ground-truth-mask-4096px.png $fold_directory/annotation-boxes 34 | done 35 | 36 | 37 | # SETUP DIRECTORY STRUCTURE 38 | cross_validation_directory=$output/input 39 | echo "Setup cross validation directory structure in $cross_validation_directory" 40 | 41 | function link_fold() { 42 | local cross_validation_fold=$1 43 | local cross_validation_fold_type=$2 44 | local original_fold=$3 45 | 46 | for input_type in screenshots text-boxes annotation-boxes;do 47 | links_directory=$cross_validation_directory/$cross_validation_fold_type/$input_type/cross-val-$cross_validation_fold 48 | mkdir -p $links_directory 49 | pushd $links_directory 50 | ln -s ../../../../folds/fold$original_fold/$input_type/* . 51 | popd > /dev/zero 52 | done 53 | } 54 | 55 | for cross_validation_fold in $(seq 0 9);do 56 | validation_fold=$cross_validation_fold 57 | test_fold=$((($cross_validation_fold + 1) % 10)) 58 | 59 | link_fold $cross_validation_fold validation $validation_fold 60 | link_fold $cross_validation_fold test $test_fold 61 | for training_fold in $(seq 0 9 | grep -v "^$validation_fold$" | grep -v "^$test_fold$");do 62 | link_fold $cross_validation_fold training $training_fold 63 | done 64 | done 65 | 66 | -------------------------------------------------------------------------------- /algorithms/mmdetection/.gitignore: -------------------------------------------------------------------------------- 1 | htc_dconv_c3-c5_mstrain_400_1400_x101_64x4d_fpn_20e_20190408-0e50669c.pth 2 | -------------------------------------------------------------------------------- /algorithms/mmdetection/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG PYTORCH="nightly-devel" 2 | ARG CUDA="9.2" 3 | ARG CUDNN="7" 4 | 5 | FROM pytorch/pytorch:${PYTORCH}-cuda${CUDA}-cudnn${CUDNN} 6 | 7 | RUN apt-get update && apt-get install -y ffmpeg libglib2.0-0 libsm6 libxrender-dev libxext6 \ 8 | && apt-get clean \ 9 | && rm -rf /var/lib/apt/lists/* 10 | 11 | # Install mmdetection 12 | RUN conda update -n base -c defaults conda 13 | RUN conda install cython -y && conda clean --all 14 | RUN git clone https://github.com/open-mmlab/mmdetection.git /mmdetection 15 | WORKDIR /mmdetection 16 | RUN git checkout 8d010d7de9c2643e715aaf6033ff7fd5c60ebdc2 17 | COPY ./requirements.txt ./infer.py ./infer_single.py /mmdetection/ 18 | ENV CUDA_HOME=/usr/local/cuda LD_LIBRARY_PATH=/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} 19 | RUN pip install --no-cache-dir -e . 20 | -------------------------------------------------------------------------------- /algorithms/mmdetection/Dockerfile-with-model: -------------------------------------------------------------------------------- 1 | # Based on https://github.com/open-mmlab/mmdetection/blob/master/docker/Dockerfile 2 | 3 | FROM webis/mmdetection19-web-page-segmentation 4 | 5 | COPY htc_dconv_c3-c5_mstrain_400_1400_x101_64x4d_fpn_20e_20190408-0e50669c.pth /resources/checkpoints/htc_dconv_c3-c5_mstrain_400_1400_x101_64x4d_fpn_20e_20190408-0e50669c.pth 6 | 7 | -------------------------------------------------------------------------------- /algorithms/mmdetection/infer.py: -------------------------------------------------------------------------------- 1 | from mmdet.apis import init_detector, inference_detector, show_result 2 | import pycocotools.mask as maskUtils 3 | import numpy as np 4 | import mmcv 5 | import sys 6 | import os 7 | import json 8 | from threading import Thread, Lock 9 | 10 | def get_segm_left(mask): 11 | return mask.any(0).argmax() 12 | 13 | def get_segm_right(mask): 14 | return mask.shape[1] - np.fliplr(mask).any(0).argmax() 15 | 16 | def get_segm_top(mask): 17 | return mask.any(1).argmax() 18 | 19 | def get_segm_bottom(mask): 20 | return mask.shape[0] - np.flipud(mask).any(1).argmax() 21 | 22 | def get_segm_bounds(mask): 23 | left = get_segm_left(mask) 24 | right = get_segm_right(mask) 25 | top = get_segm_top(mask) 26 | bottom = get_segm_bottom(mask) 27 | if left is not None and right is not None and top is not None and bottom is not None: 28 | return left, right, top, bottom 29 | else: 30 | raise ValueError('Could not determine bounds for segment') 31 | 32 | lock = Lock() 33 | 34 | def infer(model, imgfile, id): 35 | outfile = open("/out/" + id + ".json", 'w') 36 | img = mmcv.imread(imgfile) 37 | 38 | lock.acquire() 39 | result = inference_detector(model, img) 40 | lock.release() 41 | 42 | if isinstance(result, tuple): 43 | bbox_result, segm_result = result 44 | else: 45 | bbox_result, segm_result = result, None 46 | 47 | bboxes = np.vstack(bbox_result) 48 | segm_polygon_list = [] 49 | bbox_polygon_list = [] 50 | 51 | if segm_result is not None: 52 | segms = mmcv.concat_list(segm_result) 53 | inds = np.where(bboxes[:, -1] > 0.0)[0] 54 | for i in inds: 55 | mask = maskUtils.decode(segms[i]).astype(np.bool) 56 | try: 57 | left, right, top, bottom = get_segm_bounds(mask) 58 | 59 | if left is not None and right is not None and top is not None and bottom is not None: 60 | segm_polygon_list.append([[[[left.item(), top.item()], [left.item(), bottom.item()], [right.item(), bottom.item()], [right.item(), top.item()], [left.item(), top.item()]]]]) 61 | except ValueError: 62 | print() 63 | 64 | for bbox in bboxes: 65 | bbox_int = bbox.astype(np.int32) 66 | left = bbox_int[0] 67 | top = bbox_int[1] 68 | right = bbox_int[2] 69 | bottom = bbox_int[3] 70 | 71 | bbox_polygon_list.append([[[[left.item(), top.item()], [left.item(), bottom.item()], [right.item(), bottom.item()], [right.item(), top.item()], [left.item(), top.item()]]]]) 72 | 73 | out_obj = dict(height=img.shape[0], width=img.shape[1], id=id, segmentations=dict(mmdetection_bboxes=bbox_polygon_list, mmdetection_segms=segm_polygon_list)) 74 | json.dump(out_obj, outfile) 75 | 76 | 77 | directory = os.fsencode("/pages") 78 | config_file = '/mmdetection/configs/htc/htc_dconv_c3-c5_mstrain_400_1400_x101_64x4d_fpn_20e.py' 79 | checkpoint_file = '/resources/checkpoints/htc_dconv_c3-c5_mstrain_400_1400_x101_64x4d_fpn_20e_20190408-0e50669c.pth' 80 | model = init_detector(config_file, checkpoint_file, device='cuda:0') 81 | 82 | ids = [d.name for d in os.scandir(directory) if d.is_dir()] 83 | 84 | for id in ids: 85 | infer(model, os.path.join(directory, id, "screenshot.png"), id) 86 | 87 | # i = 0 88 | # files_ids = [] 89 | # for f in os.listdir(directory): 90 | # filename = os.fsdecode(f) 91 | # if filename.endswith(".png"): 92 | # id = os.path.splitext(filename)[0] 93 | # files_ids.append((os.path.join(directory, f), id)) 94 | # i += 1; 95 | # 96 | # if i == 8: 97 | # threads = [] 98 | # for j in range(i): 99 | # print("Inferring for page " + str(files_ids[j][1]) + ", file " + str(files_ids[j][0].decode("utf-8"))) 100 | # t = Thread(target=infer, args=(model, files_ids[j][0].decode("utf-8"), files_ids[j][1])) 101 | # threads += [t] 102 | # t.start() 103 | # 104 | # for thread in threads: 105 | # thread.join() 106 | # 107 | # i = 0 108 | # files_ids = [] 109 | -------------------------------------------------------------------------------- /algorithms/mmdetection/infer_single.py: -------------------------------------------------------------------------------- 1 | from mmdet.apis import init_detector, inference_detector, show_result 2 | import pycocotools.mask as maskUtils 3 | import numpy as np 4 | import mmcv 5 | import sys 6 | import os 7 | import json 8 | from threading import Thread, Lock 9 | 10 | def get_segm_left(mask): 11 | return mask.any(0).argmax() 12 | 13 | def get_segm_right(mask): 14 | return mask.shape[1] - np.fliplr(mask).any(0).argmax() 15 | 16 | def get_segm_top(mask): 17 | return mask.any(1).argmax() 18 | 19 | def get_segm_bottom(mask): 20 | return mask.shape[0] - np.flipud(mask).any(1).argmax() 21 | 22 | def get_segm_bounds(mask): 23 | left = get_segm_left(mask) 24 | right = get_segm_right(mask) 25 | top = get_segm_top(mask) 26 | bottom = get_segm_bottom(mask) 27 | if left is not None and right is not None and top is not None and bottom is not None: 28 | return left, right, top, bottom 29 | else: 30 | raise ValueError('Could not determine bounds for segment') 31 | 32 | lock = Lock() 33 | 34 | def infer(model, imgfile, id): 35 | outfile = open("/out/" + id + ".json", 'w') 36 | img = mmcv.imread(imgfile) 37 | 38 | lock.acquire() 39 | result = inference_detector(model, img) 40 | lock.release() 41 | 42 | if isinstance(result, tuple): 43 | bbox_result, segm_result = result 44 | else: 45 | bbox_result, segm_result = result, None 46 | 47 | bboxes = np.vstack(bbox_result) 48 | segm_polygon_list = [] 49 | bbox_polygon_list = [] 50 | 51 | if segm_result is not None: 52 | segms = mmcv.concat_list(segm_result) 53 | inds = np.where(bboxes[:, -1] > 0.0)[0] 54 | for i in inds: 55 | mask = maskUtils.decode(segms[i]).astype(np.bool) 56 | try: 57 | left, right, top, bottom = get_segm_bounds(mask) 58 | 59 | if left is not None and right is not None and top is not None and bottom is not None: 60 | segm_polygon_list.append([[[[left.item(), top.item()], [left.item(), bottom.item()], [right.item(), bottom.item()], [right.item(), top.item()], [left.item(), top.item()]]]]) 61 | except ValueError: 62 | print() 63 | 64 | for bbox in bboxes: 65 | bbox_int = bbox.astype(np.int32) 66 | left = bbox_int[0] 67 | top = bbox_int[1] 68 | right = bbox_int[2] 69 | bottom = bbox_int[3] 70 | 71 | bbox_polygon_list.append([[[[left.item(), top.item()], [left.item(), bottom.item()], [right.item(), bottom.item()], [right.item(), top.item()], [left.item(), top.item()]]]]) 72 | 73 | out_obj = dict(height=img.shape[0], width=img.shape[1], id=id, segmentations=dict(mmdetection_bboxes=bbox_polygon_list, mmdetection_segms=segm_polygon_list)) 74 | json.dump(out_obj, outfile) 75 | 76 | 77 | if len(sys.argv) != 2: 78 | print("Invalid number of arguments.") 79 | print("Usage: python3 " + sys.argv[0] + " ") 80 | sys.exit(1) 81 | 82 | # directory = os.fsencode(sys.argv[1]) 83 | config_file = '/mmdetection/configs/htc/htc_dconv_c3-c5_mstrain_400_1400_x101_64x4d_fpn_20e.py' 84 | checkpoint_file = '/resources/checkpoints/htc_dconv_c3-c5_mstrain_400_1400_x101_64x4d_fpn_20e_20190408-0e50669c.pth' 85 | model = init_detector(config_file, checkpoint_file, device='cuda:0') 86 | 87 | id = sys.argv[1] 88 | imgfile = os.path.join("/pages", id, "screenshot.png") 89 | infer(model, imgfile, id) 90 | 91 | # i = 0 92 | # files_ids = [] 93 | # for f in os.listdir(directory): 94 | # filename = os.fsdecode(f) 95 | # if filename.endswith(".png"): 96 | # id = os.path.splitext(filename)[0] 97 | # files_ids.append((os.path.join(directory, f), id)) 98 | # i += 1; 99 | # 100 | # if i == 8: 101 | # threads = [] 102 | # for j in range(i): 103 | # print("Inferring for page " + str(files_ids[j][1]) + ", file " + str(files_ids[j][0].decode("utf-8"))) 104 | # t = Thread(target=infer, args=(model, files_ids[j][0].decode("utf-8"), files_ids[j][1])) 105 | # threads += [t] 106 | # t.start() 107 | # 108 | # for thread in threads: 109 | # thread.join() 110 | # 111 | # i = 0 112 | # files_ids = [] 113 | -------------------------------------------------------------------------------- /algorithms/mmdetection/requirements.txt: -------------------------------------------------------------------------------- 1 | mmcv==0.2.13 2 | numpy==1.16.5 3 | matplotlib==3.1.1 4 | six==1.12.0 5 | terminaltables==3.1.0 6 | pycocotools==2.0.0 7 | torch==1.2.0 8 | torchvision==0.4.0 9 | imagecorruptions==1.0.0 10 | -------------------------------------------------------------------------------- /algorithms/vips/.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | webis-web-archiver.jar 3 | -------------------------------------------------------------------------------- /algorithms/vips/compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Make sure we are in the same directory as this file 4 | pushd $(dirname "${BASH_SOURCE[0]}") >/dev/null 2>&1 5 | 6 | if [ -e webis-web-archiver.jar ];then 7 | echo "Using existing webis-web-archiver.jar" 8 | else 9 | echo "Downloading JAR of the Webis Web Archiver for compilation" 10 | wget https://github.com/webis-de/webis-web-archiver/releases/download/0.1.0/webis-web-archiver.jar 11 | fi 12 | 13 | echo "Compiling script class src/main/java/VIPSScript.class" 14 | javac -cp "webis-web-archiver.jar:." \ 15 | --release 8 \ 16 | src/main/java/VIPSScript.java 17 | 18 | echo "Creating script JAR" 19 | jar cfM scripts/VIPSScript-1.0.0/VIPSScript.jar \ 20 | -C src/main/java VIPSScript.class 21 | 22 | -------------------------------------------------------------------------------- /algorithms/vips/scripts/VIPSScript-1.0.0/VIPSScript.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webis-de/ecir21-an-empirical-comparison-of-web-page-segmentation-algorithms/e9df2cc8baa757913c31a9281dc053eae4efeff0/algorithms/vips/scripts/VIPSScript-1.0.0/VIPSScript.jar -------------------------------------------------------------------------------- /algorithms/vips/scripts/VIPSScript-1.0.0/script.conf: -------------------------------------------------------------------------------- 1 | script = VIPSScript 2 | environment.name = de.webis.java 3 | environment.version = 1.0.0 4 | -------------------------------------------------------------------------------- /algorithms/vips/scripts/VIPSScript-1.0.0/vips.conf: -------------------------------------------------------------------------------- 1 | pdoc = 5 2 | -------------------------------------------------------------------------------- /algorithms/vips/src/main/java/VIPSScript.java: -------------------------------------------------------------------------------- 1 | import de.webis.webarchive.common.Version; 2 | import de.webis.webarchive.environment.browsers.Browser; 3 | import de.webis.webarchive.environment.browsers.Windows; 4 | import de.webis.webarchive.environment.scripts.InteractionScript; 5 | import org.openqa.selenium.JavascriptExecutor; 6 | import org.openqa.selenium.WebDriver; 7 | 8 | import java.io.FileOutputStream; 9 | import java.io.IOException; 10 | import java.io.InputStream; 11 | import java.io.OutputStreamWriter; 12 | import java.io.Writer; 13 | import java.nio.file.Path; 14 | import java.nio.file.Files; 15 | import java.util.Properties; 16 | import java.util.Scanner; 17 | import java.util.logging.Logger; 18 | 19 | public class VIPSScript extends InteractionScript { 20 | 21 | ////////////////////////////////////////////////////////////////////////////// 22 | // LOGGING 23 | ////////////////////////////////////////////////////////////////////////////// 24 | 25 | private static final Logger LOG = 26 | Logger.getLogger(VIPSScript.class.getName()); 27 | 28 | ////////////////////////////////////////////////////////////////////////////// 29 | // MEMBERS 30 | ////////////////////////////////////////////////////////////////////////////// 31 | 32 | private final String vipsJs; 33 | 34 | private final int pDoC; 35 | 36 | ////////////////////////////////////////////////////////////////////////////// 37 | // CONSTRUCTORS 38 | ////////////////////////////////////////////////////////////////////////////// 39 | 40 | public VIPSScript(final Path scriptDirectory) 41 | throws IOException { 42 | super(scriptDirectory); 43 | LOG.info("Loading VIPS configuration"); 44 | final Properties vipsConfiguration = new Properties(); 45 | try (final InputStream vipsConfigurationStream = 46 | Files.newInputStream(scriptDirectory.resolve("vips.conf"))) { 47 | vipsConfiguration.load(vipsConfigurationStream); 48 | } 49 | 50 | final int pDoCDefault = Integer.valueOf(vipsConfiguration.getProperty("pdoc")); 51 | final String pDoCEnv = System.getenv("PDoC"); 52 | if (pDoCEnv == null) { 53 | this.pDoC = pDoCDefault; 54 | LOG.info("Permitted Degree of Coherence is " + this.pDoC 55 | + " as per the configuration file"); 56 | } else { 57 | this.pDoC = Integer.valueOf(pDoCEnv); 58 | LOG.info("Permitted Degree of Coherence is " + this.pDoC 59 | + " as per environment variable"); 60 | } 61 | 62 | LOG.info("Loading VIPS script"); 63 | this.vipsJs = new Scanner(scriptDirectory.resolve("vips.js")).useDelimiter("\\A").next() 64 | + "\nvar tester = new VipsTester();\nreturn tester.main(\"TBFWID\"," + this.pDoC + ");"; 65 | } 66 | 67 | ////////////////////////////////////////////////////////////////////////////// 68 | // FUNCTIONALITY 69 | ////////////////////////////////////////////////////////////////////////////// 70 | 71 | @Override 72 | protected void executeInteraction( 73 | final Browser browser, final String startUrl, final Path outputDirectory) 74 | throws Throwable { 75 | final WebDriver window = browser.openWindow(startUrl); 76 | this.scrollDown(browser, window); 77 | this.executeVips(browser, window, outputDirectory); 78 | } 79 | 80 | protected void scrollDown(final Browser browser, final WebDriver window) { 81 | final long quietPeriodInSeconds = 3; 82 | final long waitTimeoutInSeconds = 10; 83 | browser.waitForQuiescence(quietPeriodInSeconds, waitTimeoutInSeconds); 84 | 85 | // Enough to reach "click for more"-button of google image search 86 | final int maxScrollings = 25; 87 | for (int scrollings = 0; scrollings < maxScrollings; ++scrollings) { 88 | final int scrollPosition = Windows.getScrollYPosition(window); 89 | final int scrollHeight = Windows.getScrollHeight(window); 90 | if (scrollPosition >= scrollHeight) { break; } 91 | 92 | LOG.info("Scrolling down " + (scrollings + 1) 93 | + " from " + scrollPosition + "/" + scrollHeight); 94 | Windows.scrollDownOneWindow(window); 95 | browser.waitForQuiescence(quietPeriodInSeconds, waitTimeoutInSeconds); 96 | } 97 | 98 | final int scrollPosition = Windows.getScrollYPosition(window); 99 | final int scrollHeight = Windows.getScrollHeight(window); 100 | LOG.info("Scrolled down to " + scrollPosition + "/" + scrollHeight); 101 | 102 | Windows.scrollToTop(window); 103 | LOG.info("Resize viewport height to " + scrollHeight); 104 | Windows.resizeViewportHeight(window, scrollHeight); 105 | browser.waitForQuiescence(quietPeriodInSeconds, waitTimeoutInSeconds); 106 | } 107 | 108 | protected void executeVips(final Browser browser, final WebDriver window, final Path outputDirectory) 109 | throws Throwable { 110 | LOG.info("Executing VIPS"); 111 | JavascriptExecutor jsExecutor = (JavascriptExecutor) window; 112 | String json = (String) jsExecutor.executeScript(this.vipsJs); 113 | LOG.info("Writing result to " + outputDirectory.toString() + "/vips.json"); 114 | try (final Writer writer = new OutputStreamWriter(new FileOutputStream( 115 | outputDirectory.resolve("vips.json").toFile()), "UTF-8")) { 116 | writer.write(json); 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /algorithms/vips/vips.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | script="VIPSScript" 4 | script_version=1.0.0 5 | scripts_directory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/scripts/ 6 | id=UNKNOWN 7 | keep=0 8 | pdoc="" 9 | 10 | OPTS=$(getopt --name $(basename $0) --options a:i:kp:r:u:o: --longoptions archive:,id:,keep,pdoc:,reproductionmode:,url:,output: -- $@) 11 | if [[ $? -ne 0 ]]; then 12 | exit 2 13 | fi 14 | eval set -- "$OPTS" 15 | 16 | while true;do 17 | case "$1" in 18 | -a|--archive) 19 | archive=$(readlink -f -- "$2") 20 | shift 2 21 | ;; 22 | -i|--id) 23 | id=$2 24 | shift 2 25 | ;; 26 | -k|--keep) 27 | keep=1 28 | shift 1 29 | ;; 30 | -p|--pdoc) 31 | pdoc="--env PDoC=$2" 32 | shift 2 33 | ;; 34 | -r|--reproductionmode) 35 | mode="-$2" 36 | shift 2 37 | ;; 38 | -u|--url) 39 | url="$2" 40 | shift 2 41 | ;; 42 | -o|--output) 43 | output="$2" 44 | shift 2 45 | ;; 46 | --) 47 | break 48 | ;; 49 | esac 50 | done 51 | 52 | if [ \( -z "$archive" \) -o \( -z "$url" \) -o \( -z "$output" \) ];then 53 | echo "USAGE" 54 | echo " $0 [OPTIONS] --archive --url --output " 55 | echo "WHERE" 56 | echo " --archive " 57 | echo " Specifies the archive directory created by archive.sh" 58 | echo " --id " 59 | echo " Specifies the page ID (used in the output JSON)" 60 | echo " --keep" 61 | echo " Specifies to keep log files" 62 | echo " --pdoc " 63 | echo " Specifies the PDoC parameter (overruling the one in vips.conf)" 64 | echo " --url " 65 | echo " Specifies the start URL for the script" 66 | echo " --output " 67 | echo " Specifies the directory to which logs and the script output are written" 68 | echo " --reproductionmode [warcprox|pywb]" 69 | echo " Changes the program used for web page reproduction from a custom" 70 | echo " implementation to warcprox or pywb" 71 | exit 1 72 | fi 73 | 74 | mkdir -p $output # Creating directory is required to give web-archiver user permission to write 75 | output=$(readlink -f -- $output) 76 | 77 | maindir=$(readlink -f -- $(dirname $0)/..) 78 | 79 | is_in_docker_group=$(groups | sed 's/ /\n/g' | grep '^docker$' | wc -l) 80 | 81 | # Mounting /dev/shm is required for taking big screenshot in chrome 82 | # /warcs/ can not be mounted read-only for warcprox mode (which does, however, not change anything, but tests that it could write on startup) 83 | command="docker run --rm --user $(id -u) --env URL=\"$url\" --env DBUS_SESSION_BUS_ADDRESS=/dev/null --env MODE="reproduce$mode" --env SCRIPT=$script --env SCRIPT_VERSION=$script_version $pdoc --volume $scripts_directory:/resources/scripts/:ro --volume $archive:/warcs/ --volume $output:/output/ --volume /dev/shm/:/dev/shm -a stdout -a stderr webis/web-archive-environment:1.2.1" 84 | if [ $is_in_docker_group -eq 0 ];then 85 | sudo $command 86 | else 87 | $command 88 | fi 89 | 90 | # Getting output segmentation (replacing ID) 91 | cat $output/script/vips.json \ 92 | | sed "s/TBFWID/$id/" \ 93 | > $output/vips.json 94 | rm -rf $output/script 95 | 96 | if [ $keep -eq 0 ];then 97 | # Removing logs 98 | rm -rf $output/logs 99 | fi 100 | 101 | -------------------------------------------------------------------------------- /src/main/bash/average-evaluations.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "algorithm,elementtype,tasks.valid,bcubed.precision,bcubed.recall,bcubed.f1,bcubed.f1.star" 4 | cat $@ \ 5 | | grep -v "^algorithm," \ 6 | | grep -v ",NA" \ 7 | | awk -F, '{ 8 | algorithm = $1 9 | elementtype = $2 10 | algorithms[algorithm] = 1 11 | elementtypes[elementtype] = 1 12 | precision = $3 13 | recall = $4 14 | f = $5 15 | 16 | key = algorithm" "elementtype 17 | counts[key] += 1 18 | precisions[key] += precision 19 | recalls[key] += recall 20 | fs[key] += f 21 | } END { 22 | for (algorithm in algorithms) { 23 | for (elementtype in elementtypes) { 24 | key = algorithm" "elementtype 25 | count = counts[key] 26 | precision = precisions[key] / count 27 | recall = recalls[key] / count 28 | f = fs[key] / count 29 | fstar = 2 * precision * recall / (precision + recall) 30 | printf "%s,%s,%d,%.2f,%.2f,%.2f,%.2f\n", algorithm, elementtype, count, precision, recall, f, fstar 31 | } 32 | } 33 | }' \ 34 | | sort 35 | -------------------------------------------------------------------------------- /src/main/bash/combine-evaluation-files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | algorithms="baseline cormier-t_l256-s_min45-unfitted cormier-t_l256-s_min45 cormier-t_l256-s_min90-unfitted cormier-t_l256-s_min90 cormier-t_l512-s_min45-unfitted cormier-t_l512-s_min45 cormier-t_l512-s_min90-unfitted cormier-t_l512-s_min90 heps meier-4096px-unfitted meier-4096px min-vote-4at1 min-vote-4at2 min-vote-4at3 min-vote-4at4 mmdetection-unfitted mmdetection vips-pdoc1 vips-pdoc2 vips-pdoc3 vips-pdoc4 vips-pdoc5 vips-pdoc6 vips-pdoc7 vips-pdoc8 vips-pdoc9 vips-pdoc10 vips-pdoc11" 4 | element_types="pixels edges-fine edges-coarse nodes chars" 5 | 6 | for directory in $@;do 7 | echo "$directory" 8 | output=$directory/evaluation.csv 9 | echo "algorithm,elementtype,bcubed.precision,bcubed.recall,bcubed.f1" > $output 10 | for algorithm in $algorithms;do 11 | for element_type in $element_types;do 12 | echo -n "$algorithm,$element_type," 13 | evaluation_file=$directory/evaluation-$algorithm-$element_type.txt 14 | evaluation_baseline_file=$directory/evaluation-baseline-$element_type.txt 15 | if [ -e $evaluation_file ];then 16 | cat $evaluation_file | grep "majority-vote" | cut -d, -f2- 17 | elif [ -e $evaluation_baseline_file ];then 18 | cat $evaluation_baseline_file | grep "majority-vote" | cut -d, -f2- 19 | else 20 | echo "NA,NA,NA" 21 | fi 22 | done 23 | done >> $output 24 | done 25 | --------------------------------------------------------------------------------